1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include <linux/debugfs.h> 7 8 #include <drm/drm_blend.h> 9 10 #include "i915_drv.h" 11 #include "i915_reg.h" 12 #include "i9xx_wm.h" 13 #include "intel_atomic.h" 14 #include "intel_atomic_plane.h" 15 #include "intel_bw.h" 16 #include "intel_cdclk.h" 17 #include "intel_crtc.h" 18 #include "intel_cursor_regs.h" 19 #include "intel_de.h" 20 #include "intel_display.h" 21 #include "intel_display_power.h" 22 #include "intel_display_types.h" 23 #include "intel_fb.h" 24 #include "intel_fixed.h" 25 #include "intel_pcode.h" 26 #include "intel_wm.h" 27 #include "skl_universal_plane_regs.h" 28 #include "skl_watermark.h" 29 #include "skl_watermark_regs.h" 30 31 /*It is expected that DSB can do posted writes to every register in 32 * the pipe and planes within 100us. For flip queue use case, the 33 * recommended DSB execution time is 100us + one SAGV block time. 34 */ 35 #define DSB_EXE_TIME 100 36 37 static void skl_sagv_disable(struct drm_i915_private *i915); 38 39 /* Stores plane specific WM parameters */ 40 struct skl_wm_params { 41 bool x_tiled, y_tiled; 42 bool rc_surface; 43 bool is_planar; 44 u32 width; 45 u8 cpp; 46 u32 plane_pixel_rate; 47 u32 y_min_scanlines; 48 u32 plane_bytes_per_line; 49 uint_fixed_16_16_t plane_blocks_per_line; 50 uint_fixed_16_16_t y_tile_minimum; 51 u32 linetime_us; 52 u32 dbuf_block_size; 53 }; 54 55 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915) 56 { 57 u8 enabled_slices = 0; 58 enum dbuf_slice slice; 59 60 for_each_dbuf_slice(i915, slice) { 61 if (intel_de_read(i915, DBUF_CTL_S(slice)) & DBUF_POWER_STATE) 62 enabled_slices |= BIT(slice); 63 } 64 65 return enabled_slices; 66 } 67 68 /* 69 * FIXME: We still don't have the proper code detect if we need to apply the WA, 70 * so assume we'll always need it in order to avoid underruns. 71 */ 72 static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915) 73 { 74 return DISPLAY_VER(i915) == 9; 75 } 76 77 bool 78 intel_has_sagv(struct drm_i915_private *i915) 79 { 80 struct intel_display *display = &i915->display; 81 82 return HAS_SAGV(display) && display->sagv.status != I915_SAGV_NOT_CONTROLLED; 83 } 84 85 static u32 86 intel_sagv_block_time(struct drm_i915_private *i915) 87 { 88 struct intel_display *display = &i915->display; 89 90 if (DISPLAY_VER(display) >= 14) { 91 u32 val; 92 93 val = intel_de_read(display, MTL_LATENCY_SAGV); 94 95 return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val); 96 } else if (DISPLAY_VER(display) >= 12) { 97 u32 val = 0; 98 int ret; 99 100 ret = snb_pcode_read(&i915->uncore, 101 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US, 102 &val, NULL); 103 if (ret) { 104 drm_dbg_kms(display->drm, "Couldn't read SAGV block time!\n"); 105 return 0; 106 } 107 108 return val; 109 } else if (DISPLAY_VER(display) == 11) { 110 return 10; 111 } else if (HAS_SAGV(display)) { 112 return 30; 113 } else { 114 return 0; 115 } 116 } 117 118 static void intel_sagv_init(struct drm_i915_private *i915) 119 { 120 struct intel_display *display = &i915->display; 121 122 if (!HAS_SAGV(display)) 123 display->sagv.status = I915_SAGV_NOT_CONTROLLED; 124 125 /* 126 * Probe to see if we have working SAGV control. 127 * For icl+ this was already determined by intel_bw_init_hw(). 128 */ 129 if (DISPLAY_VER(display) < 11) 130 skl_sagv_disable(i915); 131 132 drm_WARN_ON(display->drm, display->sagv.status == I915_SAGV_UNKNOWN); 133 134 display->sagv.block_time_us = intel_sagv_block_time(i915); 135 136 drm_dbg_kms(display->drm, "SAGV supported: %s, original SAGV block time: %u us\n", 137 str_yes_no(intel_has_sagv(i915)), display->sagv.block_time_us); 138 139 /* avoid overflow when adding with wm0 latency/etc. */ 140 if (drm_WARN(display->drm, display->sagv.block_time_us > U16_MAX, 141 "Excessive SAGV block time %u, ignoring\n", 142 display->sagv.block_time_us)) 143 display->sagv.block_time_us = 0; 144 145 if (!intel_has_sagv(i915)) 146 display->sagv.block_time_us = 0; 147 } 148 149 /* 150 * SAGV dynamically adjusts the system agent voltage and clock frequencies 151 * depending on power and performance requirements. The display engine access 152 * to system memory is blocked during the adjustment time. Because of the 153 * blocking time, having this enabled can cause full system hangs and/or pipe 154 * underruns if we don't meet all of the following requirements: 155 * 156 * - <= 1 pipe enabled 157 * - All planes can enable watermarks for latencies >= SAGV engine block time 158 * - We're not using an interlaced display configuration 159 */ 160 static void skl_sagv_enable(struct drm_i915_private *i915) 161 { 162 int ret; 163 164 if (!intel_has_sagv(i915)) 165 return; 166 167 if (i915->display.sagv.status == I915_SAGV_ENABLED) 168 return; 169 170 drm_dbg_kms(&i915->drm, "Enabling SAGV\n"); 171 ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL, 172 GEN9_SAGV_ENABLE); 173 174 /* We don't need to wait for SAGV when enabling */ 175 176 /* 177 * Some skl systems, pre-release machines in particular, 178 * don't actually have SAGV. 179 */ 180 if (IS_SKYLAKE(i915) && ret == -ENXIO) { 181 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n"); 182 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED; 183 return; 184 } else if (ret < 0) { 185 drm_err(&i915->drm, "Failed to enable SAGV\n"); 186 return; 187 } 188 189 i915->display.sagv.status = I915_SAGV_ENABLED; 190 } 191 192 static void skl_sagv_disable(struct drm_i915_private *i915) 193 { 194 int ret; 195 196 if (!intel_has_sagv(i915)) 197 return; 198 199 if (i915->display.sagv.status == I915_SAGV_DISABLED) 200 return; 201 202 drm_dbg_kms(&i915->drm, "Disabling SAGV\n"); 203 /* bspec says to keep retrying for at least 1 ms */ 204 ret = skl_pcode_request(&i915->uncore, GEN9_PCODE_SAGV_CONTROL, 205 GEN9_SAGV_DISABLE, 206 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED, 207 1); 208 /* 209 * Some skl systems, pre-release machines in particular, 210 * don't actually have SAGV. 211 */ 212 if (IS_SKYLAKE(i915) && ret == -ENXIO) { 213 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n"); 214 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED; 215 return; 216 } else if (ret < 0) { 217 drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret); 218 return; 219 } 220 221 i915->display.sagv.status = I915_SAGV_DISABLED; 222 } 223 224 static void skl_sagv_pre_plane_update(struct intel_atomic_state *state) 225 { 226 struct drm_i915_private *i915 = to_i915(state->base.dev); 227 const struct intel_bw_state *new_bw_state = 228 intel_atomic_get_new_bw_state(state); 229 230 if (!new_bw_state) 231 return; 232 233 if (!intel_can_enable_sagv(i915, new_bw_state)) 234 skl_sagv_disable(i915); 235 } 236 237 static void skl_sagv_post_plane_update(struct intel_atomic_state *state) 238 { 239 struct drm_i915_private *i915 = to_i915(state->base.dev); 240 const struct intel_bw_state *new_bw_state = 241 intel_atomic_get_new_bw_state(state); 242 243 if (!new_bw_state) 244 return; 245 246 if (intel_can_enable_sagv(i915, new_bw_state)) 247 skl_sagv_enable(i915); 248 } 249 250 static void icl_sagv_pre_plane_update(struct intel_atomic_state *state) 251 { 252 struct drm_i915_private *i915 = to_i915(state->base.dev); 253 const struct intel_bw_state *old_bw_state = 254 intel_atomic_get_old_bw_state(state); 255 const struct intel_bw_state *new_bw_state = 256 intel_atomic_get_new_bw_state(state); 257 u16 old_mask, new_mask; 258 259 if (!new_bw_state) 260 return; 261 262 old_mask = old_bw_state->qgv_points_mask; 263 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask; 264 265 if (old_mask == new_mask) 266 return; 267 268 WARN_ON(!new_bw_state->base.changed); 269 270 drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n", 271 old_mask, new_mask); 272 273 /* 274 * Restrict required qgv points before updating the configuration. 275 * According to BSpec we can't mask and unmask qgv points at the same 276 * time. Also masking should be done before updating the configuration 277 * and unmasking afterwards. 278 */ 279 icl_pcode_restrict_qgv_points(i915, new_mask); 280 } 281 282 static void icl_sagv_post_plane_update(struct intel_atomic_state *state) 283 { 284 struct drm_i915_private *i915 = to_i915(state->base.dev); 285 const struct intel_bw_state *old_bw_state = 286 intel_atomic_get_old_bw_state(state); 287 const struct intel_bw_state *new_bw_state = 288 intel_atomic_get_new_bw_state(state); 289 u16 old_mask, new_mask; 290 291 if (!new_bw_state) 292 return; 293 294 old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask; 295 new_mask = new_bw_state->qgv_points_mask; 296 297 if (old_mask == new_mask) 298 return; 299 300 WARN_ON(!new_bw_state->base.changed); 301 302 drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n", 303 old_mask, new_mask); 304 305 /* 306 * Allow required qgv points after updating the configuration. 307 * According to BSpec we can't mask and unmask qgv points at the same 308 * time. Also masking should be done before updating the configuration 309 * and unmasking afterwards. 310 */ 311 icl_pcode_restrict_qgv_points(i915, new_mask); 312 } 313 314 void intel_sagv_pre_plane_update(struct intel_atomic_state *state) 315 { 316 struct drm_i915_private *i915 = to_i915(state->base.dev); 317 318 /* 319 * Just return if we can't control SAGV or don't have it. 320 * This is different from situation when we have SAGV but just can't 321 * afford it due to DBuf limitation - in case if SAGV is completely 322 * disabled in a BIOS, we are not even allowed to send a PCode request, 323 * as it will throw an error. So have to check it here. 324 */ 325 if (!intel_has_sagv(i915)) 326 return; 327 328 if (DISPLAY_VER(i915) >= 11) 329 icl_sagv_pre_plane_update(state); 330 else 331 skl_sagv_pre_plane_update(state); 332 } 333 334 void intel_sagv_post_plane_update(struct intel_atomic_state *state) 335 { 336 struct drm_i915_private *i915 = to_i915(state->base.dev); 337 338 /* 339 * Just return if we can't control SAGV or don't have it. 340 * This is different from situation when we have SAGV but just can't 341 * afford it due to DBuf limitation - in case if SAGV is completely 342 * disabled in a BIOS, we are not even allowed to send a PCode request, 343 * as it will throw an error. So have to check it here. 344 */ 345 if (!intel_has_sagv(i915)) 346 return; 347 348 if (DISPLAY_VER(i915) >= 11) 349 icl_sagv_post_plane_update(state); 350 else 351 skl_sagv_post_plane_update(state); 352 } 353 354 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) 355 { 356 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 357 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 358 enum plane_id plane_id; 359 int max_level = INT_MAX; 360 361 if (!intel_has_sagv(i915)) 362 return false; 363 364 if (!crtc_state->hw.active) 365 return true; 366 367 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE) 368 return false; 369 370 for_each_plane_id_on_crtc(crtc, plane_id) { 371 const struct skl_plane_wm *wm = 372 &crtc_state->wm.skl.optimal.planes[plane_id]; 373 int level; 374 375 /* Skip this plane if it's not enabled */ 376 if (!wm->wm[0].enable) 377 continue; 378 379 /* Find the highest enabled wm level for this plane */ 380 for (level = i915->display.wm.num_levels - 1; 381 !wm->wm[level].enable; --level) 382 { } 383 384 /* Highest common enabled wm level for all planes */ 385 max_level = min(level, max_level); 386 } 387 388 /* No enabled planes? */ 389 if (max_level == INT_MAX) 390 return true; 391 392 for_each_plane_id_on_crtc(crtc, plane_id) { 393 const struct skl_plane_wm *wm = 394 &crtc_state->wm.skl.optimal.planes[plane_id]; 395 396 /* 397 * All enabled planes must have enabled a common wm level that 398 * can tolerate memory latencies higher than sagv_block_time_us 399 */ 400 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv) 401 return false; 402 } 403 404 return true; 405 } 406 407 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) 408 { 409 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 410 enum plane_id plane_id; 411 412 if (!crtc_state->hw.active) 413 return true; 414 415 for_each_plane_id_on_crtc(crtc, plane_id) { 416 const struct skl_plane_wm *wm = 417 &crtc_state->wm.skl.optimal.planes[plane_id]; 418 419 if (wm->wm[0].enable && !wm->sagv.wm0.enable) 420 return false; 421 } 422 423 return true; 424 } 425 426 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) 427 { 428 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 429 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 430 431 if (!i915->display.params.enable_sagv) 432 return false; 433 434 if (DISPLAY_VER(i915) >= 12) 435 return tgl_crtc_can_enable_sagv(crtc_state); 436 else 437 return skl_crtc_can_enable_sagv(crtc_state); 438 } 439 440 bool intel_can_enable_sagv(struct drm_i915_private *i915, 441 const struct intel_bw_state *bw_state) 442 { 443 if (DISPLAY_VER(i915) < 11 && 444 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes)) 445 return false; 446 447 return bw_state->pipe_sagv_reject == 0; 448 } 449 450 static int intel_compute_sagv_mask(struct intel_atomic_state *state) 451 { 452 struct intel_display *display = to_intel_display(state); 453 struct drm_i915_private *i915 = to_i915(state->base.dev); 454 int ret; 455 struct intel_crtc *crtc; 456 struct intel_crtc_state *new_crtc_state; 457 struct intel_bw_state *new_bw_state = NULL; 458 const struct intel_bw_state *old_bw_state = NULL; 459 int i; 460 461 for_each_new_intel_crtc_in_state(state, crtc, 462 new_crtc_state, i) { 463 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal; 464 465 new_bw_state = intel_atomic_get_bw_state(state); 466 if (IS_ERR(new_bw_state)) 467 return PTR_ERR(new_bw_state); 468 469 old_bw_state = intel_atomic_get_old_bw_state(state); 470 471 /* 472 * We store use_sagv_wm in the crtc state rather than relying on 473 * that bw state since we have no convenient way to get at the 474 * latter from the plane commit hooks (especially in the legacy 475 * cursor case). 476 * 477 * drm_atomic_check_only() gets upset if we pull more crtcs 478 * into the state, so we have to calculate this based on the 479 * individual intel_crtc_can_enable_sagv() rather than 480 * the overall intel_can_enable_sagv(). Otherwise the 481 * crtcs not included in the commit would not switch to the 482 * SAGV watermarks when we are about to enable SAGV, and that 483 * would lead to underruns. This does mean extra power draw 484 * when only a subset of the crtcs are blocking SAGV as the 485 * other crtcs can't be allowed to use the more optimal 486 * normal (ie. non-SAGV) watermarks. 487 */ 488 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(display) && 489 DISPLAY_VER(i915) >= 12 && 490 intel_crtc_can_enable_sagv(new_crtc_state); 491 492 if (intel_crtc_can_enable_sagv(new_crtc_state)) 493 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe); 494 else 495 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe); 496 } 497 498 if (!new_bw_state) 499 return 0; 500 501 new_bw_state->active_pipes = 502 intel_calc_active_pipes(state, old_bw_state->active_pipes); 503 504 if (new_bw_state->active_pipes != old_bw_state->active_pipes) { 505 ret = intel_atomic_lock_global_state(&new_bw_state->base); 506 if (ret) 507 return ret; 508 } 509 510 if (intel_can_enable_sagv(i915, new_bw_state) != 511 intel_can_enable_sagv(i915, old_bw_state)) { 512 ret = intel_atomic_serialize_global_state(&new_bw_state->base); 513 if (ret) 514 return ret; 515 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) { 516 ret = intel_atomic_lock_global_state(&new_bw_state->base); 517 if (ret) 518 return ret; 519 } 520 521 return 0; 522 } 523 524 static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry, 525 u16 start, u16 end) 526 { 527 entry->start = start; 528 entry->end = end; 529 530 return end; 531 } 532 533 static int intel_dbuf_slice_size(struct drm_i915_private *i915) 534 { 535 return DISPLAY_INFO(i915)->dbuf.size / 536 hweight8(DISPLAY_INFO(i915)->dbuf.slice_mask); 537 } 538 539 static void 540 skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask, 541 struct skl_ddb_entry *ddb) 542 { 543 int slice_size = intel_dbuf_slice_size(i915); 544 545 if (!slice_mask) { 546 ddb->start = 0; 547 ddb->end = 0; 548 return; 549 } 550 551 ddb->start = (ffs(slice_mask) - 1) * slice_size; 552 ddb->end = fls(slice_mask) * slice_size; 553 554 WARN_ON(ddb->start >= ddb->end); 555 WARN_ON(ddb->end > DISPLAY_INFO(i915)->dbuf.size); 556 } 557 558 static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask) 559 { 560 struct skl_ddb_entry ddb; 561 562 if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2))) 563 slice_mask = BIT(DBUF_S1); 564 else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4))) 565 slice_mask = BIT(DBUF_S3); 566 567 skl_ddb_entry_for_slices(i915, slice_mask, &ddb); 568 569 return ddb.start; 570 } 571 572 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915, 573 const struct skl_ddb_entry *entry) 574 { 575 int slice_size = intel_dbuf_slice_size(i915); 576 enum dbuf_slice start_slice, end_slice; 577 u8 slice_mask = 0; 578 579 if (!skl_ddb_entry_size(entry)) 580 return 0; 581 582 start_slice = entry->start / slice_size; 583 end_slice = (entry->end - 1) / slice_size; 584 585 /* 586 * Per plane DDB entry can in a really worst case be on multiple slices 587 * but single entry is anyway contigious. 588 */ 589 while (start_slice <= end_slice) { 590 slice_mask |= BIT(start_slice); 591 start_slice++; 592 } 593 594 return slice_mask; 595 } 596 597 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state) 598 { 599 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode; 600 int hdisplay, vdisplay; 601 602 if (!crtc_state->hw.active) 603 return 0; 604 605 /* 606 * Watermark/ddb requirement highly depends upon width of the 607 * framebuffer, So instead of allocating DDB equally among pipes 608 * distribute DDB based on resolution/width of the display. 609 */ 610 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay); 611 612 return hdisplay; 613 } 614 615 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state, 616 enum pipe for_pipe, 617 unsigned int *weight_start, 618 unsigned int *weight_end, 619 unsigned int *weight_total) 620 { 621 struct drm_i915_private *i915 = 622 to_i915(dbuf_state->base.state->base.dev); 623 enum pipe pipe; 624 625 *weight_start = 0; 626 *weight_end = 0; 627 *weight_total = 0; 628 629 for_each_pipe(i915, pipe) { 630 int weight = dbuf_state->weight[pipe]; 631 632 /* 633 * Do not account pipes using other slice sets 634 * luckily as of current BSpec slice sets do not partially 635 * intersect(pipes share either same one slice or same slice set 636 * i.e no partial intersection), so it is enough to check for 637 * equality for now. 638 */ 639 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe]) 640 continue; 641 642 *weight_total += weight; 643 if (pipe < for_pipe) { 644 *weight_start += weight; 645 *weight_end += weight; 646 } else if (pipe == for_pipe) { 647 *weight_end += weight; 648 } 649 } 650 } 651 652 static int 653 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc) 654 { 655 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 656 unsigned int weight_total, weight_start, weight_end; 657 const struct intel_dbuf_state *old_dbuf_state = 658 intel_atomic_get_old_dbuf_state(state); 659 struct intel_dbuf_state *new_dbuf_state = 660 intel_atomic_get_new_dbuf_state(state); 661 struct intel_crtc_state *crtc_state; 662 struct skl_ddb_entry ddb_slices; 663 enum pipe pipe = crtc->pipe; 664 unsigned int mbus_offset = 0; 665 u32 ddb_range_size; 666 u32 dbuf_slice_mask; 667 u32 start, end; 668 int ret; 669 670 if (new_dbuf_state->weight[pipe] == 0) { 671 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0); 672 goto out; 673 } 674 675 dbuf_slice_mask = new_dbuf_state->slices[pipe]; 676 677 skl_ddb_entry_for_slices(i915, dbuf_slice_mask, &ddb_slices); 678 mbus_offset = mbus_ddb_offset(i915, dbuf_slice_mask); 679 ddb_range_size = skl_ddb_entry_size(&ddb_slices); 680 681 intel_crtc_dbuf_weights(new_dbuf_state, pipe, 682 &weight_start, &weight_end, &weight_total); 683 684 start = ddb_range_size * weight_start / weight_total; 685 end = ddb_range_size * weight_end / weight_total; 686 687 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 688 ddb_slices.start - mbus_offset + start, 689 ddb_slices.start - mbus_offset + end); 690 691 out: 692 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] && 693 skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe], 694 &new_dbuf_state->ddb[pipe])) 695 return 0; 696 697 ret = intel_atomic_lock_global_state(&new_dbuf_state->base); 698 if (ret) 699 return ret; 700 701 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc); 702 if (IS_ERR(crtc_state)) 703 return PTR_ERR(crtc_state); 704 705 /* 706 * Used for checking overlaps, so we need absolute 707 * offsets instead of MBUS relative offsets. 708 */ 709 crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start; 710 crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end; 711 712 drm_dbg_kms(&i915->drm, 713 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n", 714 crtc->base.base.id, crtc->base.name, 715 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe], 716 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end, 717 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end, 718 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes); 719 720 return 0; 721 } 722 723 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state, 724 int width, const struct drm_format_info *format, 725 u64 modifier, unsigned int rotation, 726 u32 plane_pixel_rate, struct skl_wm_params *wp, 727 int color_plane, unsigned int pan_x); 728 729 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, 730 struct intel_plane *plane, 731 int level, 732 unsigned int latency, 733 const struct skl_wm_params *wp, 734 const struct skl_wm_level *result_prev, 735 struct skl_wm_level *result /* out */); 736 737 static unsigned int skl_wm_latency(struct drm_i915_private *i915, int level, 738 const struct skl_wm_params *wp) 739 { 740 unsigned int latency = i915->display.wm.skl_latency[level]; 741 742 if (latency == 0) 743 return 0; 744 745 /* 746 * WaIncreaseLatencyIPCEnabled: kbl,cfl 747 * Display WA #1141: kbl,cfl 748 */ 749 if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) && 750 skl_watermark_ipc_enabled(i915)) 751 latency += 4; 752 753 if (skl_needs_memory_bw_wa(i915) && wp && wp->x_tiled) 754 latency += 15; 755 756 return latency; 757 } 758 759 static unsigned int 760 skl_cursor_allocation(const struct intel_crtc_state *crtc_state, 761 int num_active) 762 { 763 struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor); 764 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); 765 struct skl_wm_level wm = {}; 766 int ret, min_ddb_alloc = 0; 767 struct skl_wm_params wp; 768 int level; 769 770 ret = skl_compute_wm_params(crtc_state, 256, 771 drm_format_info(DRM_FORMAT_ARGB8888), 772 DRM_FORMAT_MOD_LINEAR, 773 DRM_MODE_ROTATE_0, 774 crtc_state->pixel_rate, &wp, 0, 0); 775 drm_WARN_ON(&i915->drm, ret); 776 777 for (level = 0; level < i915->display.wm.num_levels; level++) { 778 unsigned int latency = skl_wm_latency(i915, level, &wp); 779 780 skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm); 781 if (wm.min_ddb_alloc == U16_MAX) 782 break; 783 784 min_ddb_alloc = wm.min_ddb_alloc; 785 } 786 787 return max(num_active == 1 ? 32 : 8, min_ddb_alloc); 788 } 789 790 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg) 791 { 792 skl_ddb_entry_init(entry, 793 REG_FIELD_GET(PLANE_BUF_START_MASK, reg), 794 REG_FIELD_GET(PLANE_BUF_END_MASK, reg)); 795 if (entry->end) 796 entry->end++; 797 } 798 799 static void 800 skl_ddb_get_hw_plane_state(struct drm_i915_private *i915, 801 const enum pipe pipe, 802 const enum plane_id plane_id, 803 struct skl_ddb_entry *ddb, 804 struct skl_ddb_entry *ddb_y, 805 u16 *min_ddb, u16 *interim_ddb) 806 { 807 struct intel_display *display = &i915->display; 808 u32 val; 809 810 /* Cursor doesn't support NV12/planar, so no extra calculation needed */ 811 if (plane_id == PLANE_CURSOR) { 812 val = intel_de_read(display, CUR_BUF_CFG(pipe)); 813 skl_ddb_entry_init_from_hw(ddb, val); 814 return; 815 } 816 817 val = intel_de_read(display, PLANE_BUF_CFG(pipe, plane_id)); 818 skl_ddb_entry_init_from_hw(ddb, val); 819 820 if (DISPLAY_VER(display) >= 30) { 821 val = intel_de_read(display, PLANE_MIN_BUF_CFG(pipe, plane_id)); 822 823 *min_ddb = REG_FIELD_GET(PLANE_MIN_DBUF_BLOCKS_MASK, val); 824 *interim_ddb = REG_FIELD_GET(PLANE_INTERIM_DBUF_BLOCKS_MASK, val); 825 } 826 827 if (DISPLAY_VER(display) >= 11) 828 return; 829 830 val = intel_de_read(display, PLANE_NV12_BUF_CFG(pipe, plane_id)); 831 skl_ddb_entry_init_from_hw(ddb_y, val); 832 } 833 834 static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc, 835 struct skl_ddb_entry *ddb, 836 struct skl_ddb_entry *ddb_y, 837 u16 *min_ddb, u16 *interim_ddb) 838 { 839 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 840 enum intel_display_power_domain power_domain; 841 enum pipe pipe = crtc->pipe; 842 intel_wakeref_t wakeref; 843 enum plane_id plane_id; 844 845 power_domain = POWER_DOMAIN_PIPE(pipe); 846 wakeref = intel_display_power_get_if_enabled(i915, power_domain); 847 if (!wakeref) 848 return; 849 850 for_each_plane_id_on_crtc(crtc, plane_id) 851 skl_ddb_get_hw_plane_state(i915, pipe, 852 plane_id, 853 &ddb[plane_id], 854 &ddb_y[plane_id], 855 &min_ddb[plane_id], 856 &interim_ddb[plane_id]); 857 858 intel_display_power_put(i915, power_domain, wakeref); 859 } 860 861 struct dbuf_slice_conf_entry { 862 u8 active_pipes; 863 u8 dbuf_mask[I915_MAX_PIPES]; 864 bool join_mbus; 865 }; 866 867 /* 868 * Table taken from Bspec 12716 869 * Pipes do have some preferred DBuf slice affinity, 870 * plus there are some hardcoded requirements on how 871 * those should be distributed for multipipe scenarios. 872 * For more DBuf slices algorithm can get even more messy 873 * and less readable, so decided to use a table almost 874 * as is from BSpec itself - that way it is at least easier 875 * to compare, change and check. 876 */ 877 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] = 878 /* Autogenerated with igt/tools/intel_dbuf_map tool: */ 879 { 880 { 881 .active_pipes = BIT(PIPE_A), 882 .dbuf_mask = { 883 [PIPE_A] = BIT(DBUF_S1), 884 }, 885 }, 886 { 887 .active_pipes = BIT(PIPE_B), 888 .dbuf_mask = { 889 [PIPE_B] = BIT(DBUF_S1), 890 }, 891 }, 892 { 893 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), 894 .dbuf_mask = { 895 [PIPE_A] = BIT(DBUF_S1), 896 [PIPE_B] = BIT(DBUF_S2), 897 }, 898 }, 899 { 900 .active_pipes = BIT(PIPE_C), 901 .dbuf_mask = { 902 [PIPE_C] = BIT(DBUF_S2), 903 }, 904 }, 905 { 906 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), 907 .dbuf_mask = { 908 [PIPE_A] = BIT(DBUF_S1), 909 [PIPE_C] = BIT(DBUF_S2), 910 }, 911 }, 912 { 913 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), 914 .dbuf_mask = { 915 [PIPE_B] = BIT(DBUF_S1), 916 [PIPE_C] = BIT(DBUF_S2), 917 }, 918 }, 919 { 920 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), 921 .dbuf_mask = { 922 [PIPE_A] = BIT(DBUF_S1), 923 [PIPE_B] = BIT(DBUF_S1), 924 [PIPE_C] = BIT(DBUF_S2), 925 }, 926 }, 927 {} 928 }; 929 930 /* 931 * Table taken from Bspec 49255 932 * Pipes do have some preferred DBuf slice affinity, 933 * plus there are some hardcoded requirements on how 934 * those should be distributed for multipipe scenarios. 935 * For more DBuf slices algorithm can get even more messy 936 * and less readable, so decided to use a table almost 937 * as is from BSpec itself - that way it is at least easier 938 * to compare, change and check. 939 */ 940 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] = 941 /* Autogenerated with igt/tools/intel_dbuf_map tool: */ 942 { 943 { 944 .active_pipes = BIT(PIPE_A), 945 .dbuf_mask = { 946 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 947 }, 948 }, 949 { 950 .active_pipes = BIT(PIPE_B), 951 .dbuf_mask = { 952 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), 953 }, 954 }, 955 { 956 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), 957 .dbuf_mask = { 958 [PIPE_A] = BIT(DBUF_S2), 959 [PIPE_B] = BIT(DBUF_S1), 960 }, 961 }, 962 { 963 .active_pipes = BIT(PIPE_C), 964 .dbuf_mask = { 965 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1), 966 }, 967 }, 968 { 969 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), 970 .dbuf_mask = { 971 [PIPE_A] = BIT(DBUF_S1), 972 [PIPE_C] = BIT(DBUF_S2), 973 }, 974 }, 975 { 976 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), 977 .dbuf_mask = { 978 [PIPE_B] = BIT(DBUF_S1), 979 [PIPE_C] = BIT(DBUF_S2), 980 }, 981 }, 982 { 983 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), 984 .dbuf_mask = { 985 [PIPE_A] = BIT(DBUF_S1), 986 [PIPE_B] = BIT(DBUF_S1), 987 [PIPE_C] = BIT(DBUF_S2), 988 }, 989 }, 990 { 991 .active_pipes = BIT(PIPE_D), 992 .dbuf_mask = { 993 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1), 994 }, 995 }, 996 { 997 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D), 998 .dbuf_mask = { 999 [PIPE_A] = BIT(DBUF_S1), 1000 [PIPE_D] = BIT(DBUF_S2), 1001 }, 1002 }, 1003 { 1004 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D), 1005 .dbuf_mask = { 1006 [PIPE_B] = BIT(DBUF_S1), 1007 [PIPE_D] = BIT(DBUF_S2), 1008 }, 1009 }, 1010 { 1011 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D), 1012 .dbuf_mask = { 1013 [PIPE_A] = BIT(DBUF_S1), 1014 [PIPE_B] = BIT(DBUF_S1), 1015 [PIPE_D] = BIT(DBUF_S2), 1016 }, 1017 }, 1018 { 1019 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D), 1020 .dbuf_mask = { 1021 [PIPE_C] = BIT(DBUF_S1), 1022 [PIPE_D] = BIT(DBUF_S2), 1023 }, 1024 }, 1025 { 1026 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D), 1027 .dbuf_mask = { 1028 [PIPE_A] = BIT(DBUF_S1), 1029 [PIPE_C] = BIT(DBUF_S2), 1030 [PIPE_D] = BIT(DBUF_S2), 1031 }, 1032 }, 1033 { 1034 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), 1035 .dbuf_mask = { 1036 [PIPE_B] = BIT(DBUF_S1), 1037 [PIPE_C] = BIT(DBUF_S2), 1038 [PIPE_D] = BIT(DBUF_S2), 1039 }, 1040 }, 1041 { 1042 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), 1043 .dbuf_mask = { 1044 [PIPE_A] = BIT(DBUF_S1), 1045 [PIPE_B] = BIT(DBUF_S1), 1046 [PIPE_C] = BIT(DBUF_S2), 1047 [PIPE_D] = BIT(DBUF_S2), 1048 }, 1049 }, 1050 {} 1051 }; 1052 1053 static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = { 1054 { 1055 .active_pipes = BIT(PIPE_A), 1056 .dbuf_mask = { 1057 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 1058 }, 1059 }, 1060 { 1061 .active_pipes = BIT(PIPE_B), 1062 .dbuf_mask = { 1063 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), 1064 }, 1065 }, 1066 { 1067 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), 1068 .dbuf_mask = { 1069 [PIPE_A] = BIT(DBUF_S1), 1070 [PIPE_B] = BIT(DBUF_S2), 1071 }, 1072 }, 1073 { 1074 .active_pipes = BIT(PIPE_C), 1075 .dbuf_mask = { 1076 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), 1077 }, 1078 }, 1079 { 1080 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), 1081 .dbuf_mask = { 1082 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 1083 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), 1084 }, 1085 }, 1086 { 1087 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), 1088 .dbuf_mask = { 1089 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), 1090 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), 1091 }, 1092 }, 1093 { 1094 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), 1095 .dbuf_mask = { 1096 [PIPE_A] = BIT(DBUF_S1), 1097 [PIPE_B] = BIT(DBUF_S2), 1098 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), 1099 }, 1100 }, 1101 { 1102 .active_pipes = BIT(PIPE_D), 1103 .dbuf_mask = { 1104 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), 1105 }, 1106 }, 1107 { 1108 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D), 1109 .dbuf_mask = { 1110 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 1111 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), 1112 }, 1113 }, 1114 { 1115 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D), 1116 .dbuf_mask = { 1117 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), 1118 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), 1119 }, 1120 }, 1121 { 1122 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D), 1123 .dbuf_mask = { 1124 [PIPE_A] = BIT(DBUF_S1), 1125 [PIPE_B] = BIT(DBUF_S2), 1126 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), 1127 }, 1128 }, 1129 { 1130 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D), 1131 .dbuf_mask = { 1132 [PIPE_C] = BIT(DBUF_S3), 1133 [PIPE_D] = BIT(DBUF_S4), 1134 }, 1135 }, 1136 { 1137 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D), 1138 .dbuf_mask = { 1139 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 1140 [PIPE_C] = BIT(DBUF_S3), 1141 [PIPE_D] = BIT(DBUF_S4), 1142 }, 1143 }, 1144 { 1145 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), 1146 .dbuf_mask = { 1147 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), 1148 [PIPE_C] = BIT(DBUF_S3), 1149 [PIPE_D] = BIT(DBUF_S4), 1150 }, 1151 }, 1152 { 1153 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), 1154 .dbuf_mask = { 1155 [PIPE_A] = BIT(DBUF_S1), 1156 [PIPE_B] = BIT(DBUF_S2), 1157 [PIPE_C] = BIT(DBUF_S3), 1158 [PIPE_D] = BIT(DBUF_S4), 1159 }, 1160 }, 1161 {} 1162 }; 1163 1164 static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = { 1165 /* 1166 * Keep the join_mbus cases first so check_mbus_joined() 1167 * will prefer them over the !join_mbus cases. 1168 */ 1169 { 1170 .active_pipes = BIT(PIPE_A), 1171 .dbuf_mask = { 1172 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4), 1173 }, 1174 .join_mbus = true, 1175 }, 1176 { 1177 .active_pipes = BIT(PIPE_B), 1178 .dbuf_mask = { 1179 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4), 1180 }, 1181 .join_mbus = true, 1182 }, 1183 { 1184 .active_pipes = BIT(PIPE_A), 1185 .dbuf_mask = { 1186 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 1187 }, 1188 .join_mbus = false, 1189 }, 1190 { 1191 .active_pipes = BIT(PIPE_B), 1192 .dbuf_mask = { 1193 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), 1194 }, 1195 .join_mbus = false, 1196 }, 1197 { 1198 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), 1199 .dbuf_mask = { 1200 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 1201 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), 1202 }, 1203 }, 1204 { 1205 .active_pipes = BIT(PIPE_C), 1206 .dbuf_mask = { 1207 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), 1208 }, 1209 }, 1210 { 1211 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), 1212 .dbuf_mask = { 1213 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 1214 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), 1215 }, 1216 }, 1217 { 1218 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), 1219 .dbuf_mask = { 1220 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), 1221 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), 1222 }, 1223 }, 1224 { 1225 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), 1226 .dbuf_mask = { 1227 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 1228 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), 1229 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), 1230 }, 1231 }, 1232 { 1233 .active_pipes = BIT(PIPE_D), 1234 .dbuf_mask = { 1235 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), 1236 }, 1237 }, 1238 { 1239 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D), 1240 .dbuf_mask = { 1241 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 1242 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), 1243 }, 1244 }, 1245 { 1246 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D), 1247 .dbuf_mask = { 1248 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), 1249 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), 1250 }, 1251 }, 1252 { 1253 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D), 1254 .dbuf_mask = { 1255 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 1256 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), 1257 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), 1258 }, 1259 }, 1260 { 1261 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D), 1262 .dbuf_mask = { 1263 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), 1264 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), 1265 }, 1266 }, 1267 { 1268 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D), 1269 .dbuf_mask = { 1270 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 1271 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), 1272 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), 1273 }, 1274 }, 1275 { 1276 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), 1277 .dbuf_mask = { 1278 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), 1279 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), 1280 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), 1281 }, 1282 }, 1283 { 1284 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), 1285 .dbuf_mask = { 1286 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), 1287 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), 1288 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), 1289 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), 1290 }, 1291 }, 1292 {} 1293 1294 }; 1295 1296 static bool check_mbus_joined(u8 active_pipes, 1297 const struct dbuf_slice_conf_entry *dbuf_slices) 1298 { 1299 int i; 1300 1301 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) { 1302 if (dbuf_slices[i].active_pipes == active_pipes) 1303 return dbuf_slices[i].join_mbus; 1304 } 1305 return false; 1306 } 1307 1308 static bool adlp_check_mbus_joined(u8 active_pipes) 1309 { 1310 return check_mbus_joined(active_pipes, adlp_allowed_dbufs); 1311 } 1312 1313 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus, 1314 const struct dbuf_slice_conf_entry *dbuf_slices) 1315 { 1316 int i; 1317 1318 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) { 1319 if (dbuf_slices[i].active_pipes == active_pipes && 1320 dbuf_slices[i].join_mbus == join_mbus) 1321 return dbuf_slices[i].dbuf_mask[pipe]; 1322 } 1323 return 0; 1324 } 1325 1326 /* 1327 * This function finds an entry with same enabled pipe configuration and 1328 * returns correspondent DBuf slice mask as stated in BSpec for particular 1329 * platform. 1330 */ 1331 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus) 1332 { 1333 /* 1334 * FIXME: For ICL this is still a bit unclear as prev BSpec revision 1335 * required calculating "pipe ratio" in order to determine 1336 * if one or two slices can be used for single pipe configurations 1337 * as additional constraint to the existing table. 1338 * However based on recent info, it should be not "pipe ratio" 1339 * but rather ratio between pixel_rate and cdclk with additional 1340 * constants, so for now we are using only table until this is 1341 * clarified. Also this is the reason why crtc_state param is 1342 * still here - we will need it once those additional constraints 1343 * pop up. 1344 */ 1345 return compute_dbuf_slices(pipe, active_pipes, join_mbus, 1346 icl_allowed_dbufs); 1347 } 1348 1349 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus) 1350 { 1351 return compute_dbuf_slices(pipe, active_pipes, join_mbus, 1352 tgl_allowed_dbufs); 1353 } 1354 1355 static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus) 1356 { 1357 return compute_dbuf_slices(pipe, active_pipes, join_mbus, 1358 adlp_allowed_dbufs); 1359 } 1360 1361 static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus) 1362 { 1363 return compute_dbuf_slices(pipe, active_pipes, join_mbus, 1364 dg2_allowed_dbufs); 1365 } 1366 1367 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus) 1368 { 1369 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 1370 enum pipe pipe = crtc->pipe; 1371 1372 if (IS_DG2(i915)) 1373 return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus); 1374 else if (DISPLAY_VER(i915) >= 13) 1375 return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus); 1376 else if (DISPLAY_VER(i915) == 12) 1377 return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus); 1378 else if (DISPLAY_VER(i915) == 11) 1379 return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus); 1380 /* 1381 * For anything else just return one slice yet. 1382 * Should be extended for other platforms. 1383 */ 1384 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0; 1385 } 1386 1387 static bool 1388 use_minimal_wm0_only(const struct intel_crtc_state *crtc_state, 1389 struct intel_plane *plane) 1390 { 1391 struct intel_display *display = to_intel_display(plane); 1392 1393 /* Xe3+ are auto minimum DDB capble. So don't force minimal wm0 */ 1394 return IS_DISPLAY_VER(display, 13, 20) && 1395 crtc_state->uapi.async_flip && 1396 plane->async_flip; 1397 } 1398 1399 unsigned int 1400 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state, 1401 struct intel_plane *plane, int width, int height, 1402 int cpp) 1403 { 1404 /* 1405 * We calculate extra ddb based on ratio plane rate/total data rate 1406 * in case, in some cases we should not allocate extra ddb for the plane, 1407 * so do not count its data rate, if this is the case. 1408 */ 1409 if (use_minimal_wm0_only(crtc_state, plane)) 1410 return 0; 1411 1412 return width * height * cpp; 1413 } 1414 1415 static u64 1416 skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state) 1417 { 1418 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 1419 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 1420 enum plane_id plane_id; 1421 u64 data_rate = 0; 1422 1423 for_each_plane_id_on_crtc(crtc, plane_id) { 1424 if (plane_id == PLANE_CURSOR) 1425 continue; 1426 1427 data_rate += crtc_state->rel_data_rate[plane_id]; 1428 1429 if (DISPLAY_VER(i915) < 11) 1430 data_rate += crtc_state->rel_data_rate_y[plane_id]; 1431 } 1432 1433 return data_rate; 1434 } 1435 1436 const struct skl_wm_level * 1437 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm, 1438 enum plane_id plane_id, 1439 int level) 1440 { 1441 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id]; 1442 1443 if (level == 0 && pipe_wm->use_sagv_wm) 1444 return &wm->sagv.wm0; 1445 1446 return &wm->wm[level]; 1447 } 1448 1449 const struct skl_wm_level * 1450 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm, 1451 enum plane_id plane_id) 1452 { 1453 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id]; 1454 1455 if (pipe_wm->use_sagv_wm) 1456 return &wm->sagv.trans_wm; 1457 1458 return &wm->trans_wm; 1459 } 1460 1461 /* 1462 * We only disable the watermarks for each plane if 1463 * they exceed the ddb allocation of said plane. This 1464 * is done so that we don't end up touching cursor 1465 * watermarks needlessly when some other plane reduces 1466 * our max possible watermark level. 1467 * 1468 * Bspec has this to say about the PLANE_WM enable bit: 1469 * "All the watermarks at this level for all enabled 1470 * planes must be enabled before the level will be used." 1471 * So this is actually safe to do. 1472 */ 1473 static void 1474 skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb) 1475 { 1476 if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) 1477 memset(wm, 0, sizeof(*wm)); 1478 } 1479 1480 static void 1481 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm, 1482 const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb) 1483 { 1484 if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) || 1485 uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) { 1486 memset(wm, 0, sizeof(*wm)); 1487 memset(uv_wm, 0, sizeof(*uv_wm)); 1488 } 1489 } 1490 1491 static bool skl_need_wm_copy_wa(struct drm_i915_private *i915, int level, 1492 const struct skl_plane_wm *wm) 1493 { 1494 /* 1495 * Wa_1408961008:icl, ehl 1496 * Wa_14012656716:tgl, adl 1497 * Wa_14017887344:icl 1498 * Wa_14017868169:adl, tgl 1499 * Due to some power saving optimizations, different subsystems 1500 * like PSR, might still use even disabled wm level registers, 1501 * for "reference", so lets keep at least the values sane. 1502 * Considering amount of WA requiring us to do similar things, was 1503 * decided to simply do it for all of the platforms, as those wm 1504 * levels are disabled, this isn't going to do harm anyway. 1505 */ 1506 return level > 0 && !wm->wm[level].enable; 1507 } 1508 1509 struct skl_plane_ddb_iter { 1510 u64 data_rate; 1511 u16 start, size; 1512 }; 1513 1514 static void 1515 skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter, 1516 struct skl_ddb_entry *ddb, 1517 const struct skl_wm_level *wm, 1518 u64 data_rate) 1519 { 1520 u16 size, extra = 0; 1521 1522 if (data_rate) { 1523 extra = min_t(u16, iter->size, 1524 DIV64_U64_ROUND_UP(iter->size * data_rate, 1525 iter->data_rate)); 1526 iter->size -= extra; 1527 iter->data_rate -= data_rate; 1528 } 1529 1530 /* 1531 * Keep ddb entry of all disabled planes explicitly zeroed 1532 * to avoid skl_ddb_add_affected_planes() adding them to 1533 * the state when other planes change their allocations. 1534 */ 1535 size = wm->min_ddb_alloc + extra; 1536 if (size) 1537 iter->start = skl_ddb_entry_init(ddb, iter->start, 1538 iter->start + size); 1539 } 1540 1541 static int 1542 skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state, 1543 struct intel_crtc *crtc) 1544 { 1545 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 1546 struct intel_crtc_state *crtc_state = 1547 intel_atomic_get_new_crtc_state(state, crtc); 1548 const struct intel_dbuf_state *dbuf_state = 1549 intel_atomic_get_new_dbuf_state(state); 1550 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe]; 1551 struct intel_display *display = to_intel_display(state); 1552 int num_active = hweight8(dbuf_state->active_pipes); 1553 struct skl_plane_ddb_iter iter; 1554 enum plane_id plane_id; 1555 u16 cursor_size; 1556 u32 blocks; 1557 int level; 1558 1559 /* Clear the partitioning for disabled planes. */ 1560 memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb)); 1561 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y)); 1562 memset(crtc_state->wm.skl.plane_min_ddb, 0, 1563 sizeof(crtc_state->wm.skl.plane_min_ddb)); 1564 memset(crtc_state->wm.skl.plane_interim_ddb, 0, 1565 sizeof(crtc_state->wm.skl.plane_interim_ddb)); 1566 1567 if (!crtc_state->hw.active) 1568 return 0; 1569 1570 iter.start = alloc->start; 1571 iter.size = skl_ddb_entry_size(alloc); 1572 if (iter.size == 0) 1573 return 0; 1574 1575 /* Allocate fixed number of blocks for cursor. */ 1576 cursor_size = skl_cursor_allocation(crtc_state, num_active); 1577 iter.size -= cursor_size; 1578 skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR], 1579 alloc->end - cursor_size, alloc->end); 1580 1581 iter.data_rate = skl_total_relative_data_rate(crtc_state); 1582 1583 /* 1584 * Find the highest watermark level for which we can satisfy the block 1585 * requirement of active planes. 1586 */ 1587 for (level = i915->display.wm.num_levels - 1; level >= 0; level--) { 1588 blocks = 0; 1589 for_each_plane_id_on_crtc(crtc, plane_id) { 1590 const struct skl_plane_wm *wm = 1591 &crtc_state->wm.skl.optimal.planes[plane_id]; 1592 1593 if (plane_id == PLANE_CURSOR) { 1594 const struct skl_ddb_entry *ddb = 1595 &crtc_state->wm.skl.plane_ddb[plane_id]; 1596 1597 if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) { 1598 drm_WARN_ON(&i915->drm, 1599 wm->wm[level].min_ddb_alloc != U16_MAX); 1600 blocks = U32_MAX; 1601 break; 1602 } 1603 continue; 1604 } 1605 1606 blocks += wm->wm[level].min_ddb_alloc; 1607 blocks += wm->uv_wm[level].min_ddb_alloc; 1608 } 1609 1610 if (blocks <= iter.size) { 1611 iter.size -= blocks; 1612 break; 1613 } 1614 } 1615 1616 if (level < 0) { 1617 drm_dbg_kms(&i915->drm, 1618 "Requested display configuration exceeds system DDB limitations"); 1619 drm_dbg_kms(&i915->drm, "minimum required %d/%d\n", 1620 blocks, iter.size); 1621 return -EINVAL; 1622 } 1623 1624 /* avoid the WARN later when we don't allocate any extra DDB */ 1625 if (iter.data_rate == 0) 1626 iter.size = 0; 1627 1628 /* 1629 * Grant each plane the blocks it requires at the highest achievable 1630 * watermark level, plus an extra share of the leftover blocks 1631 * proportional to its relative data rate. 1632 */ 1633 for_each_plane_id_on_crtc(crtc, plane_id) { 1634 struct skl_ddb_entry *ddb = 1635 &crtc_state->wm.skl.plane_ddb[plane_id]; 1636 struct skl_ddb_entry *ddb_y = 1637 &crtc_state->wm.skl.plane_ddb_y[plane_id]; 1638 u16 *min_ddb = &crtc_state->wm.skl.plane_min_ddb[plane_id]; 1639 u16 *interim_ddb = 1640 &crtc_state->wm.skl.plane_interim_ddb[plane_id]; 1641 const struct skl_plane_wm *wm = 1642 &crtc_state->wm.skl.optimal.planes[plane_id]; 1643 1644 if (plane_id == PLANE_CURSOR) 1645 continue; 1646 1647 if (DISPLAY_VER(i915) < 11 && 1648 crtc_state->nv12_planes & BIT(plane_id)) { 1649 skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level], 1650 crtc_state->rel_data_rate_y[plane_id]); 1651 skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level], 1652 crtc_state->rel_data_rate[plane_id]); 1653 } else { 1654 skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level], 1655 crtc_state->rel_data_rate[plane_id]); 1656 } 1657 1658 if (DISPLAY_VER(display) >= 30) { 1659 *min_ddb = wm->wm[0].min_ddb_alloc; 1660 *interim_ddb = wm->sagv.wm0.min_ddb_alloc; 1661 } 1662 } 1663 drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0); 1664 1665 /* 1666 * When we calculated watermark values we didn't know how high 1667 * of a level we'd actually be able to hit, so we just marked 1668 * all levels as "enabled." Go back now and disable the ones 1669 * that aren't actually possible. 1670 */ 1671 for (level++; level < i915->display.wm.num_levels; level++) { 1672 for_each_plane_id_on_crtc(crtc, plane_id) { 1673 const struct skl_ddb_entry *ddb = 1674 &crtc_state->wm.skl.plane_ddb[plane_id]; 1675 const struct skl_ddb_entry *ddb_y = 1676 &crtc_state->wm.skl.plane_ddb_y[plane_id]; 1677 struct skl_plane_wm *wm = 1678 &crtc_state->wm.skl.optimal.planes[plane_id]; 1679 1680 if (DISPLAY_VER(i915) < 11 && 1681 crtc_state->nv12_planes & BIT(plane_id)) 1682 skl_check_nv12_wm_level(&wm->wm[level], 1683 &wm->uv_wm[level], 1684 ddb_y, ddb); 1685 else 1686 skl_check_wm_level(&wm->wm[level], ddb); 1687 1688 if (skl_need_wm_copy_wa(i915, level, wm)) { 1689 wm->wm[level].blocks = wm->wm[level - 1].blocks; 1690 wm->wm[level].lines = wm->wm[level - 1].lines; 1691 wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines; 1692 } 1693 } 1694 } 1695 1696 /* 1697 * Go back and disable the transition and SAGV watermarks 1698 * if it turns out we don't have enough DDB blocks for them. 1699 */ 1700 for_each_plane_id_on_crtc(crtc, plane_id) { 1701 const struct skl_ddb_entry *ddb = 1702 &crtc_state->wm.skl.plane_ddb[plane_id]; 1703 const struct skl_ddb_entry *ddb_y = 1704 &crtc_state->wm.skl.plane_ddb_y[plane_id]; 1705 u16 *interim_ddb = 1706 &crtc_state->wm.skl.plane_interim_ddb[plane_id]; 1707 struct skl_plane_wm *wm = 1708 &crtc_state->wm.skl.optimal.planes[plane_id]; 1709 1710 if (DISPLAY_VER(i915) < 11 && 1711 crtc_state->nv12_planes & BIT(plane_id)) { 1712 skl_check_wm_level(&wm->trans_wm, ddb_y); 1713 } else { 1714 WARN_ON(skl_ddb_entry_size(ddb_y)); 1715 1716 skl_check_wm_level(&wm->trans_wm, ddb); 1717 } 1718 1719 skl_check_wm_level(&wm->sagv.wm0, ddb); 1720 if (DISPLAY_VER(display) >= 30) 1721 *interim_ddb = wm->sagv.wm0.min_ddb_alloc; 1722 1723 skl_check_wm_level(&wm->sagv.trans_wm, ddb); 1724 } 1725 1726 return 0; 1727 } 1728 1729 /* 1730 * The max latency should be 257 (max the punit can code is 255 and we add 2us 1731 * for the read latency) and cpp should always be <= 8, so that 1732 * should allow pixel_rate up to ~2 GHz which seems sufficient since max 1733 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that. 1734 */ 1735 static uint_fixed_16_16_t 1736 skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate, 1737 u8 cpp, u32 latency, u32 dbuf_block_size) 1738 { 1739 u32 wm_intermediate_val; 1740 uint_fixed_16_16_t ret; 1741 1742 if (latency == 0) 1743 return FP_16_16_MAX; 1744 1745 wm_intermediate_val = latency * pixel_rate * cpp; 1746 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size); 1747 1748 if (DISPLAY_VER(i915) >= 10) 1749 ret = add_fixed16_u32(ret, 1); 1750 1751 return ret; 1752 } 1753 1754 static uint_fixed_16_16_t 1755 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency, 1756 uint_fixed_16_16_t plane_blocks_per_line) 1757 { 1758 u32 wm_intermediate_val; 1759 uint_fixed_16_16_t ret; 1760 1761 if (latency == 0) 1762 return FP_16_16_MAX; 1763 1764 wm_intermediate_val = latency * pixel_rate; 1765 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val, 1766 pipe_htotal * 1000); 1767 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line); 1768 return ret; 1769 } 1770 1771 static uint_fixed_16_16_t 1772 intel_get_linetime_us(const struct intel_crtc_state *crtc_state) 1773 { 1774 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); 1775 u32 pixel_rate; 1776 u32 crtc_htotal; 1777 uint_fixed_16_16_t linetime_us; 1778 1779 if (!crtc_state->hw.active) 1780 return u32_to_fixed16(0); 1781 1782 pixel_rate = crtc_state->pixel_rate; 1783 1784 if (drm_WARN_ON(&i915->drm, pixel_rate == 0)) 1785 return u32_to_fixed16(0); 1786 1787 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal; 1788 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate); 1789 1790 return linetime_us; 1791 } 1792 1793 static int 1794 skl_compute_wm_params(const struct intel_crtc_state *crtc_state, 1795 int width, const struct drm_format_info *format, 1796 u64 modifier, unsigned int rotation, 1797 u32 plane_pixel_rate, struct skl_wm_params *wp, 1798 int color_plane, unsigned int pan_x) 1799 { 1800 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 1801 struct intel_display *display = to_intel_display(crtc_state); 1802 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 1803 u32 interm_pbpl; 1804 1805 /* only planar format has two planes */ 1806 if (color_plane == 1 && 1807 !intel_format_info_is_yuv_semiplanar(format, modifier)) { 1808 drm_dbg_kms(&i915->drm, 1809 "Non planar format have single plane\n"); 1810 return -EINVAL; 1811 } 1812 1813 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED; 1814 wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED && 1815 intel_fb_is_tiled_modifier(modifier); 1816 wp->rc_surface = intel_fb_is_ccs_modifier(modifier); 1817 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier); 1818 1819 wp->width = width; 1820 if (color_plane == 1 && wp->is_planar) 1821 wp->width /= 2; 1822 1823 wp->cpp = format->cpp[color_plane]; 1824 wp->plane_pixel_rate = plane_pixel_rate; 1825 1826 if (DISPLAY_VER(i915) >= 11 && 1827 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1) 1828 wp->dbuf_block_size = 256; 1829 else 1830 wp->dbuf_block_size = 512; 1831 1832 if (drm_rotation_90_or_270(rotation)) { 1833 switch (wp->cpp) { 1834 case 1: 1835 wp->y_min_scanlines = 16; 1836 break; 1837 case 2: 1838 wp->y_min_scanlines = 8; 1839 break; 1840 case 4: 1841 wp->y_min_scanlines = 4; 1842 break; 1843 default: 1844 MISSING_CASE(wp->cpp); 1845 return -EINVAL; 1846 } 1847 } else { 1848 wp->y_min_scanlines = 4; 1849 } 1850 1851 if (skl_needs_memory_bw_wa(i915)) 1852 wp->y_min_scanlines *= 2; 1853 1854 wp->plane_bytes_per_line = wp->width * wp->cpp; 1855 if (wp->y_tiled) { 1856 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line * 1857 wp->y_min_scanlines, 1858 wp->dbuf_block_size); 1859 1860 if (DISPLAY_VER(display) >= 30) 1861 interm_pbpl += (pan_x != 0); 1862 else if (DISPLAY_VER(i915) >= 10) 1863 interm_pbpl++; 1864 1865 wp->plane_blocks_per_line = div_fixed16(interm_pbpl, 1866 wp->y_min_scanlines); 1867 } else { 1868 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line, 1869 wp->dbuf_block_size); 1870 1871 if (!wp->x_tiled || DISPLAY_VER(i915) >= 10) 1872 interm_pbpl++; 1873 1874 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl); 1875 } 1876 1877 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines, 1878 wp->plane_blocks_per_line); 1879 1880 wp->linetime_us = fixed16_to_u32_round_up(intel_get_linetime_us(crtc_state)); 1881 1882 return 0; 1883 } 1884 1885 static int 1886 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state, 1887 const struct intel_plane_state *plane_state, 1888 struct skl_wm_params *wp, int color_plane) 1889 { 1890 const struct drm_framebuffer *fb = plane_state->hw.fb; 1891 int width; 1892 1893 /* 1894 * Src coordinates are already rotated by 270 degrees for 1895 * the 90/270 degree plane rotation cases (to match the 1896 * GTT mapping), hence no need to account for rotation here. 1897 */ 1898 width = drm_rect_width(&plane_state->uapi.src) >> 16; 1899 1900 return skl_compute_wm_params(crtc_state, width, 1901 fb->format, fb->modifier, 1902 plane_state->hw.rotation, 1903 intel_plane_pixel_rate(crtc_state, plane_state), 1904 wp, color_plane, 1905 plane_state->uapi.src.x1); 1906 } 1907 1908 static bool skl_wm_has_lines(struct drm_i915_private *i915, int level) 1909 { 1910 if (DISPLAY_VER(i915) >= 10) 1911 return true; 1912 1913 /* The number of lines are ignored for the level 0 watermark. */ 1914 return level > 0; 1915 } 1916 1917 static int skl_wm_max_lines(struct drm_i915_private *i915) 1918 { 1919 if (DISPLAY_VER(i915) >= 13) 1920 return 255; 1921 else 1922 return 31; 1923 } 1924 1925 static bool xe3_auto_min_alloc_capable(struct intel_plane *plane, int level) 1926 { 1927 struct intel_display *display = to_intel_display(plane); 1928 1929 return DISPLAY_VER(display) >= 30 && level == 0 && plane->id != PLANE_CURSOR; 1930 } 1931 1932 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, 1933 struct intel_plane *plane, 1934 int level, 1935 unsigned int latency, 1936 const struct skl_wm_params *wp, 1937 const struct skl_wm_level *result_prev, 1938 struct skl_wm_level *result /* out */) 1939 { 1940 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); 1941 uint_fixed_16_16_t method1, method2; 1942 uint_fixed_16_16_t selected_result; 1943 u32 blocks, lines, min_ddb_alloc = 0; 1944 1945 if (latency == 0 || 1946 (use_minimal_wm0_only(crtc_state, plane) && level > 0)) { 1947 /* reject it */ 1948 result->min_ddb_alloc = U16_MAX; 1949 return; 1950 } 1951 1952 method1 = skl_wm_method1(i915, wp->plane_pixel_rate, 1953 wp->cpp, latency, wp->dbuf_block_size); 1954 method2 = skl_wm_method2(wp->plane_pixel_rate, 1955 crtc_state->hw.pipe_mode.crtc_htotal, 1956 latency, 1957 wp->plane_blocks_per_line); 1958 1959 if (wp->y_tiled) { 1960 selected_result = max_fixed16(method2, wp->y_tile_minimum); 1961 } else { 1962 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal / 1963 wp->dbuf_block_size < 1) && 1964 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) { 1965 selected_result = method2; 1966 } else if (latency >= wp->linetime_us) { 1967 if (DISPLAY_VER(i915) == 9) 1968 selected_result = min_fixed16(method1, method2); 1969 else 1970 selected_result = method2; 1971 } else { 1972 selected_result = method1; 1973 } 1974 } 1975 1976 blocks = fixed16_to_u32_round_up(selected_result); 1977 if (DISPLAY_VER(i915) < 30) 1978 blocks++; 1979 1980 /* 1981 * Lets have blocks at minimum equivalent to plane_blocks_per_line 1982 * as there will be at minimum one line for lines configuration. This 1983 * is a work around for FIFO underruns observed with resolutions like 1984 * 4k 60 Hz in single channel DRAM configurations. 1985 * 1986 * As per the Bspec 49325, if the ddb allocation can hold at least 1987 * one plane_blocks_per_line, we should have selected method2 in 1988 * the above logic. Assuming that modern versions have enough dbuf 1989 * and method2 guarantees blocks equivalent to at least 1 line, 1990 * select the blocks as plane_blocks_per_line. 1991 * 1992 * TODO: Revisit the logic when we have better understanding on DRAM 1993 * channels' impact on the level 0 memory latency and the relevant 1994 * wm calculations. 1995 */ 1996 if (skl_wm_has_lines(i915, level)) 1997 blocks = max(blocks, 1998 fixed16_to_u32_round_up(wp->plane_blocks_per_line)); 1999 lines = div_round_up_fixed16(selected_result, 2000 wp->plane_blocks_per_line); 2001 2002 if (DISPLAY_VER(i915) == 9) { 2003 /* Display WA #1125: skl,bxt,kbl */ 2004 if (level == 0 && wp->rc_surface) 2005 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum); 2006 2007 /* Display WA #1126: skl,bxt,kbl */ 2008 if (level >= 1 && level <= 7) { 2009 if (wp->y_tiled) { 2010 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum); 2011 lines += wp->y_min_scanlines; 2012 } else { 2013 blocks++; 2014 } 2015 2016 /* 2017 * Make sure result blocks for higher latency levels are 2018 * at least as high as level below the current level. 2019 * Assumption in DDB algorithm optimization for special 2020 * cases. Also covers Display WA #1125 for RC. 2021 */ 2022 if (result_prev->blocks > blocks) 2023 blocks = result_prev->blocks; 2024 } 2025 } 2026 2027 if (DISPLAY_VER(i915) >= 11) { 2028 if (wp->y_tiled) { 2029 int extra_lines; 2030 2031 if (lines % wp->y_min_scanlines == 0) 2032 extra_lines = wp->y_min_scanlines; 2033 else 2034 extra_lines = wp->y_min_scanlines * 2 - 2035 lines % wp->y_min_scanlines; 2036 2037 min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines, 2038 wp->plane_blocks_per_line); 2039 } else { 2040 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10); 2041 } 2042 } 2043 2044 if (!skl_wm_has_lines(i915, level)) 2045 lines = 0; 2046 2047 if (lines > skl_wm_max_lines(i915)) { 2048 /* reject it */ 2049 result->min_ddb_alloc = U16_MAX; 2050 return; 2051 } 2052 2053 /* 2054 * If lines is valid, assume we can use this watermark level 2055 * for now. We'll come back and disable it after we calculate the 2056 * DDB allocation if it turns out we don't actually have enough 2057 * blocks to satisfy it. 2058 */ 2059 result->blocks = blocks; 2060 result->lines = lines; 2061 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */ 2062 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1; 2063 result->enable = true; 2064 result->auto_min_alloc_wm_enable = xe3_auto_min_alloc_capable(plane, level); 2065 2066 if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us) 2067 result->can_sagv = latency >= i915->display.sagv.block_time_us; 2068 } 2069 2070 static void 2071 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state, 2072 struct intel_plane *plane, 2073 const struct skl_wm_params *wm_params, 2074 struct skl_wm_level *levels) 2075 { 2076 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); 2077 struct skl_wm_level *result_prev = &levels[0]; 2078 int level; 2079 2080 for (level = 0; level < i915->display.wm.num_levels; level++) { 2081 struct skl_wm_level *result = &levels[level]; 2082 unsigned int latency = skl_wm_latency(i915, level, wm_params); 2083 2084 skl_compute_plane_wm(crtc_state, plane, level, latency, 2085 wm_params, result_prev, result); 2086 2087 result_prev = result; 2088 } 2089 } 2090 2091 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state, 2092 struct intel_plane *plane, 2093 const struct skl_wm_params *wm_params, 2094 struct skl_plane_wm *plane_wm) 2095 { 2096 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); 2097 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0; 2098 struct skl_wm_level *levels = plane_wm->wm; 2099 unsigned int latency = 0; 2100 2101 if (i915->display.sagv.block_time_us) 2102 latency = i915->display.sagv.block_time_us + 2103 skl_wm_latency(i915, 0, wm_params); 2104 2105 skl_compute_plane_wm(crtc_state, plane, 0, latency, 2106 wm_params, &levels[0], 2107 sagv_wm); 2108 } 2109 2110 static void skl_compute_transition_wm(struct drm_i915_private *i915, 2111 struct skl_wm_level *trans_wm, 2112 const struct skl_wm_level *wm0, 2113 const struct skl_wm_params *wp) 2114 { 2115 u16 trans_min, trans_amount, trans_y_tile_min; 2116 u16 wm0_blocks, trans_offset, blocks; 2117 2118 /* Transition WM don't make any sense if ipc is disabled */ 2119 if (!skl_watermark_ipc_enabled(i915)) 2120 return; 2121 2122 /* 2123 * WaDisableTWM:skl,kbl,cfl,bxt 2124 * Transition WM are not recommended by HW team for GEN9 2125 */ 2126 if (DISPLAY_VER(i915) == 9) 2127 return; 2128 2129 if (DISPLAY_VER(i915) >= 11) 2130 trans_min = 4; 2131 else 2132 trans_min = 14; 2133 2134 /* Display WA #1140: glk,cnl */ 2135 if (DISPLAY_VER(i915) == 10) 2136 trans_amount = 0; 2137 else 2138 trans_amount = 10; /* This is configurable amount */ 2139 2140 trans_offset = trans_min + trans_amount; 2141 2142 /* 2143 * The spec asks for Selected Result Blocks for wm0 (the real value), 2144 * not Result Blocks (the integer value). Pay attention to the capital 2145 * letters. The value wm_l0->blocks is actually Result Blocks, but 2146 * since Result Blocks is the ceiling of Selected Result Blocks plus 1, 2147 * and since we later will have to get the ceiling of the sum in the 2148 * transition watermarks calculation, we can just pretend Selected 2149 * Result Blocks is Result Blocks minus 1 and it should work for the 2150 * current platforms. 2151 */ 2152 wm0_blocks = wm0->blocks - 1; 2153 2154 if (wp->y_tiled) { 2155 trans_y_tile_min = 2156 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum); 2157 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset; 2158 } else { 2159 blocks = wm0_blocks + trans_offset; 2160 } 2161 blocks++; 2162 2163 /* 2164 * Just assume we can enable the transition watermark. After 2165 * computing the DDB we'll come back and disable it if that 2166 * assumption turns out to be false. 2167 */ 2168 trans_wm->blocks = blocks; 2169 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1); 2170 trans_wm->enable = true; 2171 } 2172 2173 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state, 2174 const struct intel_plane_state *plane_state, 2175 struct intel_plane *plane, int color_plane) 2176 { 2177 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 2178 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 2179 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id]; 2180 struct skl_wm_params wm_params; 2181 int ret; 2182 2183 ret = skl_compute_plane_wm_params(crtc_state, plane_state, 2184 &wm_params, color_plane); 2185 if (ret) 2186 return ret; 2187 2188 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm); 2189 2190 skl_compute_transition_wm(i915, &wm->trans_wm, 2191 &wm->wm[0], &wm_params); 2192 2193 if (DISPLAY_VER(i915) >= 12) { 2194 tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm); 2195 2196 skl_compute_transition_wm(i915, &wm->sagv.trans_wm, 2197 &wm->sagv.wm0, &wm_params); 2198 } 2199 2200 return 0; 2201 } 2202 2203 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state, 2204 const struct intel_plane_state *plane_state, 2205 struct intel_plane *plane) 2206 { 2207 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id]; 2208 struct skl_wm_params wm_params; 2209 int ret; 2210 2211 wm->is_planar = true; 2212 2213 /* uv plane watermarks must also be validated for NV12/Planar */ 2214 ret = skl_compute_plane_wm_params(crtc_state, plane_state, 2215 &wm_params, 1); 2216 if (ret) 2217 return ret; 2218 2219 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm); 2220 2221 return 0; 2222 } 2223 2224 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state, 2225 const struct intel_plane_state *plane_state) 2226 { 2227 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 2228 enum plane_id plane_id = plane->id; 2229 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; 2230 const struct drm_framebuffer *fb = plane_state->hw.fb; 2231 int ret; 2232 2233 memset(wm, 0, sizeof(*wm)); 2234 2235 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2236 return 0; 2237 2238 ret = skl_build_plane_wm_single(crtc_state, plane_state, 2239 plane, 0); 2240 if (ret) 2241 return ret; 2242 2243 if (fb->format->is_yuv && fb->format->num_planes > 1) { 2244 ret = skl_build_plane_wm_uv(crtc_state, plane_state, 2245 plane); 2246 if (ret) 2247 return ret; 2248 } 2249 2250 return 0; 2251 } 2252 2253 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state, 2254 const struct intel_plane_state *plane_state) 2255 { 2256 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 2257 struct drm_i915_private *i915 = to_i915(plane->base.dev); 2258 enum plane_id plane_id = plane->id; 2259 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; 2260 int ret; 2261 2262 /* Watermarks calculated in master */ 2263 if (plane_state->planar_slave) 2264 return 0; 2265 2266 memset(wm, 0, sizeof(*wm)); 2267 2268 if (plane_state->planar_linked_plane) { 2269 const struct drm_framebuffer *fb = plane_state->hw.fb; 2270 2271 drm_WARN_ON(&i915->drm, 2272 !intel_wm_plane_visible(crtc_state, plane_state)); 2273 drm_WARN_ON(&i915->drm, !fb->format->is_yuv || 2274 fb->format->num_planes == 1); 2275 2276 ret = skl_build_plane_wm_single(crtc_state, plane_state, 2277 plane_state->planar_linked_plane, 0); 2278 if (ret) 2279 return ret; 2280 2281 ret = skl_build_plane_wm_single(crtc_state, plane_state, 2282 plane, 1); 2283 if (ret) 2284 return ret; 2285 } else if (intel_wm_plane_visible(crtc_state, plane_state)) { 2286 ret = skl_build_plane_wm_single(crtc_state, plane_state, 2287 plane, 0); 2288 if (ret) 2289 return ret; 2290 } 2291 2292 return 0; 2293 } 2294 2295 static bool 2296 skl_is_vblank_too_short(const struct intel_crtc_state *crtc_state, 2297 int wm0_lines, int latency) 2298 { 2299 const struct drm_display_mode *adjusted_mode = 2300 &crtc_state->hw.adjusted_mode; 2301 2302 /* FIXME missing scaler and DSC pre-fill time */ 2303 return crtc_state->framestart_delay + 2304 intel_usecs_to_scanlines(adjusted_mode, latency) + 2305 wm0_lines > 2306 adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start; 2307 } 2308 2309 static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state) 2310 { 2311 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 2312 enum plane_id plane_id; 2313 int wm0_lines = 0; 2314 2315 for_each_plane_id_on_crtc(crtc, plane_id) { 2316 const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; 2317 2318 /* FIXME what about !skl_wm_has_lines() platforms? */ 2319 wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines); 2320 } 2321 2322 return wm0_lines; 2323 } 2324 2325 static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state, 2326 int wm0_lines) 2327 { 2328 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 2329 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 2330 int level; 2331 2332 for (level = i915->display.wm.num_levels - 1; level >= 0; level--) { 2333 int latency; 2334 2335 /* FIXME should we care about the latency w/a's? */ 2336 latency = skl_wm_latency(i915, level, NULL); 2337 if (latency == 0) 2338 continue; 2339 2340 /* FIXME is it correct to use 0 latency for wm0 here? */ 2341 if (level == 0) 2342 latency = 0; 2343 2344 if (!skl_is_vblank_too_short(crtc_state, wm0_lines, latency)) 2345 return level; 2346 } 2347 2348 return -EINVAL; 2349 } 2350 2351 static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state) 2352 { 2353 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 2354 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 2355 int wm0_lines, level; 2356 2357 if (!crtc_state->hw.active) 2358 return 0; 2359 2360 wm0_lines = skl_max_wm0_lines(crtc_state); 2361 2362 level = skl_max_wm_level_for_vblank(crtc_state, wm0_lines); 2363 if (level < 0) 2364 return level; 2365 2366 /* 2367 * PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_* 2368 * based on whether we're limited by the vblank duration. 2369 */ 2370 crtc_state->wm_level_disabled = level < i915->display.wm.num_levels - 1; 2371 2372 for (level++; level < i915->display.wm.num_levels; level++) { 2373 enum plane_id plane_id; 2374 2375 for_each_plane_id_on_crtc(crtc, plane_id) { 2376 struct skl_plane_wm *wm = 2377 &crtc_state->wm.skl.optimal.planes[plane_id]; 2378 2379 /* 2380 * FIXME just clear enable or flag the entire 2381 * thing as bad via min_ddb_alloc=U16_MAX? 2382 */ 2383 wm->wm[level].enable = false; 2384 wm->uv_wm[level].enable = false; 2385 } 2386 } 2387 2388 if (DISPLAY_VER(i915) >= 12 && 2389 i915->display.sagv.block_time_us && 2390 skl_is_vblank_too_short(crtc_state, wm0_lines, 2391 i915->display.sagv.block_time_us)) { 2392 enum plane_id plane_id; 2393 2394 for_each_plane_id_on_crtc(crtc, plane_id) { 2395 struct skl_plane_wm *wm = 2396 &crtc_state->wm.skl.optimal.planes[plane_id]; 2397 2398 wm->sagv.wm0.enable = false; 2399 wm->sagv.trans_wm.enable = false; 2400 } 2401 } 2402 2403 return 0; 2404 } 2405 2406 static int skl_build_pipe_wm(struct intel_atomic_state *state, 2407 struct intel_crtc *crtc) 2408 { 2409 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 2410 struct intel_crtc_state *crtc_state = 2411 intel_atomic_get_new_crtc_state(state, crtc); 2412 const struct intel_plane_state *plane_state; 2413 struct intel_plane *plane; 2414 int ret, i; 2415 2416 for_each_new_intel_plane_in_state(state, plane, plane_state, i) { 2417 /* 2418 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc 2419 * instead but we don't populate that correctly for NV12 Y 2420 * planes so for now hack this. 2421 */ 2422 if (plane->pipe != crtc->pipe) 2423 continue; 2424 2425 if (DISPLAY_VER(i915) >= 11) 2426 ret = icl_build_plane_wm(crtc_state, plane_state); 2427 else 2428 ret = skl_build_plane_wm(crtc_state, plane_state); 2429 if (ret) 2430 return ret; 2431 } 2432 2433 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw; 2434 2435 return skl_wm_check_vblank(crtc_state); 2436 } 2437 2438 static bool skl_wm_level_equals(const struct skl_wm_level *l1, 2439 const struct skl_wm_level *l2) 2440 { 2441 return l1->enable == l2->enable && 2442 l1->ignore_lines == l2->ignore_lines && 2443 l1->lines == l2->lines && 2444 l1->blocks == l2->blocks && 2445 l1->auto_min_alloc_wm_enable == l2->auto_min_alloc_wm_enable; 2446 } 2447 2448 static bool skl_plane_wm_equals(struct drm_i915_private *i915, 2449 const struct skl_plane_wm *wm1, 2450 const struct skl_plane_wm *wm2) 2451 { 2452 struct intel_display *display = &i915->display; 2453 int level; 2454 2455 for (level = 0; level < display->wm.num_levels; level++) { 2456 /* 2457 * We don't check uv_wm as the hardware doesn't actually 2458 * use it. It only gets used for calculating the required 2459 * ddb allocation. 2460 */ 2461 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level])) 2462 return false; 2463 } 2464 2465 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) && 2466 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) && 2467 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm); 2468 } 2469 2470 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a, 2471 const struct skl_ddb_entry *b) 2472 { 2473 return a->start < b->end && b->start < a->end; 2474 } 2475 2476 static void skl_ddb_entry_union(struct skl_ddb_entry *a, 2477 const struct skl_ddb_entry *b) 2478 { 2479 if (a->end && b->end) { 2480 a->start = min(a->start, b->start); 2481 a->end = max(a->end, b->end); 2482 } else if (b->end) { 2483 a->start = b->start; 2484 a->end = b->end; 2485 } 2486 } 2487 2488 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb, 2489 const struct skl_ddb_entry *entries, 2490 int num_entries, int ignore_idx) 2491 { 2492 int i; 2493 2494 for (i = 0; i < num_entries; i++) { 2495 if (i != ignore_idx && 2496 skl_ddb_entries_overlap(ddb, &entries[i])) 2497 return true; 2498 } 2499 2500 return false; 2501 } 2502 2503 static int 2504 skl_ddb_add_affected_planes(struct intel_atomic_state *state, 2505 struct intel_crtc *crtc) 2506 { 2507 struct drm_i915_private *i915 = to_i915(state->base.dev); 2508 const struct intel_crtc_state *old_crtc_state = 2509 intel_atomic_get_old_crtc_state(state, crtc); 2510 struct intel_crtc_state *new_crtc_state = 2511 intel_atomic_get_new_crtc_state(state, crtc); 2512 struct intel_plane *plane; 2513 2514 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { 2515 struct intel_plane_state *plane_state; 2516 enum plane_id plane_id = plane->id; 2517 2518 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id], 2519 &new_crtc_state->wm.skl.plane_ddb[plane_id]) && 2520 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id], 2521 &new_crtc_state->wm.skl.plane_ddb_y[plane_id])) 2522 continue; 2523 2524 if (new_crtc_state->do_async_flip) { 2525 drm_dbg_kms(&i915->drm, "[PLANE:%d:%s] Can't change DDB during async flip\n", 2526 plane->base.base.id, plane->base.name); 2527 return -EINVAL; 2528 } 2529 2530 plane_state = intel_atomic_get_plane_state(state, plane); 2531 if (IS_ERR(plane_state)) 2532 return PTR_ERR(plane_state); 2533 2534 new_crtc_state->update_planes |= BIT(plane_id); 2535 new_crtc_state->async_flip_planes = 0; 2536 new_crtc_state->do_async_flip = false; 2537 } 2538 2539 return 0; 2540 } 2541 2542 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state) 2543 { 2544 struct drm_i915_private *i915 = to_i915(dbuf_state->base.state->base.dev); 2545 u8 enabled_slices; 2546 enum pipe pipe; 2547 2548 /* 2549 * FIXME: For now we always enable slice S1 as per 2550 * the Bspec display initialization sequence. 2551 */ 2552 enabled_slices = BIT(DBUF_S1); 2553 2554 for_each_pipe(i915, pipe) 2555 enabled_slices |= dbuf_state->slices[pipe]; 2556 2557 return enabled_slices; 2558 } 2559 2560 static int 2561 skl_compute_ddb(struct intel_atomic_state *state) 2562 { 2563 struct intel_display *display = to_intel_display(state); 2564 struct drm_i915_private *i915 = to_i915(state->base.dev); 2565 const struct intel_dbuf_state *old_dbuf_state; 2566 struct intel_dbuf_state *new_dbuf_state = NULL; 2567 struct intel_crtc_state *new_crtc_state; 2568 struct intel_crtc *crtc; 2569 int ret, i; 2570 2571 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { 2572 new_dbuf_state = intel_atomic_get_dbuf_state(state); 2573 if (IS_ERR(new_dbuf_state)) 2574 return PTR_ERR(new_dbuf_state); 2575 2576 old_dbuf_state = intel_atomic_get_old_dbuf_state(state); 2577 break; 2578 } 2579 2580 if (!new_dbuf_state) 2581 return 0; 2582 2583 new_dbuf_state->active_pipes = 2584 intel_calc_active_pipes(state, old_dbuf_state->active_pipes); 2585 2586 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) { 2587 ret = intel_atomic_lock_global_state(&new_dbuf_state->base); 2588 if (ret) 2589 return ret; 2590 } 2591 2592 if (HAS_MBUS_JOINING(display)) { 2593 new_dbuf_state->joined_mbus = 2594 adlp_check_mbus_joined(new_dbuf_state->active_pipes); 2595 2596 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) { 2597 ret = intel_cdclk_state_set_joined_mbus(state, new_dbuf_state->joined_mbus); 2598 if (ret) 2599 return ret; 2600 } 2601 } 2602 2603 for_each_intel_crtc(&i915->drm, crtc) { 2604 enum pipe pipe = crtc->pipe; 2605 2606 new_dbuf_state->slices[pipe] = 2607 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes, 2608 new_dbuf_state->joined_mbus); 2609 2610 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe]) 2611 continue; 2612 2613 ret = intel_atomic_lock_global_state(&new_dbuf_state->base); 2614 if (ret) 2615 return ret; 2616 } 2617 2618 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state); 2619 2620 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices || 2621 old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) { 2622 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base); 2623 if (ret) 2624 return ret; 2625 2626 drm_dbg_kms(&i915->drm, 2627 "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n", 2628 old_dbuf_state->enabled_slices, 2629 new_dbuf_state->enabled_slices, 2630 DISPLAY_INFO(i915)->dbuf.slice_mask, 2631 str_yes_no(old_dbuf_state->joined_mbus), 2632 str_yes_no(new_dbuf_state->joined_mbus)); 2633 } 2634 2635 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { 2636 enum pipe pipe = crtc->pipe; 2637 2638 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state); 2639 2640 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe]) 2641 continue; 2642 2643 ret = intel_atomic_lock_global_state(&new_dbuf_state->base); 2644 if (ret) 2645 return ret; 2646 } 2647 2648 for_each_intel_crtc(&i915->drm, crtc) { 2649 ret = skl_crtc_allocate_ddb(state, crtc); 2650 if (ret) 2651 return ret; 2652 } 2653 2654 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { 2655 ret = skl_crtc_allocate_plane_ddb(state, crtc); 2656 if (ret) 2657 return ret; 2658 2659 ret = skl_ddb_add_affected_planes(state, crtc); 2660 if (ret) 2661 return ret; 2662 } 2663 2664 return 0; 2665 } 2666 2667 static char enast(bool enable) 2668 { 2669 return enable ? '*' : ' '; 2670 } 2671 2672 static void 2673 skl_print_wm_changes(struct intel_atomic_state *state) 2674 { 2675 struct drm_i915_private *i915 = to_i915(state->base.dev); 2676 const struct intel_crtc_state *old_crtc_state; 2677 const struct intel_crtc_state *new_crtc_state; 2678 struct intel_plane *plane; 2679 struct intel_crtc *crtc; 2680 int i; 2681 2682 if (!drm_debug_enabled(DRM_UT_KMS)) 2683 return; 2684 2685 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, 2686 new_crtc_state, i) { 2687 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm; 2688 2689 old_pipe_wm = &old_crtc_state->wm.skl.optimal; 2690 new_pipe_wm = &new_crtc_state->wm.skl.optimal; 2691 2692 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { 2693 enum plane_id plane_id = plane->id; 2694 const struct skl_ddb_entry *old, *new; 2695 2696 old = &old_crtc_state->wm.skl.plane_ddb[plane_id]; 2697 new = &new_crtc_state->wm.skl.plane_ddb[plane_id]; 2698 2699 if (skl_ddb_entry_equal(old, new)) 2700 continue; 2701 2702 drm_dbg_kms(&i915->drm, 2703 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n", 2704 plane->base.base.id, plane->base.name, 2705 old->start, old->end, new->start, new->end, 2706 skl_ddb_entry_size(old), skl_ddb_entry_size(new)); 2707 } 2708 2709 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { 2710 enum plane_id plane_id = plane->id; 2711 const struct skl_plane_wm *old_wm, *new_wm; 2712 2713 old_wm = &old_pipe_wm->planes[plane_id]; 2714 new_wm = &new_pipe_wm->planes[plane_id]; 2715 2716 if (skl_plane_wm_equals(i915, old_wm, new_wm)) 2717 continue; 2718 2719 drm_dbg_kms(&i915->drm, 2720 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm" 2721 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n", 2722 plane->base.base.id, plane->base.name, 2723 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable), 2724 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable), 2725 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable), 2726 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable), 2727 enast(old_wm->trans_wm.enable), 2728 enast(old_wm->sagv.wm0.enable), 2729 enast(old_wm->sagv.trans_wm.enable), 2730 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable), 2731 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable), 2732 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable), 2733 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable), 2734 enast(new_wm->trans_wm.enable), 2735 enast(new_wm->sagv.wm0.enable), 2736 enast(new_wm->sagv.trans_wm.enable)); 2737 2738 drm_dbg_kms(&i915->drm, 2739 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d" 2740 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n", 2741 plane->base.base.id, plane->base.name, 2742 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines, 2743 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines, 2744 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines, 2745 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines, 2746 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines, 2747 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines, 2748 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines, 2749 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines, 2750 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines, 2751 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines, 2752 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines, 2753 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines, 2754 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines, 2755 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines, 2756 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines, 2757 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines, 2758 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines, 2759 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines, 2760 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines, 2761 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines, 2762 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines, 2763 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines); 2764 2765 drm_dbg_kms(&i915->drm, 2766 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d" 2767 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n", 2768 plane->base.base.id, plane->base.name, 2769 old_wm->wm[0].blocks, old_wm->wm[1].blocks, 2770 old_wm->wm[2].blocks, old_wm->wm[3].blocks, 2771 old_wm->wm[4].blocks, old_wm->wm[5].blocks, 2772 old_wm->wm[6].blocks, old_wm->wm[7].blocks, 2773 old_wm->trans_wm.blocks, 2774 old_wm->sagv.wm0.blocks, 2775 old_wm->sagv.trans_wm.blocks, 2776 new_wm->wm[0].blocks, new_wm->wm[1].blocks, 2777 new_wm->wm[2].blocks, new_wm->wm[3].blocks, 2778 new_wm->wm[4].blocks, new_wm->wm[5].blocks, 2779 new_wm->wm[6].blocks, new_wm->wm[7].blocks, 2780 new_wm->trans_wm.blocks, 2781 new_wm->sagv.wm0.blocks, 2782 new_wm->sagv.trans_wm.blocks); 2783 2784 drm_dbg_kms(&i915->drm, 2785 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d" 2786 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n", 2787 plane->base.base.id, plane->base.name, 2788 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc, 2789 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc, 2790 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc, 2791 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc, 2792 old_wm->trans_wm.min_ddb_alloc, 2793 old_wm->sagv.wm0.min_ddb_alloc, 2794 old_wm->sagv.trans_wm.min_ddb_alloc, 2795 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc, 2796 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc, 2797 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc, 2798 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc, 2799 new_wm->trans_wm.min_ddb_alloc, 2800 new_wm->sagv.wm0.min_ddb_alloc, 2801 new_wm->sagv.trans_wm.min_ddb_alloc); 2802 } 2803 } 2804 } 2805 2806 static bool skl_plane_selected_wm_equals(struct intel_plane *plane, 2807 const struct skl_pipe_wm *old_pipe_wm, 2808 const struct skl_pipe_wm *new_pipe_wm) 2809 { 2810 struct intel_display *display = to_intel_display(plane); 2811 int level; 2812 2813 for (level = 0; level < display->wm.num_levels; level++) { 2814 /* 2815 * We don't check uv_wm as the hardware doesn't actually 2816 * use it. It only gets used for calculating the required 2817 * ddb allocation. 2818 */ 2819 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level), 2820 skl_plane_wm_level(new_pipe_wm, plane->id, level))) 2821 return false; 2822 } 2823 2824 if (HAS_HW_SAGV_WM(display)) { 2825 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id]; 2826 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id]; 2827 2828 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) || 2829 !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm)) 2830 return false; 2831 } 2832 2833 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id), 2834 skl_plane_trans_wm(new_pipe_wm, plane->id)); 2835 } 2836 2837 /* 2838 * To make sure the cursor watermark registers are always consistent 2839 * with our computed state the following scenario needs special 2840 * treatment: 2841 * 2842 * 1. enable cursor 2843 * 2. move cursor entirely offscreen 2844 * 3. disable cursor 2845 * 2846 * Step 2. does call .disable_plane() but does not zero the watermarks 2847 * (since we consider an offscreen cursor still active for the purposes 2848 * of watermarks). Step 3. would not normally call .disable_plane() 2849 * because the actual plane visibility isn't changing, and we don't 2850 * deallocate the cursor ddb until the pipe gets disabled. So we must 2851 * force step 3. to call .disable_plane() to update the watermark 2852 * registers properly. 2853 * 2854 * Other planes do not suffer from this issues as their watermarks are 2855 * calculated based on the actual plane visibility. The only time this 2856 * can trigger for the other planes is during the initial readout as the 2857 * default value of the watermarks registers is not zero. 2858 */ 2859 static int skl_wm_add_affected_planes(struct intel_atomic_state *state, 2860 struct intel_crtc *crtc) 2861 { 2862 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 2863 const struct intel_crtc_state *old_crtc_state = 2864 intel_atomic_get_old_crtc_state(state, crtc); 2865 struct intel_crtc_state *new_crtc_state = 2866 intel_atomic_get_new_crtc_state(state, crtc); 2867 struct intel_plane *plane; 2868 2869 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { 2870 struct intel_plane_state *plane_state; 2871 enum plane_id plane_id = plane->id; 2872 2873 /* 2874 * Force a full wm update for every plane on modeset. 2875 * Required because the reset value of the wm registers 2876 * is non-zero, whereas we want all disabled planes to 2877 * have zero watermarks. So if we turn off the relevant 2878 * power well the hardware state will go out of sync 2879 * with the software state. 2880 */ 2881 if (!intel_crtc_needs_modeset(new_crtc_state) && 2882 skl_plane_selected_wm_equals(plane, 2883 &old_crtc_state->wm.skl.optimal, 2884 &new_crtc_state->wm.skl.optimal)) 2885 continue; 2886 2887 if (new_crtc_state->do_async_flip) { 2888 drm_dbg_kms(&i915->drm, "[PLANE:%d:%s] Can't change watermarks during async flip\n", 2889 plane->base.base.id, plane->base.name); 2890 return -EINVAL; 2891 } 2892 2893 plane_state = intel_atomic_get_plane_state(state, plane); 2894 if (IS_ERR(plane_state)) 2895 return PTR_ERR(plane_state); 2896 2897 new_crtc_state->update_planes |= BIT(plane_id); 2898 new_crtc_state->async_flip_planes = 0; 2899 new_crtc_state->do_async_flip = false; 2900 } 2901 2902 return 0; 2903 } 2904 2905 /* 2906 * If Fixed Refresh Rate or For VRR case Vmin = Vmax = Flipline: 2907 * Program DEEP PKG_C_LATENCY Pkg C with highest valid latency from 2908 * watermark level1 and up and above. If watermark level 1 is 2909 * invalid program it with all 1's. 2910 * Program PKG_C_LATENCY Added Wake Time = DSB execution time 2911 * If Variable Refresh Rate where Vmin != Vmax != Flipline: 2912 * Program DEEP PKG_C_LATENCY Pkg C with all 1's. 2913 * Program PKG_C_LATENCY Added Wake Time = 0 2914 */ 2915 void 2916 intel_program_dpkgc_latency(struct intel_atomic_state *state) 2917 { 2918 struct intel_display *display = to_intel_display(state); 2919 struct drm_i915_private *i915 = to_i915(display->drm); 2920 struct intel_crtc *crtc; 2921 struct intel_crtc_state *new_crtc_state; 2922 u32 latency = LNL_PKG_C_LATENCY_MASK; 2923 u32 added_wake_time = 0; 2924 u32 max_linetime = 0; 2925 u32 clear, val; 2926 bool fixed_refresh_rate = false; 2927 int i; 2928 2929 if (DISPLAY_VER(display) < 20) 2930 return; 2931 2932 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { 2933 if (!new_crtc_state->vrr.enable || 2934 (new_crtc_state->vrr.vmin == new_crtc_state->vrr.vmax && 2935 new_crtc_state->vrr.vmin == new_crtc_state->vrr.flipline)) 2936 fixed_refresh_rate = true; 2937 2938 max_linetime = max(new_crtc_state->linetime, max_linetime); 2939 } 2940 2941 if (fixed_refresh_rate) { 2942 added_wake_time = DSB_EXE_TIME + 2943 display->sagv.block_time_us; 2944 2945 latency = skl_watermark_max_latency(i915, 1); 2946 2947 /* Wa_22020432604 */ 2948 if ((DISPLAY_VER(display) == 20 || DISPLAY_VER(display) == 30) && !latency) { 2949 latency += added_wake_time; 2950 added_wake_time = 0; 2951 } 2952 2953 /* Wa_22020299601 */ 2954 if ((latency && max_linetime) && 2955 (DISPLAY_VER(display) == 20 || DISPLAY_VER(display) == 30)) { 2956 latency = max_linetime * DIV_ROUND_UP(latency, max_linetime); 2957 } else if (!latency) { 2958 latency = LNL_PKG_C_LATENCY_MASK; 2959 } 2960 } 2961 2962 clear = LNL_ADDED_WAKE_TIME_MASK | LNL_PKG_C_LATENCY_MASK; 2963 val = REG_FIELD_PREP(LNL_PKG_C_LATENCY_MASK, latency) | 2964 REG_FIELD_PREP(LNL_ADDED_WAKE_TIME_MASK, added_wake_time); 2965 2966 intel_de_rmw(display, LNL_PKG_C_LATENCY, clear, val); 2967 } 2968 2969 static int 2970 skl_compute_wm(struct intel_atomic_state *state) 2971 { 2972 struct intel_crtc *crtc; 2973 struct intel_crtc_state __maybe_unused *new_crtc_state; 2974 int ret, i; 2975 2976 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { 2977 ret = skl_build_pipe_wm(state, crtc); 2978 if (ret) 2979 return ret; 2980 } 2981 2982 ret = skl_compute_ddb(state); 2983 if (ret) 2984 return ret; 2985 2986 ret = intel_compute_sagv_mask(state); 2987 if (ret) 2988 return ret; 2989 2990 /* 2991 * skl_compute_ddb() will have adjusted the final watermarks 2992 * based on how much ddb is available. Now we can actually 2993 * check if the final watermarks changed. 2994 */ 2995 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { 2996 ret = skl_wm_add_affected_planes(state, crtc); 2997 if (ret) 2998 return ret; 2999 } 3000 3001 skl_print_wm_changes(state); 3002 3003 return 0; 3004 } 3005 3006 static void skl_wm_level_from_reg_val(struct intel_display *display, 3007 u32 val, struct skl_wm_level *level) 3008 { 3009 level->enable = val & PLANE_WM_EN; 3010 level->ignore_lines = val & PLANE_WM_IGNORE_LINES; 3011 level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val); 3012 level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val); 3013 level->auto_min_alloc_wm_enable = DISPLAY_VER(display) >= 30 ? 3014 val & PLANE_WM_AUTO_MIN_ALLOC_EN : 0; 3015 } 3016 3017 static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc, 3018 struct skl_pipe_wm *out) 3019 { 3020 struct intel_display *display = to_intel_display(crtc); 3021 enum pipe pipe = crtc->pipe; 3022 enum plane_id plane_id; 3023 int level; 3024 u32 val; 3025 3026 for_each_plane_id_on_crtc(crtc, plane_id) { 3027 struct skl_plane_wm *wm = &out->planes[plane_id]; 3028 3029 for (level = 0; level < display->wm.num_levels; level++) { 3030 if (plane_id != PLANE_CURSOR) 3031 val = intel_de_read(display, PLANE_WM(pipe, plane_id, level)); 3032 else 3033 val = intel_de_read(display, CUR_WM(pipe, level)); 3034 3035 skl_wm_level_from_reg_val(display, val, &wm->wm[level]); 3036 } 3037 3038 if (plane_id != PLANE_CURSOR) 3039 val = intel_de_read(display, PLANE_WM_TRANS(pipe, plane_id)); 3040 else 3041 val = intel_de_read(display, CUR_WM_TRANS(pipe)); 3042 3043 skl_wm_level_from_reg_val(display, val, &wm->trans_wm); 3044 3045 if (HAS_HW_SAGV_WM(display)) { 3046 if (plane_id != PLANE_CURSOR) 3047 val = intel_de_read(display, PLANE_WM_SAGV(pipe, plane_id)); 3048 else 3049 val = intel_de_read(display, CUR_WM_SAGV(pipe)); 3050 3051 skl_wm_level_from_reg_val(display, val, &wm->sagv.wm0); 3052 3053 if (plane_id != PLANE_CURSOR) 3054 val = intel_de_read(display, PLANE_WM_SAGV_TRANS(pipe, plane_id)); 3055 else 3056 val = intel_de_read(display, CUR_WM_SAGV_TRANS(pipe)); 3057 3058 skl_wm_level_from_reg_val(display, val, &wm->sagv.trans_wm); 3059 } else if (DISPLAY_VER(display) >= 12) { 3060 wm->sagv.wm0 = wm->wm[0]; 3061 wm->sagv.trans_wm = wm->trans_wm; 3062 } 3063 } 3064 } 3065 3066 static void skl_wm_get_hw_state(struct drm_i915_private *i915) 3067 { 3068 struct intel_display *display = &i915->display; 3069 struct intel_dbuf_state *dbuf_state = 3070 to_intel_dbuf_state(i915->display.dbuf.obj.state); 3071 struct intel_crtc *crtc; 3072 3073 if (HAS_MBUS_JOINING(display)) 3074 dbuf_state->joined_mbus = intel_de_read(display, MBUS_CTL) & MBUS_JOIN; 3075 3076 dbuf_state->mdclk_cdclk_ratio = intel_mdclk_cdclk_ratio(display, &display->cdclk.hw); 3077 3078 for_each_intel_crtc(display->drm, crtc) { 3079 struct intel_crtc_state *crtc_state = 3080 to_intel_crtc_state(crtc->base.state); 3081 enum pipe pipe = crtc->pipe; 3082 unsigned int mbus_offset; 3083 enum plane_id plane_id; 3084 u8 slices; 3085 3086 memset(&crtc_state->wm.skl.optimal, 0, 3087 sizeof(crtc_state->wm.skl.optimal)); 3088 if (crtc_state->hw.active) 3089 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal); 3090 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal; 3091 3092 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe])); 3093 3094 for_each_plane_id_on_crtc(crtc, plane_id) { 3095 struct skl_ddb_entry *ddb = 3096 &crtc_state->wm.skl.plane_ddb[plane_id]; 3097 struct skl_ddb_entry *ddb_y = 3098 &crtc_state->wm.skl.plane_ddb_y[plane_id]; 3099 u16 *min_ddb = 3100 &crtc_state->wm.skl.plane_min_ddb[plane_id]; 3101 u16 *interim_ddb = 3102 &crtc_state->wm.skl.plane_interim_ddb[plane_id]; 3103 3104 if (!crtc_state->hw.active) 3105 continue; 3106 3107 skl_ddb_get_hw_plane_state(i915, crtc->pipe, 3108 plane_id, ddb, ddb_y, 3109 min_ddb, interim_ddb); 3110 3111 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb); 3112 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y); 3113 } 3114 3115 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state); 3116 3117 /* 3118 * Used for checking overlaps, so we need absolute 3119 * offsets instead of MBUS relative offsets. 3120 */ 3121 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes, 3122 dbuf_state->joined_mbus); 3123 mbus_offset = mbus_ddb_offset(i915, slices); 3124 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start; 3125 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end; 3126 3127 /* The slices actually used by the planes on the pipe */ 3128 dbuf_state->slices[pipe] = 3129 skl_ddb_dbuf_slice_mask(i915, &crtc_state->wm.skl.ddb); 3130 3131 drm_dbg_kms(display->drm, 3132 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n", 3133 crtc->base.base.id, crtc->base.name, 3134 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start, 3135 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes, 3136 str_yes_no(dbuf_state->joined_mbus)); 3137 } 3138 3139 dbuf_state->enabled_slices = display->dbuf.enabled_slices; 3140 } 3141 3142 bool skl_watermark_ipc_enabled(struct drm_i915_private *i915) 3143 { 3144 return i915->display.wm.ipc_enabled; 3145 } 3146 3147 void skl_watermark_ipc_update(struct drm_i915_private *i915) 3148 { 3149 if (!HAS_IPC(i915)) 3150 return; 3151 3152 intel_de_rmw(i915, DISP_ARB_CTL2, DISP_IPC_ENABLE, 3153 skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0); 3154 } 3155 3156 static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915) 3157 { 3158 /* Display WA #0477 WaDisableIPC: skl */ 3159 if (IS_SKYLAKE(i915)) 3160 return false; 3161 3162 /* Display WA #1141: SKL:all KBL:all CFL */ 3163 if (IS_KABYLAKE(i915) || 3164 IS_COFFEELAKE(i915) || 3165 IS_COMETLAKE(i915)) 3166 return i915->dram_info.symmetric_memory; 3167 3168 return true; 3169 } 3170 3171 void skl_watermark_ipc_init(struct drm_i915_private *i915) 3172 { 3173 if (!HAS_IPC(i915)) 3174 return; 3175 3176 i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915); 3177 3178 skl_watermark_ipc_update(i915); 3179 } 3180 3181 static void 3182 adjust_wm_latency(struct drm_i915_private *i915, 3183 u16 wm[], int num_levels, int read_latency) 3184 { 3185 bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed; 3186 int i, level; 3187 3188 /* 3189 * If a level n (n > 1) has a 0us latency, all levels m (m >= n) 3190 * need to be disabled. We make sure to sanitize the values out 3191 * of the punit to satisfy this requirement. 3192 */ 3193 for (level = 1; level < num_levels; level++) { 3194 if (wm[level] == 0) { 3195 for (i = level + 1; i < num_levels; i++) 3196 wm[i] = 0; 3197 3198 num_levels = level; 3199 break; 3200 } 3201 } 3202 3203 /* 3204 * WaWmMemoryReadLatency 3205 * 3206 * punit doesn't take into account the read latency so we need 3207 * to add proper adjustement to each valid level we retrieve 3208 * from the punit when level 0 response data is 0us. 3209 */ 3210 if (wm[0] == 0) { 3211 for (level = 0; level < num_levels; level++) 3212 wm[level] += read_latency; 3213 } 3214 3215 /* 3216 * WA Level-0 adjustment for 16GB DIMMs: SKL+ 3217 * If we could not get dimm info enable this WA to prevent from 3218 * any underrun. If not able to get Dimm info assume 16GB dimm 3219 * to avoid any underrun. 3220 */ 3221 if (wm_lv_0_adjust_needed) 3222 wm[0] += 1; 3223 } 3224 3225 static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) 3226 { 3227 int num_levels = i915->display.wm.num_levels; 3228 u32 val; 3229 3230 val = intel_de_read(i915, MTL_LATENCY_LP0_LP1); 3231 wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val); 3232 wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val); 3233 3234 val = intel_de_read(i915, MTL_LATENCY_LP2_LP3); 3235 wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val); 3236 wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val); 3237 3238 val = intel_de_read(i915, MTL_LATENCY_LP4_LP5); 3239 wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val); 3240 wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val); 3241 3242 adjust_wm_latency(i915, wm, num_levels, 6); 3243 } 3244 3245 static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) 3246 { 3247 int num_levels = i915->display.wm.num_levels; 3248 int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2; 3249 int mult = IS_DG2(i915) ? 2 : 1; 3250 u32 val; 3251 int ret; 3252 3253 /* read the first set of memory latencies[0:3] */ 3254 val = 0; /* data0 to be programmed to 0 for first set */ 3255 ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL); 3256 if (ret) { 3257 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret); 3258 return; 3259 } 3260 3261 wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult; 3262 wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult; 3263 wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult; 3264 wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult; 3265 3266 /* read the second set of memory latencies[4:7] */ 3267 val = 1; /* data0 to be programmed to 1 for second set */ 3268 ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL); 3269 if (ret) { 3270 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret); 3271 return; 3272 } 3273 3274 wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult; 3275 wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult; 3276 wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult; 3277 wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult; 3278 3279 adjust_wm_latency(i915, wm, num_levels, read_latency); 3280 } 3281 3282 static void skl_setup_wm_latency(struct drm_i915_private *i915) 3283 { 3284 struct intel_display *display = &i915->display; 3285 3286 if (HAS_HW_SAGV_WM(display)) 3287 display->wm.num_levels = 6; 3288 else 3289 display->wm.num_levels = 8; 3290 3291 if (DISPLAY_VER(display) >= 14) 3292 mtl_read_wm_latency(i915, display->wm.skl_latency); 3293 else 3294 skl_read_wm_latency(i915, display->wm.skl_latency); 3295 3296 intel_print_wm_latency(i915, "Gen9 Plane", display->wm.skl_latency); 3297 } 3298 3299 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj) 3300 { 3301 struct intel_dbuf_state *dbuf_state; 3302 3303 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL); 3304 if (!dbuf_state) 3305 return NULL; 3306 3307 return &dbuf_state->base; 3308 } 3309 3310 static void intel_dbuf_destroy_state(struct intel_global_obj *obj, 3311 struct intel_global_state *state) 3312 { 3313 kfree(state); 3314 } 3315 3316 static const struct intel_global_state_funcs intel_dbuf_funcs = { 3317 .atomic_duplicate_state = intel_dbuf_duplicate_state, 3318 .atomic_destroy_state = intel_dbuf_destroy_state, 3319 }; 3320 3321 struct intel_dbuf_state * 3322 intel_atomic_get_dbuf_state(struct intel_atomic_state *state) 3323 { 3324 struct drm_i915_private *i915 = to_i915(state->base.dev); 3325 struct intel_global_state *dbuf_state; 3326 3327 dbuf_state = intel_atomic_get_global_obj_state(state, &i915->display.dbuf.obj); 3328 if (IS_ERR(dbuf_state)) 3329 return ERR_CAST(dbuf_state); 3330 3331 return to_intel_dbuf_state(dbuf_state); 3332 } 3333 3334 int intel_dbuf_init(struct drm_i915_private *i915) 3335 { 3336 struct intel_display *display = &i915->display; 3337 struct intel_dbuf_state *dbuf_state; 3338 3339 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL); 3340 if (!dbuf_state) 3341 return -ENOMEM; 3342 3343 intel_atomic_global_obj_init(display, &display->dbuf.obj, 3344 &dbuf_state->base, &intel_dbuf_funcs); 3345 3346 return 0; 3347 } 3348 3349 static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes) 3350 { 3351 switch (pipe) { 3352 case PIPE_A: 3353 case PIPE_D: 3354 active_pipes &= BIT(PIPE_A) | BIT(PIPE_D); 3355 break; 3356 case PIPE_B: 3357 case PIPE_C: 3358 active_pipes &= BIT(PIPE_B) | BIT(PIPE_C); 3359 break; 3360 default: /* to suppress compiler warning */ 3361 MISSING_CASE(pipe); 3362 return false; 3363 } 3364 3365 return is_power_of_2(active_pipes); 3366 } 3367 3368 static u32 pipe_mbus_dbox_ctl(const struct intel_crtc *crtc, 3369 const struct intel_dbuf_state *dbuf_state) 3370 { 3371 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 3372 u32 val = 0; 3373 3374 if (DISPLAY_VER(i915) >= 14) 3375 val |= MBUS_DBOX_I_CREDIT(2); 3376 3377 if (DISPLAY_VER(i915) >= 12) { 3378 val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16); 3379 val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1); 3380 val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN; 3381 } 3382 3383 if (DISPLAY_VER(i915) >= 14) 3384 val |= dbuf_state->joined_mbus ? 3385 MBUS_DBOX_A_CREDIT(12) : MBUS_DBOX_A_CREDIT(8); 3386 else if (IS_ALDERLAKE_P(i915)) 3387 /* Wa_22010947358:adl-p */ 3388 val |= dbuf_state->joined_mbus ? 3389 MBUS_DBOX_A_CREDIT(6) : MBUS_DBOX_A_CREDIT(4); 3390 else 3391 val |= MBUS_DBOX_A_CREDIT(2); 3392 3393 if (DISPLAY_VER(i915) >= 14) { 3394 val |= MBUS_DBOX_B_CREDIT(0xA); 3395 } else if (IS_ALDERLAKE_P(i915)) { 3396 val |= MBUS_DBOX_BW_CREDIT(2); 3397 val |= MBUS_DBOX_B_CREDIT(8); 3398 } else if (DISPLAY_VER(i915) >= 12) { 3399 val |= MBUS_DBOX_BW_CREDIT(2); 3400 val |= MBUS_DBOX_B_CREDIT(12); 3401 } else { 3402 val |= MBUS_DBOX_BW_CREDIT(1); 3403 val |= MBUS_DBOX_B_CREDIT(8); 3404 } 3405 3406 if (DISPLAY_VERx100(i915) == 1400) { 3407 if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe, dbuf_state->active_pipes)) 3408 val |= MBUS_DBOX_BW_8CREDITS_MTL; 3409 else 3410 val |= MBUS_DBOX_BW_4CREDITS_MTL; 3411 } 3412 3413 return val; 3414 } 3415 3416 static void pipe_mbus_dbox_ctl_update(struct drm_i915_private *i915, 3417 const struct intel_dbuf_state *dbuf_state) 3418 { 3419 struct intel_crtc *crtc; 3420 3421 for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, dbuf_state->active_pipes) 3422 intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), 3423 pipe_mbus_dbox_ctl(crtc, dbuf_state)); 3424 } 3425 3426 static void intel_mbus_dbox_update(struct intel_atomic_state *state) 3427 { 3428 struct drm_i915_private *i915 = to_i915(state->base.dev); 3429 const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state; 3430 3431 if (DISPLAY_VER(i915) < 11) 3432 return; 3433 3434 new_dbuf_state = intel_atomic_get_new_dbuf_state(state); 3435 old_dbuf_state = intel_atomic_get_old_dbuf_state(state); 3436 if (!new_dbuf_state || 3437 (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus && 3438 new_dbuf_state->active_pipes == old_dbuf_state->active_pipes)) 3439 return; 3440 3441 pipe_mbus_dbox_ctl_update(i915, new_dbuf_state); 3442 } 3443 3444 int intel_dbuf_state_set_mdclk_cdclk_ratio(struct intel_atomic_state *state, 3445 int ratio) 3446 { 3447 struct intel_dbuf_state *dbuf_state; 3448 3449 dbuf_state = intel_atomic_get_dbuf_state(state); 3450 if (IS_ERR(dbuf_state)) 3451 return PTR_ERR(dbuf_state); 3452 3453 dbuf_state->mdclk_cdclk_ratio = ratio; 3454 3455 return intel_atomic_lock_global_state(&dbuf_state->base); 3456 } 3457 3458 void intel_dbuf_mdclk_cdclk_ratio_update(struct drm_i915_private *i915, 3459 int ratio, bool joined_mbus) 3460 { 3461 struct intel_display *display = &i915->display; 3462 enum dbuf_slice slice; 3463 3464 if (!HAS_MBUS_JOINING(display)) 3465 return; 3466 3467 if (DISPLAY_VER(display) >= 20) 3468 intel_de_rmw(display, MBUS_CTL, MBUS_TRANSLATION_THROTTLE_MIN_MASK, 3469 MBUS_TRANSLATION_THROTTLE_MIN(ratio - 1)); 3470 3471 if (joined_mbus) 3472 ratio *= 2; 3473 3474 drm_dbg_kms(display->drm, "Updating dbuf ratio to %d (mbus joined: %s)\n", 3475 ratio, str_yes_no(joined_mbus)); 3476 3477 for_each_dbuf_slice(display, slice) 3478 intel_de_rmw(display, DBUF_CTL_S(slice), 3479 DBUF_MIN_TRACKER_STATE_SERVICE_MASK, 3480 DBUF_MIN_TRACKER_STATE_SERVICE(ratio - 1)); 3481 } 3482 3483 static void intel_dbuf_mdclk_min_tracker_update(struct intel_atomic_state *state) 3484 { 3485 struct drm_i915_private *i915 = to_i915(state->base.dev); 3486 const struct intel_dbuf_state *old_dbuf_state = 3487 intel_atomic_get_old_dbuf_state(state); 3488 const struct intel_dbuf_state *new_dbuf_state = 3489 intel_atomic_get_new_dbuf_state(state); 3490 int mdclk_cdclk_ratio; 3491 3492 if (intel_cdclk_is_decreasing_later(state)) { 3493 /* cdclk/mdclk will be changed later by intel_set_cdclk_post_plane_update() */ 3494 mdclk_cdclk_ratio = old_dbuf_state->mdclk_cdclk_ratio; 3495 } else { 3496 /* cdclk/mdclk already changed by intel_set_cdclk_pre_plane_update() */ 3497 mdclk_cdclk_ratio = new_dbuf_state->mdclk_cdclk_ratio; 3498 } 3499 3500 intel_dbuf_mdclk_cdclk_ratio_update(i915, mdclk_cdclk_ratio, 3501 new_dbuf_state->joined_mbus); 3502 } 3503 3504 static enum pipe intel_mbus_joined_pipe(struct intel_atomic_state *state, 3505 const struct intel_dbuf_state *dbuf_state) 3506 { 3507 struct intel_display *display = to_intel_display(state); 3508 struct drm_i915_private *i915 = to_i915(state->base.dev); 3509 enum pipe pipe = ffs(dbuf_state->active_pipes) - 1; 3510 const struct intel_crtc_state *new_crtc_state; 3511 struct intel_crtc *crtc; 3512 3513 drm_WARN_ON(&i915->drm, !dbuf_state->joined_mbus); 3514 drm_WARN_ON(&i915->drm, !is_power_of_2(dbuf_state->active_pipes)); 3515 3516 crtc = intel_crtc_for_pipe(display, pipe); 3517 new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc); 3518 3519 if (new_crtc_state && !intel_crtc_needs_modeset(new_crtc_state)) 3520 return pipe; 3521 else 3522 return INVALID_PIPE; 3523 } 3524 3525 static void mbus_ctl_join_update(struct drm_i915_private *i915, 3526 const struct intel_dbuf_state *dbuf_state, 3527 enum pipe pipe) 3528 { 3529 u32 mbus_ctl; 3530 3531 if (dbuf_state->joined_mbus) 3532 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN; 3533 else 3534 mbus_ctl = MBUS_HASHING_MODE_2x2; 3535 3536 if (pipe != INVALID_PIPE) 3537 mbus_ctl |= MBUS_JOIN_PIPE_SELECT(pipe); 3538 else 3539 mbus_ctl |= MBUS_JOIN_PIPE_SELECT_NONE; 3540 3541 intel_de_rmw(i915, MBUS_CTL, 3542 MBUS_HASHING_MODE_MASK | MBUS_JOIN | 3543 MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl); 3544 } 3545 3546 static void intel_dbuf_mbus_join_update(struct intel_atomic_state *state, 3547 enum pipe pipe) 3548 { 3549 struct drm_i915_private *i915 = to_i915(state->base.dev); 3550 const struct intel_dbuf_state *old_dbuf_state = 3551 intel_atomic_get_old_dbuf_state(state); 3552 const struct intel_dbuf_state *new_dbuf_state = 3553 intel_atomic_get_new_dbuf_state(state); 3554 3555 drm_dbg_kms(&i915->drm, "Changing mbus joined: %s -> %s (pipe: %c)\n", 3556 str_yes_no(old_dbuf_state->joined_mbus), 3557 str_yes_no(new_dbuf_state->joined_mbus), 3558 pipe != INVALID_PIPE ? pipe_name(pipe) : '*'); 3559 3560 mbus_ctl_join_update(i915, new_dbuf_state, pipe); 3561 } 3562 3563 void intel_dbuf_mbus_pre_ddb_update(struct intel_atomic_state *state) 3564 { 3565 const struct intel_dbuf_state *new_dbuf_state = 3566 intel_atomic_get_new_dbuf_state(state); 3567 const struct intel_dbuf_state *old_dbuf_state = 3568 intel_atomic_get_old_dbuf_state(state); 3569 3570 if (!new_dbuf_state) 3571 return; 3572 3573 if (!old_dbuf_state->joined_mbus && new_dbuf_state->joined_mbus) { 3574 enum pipe pipe = intel_mbus_joined_pipe(state, new_dbuf_state); 3575 3576 WARN_ON(!new_dbuf_state->base.changed); 3577 3578 intel_dbuf_mbus_join_update(state, pipe); 3579 intel_mbus_dbox_update(state); 3580 intel_dbuf_mdclk_min_tracker_update(state); 3581 } 3582 } 3583 3584 void intel_dbuf_mbus_post_ddb_update(struct intel_atomic_state *state) 3585 { 3586 struct intel_display *display = to_intel_display(state); 3587 const struct intel_dbuf_state *new_dbuf_state = 3588 intel_atomic_get_new_dbuf_state(state); 3589 const struct intel_dbuf_state *old_dbuf_state = 3590 intel_atomic_get_old_dbuf_state(state); 3591 3592 if (!new_dbuf_state) 3593 return; 3594 3595 if (old_dbuf_state->joined_mbus && !new_dbuf_state->joined_mbus) { 3596 enum pipe pipe = intel_mbus_joined_pipe(state, old_dbuf_state); 3597 3598 WARN_ON(!new_dbuf_state->base.changed); 3599 3600 intel_dbuf_mdclk_min_tracker_update(state); 3601 intel_mbus_dbox_update(state); 3602 intel_dbuf_mbus_join_update(state, pipe); 3603 3604 if (pipe != INVALID_PIPE) { 3605 struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe); 3606 3607 intel_crtc_wait_for_next_vblank(crtc); 3608 } 3609 } else if (old_dbuf_state->joined_mbus == new_dbuf_state->joined_mbus && 3610 old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) { 3611 WARN_ON(!new_dbuf_state->base.changed); 3612 3613 intel_dbuf_mdclk_min_tracker_update(state); 3614 intel_mbus_dbox_update(state); 3615 } 3616 3617 } 3618 3619 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state) 3620 { 3621 struct drm_i915_private *i915 = to_i915(state->base.dev); 3622 const struct intel_dbuf_state *new_dbuf_state = 3623 intel_atomic_get_new_dbuf_state(state); 3624 const struct intel_dbuf_state *old_dbuf_state = 3625 intel_atomic_get_old_dbuf_state(state); 3626 u8 old_slices, new_slices; 3627 3628 if (!new_dbuf_state) 3629 return; 3630 3631 old_slices = old_dbuf_state->enabled_slices; 3632 new_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices; 3633 3634 if (old_slices == new_slices) 3635 return; 3636 3637 WARN_ON(!new_dbuf_state->base.changed); 3638 3639 gen9_dbuf_slices_update(i915, new_slices); 3640 } 3641 3642 void intel_dbuf_post_plane_update(struct intel_atomic_state *state) 3643 { 3644 struct drm_i915_private *i915 = to_i915(state->base.dev); 3645 const struct intel_dbuf_state *new_dbuf_state = 3646 intel_atomic_get_new_dbuf_state(state); 3647 const struct intel_dbuf_state *old_dbuf_state = 3648 intel_atomic_get_old_dbuf_state(state); 3649 u8 old_slices, new_slices; 3650 3651 if (!new_dbuf_state) 3652 return; 3653 3654 old_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices; 3655 new_slices = new_dbuf_state->enabled_slices; 3656 3657 if (old_slices == new_slices) 3658 return; 3659 3660 WARN_ON(!new_dbuf_state->base.changed); 3661 3662 gen9_dbuf_slices_update(i915, new_slices); 3663 } 3664 3665 static void skl_mbus_sanitize(struct drm_i915_private *i915) 3666 { 3667 struct intel_display *display = &i915->display; 3668 struct intel_dbuf_state *dbuf_state = 3669 to_intel_dbuf_state(display->dbuf.obj.state); 3670 3671 if (!HAS_MBUS_JOINING(display)) 3672 return; 3673 3674 if (!dbuf_state->joined_mbus || 3675 adlp_check_mbus_joined(dbuf_state->active_pipes)) 3676 return; 3677 3678 drm_dbg_kms(display->drm, "Disabling redundant MBUS joining (active pipes 0x%x)\n", 3679 dbuf_state->active_pipes); 3680 3681 dbuf_state->joined_mbus = false; 3682 intel_dbuf_mdclk_cdclk_ratio_update(i915, 3683 dbuf_state->mdclk_cdclk_ratio, 3684 dbuf_state->joined_mbus); 3685 pipe_mbus_dbox_ctl_update(i915, dbuf_state); 3686 mbus_ctl_join_update(i915, dbuf_state, INVALID_PIPE); 3687 } 3688 3689 static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915) 3690 { 3691 const struct intel_dbuf_state *dbuf_state = 3692 to_intel_dbuf_state(i915->display.dbuf.obj.state); 3693 struct skl_ddb_entry entries[I915_MAX_PIPES] = {}; 3694 struct intel_crtc *crtc; 3695 3696 for_each_intel_crtc(&i915->drm, crtc) { 3697 const struct intel_crtc_state *crtc_state = 3698 to_intel_crtc_state(crtc->base.state); 3699 3700 entries[crtc->pipe] = crtc_state->wm.skl.ddb; 3701 } 3702 3703 for_each_intel_crtc(&i915->drm, crtc) { 3704 const struct intel_crtc_state *crtc_state = 3705 to_intel_crtc_state(crtc->base.state); 3706 u8 slices; 3707 3708 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes, 3709 dbuf_state->joined_mbus); 3710 if (dbuf_state->slices[crtc->pipe] & ~slices) 3711 return true; 3712 3713 if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries, 3714 I915_MAX_PIPES, crtc->pipe)) 3715 return true; 3716 } 3717 3718 return false; 3719 } 3720 3721 static void skl_dbuf_sanitize(struct drm_i915_private *i915) 3722 { 3723 struct intel_crtc *crtc; 3724 3725 /* 3726 * On TGL/RKL (at least) the BIOS likes to assign the planes 3727 * to the wrong DBUF slices. This will cause an infinite loop 3728 * in skl_commit_modeset_enables() as it can't find a way to 3729 * transition between the old bogus DBUF layout to the new 3730 * proper DBUF layout without DBUF allocation overlaps between 3731 * the planes (which cannot be allowed or else the hardware 3732 * may hang). If we detect a bogus DBUF layout just turn off 3733 * all the planes so that skl_commit_modeset_enables() can 3734 * simply ignore them. 3735 */ 3736 if (!skl_dbuf_is_misconfigured(i915)) 3737 return; 3738 3739 drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n"); 3740 3741 for_each_intel_crtc(&i915->drm, crtc) { 3742 struct intel_plane *plane = to_intel_plane(crtc->base.primary); 3743 const struct intel_plane_state *plane_state = 3744 to_intel_plane_state(plane->base.state); 3745 struct intel_crtc_state *crtc_state = 3746 to_intel_crtc_state(crtc->base.state); 3747 3748 if (plane_state->uapi.visible) 3749 intel_plane_disable_noatomic(crtc, plane); 3750 3751 drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0); 3752 3753 memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb)); 3754 } 3755 } 3756 3757 static void skl_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915) 3758 { 3759 skl_wm_get_hw_state(i915); 3760 3761 skl_mbus_sanitize(i915); 3762 skl_dbuf_sanitize(i915); 3763 } 3764 3765 void intel_wm_state_verify(struct intel_atomic_state *state, 3766 struct intel_crtc *crtc) 3767 { 3768 struct intel_display *display = to_intel_display(state); 3769 struct drm_i915_private *i915 = to_i915(state->base.dev); 3770 const struct intel_crtc_state *new_crtc_state = 3771 intel_atomic_get_new_crtc_state(state, crtc); 3772 struct skl_hw_state { 3773 struct skl_ddb_entry ddb[I915_MAX_PLANES]; 3774 struct skl_ddb_entry ddb_y[I915_MAX_PLANES]; 3775 u16 min_ddb[I915_MAX_PLANES]; 3776 u16 interim_ddb[I915_MAX_PLANES]; 3777 struct skl_pipe_wm wm; 3778 } *hw; 3779 const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal; 3780 struct intel_plane *plane; 3781 u8 hw_enabled_slices; 3782 int level; 3783 3784 if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active) 3785 return; 3786 3787 hw = kzalloc(sizeof(*hw), GFP_KERNEL); 3788 if (!hw) 3789 return; 3790 3791 skl_pipe_wm_get_hw_state(crtc, &hw->wm); 3792 3793 skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y, hw->min_ddb, hw->interim_ddb); 3794 3795 hw_enabled_slices = intel_enabled_dbuf_slices_mask(i915); 3796 3797 if (DISPLAY_VER(i915) >= 11 && 3798 hw_enabled_slices != i915->display.dbuf.enabled_slices) 3799 drm_err(&i915->drm, 3800 "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n", 3801 i915->display.dbuf.enabled_slices, 3802 hw_enabled_slices); 3803 3804 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { 3805 const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry; 3806 const struct skl_wm_level *hw_wm_level, *sw_wm_level; 3807 3808 /* Watermarks */ 3809 for (level = 0; level < i915->display.wm.num_levels; level++) { 3810 hw_wm_level = &hw->wm.planes[plane->id].wm[level]; 3811 sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level); 3812 3813 if (skl_wm_level_equals(hw_wm_level, sw_wm_level)) 3814 continue; 3815 3816 drm_err(&i915->drm, 3817 "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", 3818 plane->base.base.id, plane->base.name, level, 3819 sw_wm_level->enable, 3820 sw_wm_level->blocks, 3821 sw_wm_level->lines, 3822 hw_wm_level->enable, 3823 hw_wm_level->blocks, 3824 hw_wm_level->lines); 3825 } 3826 3827 hw_wm_level = &hw->wm.planes[plane->id].trans_wm; 3828 sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id); 3829 3830 if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) { 3831 drm_err(&i915->drm, 3832 "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", 3833 plane->base.base.id, plane->base.name, 3834 sw_wm_level->enable, 3835 sw_wm_level->blocks, 3836 sw_wm_level->lines, 3837 hw_wm_level->enable, 3838 hw_wm_level->blocks, 3839 hw_wm_level->lines); 3840 } 3841 3842 hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0; 3843 sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0; 3844 3845 if (HAS_HW_SAGV_WM(display) && 3846 !skl_wm_level_equals(hw_wm_level, sw_wm_level)) { 3847 drm_err(&i915->drm, 3848 "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", 3849 plane->base.base.id, plane->base.name, 3850 sw_wm_level->enable, 3851 sw_wm_level->blocks, 3852 sw_wm_level->lines, 3853 hw_wm_level->enable, 3854 hw_wm_level->blocks, 3855 hw_wm_level->lines); 3856 } 3857 3858 hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm; 3859 sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm; 3860 3861 if (HAS_HW_SAGV_WM(display) && 3862 !skl_wm_level_equals(hw_wm_level, sw_wm_level)) { 3863 drm_err(&i915->drm, 3864 "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", 3865 plane->base.base.id, plane->base.name, 3866 sw_wm_level->enable, 3867 sw_wm_level->blocks, 3868 sw_wm_level->lines, 3869 hw_wm_level->enable, 3870 hw_wm_level->blocks, 3871 hw_wm_level->lines); 3872 } 3873 3874 /* DDB */ 3875 hw_ddb_entry = &hw->ddb[PLANE_CURSOR]; 3876 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR]; 3877 3878 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) { 3879 drm_err(&i915->drm, 3880 "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n", 3881 plane->base.base.id, plane->base.name, 3882 sw_ddb_entry->start, sw_ddb_entry->end, 3883 hw_ddb_entry->start, hw_ddb_entry->end); 3884 } 3885 } 3886 3887 kfree(hw); 3888 } 3889 3890 static const struct intel_wm_funcs skl_wm_funcs = { 3891 .compute_global_watermarks = skl_compute_wm, 3892 .get_hw_state = skl_wm_get_hw_state_and_sanitize, 3893 }; 3894 3895 void skl_wm_init(struct drm_i915_private *i915) 3896 { 3897 intel_sagv_init(i915); 3898 3899 skl_setup_wm_latency(i915); 3900 3901 i915->display.funcs.wm = &skl_wm_funcs; 3902 } 3903 3904 static int skl_watermark_ipc_status_show(struct seq_file *m, void *data) 3905 { 3906 struct drm_i915_private *i915 = m->private; 3907 3908 seq_printf(m, "Isochronous Priority Control: %s\n", 3909 str_yes_no(skl_watermark_ipc_enabled(i915))); 3910 return 0; 3911 } 3912 3913 static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file) 3914 { 3915 struct drm_i915_private *i915 = inode->i_private; 3916 3917 return single_open(file, skl_watermark_ipc_status_show, i915); 3918 } 3919 3920 static ssize_t skl_watermark_ipc_status_write(struct file *file, 3921 const char __user *ubuf, 3922 size_t len, loff_t *offp) 3923 { 3924 struct seq_file *m = file->private_data; 3925 struct drm_i915_private *i915 = m->private; 3926 intel_wakeref_t wakeref; 3927 bool enable; 3928 int ret; 3929 3930 ret = kstrtobool_from_user(ubuf, len, &enable); 3931 if (ret < 0) 3932 return ret; 3933 3934 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 3935 if (!skl_watermark_ipc_enabled(i915) && enable) 3936 drm_info(&i915->drm, 3937 "Enabling IPC: WM will be proper only after next commit\n"); 3938 i915->display.wm.ipc_enabled = enable; 3939 skl_watermark_ipc_update(i915); 3940 } 3941 3942 return len; 3943 } 3944 3945 static const struct file_operations skl_watermark_ipc_status_fops = { 3946 .owner = THIS_MODULE, 3947 .open = skl_watermark_ipc_status_open, 3948 .read = seq_read, 3949 .llseek = seq_lseek, 3950 .release = single_release, 3951 .write = skl_watermark_ipc_status_write 3952 }; 3953 3954 static int intel_sagv_status_show(struct seq_file *m, void *unused) 3955 { 3956 struct drm_i915_private *i915 = m->private; 3957 static const char * const sagv_status[] = { 3958 [I915_SAGV_UNKNOWN] = "unknown", 3959 [I915_SAGV_DISABLED] = "disabled", 3960 [I915_SAGV_ENABLED] = "enabled", 3961 [I915_SAGV_NOT_CONTROLLED] = "not controlled", 3962 }; 3963 3964 seq_printf(m, "SAGV available: %s\n", str_yes_no(intel_has_sagv(i915))); 3965 seq_printf(m, "SAGV modparam: %s\n", 3966 str_enabled_disabled(i915->display.params.enable_sagv)); 3967 seq_printf(m, "SAGV status: %s\n", sagv_status[i915->display.sagv.status]); 3968 seq_printf(m, "SAGV block time: %d usec\n", i915->display.sagv.block_time_us); 3969 3970 return 0; 3971 } 3972 3973 DEFINE_SHOW_ATTRIBUTE(intel_sagv_status); 3974 3975 void skl_watermark_debugfs_register(struct drm_i915_private *i915) 3976 { 3977 struct intel_display *display = &i915->display; 3978 struct drm_minor *minor = display->drm->primary; 3979 3980 if (HAS_IPC(display)) 3981 debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915, 3982 &skl_watermark_ipc_status_fops); 3983 3984 if (HAS_SAGV(display)) 3985 debugfs_create_file("i915_sagv_status", 0444, minor->debugfs_root, i915, 3986 &intel_sagv_status_fops); 3987 } 3988 3989 unsigned int skl_watermark_max_latency(struct drm_i915_private *i915, int initial_wm_level) 3990 { 3991 int level; 3992 3993 for (level = i915->display.wm.num_levels - 1; level >= initial_wm_level; level--) { 3994 unsigned int latency = skl_wm_latency(i915, level, NULL); 3995 3996 if (latency) 3997 return latency; 3998 } 3999 4000 return 0; 4001 } 4002