1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include <linux/iopoll.h> 7 8 #include <drm/drm_print.h> 9 10 #include "soc/intel_dram.h" 11 12 #include "i915_drv.h" 13 #include "i915_reg.h" 14 #include "i9xx_wm.h" 15 #include "i9xx_wm_regs.h" 16 #include "intel_atomic.h" 17 #include "intel_bo.h" 18 #include "intel_de.h" 19 #include "intel_display.h" 20 #include "intel_display_regs.h" 21 #include "intel_display_trace.h" 22 #include "intel_fb.h" 23 #include "intel_mchbar_regs.h" 24 #include "intel_wm.h" 25 #include "skl_watermark.h" 26 #include "vlv_sideband.h" 27 28 struct intel_watermark_params { 29 u16 fifo_size; 30 u16 max_wm; 31 u8 default_wm; 32 u8 guard_size; 33 u8 cacheline_size; 34 }; 35 36 /* used in computing the new watermarks state */ 37 struct intel_wm_config { 38 unsigned int num_pipes_active; 39 bool sprites_enabled; 40 bool sprites_scaled; 41 }; 42 43 struct cxsr_latency { 44 bool is_desktop : 1; 45 bool is_ddr3 : 1; 46 u16 fsb_freq; 47 u16 mem_freq; 48 u16 display_sr; 49 u16 display_hpll_disable; 50 u16 cursor_sr; 51 u16 cursor_hpll_disable; 52 }; 53 54 static const struct cxsr_latency cxsr_latency_table[] = { 55 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */ 56 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */ 57 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */ 58 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */ 59 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */ 60 61 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */ 62 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */ 63 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */ 64 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */ 65 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */ 66 67 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */ 68 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */ 69 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */ 70 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */ 71 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */ 72 73 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */ 74 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */ 75 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */ 76 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */ 77 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */ 78 79 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */ 80 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */ 81 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */ 82 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */ 83 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */ 84 85 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */ 86 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */ 87 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */ 88 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */ 89 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */ 90 }; 91 92 static const struct cxsr_latency *pnv_get_cxsr_latency(struct intel_display *display) 93 { 94 const struct dram_info *dram_info = intel_dram_info(display->drm); 95 bool is_ddr3 = dram_info->type == INTEL_DRAM_DDR3; 96 int i; 97 98 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) { 99 const struct cxsr_latency *latency = &cxsr_latency_table[i]; 100 bool is_desktop = !display->platform.mobile; 101 102 if (is_desktop == latency->is_desktop && 103 is_ddr3 == latency->is_ddr3 && 104 DIV_ROUND_CLOSEST(dram_info->fsb_freq, 1000) == latency->fsb_freq && 105 DIV_ROUND_CLOSEST(dram_info->mem_freq, 1000) == latency->mem_freq) 106 return latency; 107 } 108 109 drm_dbg_kms(display->drm, 110 "Could not find CxSR latency for %s, FSB %u kHz, MEM %u kHz\n", 111 intel_dram_type_str(dram_info->type), 112 dram_info->fsb_freq, dram_info->mem_freq); 113 114 return NULL; 115 } 116 117 static void chv_set_memory_dvfs(struct intel_display *display, bool enable) 118 { 119 u32 val; 120 int ret; 121 122 vlv_punit_get(display->drm); 123 124 val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2); 125 if (enable) 126 val &= ~FORCE_DDR_HIGH_FREQ; 127 else 128 val |= FORCE_DDR_HIGH_FREQ; 129 val &= ~FORCE_DDR_LOW_FREQ; 130 val |= FORCE_DDR_FREQ_REQ_ACK; 131 vlv_punit_write(display->drm, PUNIT_REG_DDR_SETUP2, val); 132 133 ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2), 134 (val & FORCE_DDR_FREQ_REQ_ACK) == 0, 135 500, 3000, false); 136 if (ret) 137 drm_err(display->drm, 138 "timed out waiting for Punit DDR DVFS request\n"); 139 140 vlv_punit_put(display->drm); 141 } 142 143 static void chv_set_memory_pm5(struct intel_display *display, bool enable) 144 { 145 u32 val; 146 147 vlv_punit_get(display->drm); 148 149 val = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM); 150 if (enable) 151 val |= DSP_MAXFIFO_PM5_ENABLE; 152 else 153 val &= ~DSP_MAXFIFO_PM5_ENABLE; 154 vlv_punit_write(display->drm, PUNIT_REG_DSPSSPM, val); 155 156 vlv_punit_put(display->drm); 157 } 158 159 #define FW_WM(value, plane) \ 160 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK) 161 162 static bool _intel_set_memory_cxsr(struct intel_display *display, bool enable) 163 { 164 bool was_enabled; 165 u32 val; 166 167 if (display->platform.valleyview || display->platform.cherryview) { 168 was_enabled = intel_de_read(display, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; 169 intel_de_write(display, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0); 170 intel_de_posting_read(display, FW_BLC_SELF_VLV); 171 } else if (display->platform.g4x || display->platform.i965gm) { 172 was_enabled = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN; 173 intel_de_write(display, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0); 174 intel_de_posting_read(display, FW_BLC_SELF); 175 } else if (display->platform.pineview) { 176 val = intel_de_read(display, DSPFW3(display)); 177 was_enabled = val & PINEVIEW_SELF_REFRESH_EN; 178 if (enable) 179 val |= PINEVIEW_SELF_REFRESH_EN; 180 else 181 val &= ~PINEVIEW_SELF_REFRESH_EN; 182 intel_de_write(display, DSPFW3(display), val); 183 intel_de_posting_read(display, DSPFW3(display)); 184 } else if (display->platform.i945g || display->platform.i945gm) { 185 was_enabled = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN; 186 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) : 187 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN); 188 intel_de_write(display, FW_BLC_SELF, val); 189 intel_de_posting_read(display, FW_BLC_SELF); 190 } else if (display->platform.i915gm) { 191 /* 192 * FIXME can't find a bit like this for 915G, and 193 * yet it does have the related watermark in 194 * FW_BLC_SELF. What's going on? 195 */ 196 was_enabled = intel_de_read(display, INSTPM) & INSTPM_SELF_EN; 197 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) : 198 _MASKED_BIT_DISABLE(INSTPM_SELF_EN); 199 intel_de_write(display, INSTPM, val); 200 intel_de_posting_read(display, INSTPM); 201 } else { 202 return false; 203 } 204 205 trace_intel_memory_cxsr(display, was_enabled, enable); 206 207 drm_dbg_kms(display->drm, "memory self-refresh is %s (was %s)\n", 208 str_enabled_disabled(enable), 209 str_enabled_disabled(was_enabled)); 210 211 return was_enabled; 212 } 213 214 /** 215 * intel_set_memory_cxsr - Configure CxSR state 216 * @display: display device 217 * @enable: Allow vs. disallow CxSR 218 * 219 * Allow or disallow the system to enter a special CxSR 220 * (C-state self refresh) state. What typically happens in CxSR mode 221 * is that several display FIFOs may get combined into a single larger 222 * FIFO for a particular plane (so called max FIFO mode) to allow the 223 * system to defer memory fetches longer, and the memory will enter 224 * self refresh. 225 * 226 * Note that enabling CxSR does not guarantee that the system enter 227 * this special mode, nor does it guarantee that the system stays 228 * in that mode once entered. So this just allows/disallows the system 229 * to autonomously utilize the CxSR mode. Other factors such as core 230 * C-states will affect when/if the system actually enters/exits the 231 * CxSR mode. 232 * 233 * Note that on VLV/CHV this actually only controls the max FIFO mode, 234 * and the system is free to enter/exit memory self refresh at any time 235 * even when the use of CxSR has been disallowed. 236 * 237 * While the system is actually in the CxSR/max FIFO mode, some plane 238 * control registers will not get latched on vblank. Thus in order to 239 * guarantee the system will respond to changes in the plane registers 240 * we must always disallow CxSR prior to making changes to those registers. 241 * Unfortunately the system will re-evaluate the CxSR conditions at 242 * frame start which happens after vblank start (which is when the plane 243 * registers would get latched), so we can't proceed with the plane update 244 * during the same frame where we disallowed CxSR. 245 * 246 * Certain platforms also have a deeper HPLL SR mode. Fortunately the 247 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold 248 * the hardware w.r.t. HPLL SR when writing to plane registers. 249 * Disallowing just CxSR is sufficient. 250 */ 251 bool intel_set_memory_cxsr(struct intel_display *display, bool enable) 252 { 253 bool ret; 254 255 mutex_lock(&display->wm.wm_mutex); 256 ret = _intel_set_memory_cxsr(display, enable); 257 if (display->platform.valleyview || display->platform.cherryview) 258 display->wm.vlv.cxsr = enable; 259 else if (display->platform.g4x) 260 display->wm.g4x.cxsr = enable; 261 mutex_unlock(&display->wm.wm_mutex); 262 263 return ret; 264 } 265 266 /* 267 * Latency for FIFO fetches is dependent on several factors: 268 * - memory configuration (speed, channels) 269 * - chipset 270 * - current MCH state 271 * It can be fairly high in some situations, so here we assume a fairly 272 * pessimal value. It's a tradeoff between extra memory fetches (if we 273 * set this value too high, the FIFO will fetch frequently to stay full) 274 * and power consumption (set it too low to save power and we might see 275 * FIFO underruns and display "flicker"). 276 * 277 * A value of 5us seems to be a good balance; safe for very low end 278 * platforms but not overly aggressive on lower latency configs. 279 */ 280 static const int pessimal_latency_ns = 5000; 281 282 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \ 283 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8)) 284 285 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state) 286 { 287 struct intel_display *display = to_intel_display(crtc_state); 288 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 289 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state; 290 enum pipe pipe = crtc->pipe; 291 int sprite0_start, sprite1_start; 292 u32 dsparb, dsparb2, dsparb3; 293 294 switch (pipe) { 295 case PIPE_A: 296 dsparb = intel_de_read(display, DSPARB(display)); 297 dsparb2 = intel_de_read(display, DSPARB2); 298 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0); 299 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4); 300 break; 301 case PIPE_B: 302 dsparb = intel_de_read(display, DSPARB(display)); 303 dsparb2 = intel_de_read(display, DSPARB2); 304 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8); 305 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12); 306 break; 307 case PIPE_C: 308 dsparb2 = intel_de_read(display, DSPARB2); 309 dsparb3 = intel_de_read(display, DSPARB3); 310 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16); 311 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20); 312 break; 313 default: 314 MISSING_CASE(pipe); 315 return; 316 } 317 318 fifo_state->plane[PLANE_PRIMARY] = sprite0_start; 319 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start; 320 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start; 321 fifo_state->plane[PLANE_CURSOR] = 63; 322 } 323 324 static int i9xx_get_fifo_size(struct intel_display *display, 325 enum i9xx_plane_id i9xx_plane) 326 { 327 u32 dsparb = intel_de_read(display, DSPARB(display)); 328 int size; 329 330 size = dsparb & 0x7f; 331 if (i9xx_plane == PLANE_B) 332 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size; 333 334 drm_dbg_kms(display->drm, "FIFO size - (0x%08x) %c: %d\n", 335 dsparb, plane_name(i9xx_plane), size); 336 337 return size; 338 } 339 340 static int i830_get_fifo_size(struct intel_display *display, 341 enum i9xx_plane_id i9xx_plane) 342 { 343 u32 dsparb = intel_de_read(display, DSPARB(display)); 344 int size; 345 346 size = dsparb & 0x1ff; 347 if (i9xx_plane == PLANE_B) 348 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size; 349 size >>= 1; /* Convert to cachelines */ 350 351 drm_dbg_kms(display->drm, "FIFO size - (0x%08x) %c: %d\n", 352 dsparb, plane_name(i9xx_plane), size); 353 354 return size; 355 } 356 357 static int i845_get_fifo_size(struct intel_display *display, 358 enum i9xx_plane_id i9xx_plane) 359 { 360 u32 dsparb = intel_de_read(display, DSPARB(display)); 361 int size; 362 363 size = dsparb & 0x7f; 364 size >>= 2; /* Convert to cachelines */ 365 366 drm_dbg_kms(display->drm, "FIFO size - (0x%08x) %c: %d\n", 367 dsparb, plane_name(i9xx_plane), size); 368 369 return size; 370 } 371 372 /* Pineview has different values for various configs */ 373 static const struct intel_watermark_params pnv_display_wm = { 374 .fifo_size = PINEVIEW_DISPLAY_FIFO, 375 .max_wm = PINEVIEW_MAX_WM, 376 .default_wm = PINEVIEW_DFT_WM, 377 .guard_size = PINEVIEW_GUARD_WM, 378 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 379 }; 380 381 static const struct intel_watermark_params pnv_display_hplloff_wm = { 382 .fifo_size = PINEVIEW_DISPLAY_FIFO, 383 .max_wm = PINEVIEW_MAX_WM, 384 .default_wm = PINEVIEW_DFT_HPLLOFF_WM, 385 .guard_size = PINEVIEW_GUARD_WM, 386 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 387 }; 388 389 static const struct intel_watermark_params pnv_cursor_wm = { 390 .fifo_size = PINEVIEW_CURSOR_FIFO, 391 .max_wm = PINEVIEW_CURSOR_MAX_WM, 392 .default_wm = PINEVIEW_CURSOR_DFT_WM, 393 .guard_size = PINEVIEW_CURSOR_GUARD_WM, 394 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 395 }; 396 397 static const struct intel_watermark_params pnv_cursor_hplloff_wm = { 398 .fifo_size = PINEVIEW_CURSOR_FIFO, 399 .max_wm = PINEVIEW_CURSOR_MAX_WM, 400 .default_wm = PINEVIEW_CURSOR_DFT_WM, 401 .guard_size = PINEVIEW_CURSOR_GUARD_WM, 402 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 403 }; 404 405 static const struct intel_watermark_params i965_cursor_wm_info = { 406 .fifo_size = I965_CURSOR_FIFO, 407 .max_wm = I965_CURSOR_MAX_WM, 408 .default_wm = I965_CURSOR_DFT_WM, 409 .guard_size = 2, 410 .cacheline_size = I915_FIFO_LINE_SIZE, 411 }; 412 413 static const struct intel_watermark_params i945_wm_info = { 414 .fifo_size = I945_FIFO_SIZE, 415 .max_wm = I915_MAX_WM, 416 .default_wm = 1, 417 .guard_size = 2, 418 .cacheline_size = I915_FIFO_LINE_SIZE, 419 }; 420 421 static const struct intel_watermark_params i915_wm_info = { 422 .fifo_size = I915_FIFO_SIZE, 423 .max_wm = I915_MAX_WM, 424 .default_wm = 1, 425 .guard_size = 2, 426 .cacheline_size = I915_FIFO_LINE_SIZE, 427 }; 428 429 static const struct intel_watermark_params i830_a_wm_info = { 430 .fifo_size = I855GM_FIFO_SIZE, 431 .max_wm = I915_MAX_WM, 432 .default_wm = 1, 433 .guard_size = 2, 434 .cacheline_size = I830_FIFO_LINE_SIZE, 435 }; 436 437 static const struct intel_watermark_params i830_bc_wm_info = { 438 .fifo_size = I855GM_FIFO_SIZE, 439 .max_wm = I915_MAX_WM / 2, 440 .default_wm = 1, 441 .guard_size = 2, 442 .cacheline_size = I830_FIFO_LINE_SIZE, 443 }; 444 445 static const struct intel_watermark_params i845_wm_info = { 446 .fifo_size = I830_FIFO_SIZE, 447 .max_wm = I915_MAX_WM, 448 .default_wm = 1, 449 .guard_size = 2, 450 .cacheline_size = I830_FIFO_LINE_SIZE, 451 }; 452 453 /** 454 * intel_wm_method1 - Method 1 / "small buffer" watermark formula 455 * @pixel_rate: Pipe pixel rate in kHz 456 * @cpp: Plane bytes per pixel 457 * @latency: Memory wakeup latency in 0.1us units 458 * 459 * Compute the watermark using the method 1 or "small buffer" 460 * formula. The caller may additionally add extra cachelines 461 * to account for TLB misses and clock crossings. 462 * 463 * This method is concerned with the short term drain rate 464 * of the FIFO, ie. it does not account for blanking periods 465 * which would effectively reduce the average drain rate across 466 * a longer period. The name "small" refers to the fact the 467 * FIFO is relatively small compared to the amount of data 468 * fetched. 469 * 470 * The FIFO level vs. time graph might look something like: 471 * 472 * |\ |\ 473 * | \ | \ 474 * __---__---__ (- plane active, _ blanking) 475 * -> time 476 * 477 * or perhaps like this: 478 * 479 * |\|\ |\|\ 480 * __----__----__ (- plane active, _ blanking) 481 * -> time 482 * 483 * Returns: 484 * The watermark in bytes 485 */ 486 static unsigned int intel_wm_method1(unsigned int pixel_rate, 487 unsigned int cpp, 488 unsigned int latency) 489 { 490 u64 ret; 491 492 ret = mul_u32_u32(pixel_rate, cpp * latency); 493 ret = DIV_ROUND_UP_ULL(ret, 10000); 494 495 return ret; 496 } 497 498 /** 499 * intel_wm_method2 - Method 2 / "large buffer" watermark formula 500 * @pixel_rate: Pipe pixel rate in kHz 501 * @htotal: Pipe horizontal total 502 * @width: Plane width in pixels 503 * @cpp: Plane bytes per pixel 504 * @latency: Memory wakeup latency in 0.1us units 505 * 506 * Compute the watermark using the method 2 or "large buffer" 507 * formula. The caller may additionally add extra cachelines 508 * to account for TLB misses and clock crossings. 509 * 510 * This method is concerned with the long term drain rate 511 * of the FIFO, ie. it does account for blanking periods 512 * which effectively reduce the average drain rate across 513 * a longer period. The name "large" refers to the fact the 514 * FIFO is relatively large compared to the amount of data 515 * fetched. 516 * 517 * The FIFO level vs. time graph might look something like: 518 * 519 * |\___ |\___ 520 * | \___ | \___ 521 * | \ | \ 522 * __ --__--__--__--__--__--__ (- plane active, _ blanking) 523 * -> time 524 * 525 * Returns: 526 * The watermark in bytes 527 */ 528 static unsigned int intel_wm_method2(unsigned int pixel_rate, 529 unsigned int htotal, 530 unsigned int width, 531 unsigned int cpp, 532 unsigned int latency) 533 { 534 unsigned int ret; 535 536 /* 537 * FIXME remove once all users are computing 538 * watermarks in the correct place. 539 */ 540 if (WARN_ON_ONCE(htotal == 0)) 541 htotal = 1; 542 543 ret = (latency * pixel_rate) / (htotal * 10000); 544 ret = (ret + 1) * width * cpp; 545 546 return ret; 547 } 548 549 /** 550 * intel_calculate_wm - calculate watermark level 551 * @display: display device 552 * @pixel_rate: pixel clock 553 * @wm: chip FIFO params 554 * @fifo_size: size of the FIFO buffer 555 * @cpp: bytes per pixel 556 * @latency_ns: memory latency for the platform 557 * 558 * Calculate the watermark level (the level at which the display plane will 559 * start fetching from memory again). Each chip has a different display 560 * FIFO size and allocation, so the caller needs to figure that out and pass 561 * in the correct intel_watermark_params structure. 562 * 563 * As the pixel clock runs, the FIFO will be drained at a rate that depends 564 * on the pixel size. When it reaches the watermark level, it'll start 565 * fetching FIFO line sized based chunks from memory until the FIFO fills 566 * past the watermark point. If the FIFO drains completely, a FIFO underrun 567 * will occur, and a display engine hang could result. 568 */ 569 static unsigned int intel_calculate_wm(struct intel_display *display, 570 int pixel_rate, 571 const struct intel_watermark_params *wm, 572 int fifo_size, int cpp, 573 unsigned int latency_ns) 574 { 575 int entries, wm_size; 576 577 /* 578 * Note: we need to make sure we don't overflow for various clock & 579 * latency values. 580 * clocks go from a few thousand to several hundred thousand. 581 * latency is usually a few thousand 582 */ 583 entries = intel_wm_method1(pixel_rate, cpp, 584 latency_ns / 100); 585 entries = DIV_ROUND_UP(entries, wm->cacheline_size) + 586 wm->guard_size; 587 drm_dbg_kms(display->drm, "FIFO entries required for mode: %d\n", entries); 588 589 wm_size = fifo_size - entries; 590 drm_dbg_kms(display->drm, "FIFO watermark level: %d\n", wm_size); 591 592 /* Don't promote wm_size to unsigned... */ 593 if (wm_size > wm->max_wm) 594 wm_size = wm->max_wm; 595 if (wm_size <= 0) 596 wm_size = wm->default_wm; 597 598 /* 599 * Bspec seems to indicate that the value shouldn't be lower than 600 * 'burst size + 1'. Certainly 830 is quite unhappy with low values. 601 * Lets go for 8 which is the burst size since certain platforms 602 * already use a hardcoded 8 (which is what the spec says should be 603 * done). 604 */ 605 if (wm_size <= 8) 606 wm_size = 8; 607 608 return wm_size; 609 } 610 611 static bool is_disabling(int old, int new, int threshold) 612 { 613 return old >= threshold && new < threshold; 614 } 615 616 static bool is_enabling(int old, int new, int threshold) 617 { 618 return old < threshold && new >= threshold; 619 } 620 621 static bool intel_crtc_active(struct intel_crtc *crtc) 622 { 623 /* Be paranoid as we can arrive here with only partial 624 * state retrieved from the hardware during setup. 625 * 626 * We can ditch the adjusted_mode.crtc_clock check as soon 627 * as Haswell has gained clock readout/fastboot support. 628 * 629 * We can ditch the crtc->primary->state->fb check as soon as we can 630 * properly reconstruct framebuffers. 631 * 632 * FIXME: The intel_crtc->active here should be switched to 633 * crtc->state->active once we have proper CRTC states wired up 634 * for atomic. 635 */ 636 return crtc->active && crtc->base.primary->state->fb && 637 crtc->config->hw.adjusted_mode.crtc_clock; 638 } 639 640 static struct intel_crtc *single_enabled_crtc(struct intel_display *display) 641 { 642 struct intel_crtc *crtc, *enabled = NULL; 643 644 for_each_intel_crtc(display->drm, crtc) { 645 if (intel_crtc_active(crtc)) { 646 if (enabled) 647 return NULL; 648 enabled = crtc; 649 } 650 } 651 652 return enabled; 653 } 654 655 static void pnv_update_wm(struct intel_display *display) 656 { 657 struct intel_crtc *crtc; 658 const struct cxsr_latency *latency; 659 u32 reg; 660 unsigned int wm; 661 662 latency = pnv_get_cxsr_latency(display); 663 if (!latency) { 664 drm_dbg_kms(display->drm, "Unknown FSB/MEM, disabling CxSR\n"); 665 intel_set_memory_cxsr(display, false); 666 return; 667 } 668 669 crtc = single_enabled_crtc(display); 670 if (crtc) { 671 const struct drm_framebuffer *fb = 672 crtc->base.primary->state->fb; 673 int pixel_rate = crtc->config->pixel_rate; 674 int cpp = fb->format->cpp[0]; 675 676 /* Display SR */ 677 wm = intel_calculate_wm(display, pixel_rate, 678 &pnv_display_wm, 679 pnv_display_wm.fifo_size, 680 cpp, latency->display_sr); 681 reg = intel_de_read(display, DSPFW1(display)); 682 reg &= ~DSPFW_SR_MASK; 683 reg |= FW_WM(wm, SR); 684 intel_de_write(display, DSPFW1(display), reg); 685 drm_dbg_kms(display->drm, "DSPFW1 register is %x\n", reg); 686 687 /* cursor SR */ 688 wm = intel_calculate_wm(display, pixel_rate, 689 &pnv_cursor_wm, 690 pnv_display_wm.fifo_size, 691 4, latency->cursor_sr); 692 intel_de_rmw(display, DSPFW3(display), 693 DSPFW_CURSOR_SR_MASK, FW_WM(wm, CURSOR_SR)); 694 695 /* Display HPLL off SR */ 696 wm = intel_calculate_wm(display, pixel_rate, 697 &pnv_display_hplloff_wm, 698 pnv_display_hplloff_wm.fifo_size, 699 cpp, latency->display_hpll_disable); 700 intel_de_rmw(display, DSPFW3(display), 701 DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR)); 702 703 /* cursor HPLL off SR */ 704 wm = intel_calculate_wm(display, pixel_rate, 705 &pnv_cursor_hplloff_wm, 706 pnv_display_hplloff_wm.fifo_size, 707 4, latency->cursor_hpll_disable); 708 reg = intel_de_read(display, DSPFW3(display)); 709 reg &= ~DSPFW_HPLL_CURSOR_MASK; 710 reg |= FW_WM(wm, HPLL_CURSOR); 711 intel_de_write(display, DSPFW3(display), reg); 712 drm_dbg_kms(display->drm, "DSPFW3 register is %x\n", reg); 713 714 intel_set_memory_cxsr(display, true); 715 } else { 716 intel_set_memory_cxsr(display, false); 717 } 718 } 719 720 static bool i9xx_wm_need_update(const struct intel_plane_state *old_plane_state, 721 const struct intel_plane_state *new_plane_state) 722 { 723 /* Update watermarks on tiling or size changes. */ 724 if (old_plane_state->uapi.visible != new_plane_state->uapi.visible) 725 return true; 726 727 if (!old_plane_state->hw.fb || !new_plane_state->hw.fb) 728 return false; 729 730 if (old_plane_state->hw.fb->modifier != new_plane_state->hw.fb->modifier || 731 old_plane_state->hw.rotation != new_plane_state->hw.rotation || 732 drm_rect_width(&old_plane_state->uapi.src) != drm_rect_width(&new_plane_state->uapi.src) || 733 drm_rect_height(&old_plane_state->uapi.src) != drm_rect_height(&new_plane_state->uapi.src) || 734 drm_rect_width(&old_plane_state->uapi.dst) != drm_rect_width(&new_plane_state->uapi.dst) || 735 drm_rect_height(&old_plane_state->uapi.dst) != drm_rect_height(&new_plane_state->uapi.dst)) 736 return true; 737 738 return false; 739 } 740 741 static void i9xx_wm_compute(struct intel_crtc_state *new_crtc_state, 742 const struct intel_plane_state *old_plane_state, 743 const struct intel_plane_state *new_plane_state) 744 { 745 bool turn_off, turn_on, visible, was_visible, mode_changed; 746 747 mode_changed = intel_crtc_needs_modeset(new_crtc_state); 748 was_visible = old_plane_state->uapi.visible; 749 visible = new_plane_state->uapi.visible; 750 751 if (!was_visible && !visible) 752 return; 753 754 turn_off = was_visible && (!visible || mode_changed); 755 turn_on = visible && (!was_visible || mode_changed); 756 757 /* FIXME nuke when all wm code is atomic */ 758 if (turn_on) { 759 new_crtc_state->update_wm_pre = true; 760 } else if (turn_off) { 761 new_crtc_state->update_wm_post = true; 762 } else if (i9xx_wm_need_update(old_plane_state, new_plane_state)) { 763 /* FIXME bollocks */ 764 new_crtc_state->update_wm_pre = true; 765 new_crtc_state->update_wm_post = true; 766 } 767 } 768 769 static int i9xx_compute_watermarks(struct intel_atomic_state *state, 770 struct intel_crtc *crtc) 771 { 772 struct intel_crtc_state *new_crtc_state = 773 intel_atomic_get_new_crtc_state(state, crtc); 774 const struct intel_plane_state *old_plane_state; 775 const struct intel_plane_state *new_plane_state; 776 struct intel_plane *plane; 777 int i; 778 779 for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state, 780 new_plane_state, i) { 781 if (plane->pipe != crtc->pipe) 782 continue; 783 784 i9xx_wm_compute(new_crtc_state, old_plane_state, new_plane_state); 785 } 786 787 return 0; 788 } 789 790 /* 791 * Documentation says: 792 * "If the line size is small, the TLB fetches can get in the way of the 793 * data fetches, causing some lag in the pixel data return which is not 794 * accounted for in the above formulas. The following adjustment only 795 * needs to be applied if eight whole lines fit in the buffer at once. 796 * The WM is adjusted upwards by the difference between the FIFO size 797 * and the size of 8 whole lines. This adjustment is always performed 798 * in the actual pixel depth regardless of whether FBC is enabled or not." 799 */ 800 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp) 801 { 802 int tlb_miss = fifo_size * 64 - width * cpp * 8; 803 804 return max(0, tlb_miss); 805 } 806 807 static void g4x_write_wm_values(struct intel_display *display, 808 const struct g4x_wm_values *wm) 809 { 810 enum pipe pipe; 811 812 for_each_pipe(display, pipe) 813 trace_g4x_wm(intel_crtc_for_pipe(display, pipe), wm); 814 815 intel_de_write(display, DSPFW1(display), 816 FW_WM(wm->sr.plane, SR) | 817 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) | 818 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) | 819 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA)); 820 intel_de_write(display, DSPFW2(display), 821 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) | 822 FW_WM(wm->sr.fbc, FBC_SR) | 823 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) | 824 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) | 825 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) | 826 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA)); 827 intel_de_write(display, DSPFW3(display), 828 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) | 829 FW_WM(wm->sr.cursor, CURSOR_SR) | 830 FW_WM(wm->hpll.cursor, HPLL_CURSOR) | 831 FW_WM(wm->hpll.plane, HPLL_SR)); 832 833 intel_de_posting_read(display, DSPFW1(display)); 834 } 835 836 #define FW_WM_VLV(value, plane) \ 837 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV) 838 839 static void vlv_write_wm_values(struct intel_display *display, 840 const struct vlv_wm_values *wm) 841 { 842 enum pipe pipe; 843 844 for_each_pipe(display, pipe) { 845 trace_vlv_wm(intel_crtc_for_pipe(display, pipe), wm); 846 847 intel_de_write(display, VLV_DDL(pipe), 848 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) | 849 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) | 850 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) | 851 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT)); 852 } 853 854 /* 855 * Zero the (unused) WM1 watermarks, and also clear all the 856 * high order bits so that there are no out of bounds values 857 * present in the registers during the reprogramming. 858 */ 859 intel_de_write(display, DSPHOWM, 0); 860 intel_de_write(display, DSPHOWM1, 0); 861 intel_de_write(display, DSPFW4, 0); 862 intel_de_write(display, DSPFW5, 0); 863 intel_de_write(display, DSPFW6, 0); 864 865 intel_de_write(display, DSPFW1(display), 866 FW_WM(wm->sr.plane, SR) | 867 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) | 868 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) | 869 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA)); 870 intel_de_write(display, DSPFW2(display), 871 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) | 872 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) | 873 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA)); 874 intel_de_write(display, DSPFW3(display), 875 FW_WM(wm->sr.cursor, CURSOR_SR)); 876 877 if (display->platform.cherryview) { 878 intel_de_write(display, DSPFW7_CHV, 879 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) | 880 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC)); 881 intel_de_write(display, DSPFW8_CHV, 882 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) | 883 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE)); 884 intel_de_write(display, DSPFW9_CHV, 885 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) | 886 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC)); 887 intel_de_write(display, DSPHOWM, 888 FW_WM(wm->sr.plane >> 9, SR_HI) | 889 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) | 890 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) | 891 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) | 892 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) | 893 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) | 894 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) | 895 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) | 896 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) | 897 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI)); 898 } else { 899 intel_de_write(display, DSPFW7, 900 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) | 901 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC)); 902 intel_de_write(display, DSPHOWM, 903 FW_WM(wm->sr.plane >> 9, SR_HI) | 904 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) | 905 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) | 906 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) | 907 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) | 908 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) | 909 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI)); 910 } 911 912 intel_de_posting_read(display, DSPFW1(display)); 913 } 914 915 #undef FW_WM_VLV 916 917 static void g4x_setup_wm_latency(struct intel_display *display) 918 { 919 /* all latencies in usec */ 920 display->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5; 921 display->wm.pri_latency[G4X_WM_LEVEL_SR] = 12; 922 display->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35; 923 924 display->wm.num_levels = G4X_WM_LEVEL_HPLL + 1; 925 } 926 927 static int g4x_plane_fifo_size(enum plane_id plane_id, int level) 928 { 929 /* 930 * DSPCNTR[13] supposedly controls whether the 931 * primary plane can use the FIFO space otherwise 932 * reserved for the sprite plane. It's not 100% clear 933 * what the actual FIFO size is, but it looks like we 934 * can happily set both primary and sprite watermarks 935 * up to 127 cachelines. So that would seem to mean 936 * that either DSPCNTR[13] doesn't do anything, or that 937 * the total FIFO is >= 256 cachelines in size. Either 938 * way, we don't seem to have to worry about this 939 * repartitioning as the maximum watermark value the 940 * register can hold for each plane is lower than the 941 * minimum FIFO size. 942 */ 943 switch (plane_id) { 944 case PLANE_CURSOR: 945 return 63; 946 case PLANE_PRIMARY: 947 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511; 948 case PLANE_SPRITE0: 949 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0; 950 default: 951 MISSING_CASE(plane_id); 952 return 0; 953 } 954 } 955 956 static int g4x_fbc_fifo_size(int level) 957 { 958 switch (level) { 959 case G4X_WM_LEVEL_SR: 960 return 7; 961 case G4X_WM_LEVEL_HPLL: 962 return 15; 963 default: 964 MISSING_CASE(level); 965 return 0; 966 } 967 } 968 969 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, 970 const struct intel_plane_state *plane_state, 971 int level) 972 { 973 struct intel_display *display = to_intel_display(plane_state); 974 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 975 const struct drm_display_mode *pipe_mode = 976 &crtc_state->hw.pipe_mode; 977 unsigned int latency = display->wm.pri_latency[level] * 10; 978 unsigned int pixel_rate, htotal, cpp, width, wm; 979 980 if (latency == 0) 981 return USHRT_MAX; 982 983 if (!intel_wm_plane_visible(crtc_state, plane_state)) 984 return 0; 985 986 cpp = plane_state->hw.fb->format->cpp[0]; 987 988 /* 989 * WaUse32BppForSRWM:ctg,elk 990 * 991 * The spec fails to list this restriction for the 992 * HPLL watermark, which seems a little strange. 993 * Let's use 32bpp for the HPLL watermark as well. 994 */ 995 if (plane->id == PLANE_PRIMARY && 996 level != G4X_WM_LEVEL_NORMAL) 997 cpp = max(cpp, 4u); 998 999 pixel_rate = crtc_state->pixel_rate; 1000 htotal = pipe_mode->crtc_htotal; 1001 width = drm_rect_width(&plane_state->uapi.src) >> 16; 1002 1003 if (plane->id == PLANE_CURSOR) { 1004 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency); 1005 } else if (plane->id == PLANE_PRIMARY && 1006 level == G4X_WM_LEVEL_NORMAL) { 1007 wm = intel_wm_method1(pixel_rate, cpp, latency); 1008 } else { 1009 unsigned int small, large; 1010 1011 small = intel_wm_method1(pixel_rate, cpp, latency); 1012 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency); 1013 1014 wm = min(small, large); 1015 } 1016 1017 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level), 1018 width, cpp); 1019 1020 wm = DIV_ROUND_UP(wm, 64) + 2; 1021 1022 return min_t(unsigned int, wm, USHRT_MAX); 1023 } 1024 1025 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state, 1026 int level, enum plane_id plane_id, u16 value) 1027 { 1028 struct intel_display *display = to_intel_display(crtc_state); 1029 bool dirty = false; 1030 1031 for (; level < display->wm.num_levels; level++) { 1032 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level]; 1033 1034 dirty |= raw->plane[plane_id] != value; 1035 raw->plane[plane_id] = value; 1036 } 1037 1038 return dirty; 1039 } 1040 1041 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state, 1042 int level, u16 value) 1043 { 1044 struct intel_display *display = to_intel_display(crtc_state); 1045 bool dirty = false; 1046 1047 /* NORMAL level doesn't have an FBC watermark */ 1048 level = max(level, G4X_WM_LEVEL_SR); 1049 1050 for (; level < display->wm.num_levels; level++) { 1051 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level]; 1052 1053 dirty |= raw->fbc != value; 1054 raw->fbc = value; 1055 } 1056 1057 return dirty; 1058 } 1059 1060 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state, 1061 const struct intel_plane_state *plane_state, 1062 u32 pri_val); 1063 1064 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state, 1065 const struct intel_plane_state *plane_state) 1066 { 1067 struct intel_display *display = to_intel_display(crtc_state); 1068 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 1069 enum plane_id plane_id = plane->id; 1070 bool dirty = false; 1071 int level; 1072 1073 if (!intel_wm_plane_visible(crtc_state, plane_state)) { 1074 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0); 1075 if (plane_id == PLANE_PRIMARY) 1076 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0); 1077 goto out; 1078 } 1079 1080 for (level = 0; level < display->wm.num_levels; level++) { 1081 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level]; 1082 int wm, max_wm; 1083 1084 wm = g4x_compute_wm(crtc_state, plane_state, level); 1085 max_wm = g4x_plane_fifo_size(plane_id, level); 1086 1087 if (wm > max_wm) 1088 break; 1089 1090 dirty |= raw->plane[plane_id] != wm; 1091 raw->plane[plane_id] = wm; 1092 1093 if (plane_id != PLANE_PRIMARY || 1094 level == G4X_WM_LEVEL_NORMAL) 1095 continue; 1096 1097 wm = ilk_compute_fbc_wm(crtc_state, plane_state, 1098 raw->plane[plane_id]); 1099 max_wm = g4x_fbc_fifo_size(level); 1100 1101 /* 1102 * FBC wm is not mandatory as we 1103 * can always just disable its use. 1104 */ 1105 if (wm > max_wm) 1106 wm = USHRT_MAX; 1107 1108 dirty |= raw->fbc != wm; 1109 raw->fbc = wm; 1110 } 1111 1112 /* mark watermarks as invalid */ 1113 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX); 1114 1115 if (plane_id == PLANE_PRIMARY) 1116 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX); 1117 1118 out: 1119 if (dirty) { 1120 drm_dbg_kms(display->drm, 1121 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n", 1122 plane->base.name, 1123 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id], 1124 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id], 1125 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]); 1126 1127 if (plane_id == PLANE_PRIMARY) 1128 drm_dbg_kms(display->drm, 1129 "FBC watermarks: SR=%d, HPLL=%d\n", 1130 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc, 1131 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc); 1132 } 1133 1134 return dirty; 1135 } 1136 1137 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state, 1138 enum plane_id plane_id, int level) 1139 { 1140 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level]; 1141 1142 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level); 1143 } 1144 1145 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, 1146 int level) 1147 { 1148 struct intel_display *display = to_intel_display(crtc_state); 1149 1150 if (level >= display->wm.num_levels) 1151 return false; 1152 1153 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) && 1154 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) && 1155 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level); 1156 } 1157 1158 /* mark all levels starting from 'level' as invalid */ 1159 static void g4x_invalidate_wms(struct intel_crtc *crtc, 1160 struct g4x_wm_state *wm_state, int level) 1161 { 1162 if (level <= G4X_WM_LEVEL_NORMAL) { 1163 enum plane_id plane_id; 1164 1165 for_each_plane_id_on_crtc(crtc, plane_id) 1166 wm_state->wm.plane[plane_id] = USHRT_MAX; 1167 } 1168 1169 if (level <= G4X_WM_LEVEL_SR) { 1170 wm_state->cxsr = false; 1171 wm_state->sr.cursor = USHRT_MAX; 1172 wm_state->sr.plane = USHRT_MAX; 1173 wm_state->sr.fbc = USHRT_MAX; 1174 } 1175 1176 if (level <= G4X_WM_LEVEL_HPLL) { 1177 wm_state->hpll_en = false; 1178 wm_state->hpll.cursor = USHRT_MAX; 1179 wm_state->hpll.plane = USHRT_MAX; 1180 wm_state->hpll.fbc = USHRT_MAX; 1181 } 1182 } 1183 1184 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state, 1185 int level) 1186 { 1187 if (level < G4X_WM_LEVEL_SR) 1188 return false; 1189 1190 if (level >= G4X_WM_LEVEL_SR && 1191 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR)) 1192 return false; 1193 1194 if (level >= G4X_WM_LEVEL_HPLL && 1195 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL)) 1196 return false; 1197 1198 return true; 1199 } 1200 1201 static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state) 1202 { 1203 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 1204 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal; 1205 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR); 1206 const struct g4x_pipe_wm *raw; 1207 enum plane_id plane_id; 1208 int level; 1209 1210 level = G4X_WM_LEVEL_NORMAL; 1211 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level)) 1212 goto out; 1213 1214 raw = &crtc_state->wm.g4x.raw[level]; 1215 for_each_plane_id_on_crtc(crtc, plane_id) 1216 wm_state->wm.plane[plane_id] = raw->plane[plane_id]; 1217 1218 level = G4X_WM_LEVEL_SR; 1219 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level)) 1220 goto out; 1221 1222 raw = &crtc_state->wm.g4x.raw[level]; 1223 wm_state->sr.plane = raw->plane[PLANE_PRIMARY]; 1224 wm_state->sr.cursor = raw->plane[PLANE_CURSOR]; 1225 wm_state->sr.fbc = raw->fbc; 1226 1227 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY); 1228 1229 level = G4X_WM_LEVEL_HPLL; 1230 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level)) 1231 goto out; 1232 1233 raw = &crtc_state->wm.g4x.raw[level]; 1234 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY]; 1235 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR]; 1236 wm_state->hpll.fbc = raw->fbc; 1237 1238 wm_state->hpll_en = wm_state->cxsr; 1239 1240 level++; 1241 1242 out: 1243 if (level == G4X_WM_LEVEL_NORMAL) 1244 return -EINVAL; 1245 1246 /* invalidate the higher levels */ 1247 g4x_invalidate_wms(crtc, wm_state, level); 1248 1249 /* 1250 * Determine if the FBC watermark(s) can be used. IF 1251 * this isn't the case we prefer to disable the FBC 1252 * watermark(s) rather than disable the SR/HPLL 1253 * level(s) entirely. 'level-1' is the highest valid 1254 * level here. 1255 */ 1256 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1); 1257 1258 return 0; 1259 } 1260 1261 static int g4x_compute_pipe_wm(struct intel_atomic_state *state, 1262 struct intel_crtc *crtc) 1263 { 1264 struct intel_crtc_state *crtc_state = 1265 intel_atomic_get_new_crtc_state(state, crtc); 1266 const struct intel_plane_state *old_plane_state; 1267 const struct intel_plane_state *new_plane_state; 1268 struct intel_plane *plane; 1269 unsigned int dirty = 0; 1270 int i; 1271 1272 for_each_oldnew_intel_plane_in_state(state, plane, 1273 old_plane_state, 1274 new_plane_state, i) { 1275 if (new_plane_state->hw.crtc != &crtc->base && 1276 old_plane_state->hw.crtc != &crtc->base) 1277 continue; 1278 1279 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state)) 1280 dirty |= BIT(plane->id); 1281 } 1282 1283 if (!dirty) 1284 return 0; 1285 1286 return _g4x_compute_pipe_wm(crtc_state); 1287 } 1288 1289 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state, 1290 struct intel_crtc *crtc) 1291 { 1292 struct intel_display *display = to_intel_display(state); 1293 struct intel_crtc_state *new_crtc_state = 1294 intel_atomic_get_new_crtc_state(state, crtc); 1295 const struct intel_crtc_state *old_crtc_state = 1296 intel_atomic_get_old_crtc_state(state, crtc); 1297 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate; 1298 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal; 1299 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal; 1300 enum plane_id plane_id; 1301 1302 if (!new_crtc_state->hw.active || 1303 intel_crtc_needs_modeset(new_crtc_state)) { 1304 *intermediate = *optimal; 1305 1306 intermediate->cxsr = false; 1307 intermediate->hpll_en = false; 1308 goto out; 1309 } 1310 1311 intermediate->cxsr = optimal->cxsr && active->cxsr && 1312 !new_crtc_state->disable_cxsr; 1313 intermediate->hpll_en = optimal->hpll_en && active->hpll_en && 1314 !new_crtc_state->disable_cxsr; 1315 intermediate->fbc_en = optimal->fbc_en && active->fbc_en; 1316 1317 for_each_plane_id_on_crtc(crtc, plane_id) { 1318 intermediate->wm.plane[plane_id] = 1319 max(optimal->wm.plane[plane_id], 1320 active->wm.plane[plane_id]); 1321 1322 drm_WARN_ON(display->drm, intermediate->wm.plane[plane_id] > 1323 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL)); 1324 } 1325 1326 intermediate->sr.plane = max(optimal->sr.plane, 1327 active->sr.plane); 1328 intermediate->sr.cursor = max(optimal->sr.cursor, 1329 active->sr.cursor); 1330 intermediate->sr.fbc = max(optimal->sr.fbc, 1331 active->sr.fbc); 1332 1333 intermediate->hpll.plane = max(optimal->hpll.plane, 1334 active->hpll.plane); 1335 intermediate->hpll.cursor = max(optimal->hpll.cursor, 1336 active->hpll.cursor); 1337 intermediate->hpll.fbc = max(optimal->hpll.fbc, 1338 active->hpll.fbc); 1339 1340 drm_WARN_ON(display->drm, 1341 (intermediate->sr.plane > 1342 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) || 1343 intermediate->sr.cursor > 1344 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) && 1345 intermediate->cxsr); 1346 drm_WARN_ON(display->drm, 1347 (intermediate->sr.plane > 1348 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) || 1349 intermediate->sr.cursor > 1350 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) && 1351 intermediate->hpll_en); 1352 1353 drm_WARN_ON(display->drm, 1354 intermediate->sr.fbc > g4x_fbc_fifo_size(1) && 1355 intermediate->fbc_en && intermediate->cxsr); 1356 drm_WARN_ON(display->drm, 1357 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) && 1358 intermediate->fbc_en && intermediate->hpll_en); 1359 1360 out: 1361 /* 1362 * If our intermediate WM are identical to the final WM, then we can 1363 * omit the post-vblank programming; only update if it's different. 1364 */ 1365 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0) 1366 new_crtc_state->wm.need_postvbl_update = true; 1367 1368 return 0; 1369 } 1370 1371 static int g4x_compute_watermarks(struct intel_atomic_state *state, 1372 struct intel_crtc *crtc) 1373 { 1374 int ret; 1375 1376 ret = g4x_compute_pipe_wm(state, crtc); 1377 if (ret) 1378 return ret; 1379 1380 ret = g4x_compute_intermediate_wm(state, crtc); 1381 if (ret) 1382 return ret; 1383 1384 return 0; 1385 } 1386 1387 static void g4x_merge_wm(struct intel_display *display, 1388 struct g4x_wm_values *wm) 1389 { 1390 struct intel_crtc *crtc; 1391 int num_active_pipes = 0; 1392 1393 wm->cxsr = true; 1394 wm->hpll_en = true; 1395 wm->fbc_en = true; 1396 1397 for_each_intel_crtc(display->drm, crtc) { 1398 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x; 1399 1400 if (!crtc->active) 1401 continue; 1402 1403 if (!wm_state->cxsr) 1404 wm->cxsr = false; 1405 if (!wm_state->hpll_en) 1406 wm->hpll_en = false; 1407 if (!wm_state->fbc_en) 1408 wm->fbc_en = false; 1409 1410 num_active_pipes++; 1411 } 1412 1413 if (num_active_pipes != 1) { 1414 wm->cxsr = false; 1415 wm->hpll_en = false; 1416 wm->fbc_en = false; 1417 } 1418 1419 for_each_intel_crtc(display->drm, crtc) { 1420 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x; 1421 enum pipe pipe = crtc->pipe; 1422 1423 wm->pipe[pipe] = wm_state->wm; 1424 if (crtc->active && wm->cxsr) 1425 wm->sr = wm_state->sr; 1426 if (crtc->active && wm->hpll_en) 1427 wm->hpll = wm_state->hpll; 1428 } 1429 } 1430 1431 static void g4x_program_watermarks(struct intel_display *display) 1432 { 1433 struct g4x_wm_values *old_wm = &display->wm.g4x; 1434 struct g4x_wm_values new_wm = {}; 1435 1436 g4x_merge_wm(display, &new_wm); 1437 1438 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0) 1439 return; 1440 1441 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true)) 1442 _intel_set_memory_cxsr(display, false); 1443 1444 g4x_write_wm_values(display, &new_wm); 1445 1446 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true)) 1447 _intel_set_memory_cxsr(display, true); 1448 1449 *old_wm = new_wm; 1450 } 1451 1452 static void g4x_initial_watermarks(struct intel_atomic_state *state, 1453 struct intel_crtc *crtc) 1454 { 1455 struct intel_display *display = to_intel_display(crtc); 1456 const struct intel_crtc_state *crtc_state = 1457 intel_atomic_get_new_crtc_state(state, crtc); 1458 1459 mutex_lock(&display->wm.wm_mutex); 1460 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate; 1461 g4x_program_watermarks(display); 1462 mutex_unlock(&display->wm.wm_mutex); 1463 } 1464 1465 static void g4x_optimize_watermarks(struct intel_atomic_state *state, 1466 struct intel_crtc *crtc) 1467 { 1468 struct intel_display *display = to_intel_display(crtc); 1469 const struct intel_crtc_state *crtc_state = 1470 intel_atomic_get_new_crtc_state(state, crtc); 1471 1472 if (!crtc_state->wm.need_postvbl_update) 1473 return; 1474 1475 mutex_lock(&display->wm.wm_mutex); 1476 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal; 1477 g4x_program_watermarks(display); 1478 mutex_unlock(&display->wm.wm_mutex); 1479 } 1480 1481 /* latency must be in 0.1us units. */ 1482 static unsigned int vlv_wm_method2(unsigned int pixel_rate, 1483 unsigned int htotal, 1484 unsigned int width, 1485 unsigned int cpp, 1486 unsigned int latency) 1487 { 1488 unsigned int ret; 1489 1490 ret = intel_wm_method2(pixel_rate, htotal, 1491 width, cpp, latency); 1492 ret = DIV_ROUND_UP(ret, 64); 1493 1494 return ret; 1495 } 1496 1497 static void vlv_setup_wm_latency(struct intel_display *display) 1498 { 1499 /* all latencies in usec */ 1500 display->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3; 1501 1502 display->wm.num_levels = VLV_WM_LEVEL_PM2 + 1; 1503 1504 if (display->platform.cherryview) { 1505 display->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12; 1506 display->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33; 1507 1508 display->wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1; 1509 } 1510 } 1511 1512 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, 1513 const struct intel_plane_state *plane_state, 1514 int level) 1515 { 1516 struct intel_display *display = to_intel_display(plane_state); 1517 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 1518 const struct drm_display_mode *pipe_mode = 1519 &crtc_state->hw.pipe_mode; 1520 unsigned int pixel_rate, htotal, cpp, width, wm; 1521 1522 if (display->wm.pri_latency[level] == 0) 1523 return USHRT_MAX; 1524 1525 if (!intel_wm_plane_visible(crtc_state, plane_state)) 1526 return 0; 1527 1528 cpp = plane_state->hw.fb->format->cpp[0]; 1529 pixel_rate = crtc_state->pixel_rate; 1530 htotal = pipe_mode->crtc_htotal; 1531 width = drm_rect_width(&plane_state->uapi.src) >> 16; 1532 1533 if (plane->id == PLANE_CURSOR) { 1534 /* 1535 * FIXME the formula gives values that are 1536 * too big for the cursor FIFO, and hence we 1537 * would never be able to use cursors. For 1538 * now just hardcode the watermark. 1539 */ 1540 wm = 63; 1541 } else { 1542 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp, 1543 display->wm.pri_latency[level] * 10); 1544 } 1545 1546 return min_t(unsigned int, wm, USHRT_MAX); 1547 } 1548 1549 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes) 1550 { 1551 return (active_planes & (BIT(PLANE_SPRITE0) | 1552 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1); 1553 } 1554 1555 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state) 1556 { 1557 struct intel_display *display = to_intel_display(crtc_state); 1558 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 1559 const struct g4x_pipe_wm *raw = 1560 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2]; 1561 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state; 1562 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR); 1563 int num_active_planes = hweight8(active_planes); 1564 const int fifo_size = 511; 1565 int fifo_extra, fifo_left = fifo_size; 1566 int sprite0_fifo_extra = 0; 1567 unsigned int total_rate; 1568 enum plane_id plane_id; 1569 1570 /* 1571 * When enabling sprite0 after sprite1 has already been enabled 1572 * we tend to get an underrun unless sprite0 already has some 1573 * FIFO space allocated. Hence we always allocate at least one 1574 * cacheline for sprite0 whenever sprite1 is enabled. 1575 * 1576 * All other plane enable sequences appear immune to this problem. 1577 */ 1578 if (vlv_need_sprite0_fifo_workaround(active_planes)) 1579 sprite0_fifo_extra = 1; 1580 1581 total_rate = raw->plane[PLANE_PRIMARY] + 1582 raw->plane[PLANE_SPRITE0] + 1583 raw->plane[PLANE_SPRITE1] + 1584 sprite0_fifo_extra; 1585 1586 if (total_rate > fifo_size) 1587 return -EINVAL; 1588 1589 if (total_rate == 0) 1590 total_rate = 1; 1591 1592 for_each_plane_id_on_crtc(crtc, plane_id) { 1593 unsigned int rate; 1594 1595 if ((active_planes & BIT(plane_id)) == 0) { 1596 fifo_state->plane[plane_id] = 0; 1597 continue; 1598 } 1599 1600 rate = raw->plane[plane_id]; 1601 fifo_state->plane[plane_id] = fifo_size * rate / total_rate; 1602 fifo_left -= fifo_state->plane[plane_id]; 1603 } 1604 1605 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra; 1606 fifo_left -= sprite0_fifo_extra; 1607 1608 fifo_state->plane[PLANE_CURSOR] = 63; 1609 1610 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1); 1611 1612 /* spread the remainder evenly */ 1613 for_each_plane_id_on_crtc(crtc, plane_id) { 1614 int plane_extra; 1615 1616 if (fifo_left == 0) 1617 break; 1618 1619 if ((active_planes & BIT(plane_id)) == 0) 1620 continue; 1621 1622 plane_extra = min(fifo_extra, fifo_left); 1623 fifo_state->plane[plane_id] += plane_extra; 1624 fifo_left -= plane_extra; 1625 } 1626 1627 drm_WARN_ON(display->drm, active_planes != 0 && fifo_left != 0); 1628 1629 /* give it all to the first plane if none are active */ 1630 if (active_planes == 0) { 1631 drm_WARN_ON(display->drm, fifo_left != fifo_size); 1632 fifo_state->plane[PLANE_PRIMARY] = fifo_left; 1633 } 1634 1635 return 0; 1636 } 1637 1638 /* mark all levels starting from 'level' as invalid */ 1639 static void vlv_invalidate_wms(struct intel_crtc *crtc, 1640 struct vlv_wm_state *wm_state, int level) 1641 { 1642 struct intel_display *display = to_intel_display(crtc); 1643 1644 for (; level < display->wm.num_levels; level++) { 1645 enum plane_id plane_id; 1646 1647 for_each_plane_id_on_crtc(crtc, plane_id) 1648 wm_state->wm[level].plane[plane_id] = USHRT_MAX; 1649 1650 wm_state->sr[level].cursor = USHRT_MAX; 1651 wm_state->sr[level].plane = USHRT_MAX; 1652 } 1653 } 1654 1655 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size) 1656 { 1657 if (wm > fifo_size) 1658 return USHRT_MAX; 1659 else 1660 return fifo_size - wm; 1661 } 1662 1663 /* 1664 * Starting from 'level' set all higher 1665 * levels to 'value' in the "raw" watermarks. 1666 */ 1667 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state, 1668 int level, enum plane_id plane_id, u16 value) 1669 { 1670 struct intel_display *display = to_intel_display(crtc_state); 1671 bool dirty = false; 1672 1673 for (; level < display->wm.num_levels; level++) { 1674 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level]; 1675 1676 dirty |= raw->plane[plane_id] != value; 1677 raw->plane[plane_id] = value; 1678 } 1679 1680 return dirty; 1681 } 1682 1683 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state, 1684 const struct intel_plane_state *plane_state) 1685 { 1686 struct intel_display *display = to_intel_display(crtc_state); 1687 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 1688 enum plane_id plane_id = plane->id; 1689 int level; 1690 bool dirty = false; 1691 1692 if (!intel_wm_plane_visible(crtc_state, plane_state)) { 1693 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0); 1694 goto out; 1695 } 1696 1697 for (level = 0; level < display->wm.num_levels; level++) { 1698 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level]; 1699 int wm = vlv_compute_wm_level(crtc_state, plane_state, level); 1700 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511; 1701 1702 if (wm > max_wm) 1703 break; 1704 1705 dirty |= raw->plane[plane_id] != wm; 1706 raw->plane[plane_id] = wm; 1707 } 1708 1709 /* mark all higher levels as invalid */ 1710 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX); 1711 1712 out: 1713 if (dirty) 1714 drm_dbg_kms(display->drm, 1715 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n", 1716 plane->base.name, 1717 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id], 1718 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id], 1719 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]); 1720 1721 return dirty; 1722 } 1723 1724 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state, 1725 enum plane_id plane_id, int level) 1726 { 1727 const struct g4x_pipe_wm *raw = 1728 &crtc_state->wm.vlv.raw[level]; 1729 const struct vlv_fifo_state *fifo_state = 1730 &crtc_state->wm.vlv.fifo_state; 1731 1732 return raw->plane[plane_id] <= fifo_state->plane[plane_id]; 1733 } 1734 1735 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level) 1736 { 1737 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) && 1738 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) && 1739 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) && 1740 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level); 1741 } 1742 1743 static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state) 1744 { 1745 struct intel_display *display = to_intel_display(crtc_state); 1746 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 1747 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal; 1748 const struct vlv_fifo_state *fifo_state = 1749 &crtc_state->wm.vlv.fifo_state; 1750 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR); 1751 int num_active_planes = hweight8(active_planes); 1752 enum plane_id plane_id; 1753 int level; 1754 1755 /* initially allow all levels */ 1756 wm_state->num_levels = display->wm.num_levels; 1757 /* 1758 * Note that enabling cxsr with no primary/sprite planes 1759 * enabled can wedge the pipe. Hence we only allow cxsr 1760 * with exactly one enabled primary/sprite plane. 1761 */ 1762 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1; 1763 1764 for (level = 0; level < wm_state->num_levels; level++) { 1765 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level]; 1766 const int sr_fifo_size = INTEL_NUM_PIPES(display) * 512 - 1; 1767 1768 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level)) 1769 break; 1770 1771 for_each_plane_id_on_crtc(crtc, plane_id) { 1772 wm_state->wm[level].plane[plane_id] = 1773 vlv_invert_wm_value(raw->plane[plane_id], 1774 fifo_state->plane[plane_id]); 1775 } 1776 1777 wm_state->sr[level].plane = 1778 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY], 1779 raw->plane[PLANE_SPRITE0], 1780 raw->plane[PLANE_SPRITE1]), 1781 sr_fifo_size); 1782 1783 wm_state->sr[level].cursor = 1784 vlv_invert_wm_value(raw->plane[PLANE_CURSOR], 1785 63); 1786 } 1787 1788 if (level == 0) 1789 return -EINVAL; 1790 1791 /* limit to only levels we can actually handle */ 1792 wm_state->num_levels = level; 1793 1794 /* invalidate the higher levels */ 1795 vlv_invalidate_wms(crtc, wm_state, level); 1796 1797 return 0; 1798 } 1799 1800 static int vlv_compute_pipe_wm(struct intel_atomic_state *state, 1801 struct intel_crtc *crtc) 1802 { 1803 struct intel_crtc_state *crtc_state = 1804 intel_atomic_get_new_crtc_state(state, crtc); 1805 const struct intel_plane_state *old_plane_state; 1806 const struct intel_plane_state *new_plane_state; 1807 struct intel_plane *plane; 1808 unsigned int dirty = 0; 1809 int i; 1810 1811 for_each_oldnew_intel_plane_in_state(state, plane, 1812 old_plane_state, 1813 new_plane_state, i) { 1814 if (new_plane_state->hw.crtc != &crtc->base && 1815 old_plane_state->hw.crtc != &crtc->base) 1816 continue; 1817 1818 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state)) 1819 dirty |= BIT(plane->id); 1820 } 1821 1822 /* 1823 * DSPARB registers may have been reset due to the 1824 * power well being turned off. Make sure we restore 1825 * them to a consistent state even if no primary/sprite 1826 * planes are initially active. We also force a FIFO 1827 * recomputation so that we are sure to sanitize the 1828 * FIFO setting we took over from the BIOS even if there 1829 * are no active planes on the crtc. 1830 */ 1831 if (intel_crtc_needs_modeset(crtc_state)) 1832 dirty = ~0; 1833 1834 if (!dirty) 1835 return 0; 1836 1837 /* cursor changes don't warrant a FIFO recompute */ 1838 if (dirty & ~BIT(PLANE_CURSOR)) { 1839 const struct intel_crtc_state *old_crtc_state = 1840 intel_atomic_get_old_crtc_state(state, crtc); 1841 const struct vlv_fifo_state *old_fifo_state = 1842 &old_crtc_state->wm.vlv.fifo_state; 1843 const struct vlv_fifo_state *new_fifo_state = 1844 &crtc_state->wm.vlv.fifo_state; 1845 int ret; 1846 1847 ret = vlv_compute_fifo(crtc_state); 1848 if (ret) 1849 return ret; 1850 1851 if (intel_crtc_needs_modeset(crtc_state) || 1852 memcmp(old_fifo_state, new_fifo_state, 1853 sizeof(*new_fifo_state)) != 0) 1854 crtc_state->fifo_changed = true; 1855 } 1856 1857 return _vlv_compute_pipe_wm(crtc_state); 1858 } 1859 1860 #define VLV_FIFO(plane, value) \ 1861 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV) 1862 1863 static void vlv_atomic_update_fifo(struct intel_atomic_state *state, 1864 struct intel_crtc *crtc) 1865 { 1866 struct intel_display *display = to_intel_display(crtc); 1867 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 1868 struct intel_uncore *uncore = &dev_priv->uncore; 1869 const struct intel_crtc_state *crtc_state = 1870 intel_atomic_get_new_crtc_state(state, crtc); 1871 const struct vlv_fifo_state *fifo_state = 1872 &crtc_state->wm.vlv.fifo_state; 1873 int sprite0_start, sprite1_start, fifo_size; 1874 u32 dsparb, dsparb2, dsparb3; 1875 1876 if (!crtc_state->fifo_changed) 1877 return; 1878 1879 sprite0_start = fifo_state->plane[PLANE_PRIMARY]; 1880 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start; 1881 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start; 1882 1883 drm_WARN_ON(display->drm, fifo_state->plane[PLANE_CURSOR] != 63); 1884 drm_WARN_ON(display->drm, fifo_size != 511); 1885 1886 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size); 1887 1888 /* 1889 * uncore.lock serves a double purpose here. It allows us to 1890 * use the less expensive I915_{READ,WRITE}_FW() functions, and 1891 * it protects the DSPARB registers from getting clobbered by 1892 * parallel updates from multiple pipes. 1893 * 1894 * intel_pipe_update_start() has already disabled interrupts 1895 * for us, so a plain spin_lock() is sufficient here. 1896 */ 1897 spin_lock(&uncore->lock); 1898 1899 switch (crtc->pipe) { 1900 case PIPE_A: 1901 dsparb = intel_de_read_fw(display, DSPARB(display)); 1902 dsparb2 = intel_de_read_fw(display, DSPARB2); 1903 1904 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) | 1905 VLV_FIFO(SPRITEB, 0xff)); 1906 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) | 1907 VLV_FIFO(SPRITEB, sprite1_start)); 1908 1909 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) | 1910 VLV_FIFO(SPRITEB_HI, 0x1)); 1911 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) | 1912 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8)); 1913 1914 intel_de_write_fw(display, DSPARB(display), dsparb); 1915 intel_de_write_fw(display, DSPARB2, dsparb2); 1916 break; 1917 case PIPE_B: 1918 dsparb = intel_de_read_fw(display, DSPARB(display)); 1919 dsparb2 = intel_de_read_fw(display, DSPARB2); 1920 1921 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) | 1922 VLV_FIFO(SPRITED, 0xff)); 1923 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) | 1924 VLV_FIFO(SPRITED, sprite1_start)); 1925 1926 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) | 1927 VLV_FIFO(SPRITED_HI, 0xff)); 1928 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) | 1929 VLV_FIFO(SPRITED_HI, sprite1_start >> 8)); 1930 1931 intel_de_write_fw(display, DSPARB(display), dsparb); 1932 intel_de_write_fw(display, DSPARB2, dsparb2); 1933 break; 1934 case PIPE_C: 1935 dsparb3 = intel_de_read_fw(display, DSPARB3); 1936 dsparb2 = intel_de_read_fw(display, DSPARB2); 1937 1938 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) | 1939 VLV_FIFO(SPRITEF, 0xff)); 1940 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) | 1941 VLV_FIFO(SPRITEF, sprite1_start)); 1942 1943 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) | 1944 VLV_FIFO(SPRITEF_HI, 0xff)); 1945 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) | 1946 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8)); 1947 1948 intel_de_write_fw(display, DSPARB3, dsparb3); 1949 intel_de_write_fw(display, DSPARB2, dsparb2); 1950 break; 1951 default: 1952 break; 1953 } 1954 1955 intel_de_read_fw(display, DSPARB(display)); 1956 1957 spin_unlock(&uncore->lock); 1958 } 1959 1960 #undef VLV_FIFO 1961 1962 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state, 1963 struct intel_crtc *crtc) 1964 { 1965 struct intel_crtc_state *new_crtc_state = 1966 intel_atomic_get_new_crtc_state(state, crtc); 1967 const struct intel_crtc_state *old_crtc_state = 1968 intel_atomic_get_old_crtc_state(state, crtc); 1969 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate; 1970 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal; 1971 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal; 1972 int level; 1973 1974 if (!new_crtc_state->hw.active || 1975 intel_crtc_needs_modeset(new_crtc_state)) { 1976 *intermediate = *optimal; 1977 1978 intermediate->cxsr = false; 1979 goto out; 1980 } 1981 1982 intermediate->num_levels = min(optimal->num_levels, active->num_levels); 1983 intermediate->cxsr = optimal->cxsr && active->cxsr && 1984 !new_crtc_state->disable_cxsr; 1985 1986 for (level = 0; level < intermediate->num_levels; level++) { 1987 enum plane_id plane_id; 1988 1989 for_each_plane_id_on_crtc(crtc, plane_id) { 1990 intermediate->wm[level].plane[plane_id] = 1991 min(optimal->wm[level].plane[plane_id], 1992 active->wm[level].plane[plane_id]); 1993 } 1994 1995 intermediate->sr[level].plane = min(optimal->sr[level].plane, 1996 active->sr[level].plane); 1997 intermediate->sr[level].cursor = min(optimal->sr[level].cursor, 1998 active->sr[level].cursor); 1999 } 2000 2001 vlv_invalidate_wms(crtc, intermediate, level); 2002 2003 out: 2004 /* 2005 * If our intermediate WM are identical to the final WM, then we can 2006 * omit the post-vblank programming; only update if it's different. 2007 */ 2008 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0) 2009 new_crtc_state->wm.need_postvbl_update = true; 2010 2011 return 0; 2012 } 2013 2014 static int vlv_compute_watermarks(struct intel_atomic_state *state, 2015 struct intel_crtc *crtc) 2016 { 2017 int ret; 2018 2019 ret = vlv_compute_pipe_wm(state, crtc); 2020 if (ret) 2021 return ret; 2022 2023 ret = vlv_compute_intermediate_wm(state, crtc); 2024 if (ret) 2025 return ret; 2026 2027 return 0; 2028 } 2029 2030 static void vlv_merge_wm(struct intel_display *display, 2031 struct vlv_wm_values *wm) 2032 { 2033 struct intel_crtc *crtc; 2034 int num_active_pipes = 0; 2035 2036 wm->level = display->wm.num_levels - 1; 2037 wm->cxsr = true; 2038 2039 for_each_intel_crtc(display->drm, crtc) { 2040 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv; 2041 2042 if (!crtc->active) 2043 continue; 2044 2045 if (!wm_state->cxsr) 2046 wm->cxsr = false; 2047 2048 num_active_pipes++; 2049 wm->level = min_t(int, wm->level, wm_state->num_levels - 1); 2050 } 2051 2052 if (num_active_pipes != 1) 2053 wm->cxsr = false; 2054 2055 if (num_active_pipes > 1) 2056 wm->level = VLV_WM_LEVEL_PM2; 2057 2058 for_each_intel_crtc(display->drm, crtc) { 2059 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv; 2060 enum pipe pipe = crtc->pipe; 2061 2062 wm->pipe[pipe] = wm_state->wm[wm->level]; 2063 if (crtc->active && wm->cxsr) 2064 wm->sr = wm_state->sr[wm->level]; 2065 2066 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2; 2067 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2; 2068 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2; 2069 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2; 2070 } 2071 } 2072 2073 static void vlv_program_watermarks(struct intel_display *display) 2074 { 2075 struct vlv_wm_values *old_wm = &display->wm.vlv; 2076 struct vlv_wm_values new_wm = {}; 2077 2078 vlv_merge_wm(display, &new_wm); 2079 2080 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0) 2081 return; 2082 2083 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS)) 2084 chv_set_memory_dvfs(display, false); 2085 2086 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5)) 2087 chv_set_memory_pm5(display, false); 2088 2089 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true)) 2090 _intel_set_memory_cxsr(display, false); 2091 2092 vlv_write_wm_values(display, &new_wm); 2093 2094 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true)) 2095 _intel_set_memory_cxsr(display, true); 2096 2097 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5)) 2098 chv_set_memory_pm5(display, true); 2099 2100 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS)) 2101 chv_set_memory_dvfs(display, true); 2102 2103 *old_wm = new_wm; 2104 } 2105 2106 static void vlv_initial_watermarks(struct intel_atomic_state *state, 2107 struct intel_crtc *crtc) 2108 { 2109 struct intel_display *display = to_intel_display(crtc); 2110 const struct intel_crtc_state *crtc_state = 2111 intel_atomic_get_new_crtc_state(state, crtc); 2112 2113 mutex_lock(&display->wm.wm_mutex); 2114 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate; 2115 vlv_program_watermarks(display); 2116 mutex_unlock(&display->wm.wm_mutex); 2117 } 2118 2119 static void vlv_optimize_watermarks(struct intel_atomic_state *state, 2120 struct intel_crtc *crtc) 2121 { 2122 struct intel_display *display = to_intel_display(crtc); 2123 const struct intel_crtc_state *crtc_state = 2124 intel_atomic_get_new_crtc_state(state, crtc); 2125 2126 if (!crtc_state->wm.need_postvbl_update) 2127 return; 2128 2129 mutex_lock(&display->wm.wm_mutex); 2130 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal; 2131 vlv_program_watermarks(display); 2132 mutex_unlock(&display->wm.wm_mutex); 2133 } 2134 2135 static void i965_update_wm(struct intel_display *display) 2136 { 2137 struct intel_crtc *crtc; 2138 int srwm = 1; 2139 int cursor_sr = 16; 2140 bool cxsr_enabled; 2141 2142 /* Calc sr entries for one plane configs */ 2143 crtc = single_enabled_crtc(display); 2144 if (crtc) { 2145 /* self-refresh has much higher latency */ 2146 static const int sr_latency_ns = 12000; 2147 const struct drm_display_mode *pipe_mode = 2148 &crtc->config->hw.pipe_mode; 2149 const struct drm_framebuffer *fb = 2150 crtc->base.primary->state->fb; 2151 int pixel_rate = crtc->config->pixel_rate; 2152 int htotal = pipe_mode->crtc_htotal; 2153 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16; 2154 int cpp = fb->format->cpp[0]; 2155 int entries; 2156 2157 entries = intel_wm_method2(pixel_rate, htotal, 2158 width, cpp, sr_latency_ns / 100); 2159 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE); 2160 srwm = I965_FIFO_SIZE - entries; 2161 if (srwm < 0) 2162 srwm = 1; 2163 srwm &= 0x1ff; 2164 drm_dbg_kms(display->drm, 2165 "self-refresh entries: %d, wm: %d\n", 2166 entries, srwm); 2167 2168 entries = intel_wm_method2(pixel_rate, htotal, 2169 crtc->base.cursor->state->crtc_w, 4, 2170 sr_latency_ns / 100); 2171 entries = DIV_ROUND_UP(entries, 2172 i965_cursor_wm_info.cacheline_size) + 2173 i965_cursor_wm_info.guard_size; 2174 2175 cursor_sr = i965_cursor_wm_info.fifo_size - entries; 2176 if (cursor_sr > i965_cursor_wm_info.max_wm) 2177 cursor_sr = i965_cursor_wm_info.max_wm; 2178 2179 drm_dbg_kms(display->drm, 2180 "self-refresh watermark: display plane %d " 2181 "cursor %d\n", srwm, cursor_sr); 2182 2183 cxsr_enabled = true; 2184 } else { 2185 cxsr_enabled = false; 2186 /* Turn off self refresh if both pipes are enabled */ 2187 intel_set_memory_cxsr(display, false); 2188 } 2189 2190 drm_dbg_kms(display->drm, 2191 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n", 2192 srwm); 2193 2194 /* 965 has limitations... */ 2195 intel_de_write(display, DSPFW1(display), 2196 FW_WM(srwm, SR) | 2197 FW_WM(8, CURSORB) | 2198 FW_WM(8, PLANEB) | 2199 FW_WM(8, PLANEA)); 2200 intel_de_write(display, DSPFW2(display), 2201 FW_WM(8, CURSORA) | 2202 FW_WM(8, PLANEC_OLD)); 2203 /* update cursor SR watermark */ 2204 intel_de_write(display, DSPFW3(display), 2205 FW_WM(cursor_sr, CURSOR_SR)); 2206 2207 if (cxsr_enabled) 2208 intel_set_memory_cxsr(display, true); 2209 } 2210 2211 #undef FW_WM 2212 2213 static struct intel_crtc *intel_crtc_for_plane(struct intel_display *display, 2214 enum i9xx_plane_id i9xx_plane) 2215 { 2216 struct intel_plane *plane; 2217 2218 for_each_intel_plane(display->drm, plane) { 2219 if (plane->id == PLANE_PRIMARY && 2220 plane->i9xx_plane == i9xx_plane) 2221 return intel_crtc_for_pipe(display, plane->pipe); 2222 } 2223 2224 return NULL; 2225 } 2226 2227 static void i9xx_update_wm(struct intel_display *display) 2228 { 2229 const struct intel_watermark_params *wm_info; 2230 u32 fwater_lo; 2231 u32 fwater_hi; 2232 int cwm, srwm = 1; 2233 int fifo_size; 2234 int planea_wm, planeb_wm; 2235 struct intel_crtc *crtc; 2236 2237 if (display->platform.i945gm) 2238 wm_info = &i945_wm_info; 2239 else if (DISPLAY_VER(display) != 2) 2240 wm_info = &i915_wm_info; 2241 else 2242 wm_info = &i830_a_wm_info; 2243 2244 if (DISPLAY_VER(display) == 2) 2245 fifo_size = i830_get_fifo_size(display, PLANE_A); 2246 else 2247 fifo_size = i9xx_get_fifo_size(display, PLANE_A); 2248 crtc = intel_crtc_for_plane(display, PLANE_A); 2249 if (intel_crtc_active(crtc)) { 2250 const struct drm_framebuffer *fb = 2251 crtc->base.primary->state->fb; 2252 int cpp; 2253 2254 if (DISPLAY_VER(display) == 2) 2255 cpp = 4; 2256 else 2257 cpp = fb->format->cpp[0]; 2258 2259 planea_wm = intel_calculate_wm(display, crtc->config->pixel_rate, 2260 wm_info, fifo_size, cpp, 2261 pessimal_latency_ns); 2262 } else { 2263 planea_wm = fifo_size - wm_info->guard_size; 2264 if (planea_wm > (long)wm_info->max_wm) 2265 planea_wm = wm_info->max_wm; 2266 } 2267 2268 if (DISPLAY_VER(display) == 2) 2269 wm_info = &i830_bc_wm_info; 2270 2271 if (DISPLAY_VER(display) == 2) 2272 fifo_size = i830_get_fifo_size(display, PLANE_B); 2273 else 2274 fifo_size = i9xx_get_fifo_size(display, PLANE_B); 2275 crtc = intel_crtc_for_plane(display, PLANE_B); 2276 if (intel_crtc_active(crtc)) { 2277 const struct drm_framebuffer *fb = 2278 crtc->base.primary->state->fb; 2279 int cpp; 2280 2281 if (DISPLAY_VER(display) == 2) 2282 cpp = 4; 2283 else 2284 cpp = fb->format->cpp[0]; 2285 2286 planeb_wm = intel_calculate_wm(display, crtc->config->pixel_rate, 2287 wm_info, fifo_size, cpp, 2288 pessimal_latency_ns); 2289 } else { 2290 planeb_wm = fifo_size - wm_info->guard_size; 2291 if (planeb_wm > (long)wm_info->max_wm) 2292 planeb_wm = wm_info->max_wm; 2293 } 2294 2295 drm_dbg_kms(display->drm, 2296 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm); 2297 2298 crtc = single_enabled_crtc(display); 2299 if (display->platform.i915gm && crtc) { 2300 const struct drm_framebuffer *fb = 2301 crtc->base.primary->state->fb; 2302 2303 /* self-refresh seems busted with untiled */ 2304 if (fb->modifier == DRM_FORMAT_MOD_LINEAR) 2305 crtc = NULL; 2306 } 2307 2308 /* 2309 * Overlay gets an aggressive default since video jitter is bad. 2310 */ 2311 cwm = 2; 2312 2313 /* Play safe and disable self-refresh before adjusting watermarks. */ 2314 intel_set_memory_cxsr(display, false); 2315 2316 /* Calc sr entries for one plane configs */ 2317 if (HAS_FW_BLC(display) && crtc) { 2318 /* self-refresh has much higher latency */ 2319 static const int sr_latency_ns = 6000; 2320 const struct drm_display_mode *pipe_mode = 2321 &crtc->config->hw.pipe_mode; 2322 const struct drm_framebuffer *fb = 2323 crtc->base.primary->state->fb; 2324 int pixel_rate = crtc->config->pixel_rate; 2325 int htotal = pipe_mode->crtc_htotal; 2326 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16; 2327 int cpp; 2328 int entries; 2329 2330 if (display->platform.i915gm || display->platform.i945gm) 2331 cpp = 4; 2332 else 2333 cpp = fb->format->cpp[0]; 2334 2335 entries = intel_wm_method2(pixel_rate, htotal, width, cpp, 2336 sr_latency_ns / 100); 2337 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size); 2338 drm_dbg_kms(display->drm, 2339 "self-refresh entries: %d\n", entries); 2340 srwm = wm_info->fifo_size - entries; 2341 if (srwm < 0) 2342 srwm = 1; 2343 2344 if (display->platform.i945g || display->platform.i945gm) 2345 intel_de_write(display, FW_BLC_SELF, 2346 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff)); 2347 else 2348 intel_de_write(display, FW_BLC_SELF, srwm & 0x3f); 2349 } 2350 2351 drm_dbg_kms(display->drm, 2352 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n", 2353 planea_wm, planeb_wm, cwm, srwm); 2354 2355 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f); 2356 fwater_hi = (cwm & 0x1f); 2357 2358 /* Set request length to 8 cachelines per fetch */ 2359 fwater_lo = fwater_lo | (1 << 24) | (1 << 8); 2360 fwater_hi = fwater_hi | (1 << 8); 2361 2362 intel_de_write(display, FW_BLC, fwater_lo); 2363 intel_de_write(display, FW_BLC2, fwater_hi); 2364 2365 if (crtc) 2366 intel_set_memory_cxsr(display, true); 2367 } 2368 2369 static void i845_update_wm(struct intel_display *display) 2370 { 2371 struct intel_crtc *crtc; 2372 u32 fwater_lo; 2373 int planea_wm; 2374 2375 crtc = single_enabled_crtc(display); 2376 if (crtc == NULL) 2377 return; 2378 2379 planea_wm = intel_calculate_wm(display, crtc->config->pixel_rate, 2380 &i845_wm_info, 2381 i845_get_fifo_size(display, PLANE_A), 2382 4, pessimal_latency_ns); 2383 fwater_lo = intel_de_read(display, FW_BLC) & ~0xfff; 2384 fwater_lo |= (3<<8) | planea_wm; 2385 2386 drm_dbg_kms(display->drm, 2387 "Setting FIFO watermarks - A: %d\n", planea_wm); 2388 2389 intel_de_write(display, FW_BLC, fwater_lo); 2390 } 2391 2392 /* latency must be in 0.1us units. */ 2393 static unsigned int ilk_wm_method1(unsigned int pixel_rate, 2394 unsigned int cpp, 2395 unsigned int latency) 2396 { 2397 unsigned int ret; 2398 2399 ret = intel_wm_method1(pixel_rate, cpp, latency); 2400 ret = DIV_ROUND_UP(ret, 64) + 2; 2401 2402 return ret; 2403 } 2404 2405 /* latency must be in 0.1us units. */ 2406 static unsigned int ilk_wm_method2(unsigned int pixel_rate, 2407 unsigned int htotal, 2408 unsigned int width, 2409 unsigned int cpp, 2410 unsigned int latency) 2411 { 2412 unsigned int ret; 2413 2414 ret = intel_wm_method2(pixel_rate, htotal, 2415 width, cpp, latency); 2416 ret = DIV_ROUND_UP(ret, 64) + 2; 2417 2418 return ret; 2419 } 2420 2421 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp) 2422 { 2423 /* 2424 * Neither of these should be possible since this function shouldn't be 2425 * called if the CRTC is off or the plane is invisible. But let's be 2426 * extra paranoid to avoid a potential divide-by-zero if we screw up 2427 * elsewhere in the driver. 2428 */ 2429 if (WARN_ON(!cpp)) 2430 return 0; 2431 if (WARN_ON(!horiz_pixels)) 2432 return 0; 2433 2434 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2; 2435 } 2436 2437 struct ilk_wm_maximums { 2438 u16 pri; 2439 u16 spr; 2440 u16 cur; 2441 u16 fbc; 2442 }; 2443 2444 /* 2445 * For both WM_PIPE and WM_LP. 2446 * mem_value must be in 0.1us units. 2447 */ 2448 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state, 2449 const struct intel_plane_state *plane_state, 2450 u32 mem_value, bool is_lp) 2451 { 2452 u32 method1, method2; 2453 int cpp; 2454 2455 if (mem_value == 0) 2456 return U32_MAX; 2457 2458 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2459 return 0; 2460 2461 cpp = plane_state->hw.fb->format->cpp[0]; 2462 2463 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value); 2464 2465 if (!is_lp) 2466 return method1; 2467 2468 method2 = ilk_wm_method2(crtc_state->pixel_rate, 2469 crtc_state->hw.pipe_mode.crtc_htotal, 2470 drm_rect_width(&plane_state->uapi.src) >> 16, 2471 cpp, mem_value); 2472 2473 return min(method1, method2); 2474 } 2475 2476 /* 2477 * For both WM_PIPE and WM_LP. 2478 * mem_value must be in 0.1us units. 2479 */ 2480 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state, 2481 const struct intel_plane_state *plane_state, 2482 u32 mem_value) 2483 { 2484 u32 method1, method2; 2485 int cpp; 2486 2487 if (mem_value == 0) 2488 return U32_MAX; 2489 2490 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2491 return 0; 2492 2493 cpp = plane_state->hw.fb->format->cpp[0]; 2494 2495 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value); 2496 method2 = ilk_wm_method2(crtc_state->pixel_rate, 2497 crtc_state->hw.pipe_mode.crtc_htotal, 2498 drm_rect_width(&plane_state->uapi.src) >> 16, 2499 cpp, mem_value); 2500 return min(method1, method2); 2501 } 2502 2503 /* 2504 * For both WM_PIPE and WM_LP. 2505 * mem_value must be in 0.1us units. 2506 */ 2507 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state, 2508 const struct intel_plane_state *plane_state, 2509 u32 mem_value) 2510 { 2511 int cpp; 2512 2513 if (mem_value == 0) 2514 return U32_MAX; 2515 2516 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2517 return 0; 2518 2519 cpp = plane_state->hw.fb->format->cpp[0]; 2520 2521 return ilk_wm_method2(crtc_state->pixel_rate, 2522 crtc_state->hw.pipe_mode.crtc_htotal, 2523 drm_rect_width(&plane_state->uapi.src) >> 16, 2524 cpp, mem_value); 2525 } 2526 2527 /* Only for WM_LP. */ 2528 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state, 2529 const struct intel_plane_state *plane_state, 2530 u32 pri_val) 2531 { 2532 int cpp; 2533 2534 if (!intel_wm_plane_visible(crtc_state, plane_state)) 2535 return 0; 2536 2537 cpp = plane_state->hw.fb->format->cpp[0]; 2538 2539 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16, 2540 cpp); 2541 } 2542 2543 static unsigned int 2544 ilk_display_fifo_size(struct intel_display *display) 2545 { 2546 if (DISPLAY_VER(display) >= 8) 2547 return 3072; 2548 else if (DISPLAY_VER(display) >= 7) 2549 return 768; 2550 else 2551 return 512; 2552 } 2553 2554 static unsigned int 2555 ilk_plane_wm_reg_max(struct intel_display *display, 2556 int level, bool is_sprite) 2557 { 2558 if (DISPLAY_VER(display) >= 8) 2559 /* BDW primary/sprite plane watermarks */ 2560 return level == 0 ? 255 : 2047; 2561 else if (DISPLAY_VER(display) >= 7) 2562 /* IVB/HSW primary/sprite plane watermarks */ 2563 return level == 0 ? 127 : 1023; 2564 else if (!is_sprite) 2565 /* ILK/SNB primary plane watermarks */ 2566 return level == 0 ? 127 : 511; 2567 else 2568 /* ILK/SNB sprite plane watermarks */ 2569 return level == 0 ? 63 : 255; 2570 } 2571 2572 static unsigned int 2573 ilk_cursor_wm_reg_max(struct intel_display *display, int level) 2574 { 2575 if (DISPLAY_VER(display) >= 7) 2576 return level == 0 ? 63 : 255; 2577 else 2578 return level == 0 ? 31 : 63; 2579 } 2580 2581 static unsigned int ilk_fbc_wm_reg_max(struct intel_display *display) 2582 { 2583 if (DISPLAY_VER(display) >= 8) 2584 return 31; 2585 else 2586 return 15; 2587 } 2588 2589 /* Calculate the maximum primary/sprite plane watermark */ 2590 static unsigned int ilk_plane_wm_max(struct intel_display *display, 2591 int level, 2592 const struct intel_wm_config *config, 2593 enum intel_ddb_partitioning ddb_partitioning, 2594 bool is_sprite) 2595 { 2596 unsigned int fifo_size = ilk_display_fifo_size(display); 2597 2598 /* if sprites aren't enabled, sprites get nothing */ 2599 if (is_sprite && !config->sprites_enabled) 2600 return 0; 2601 2602 /* HSW allows LP1+ watermarks even with multiple pipes */ 2603 if (level == 0 || config->num_pipes_active > 1) { 2604 fifo_size /= INTEL_NUM_PIPES(display); 2605 2606 /* 2607 * For some reason the non self refresh 2608 * FIFO size is only half of the self 2609 * refresh FIFO size on ILK/SNB. 2610 */ 2611 if (DISPLAY_VER(display) < 7) 2612 fifo_size /= 2; 2613 } 2614 2615 if (config->sprites_enabled) { 2616 /* level 0 is always calculated with 1:1 split */ 2617 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) { 2618 if (is_sprite) 2619 fifo_size *= 5; 2620 fifo_size /= 6; 2621 } else { 2622 fifo_size /= 2; 2623 } 2624 } 2625 2626 /* clamp to max that the registers can hold */ 2627 return min(fifo_size, ilk_plane_wm_reg_max(display, level, is_sprite)); 2628 } 2629 2630 /* Calculate the maximum cursor plane watermark */ 2631 static unsigned int ilk_cursor_wm_max(struct intel_display *display, 2632 int level, 2633 const struct intel_wm_config *config) 2634 { 2635 /* HSW LP1+ watermarks w/ multiple pipes */ 2636 if (level > 0 && config->num_pipes_active > 1) 2637 return 64; 2638 2639 /* otherwise just report max that registers can hold */ 2640 return ilk_cursor_wm_reg_max(display, level); 2641 } 2642 2643 static void ilk_compute_wm_maximums(struct intel_display *display, 2644 int level, 2645 const struct intel_wm_config *config, 2646 enum intel_ddb_partitioning ddb_partitioning, 2647 struct ilk_wm_maximums *max) 2648 { 2649 max->pri = ilk_plane_wm_max(display, level, config, ddb_partitioning, false); 2650 max->spr = ilk_plane_wm_max(display, level, config, ddb_partitioning, true); 2651 max->cur = ilk_cursor_wm_max(display, level, config); 2652 max->fbc = ilk_fbc_wm_reg_max(display); 2653 } 2654 2655 static void ilk_compute_wm_reg_maximums(struct intel_display *display, 2656 int level, 2657 struct ilk_wm_maximums *max) 2658 { 2659 max->pri = ilk_plane_wm_reg_max(display, level, false); 2660 max->spr = ilk_plane_wm_reg_max(display, level, true); 2661 max->cur = ilk_cursor_wm_reg_max(display, level); 2662 max->fbc = ilk_fbc_wm_reg_max(display); 2663 } 2664 2665 static bool ilk_validate_wm_level(struct intel_display *display, 2666 int level, 2667 const struct ilk_wm_maximums *max, 2668 struct intel_wm_level *result) 2669 { 2670 bool ret; 2671 2672 /* already determined to be invalid? */ 2673 if (!result->enable) 2674 return false; 2675 2676 result->enable = result->pri_val <= max->pri && 2677 result->spr_val <= max->spr && 2678 result->cur_val <= max->cur; 2679 2680 ret = result->enable; 2681 2682 /* 2683 * HACK until we can pre-compute everything, 2684 * and thus fail gracefully if LP0 watermarks 2685 * are exceeded... 2686 */ 2687 if (level == 0 && !result->enable) { 2688 if (result->pri_val > max->pri) 2689 drm_dbg_kms(display->drm, 2690 "Primary WM%d too large %u (max %u)\n", 2691 level, result->pri_val, max->pri); 2692 if (result->spr_val > max->spr) 2693 drm_dbg_kms(display->drm, 2694 "Sprite WM%d too large %u (max %u)\n", 2695 level, result->spr_val, max->spr); 2696 if (result->cur_val > max->cur) 2697 drm_dbg_kms(display->drm, 2698 "Cursor WM%d too large %u (max %u)\n", 2699 level, result->cur_val, max->cur); 2700 2701 result->pri_val = min_t(u32, result->pri_val, max->pri); 2702 result->spr_val = min_t(u32, result->spr_val, max->spr); 2703 result->cur_val = min_t(u32, result->cur_val, max->cur); 2704 result->enable = true; 2705 } 2706 2707 return ret; 2708 } 2709 2710 static void ilk_compute_wm_level(struct intel_display *display, 2711 const struct intel_crtc *crtc, 2712 int level, 2713 struct intel_crtc_state *crtc_state, 2714 const struct intel_plane_state *pristate, 2715 const struct intel_plane_state *sprstate, 2716 const struct intel_plane_state *curstate, 2717 struct intel_wm_level *result) 2718 { 2719 u16 pri_latency = display->wm.pri_latency[level]; 2720 u16 spr_latency = display->wm.spr_latency[level]; 2721 u16 cur_latency = display->wm.cur_latency[level]; 2722 2723 /* WM1+ latency values stored in 0.5us units */ 2724 if (level > 0) { 2725 pri_latency *= 5; 2726 spr_latency *= 5; 2727 cur_latency *= 5; 2728 } 2729 2730 if (pristate) { 2731 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate, 2732 pri_latency, level); 2733 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val); 2734 } 2735 2736 if (sprstate) 2737 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency); 2738 2739 if (curstate) 2740 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency); 2741 2742 result->enable = true; 2743 } 2744 2745 static void hsw_read_wm_latency(struct intel_display *display, u16 wm[]) 2746 { 2747 struct drm_i915_private *i915 = to_i915(display->drm); 2748 u64 sskpd; 2749 2750 display->wm.num_levels = 5; 2751 2752 sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD); 2753 2754 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd); 2755 if (wm[0] == 0) 2756 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd); 2757 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd); 2758 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd); 2759 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd); 2760 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd); 2761 } 2762 2763 static void snb_read_wm_latency(struct intel_display *display, u16 wm[]) 2764 { 2765 struct drm_i915_private *i915 = to_i915(display->drm); 2766 u32 sskpd; 2767 2768 display->wm.num_levels = 4; 2769 2770 sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD); 2771 2772 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd); 2773 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd); 2774 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd); 2775 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd); 2776 } 2777 2778 static void ilk_read_wm_latency(struct intel_display *display, u16 wm[]) 2779 { 2780 struct drm_i915_private *i915 = to_i915(display->drm); 2781 u32 mltr; 2782 2783 display->wm.num_levels = 3; 2784 2785 mltr = intel_uncore_read(&i915->uncore, MLTR_ILK); 2786 2787 /* ILK primary LP0 latency is 700 ns */ 2788 wm[0] = 7; 2789 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr); 2790 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr); 2791 } 2792 2793 static void intel_fixup_spr_wm_latency(struct intel_display *display, u16 wm[5]) 2794 { 2795 /* ILK sprite LP0 latency is 1300 ns */ 2796 if (DISPLAY_VER(display) == 5) 2797 wm[0] = 13; 2798 } 2799 2800 static void intel_fixup_cur_wm_latency(struct intel_display *display, u16 wm[5]) 2801 { 2802 /* ILK cursor LP0 latency is 1300 ns */ 2803 if (DISPLAY_VER(display) == 5) 2804 wm[0] = 13; 2805 } 2806 2807 static bool ilk_increase_wm_latency(struct intel_display *display, u16 wm[5], u16 min) 2808 { 2809 int level; 2810 2811 if (wm[0] >= min) 2812 return false; 2813 2814 wm[0] = max(wm[0], min); 2815 for (level = 1; level < display->wm.num_levels; level++) 2816 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5)); 2817 2818 return true; 2819 } 2820 2821 static void snb_wm_latency_quirk(struct intel_display *display) 2822 { 2823 bool changed; 2824 2825 /* 2826 * The BIOS provided WM memory latency values are often 2827 * inadequate for high resolution displays. Adjust them. 2828 */ 2829 changed = ilk_increase_wm_latency(display, display->wm.pri_latency, 12); 2830 changed |= ilk_increase_wm_latency(display, display->wm.spr_latency, 12); 2831 changed |= ilk_increase_wm_latency(display, display->wm.cur_latency, 12); 2832 2833 if (!changed) 2834 return; 2835 2836 drm_dbg_kms(display->drm, 2837 "WM latency values increased to avoid potential underruns\n"); 2838 intel_print_wm_latency(display, "Primary", display->wm.pri_latency); 2839 intel_print_wm_latency(display, "Sprite", display->wm.spr_latency); 2840 intel_print_wm_latency(display, "Cursor", display->wm.cur_latency); 2841 } 2842 2843 static void snb_wm_lp3_irq_quirk(struct intel_display *display) 2844 { 2845 /* 2846 * On some SNB machines (Thinkpad X220 Tablet at least) 2847 * LP3 usage can cause vblank interrupts to be lost. 2848 * The DEIIR bit will go high but it looks like the CPU 2849 * never gets interrupted. 2850 * 2851 * It's not clear whether other interrupt source could 2852 * be affected or if this is somehow limited to vblank 2853 * interrupts only. To play it safe we disable LP3 2854 * watermarks entirely. 2855 */ 2856 if (display->wm.pri_latency[3] == 0 && 2857 display->wm.spr_latency[3] == 0 && 2858 display->wm.cur_latency[3] == 0) 2859 return; 2860 2861 display->wm.pri_latency[3] = 0; 2862 display->wm.spr_latency[3] = 0; 2863 display->wm.cur_latency[3] = 0; 2864 2865 drm_dbg_kms(display->drm, 2866 "LP3 watermarks disabled due to potential for lost interrupts\n"); 2867 intel_print_wm_latency(display, "Primary", display->wm.pri_latency); 2868 intel_print_wm_latency(display, "Sprite", display->wm.spr_latency); 2869 intel_print_wm_latency(display, "Cursor", display->wm.cur_latency); 2870 } 2871 2872 static void ilk_setup_wm_latency(struct intel_display *display) 2873 { 2874 if (display->platform.broadwell || display->platform.haswell) 2875 hsw_read_wm_latency(display, display->wm.pri_latency); 2876 else if (DISPLAY_VER(display) >= 6) 2877 snb_read_wm_latency(display, display->wm.pri_latency); 2878 else 2879 ilk_read_wm_latency(display, display->wm.pri_latency); 2880 2881 memcpy(display->wm.spr_latency, display->wm.pri_latency, 2882 sizeof(display->wm.pri_latency)); 2883 memcpy(display->wm.cur_latency, display->wm.pri_latency, 2884 sizeof(display->wm.pri_latency)); 2885 2886 intel_fixup_spr_wm_latency(display, display->wm.spr_latency); 2887 intel_fixup_cur_wm_latency(display, display->wm.cur_latency); 2888 2889 intel_print_wm_latency(display, "Primary", display->wm.pri_latency); 2890 intel_print_wm_latency(display, "Sprite", display->wm.spr_latency); 2891 intel_print_wm_latency(display, "Cursor", display->wm.cur_latency); 2892 2893 if (DISPLAY_VER(display) == 6) { 2894 snb_wm_latency_quirk(display); 2895 snb_wm_lp3_irq_quirk(display); 2896 } 2897 } 2898 2899 static bool ilk_validate_pipe_wm(struct intel_display *display, 2900 struct intel_pipe_wm *pipe_wm) 2901 { 2902 /* LP0 watermark maximums depend on this pipe alone */ 2903 const struct intel_wm_config config = { 2904 .num_pipes_active = 1, 2905 .sprites_enabled = pipe_wm->sprites_enabled, 2906 .sprites_scaled = pipe_wm->sprites_scaled, 2907 }; 2908 struct ilk_wm_maximums max; 2909 2910 /* LP0 watermarks always use 1/2 DDB partitioning */ 2911 ilk_compute_wm_maximums(display, 0, &config, INTEL_DDB_PART_1_2, &max); 2912 2913 /* At least LP0 must be valid */ 2914 if (!ilk_validate_wm_level(display, 0, &max, &pipe_wm->wm[0])) { 2915 drm_dbg_kms(display->drm, "LP0 watermark invalid\n"); 2916 return false; 2917 } 2918 2919 return true; 2920 } 2921 2922 /* Compute new watermarks for the pipe */ 2923 static int ilk_compute_pipe_wm(struct intel_atomic_state *state, 2924 struct intel_crtc *crtc) 2925 { 2926 struct intel_display *display = to_intel_display(state); 2927 struct intel_crtc_state *crtc_state = 2928 intel_atomic_get_new_crtc_state(state, crtc); 2929 struct intel_pipe_wm *pipe_wm; 2930 struct intel_plane *plane; 2931 const struct intel_plane_state *plane_state; 2932 const struct intel_plane_state *pristate = NULL; 2933 const struct intel_plane_state *sprstate = NULL; 2934 const struct intel_plane_state *curstate = NULL; 2935 struct ilk_wm_maximums max; 2936 int level, usable_level; 2937 2938 pipe_wm = &crtc_state->wm.ilk.optimal; 2939 2940 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { 2941 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY) 2942 pristate = plane_state; 2943 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY) 2944 sprstate = plane_state; 2945 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR) 2946 curstate = plane_state; 2947 } 2948 2949 pipe_wm->pipe_enabled = crtc_state->hw.active; 2950 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0); 2951 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0); 2952 2953 usable_level = display->wm.num_levels - 1; 2954 2955 /* ILK/SNB: LP2+ watermarks only w/o sprites */ 2956 if (DISPLAY_VER(display) < 7 && pipe_wm->sprites_enabled) 2957 usable_level = 1; 2958 2959 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */ 2960 if (pipe_wm->sprites_scaled) 2961 usable_level = 0; 2962 2963 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm)); 2964 ilk_compute_wm_level(display, crtc, 0, crtc_state, 2965 pristate, sprstate, curstate, &pipe_wm->wm[0]); 2966 2967 if (!ilk_validate_pipe_wm(display, pipe_wm)) 2968 return -EINVAL; 2969 2970 ilk_compute_wm_reg_maximums(display, 1, &max); 2971 2972 for (level = 1; level <= usable_level; level++) { 2973 struct intel_wm_level *wm = &pipe_wm->wm[level]; 2974 2975 ilk_compute_wm_level(display, crtc, level, crtc_state, 2976 pristate, sprstate, curstate, wm); 2977 2978 /* 2979 * Disable any watermark level that exceeds the 2980 * register maximums since such watermarks are 2981 * always invalid. 2982 */ 2983 if (!ilk_validate_wm_level(display, level, &max, wm)) { 2984 memset(wm, 0, sizeof(*wm)); 2985 break; 2986 } 2987 } 2988 2989 return 0; 2990 } 2991 2992 /* 2993 * Build a set of 'intermediate' watermark values that satisfy both the old 2994 * state and the new state. These can be programmed to the hardware 2995 * immediately. 2996 */ 2997 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state, 2998 struct intel_crtc *crtc) 2999 { 3000 struct intel_display *display = to_intel_display(crtc); 3001 struct intel_crtc_state *new_crtc_state = 3002 intel_atomic_get_new_crtc_state(state, crtc); 3003 const struct intel_crtc_state *old_crtc_state = 3004 intel_atomic_get_old_crtc_state(state, crtc); 3005 struct intel_pipe_wm *intermediate = &new_crtc_state->wm.ilk.intermediate; 3006 const struct intel_pipe_wm *optimal = &new_crtc_state->wm.ilk.optimal; 3007 const struct intel_pipe_wm *active = &old_crtc_state->wm.ilk.optimal; 3008 int level; 3009 3010 /* 3011 * Start with the final, target watermarks, then combine with the 3012 * currently active watermarks to get values that are safe both before 3013 * and after the vblank. 3014 */ 3015 *intermediate = *optimal; 3016 if (!new_crtc_state->hw.active || 3017 intel_crtc_needs_modeset(new_crtc_state) || 3018 state->skip_intermediate_wm) 3019 return 0; 3020 3021 intermediate->pipe_enabled |= active->pipe_enabled; 3022 intermediate->sprites_enabled |= active->sprites_enabled; 3023 intermediate->sprites_scaled |= active->sprites_scaled; 3024 3025 for (level = 0; level < display->wm.num_levels; level++) { 3026 struct intel_wm_level *intermediate_wm = &intermediate->wm[level]; 3027 const struct intel_wm_level *active_wm = &active->wm[level]; 3028 3029 intermediate_wm->enable &= active_wm->enable; 3030 intermediate_wm->pri_val = max(intermediate_wm->pri_val, 3031 active_wm->pri_val); 3032 intermediate_wm->spr_val = max(intermediate_wm->spr_val, 3033 active_wm->spr_val); 3034 intermediate_wm->cur_val = max(intermediate_wm->cur_val, 3035 active_wm->cur_val); 3036 intermediate_wm->fbc_val = max(intermediate_wm->fbc_val, 3037 active_wm->fbc_val); 3038 } 3039 3040 /* 3041 * We need to make sure that these merged watermark values are 3042 * actually a valid configuration themselves. If they're not, 3043 * there's no safe way to transition from the old state to 3044 * the new state, so we need to fail the atomic transaction. 3045 */ 3046 if (!ilk_validate_pipe_wm(display, intermediate)) 3047 return -EINVAL; 3048 3049 /* 3050 * If our intermediate WM are identical to the final WM, then we can 3051 * omit the post-vblank programming; only update if it's different. 3052 */ 3053 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0) 3054 new_crtc_state->wm.need_postvbl_update = true; 3055 3056 return 0; 3057 } 3058 3059 static int ilk_compute_watermarks(struct intel_atomic_state *state, 3060 struct intel_crtc *crtc) 3061 { 3062 int ret; 3063 3064 ret = ilk_compute_pipe_wm(state, crtc); 3065 if (ret) 3066 return ret; 3067 3068 ret = ilk_compute_intermediate_wm(state, crtc); 3069 if (ret) 3070 return ret; 3071 3072 return 0; 3073 } 3074 3075 /* 3076 * Merge the watermarks from all active pipes for a specific level. 3077 */ 3078 static void ilk_merge_wm_level(struct intel_display *display, 3079 int level, 3080 struct intel_wm_level *ret_wm) 3081 { 3082 const struct intel_crtc *crtc; 3083 3084 ret_wm->enable = true; 3085 3086 for_each_intel_crtc(display->drm, crtc) { 3087 const struct intel_pipe_wm *active = &crtc->wm.active.ilk; 3088 const struct intel_wm_level *wm = &active->wm[level]; 3089 3090 if (!active->pipe_enabled) 3091 continue; 3092 3093 /* 3094 * The watermark values may have been used in the past, 3095 * so we must maintain them in the registers for some 3096 * time even if the level is now disabled. 3097 */ 3098 if (!wm->enable) 3099 ret_wm->enable = false; 3100 3101 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val); 3102 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val); 3103 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val); 3104 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val); 3105 } 3106 } 3107 3108 /* 3109 * Merge all low power watermarks for all active pipes. 3110 */ 3111 static void ilk_wm_merge(struct intel_display *display, 3112 const struct intel_wm_config *config, 3113 const struct ilk_wm_maximums *max, 3114 struct intel_pipe_wm *merged) 3115 { 3116 int level, num_levels = display->wm.num_levels; 3117 int last_enabled_level = num_levels - 1; 3118 3119 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */ 3120 if ((DISPLAY_VER(display) < 7 || display->platform.ivybridge) && 3121 config->num_pipes_active > 1) 3122 last_enabled_level = 0; 3123 3124 /* ILK: FBC WM must be disabled always */ 3125 merged->fbc_wm_enabled = DISPLAY_VER(display) >= 6; 3126 3127 /* merge each WM1+ level */ 3128 for (level = 1; level < num_levels; level++) { 3129 struct intel_wm_level *wm = &merged->wm[level]; 3130 3131 ilk_merge_wm_level(display, level, wm); 3132 3133 if (level > last_enabled_level) 3134 wm->enable = false; 3135 else if (!ilk_validate_wm_level(display, level, max, wm)) 3136 /* make sure all following levels get disabled */ 3137 last_enabled_level = level - 1; 3138 3139 /* 3140 * The spec says it is preferred to disable 3141 * FBC WMs instead of disabling a WM level. 3142 */ 3143 if (wm->fbc_val > max->fbc) { 3144 if (wm->enable) 3145 merged->fbc_wm_enabled = false; 3146 wm->fbc_val = 0; 3147 } 3148 } 3149 3150 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */ 3151 if (DISPLAY_VER(display) == 5 && HAS_FBC(display) && 3152 display->params.enable_fbc && !merged->fbc_wm_enabled) { 3153 for (level = 2; level < num_levels; level++) { 3154 struct intel_wm_level *wm = &merged->wm[level]; 3155 3156 wm->enable = false; 3157 } 3158 } 3159 } 3160 3161 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm) 3162 { 3163 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */ 3164 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable); 3165 } 3166 3167 /* The value we need to program into the WM_LPx latency field */ 3168 static unsigned int ilk_wm_lp_latency(struct intel_display *display, 3169 int level) 3170 { 3171 if (display->platform.haswell || display->platform.broadwell) 3172 return 2 * level; 3173 else 3174 return display->wm.pri_latency[level]; 3175 } 3176 3177 static void ilk_compute_wm_results(struct intel_display *display, 3178 const struct intel_pipe_wm *merged, 3179 enum intel_ddb_partitioning partitioning, 3180 struct ilk_wm_values *results) 3181 { 3182 struct intel_crtc *crtc; 3183 int level, wm_lp; 3184 3185 results->enable_fbc_wm = merged->fbc_wm_enabled; 3186 results->partitioning = partitioning; 3187 3188 /* LP1+ register values */ 3189 for (wm_lp = 1; wm_lp <= 3; wm_lp++) { 3190 const struct intel_wm_level *r; 3191 3192 level = ilk_wm_lp_to_level(wm_lp, merged); 3193 3194 r = &merged->wm[level]; 3195 3196 /* 3197 * Maintain the watermark values even if the level is 3198 * disabled. Doing otherwise could cause underruns. 3199 */ 3200 results->wm_lp[wm_lp - 1] = 3201 WM_LP_LATENCY(ilk_wm_lp_latency(display, level)) | 3202 WM_LP_PRIMARY(r->pri_val) | 3203 WM_LP_CURSOR(r->cur_val); 3204 3205 if (r->enable) 3206 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE; 3207 3208 if (DISPLAY_VER(display) >= 8) 3209 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val); 3210 else 3211 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val); 3212 3213 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val); 3214 3215 /* 3216 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the 3217 * level is disabled. Doing otherwise could cause underruns. 3218 */ 3219 if (DISPLAY_VER(display) < 7 && r->spr_val) { 3220 drm_WARN_ON(display->drm, wm_lp != 1); 3221 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE; 3222 } 3223 } 3224 3225 /* LP0 register values */ 3226 for_each_intel_crtc(display->drm, crtc) { 3227 enum pipe pipe = crtc->pipe; 3228 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk; 3229 const struct intel_wm_level *r = &pipe_wm->wm[0]; 3230 3231 if (drm_WARN_ON(display->drm, !r->enable)) 3232 continue; 3233 3234 results->wm_pipe[pipe] = 3235 WM0_PIPE_PRIMARY(r->pri_val) | 3236 WM0_PIPE_SPRITE(r->spr_val) | 3237 WM0_PIPE_CURSOR(r->cur_val); 3238 } 3239 } 3240 3241 /* 3242 * Find the result with the highest level enabled. Check for enable_fbc_wm in 3243 * case both are at the same level. Prefer r1 in case they're the same. 3244 */ 3245 static struct intel_pipe_wm * 3246 ilk_find_best_result(struct intel_display *display, 3247 struct intel_pipe_wm *r1, 3248 struct intel_pipe_wm *r2) 3249 { 3250 int level, level1 = 0, level2 = 0; 3251 3252 for (level = 1; level < display->wm.num_levels; level++) { 3253 if (r1->wm[level].enable) 3254 level1 = level; 3255 if (r2->wm[level].enable) 3256 level2 = level; 3257 } 3258 3259 if (level1 == level2) { 3260 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled) 3261 return r2; 3262 else 3263 return r1; 3264 } else if (level1 > level2) { 3265 return r1; 3266 } else { 3267 return r2; 3268 } 3269 } 3270 3271 /* dirty bits used to track which watermarks need changes */ 3272 #define WM_DIRTY_PIPE(pipe) (1 << (pipe)) 3273 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp))) 3274 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3)) 3275 #define WM_DIRTY_FBC (1 << 24) 3276 #define WM_DIRTY_DDB (1 << 25) 3277 3278 static unsigned int ilk_compute_wm_dirty(struct intel_display *display, 3279 const struct ilk_wm_values *old, 3280 const struct ilk_wm_values *new) 3281 { 3282 unsigned int dirty = 0; 3283 enum pipe pipe; 3284 int wm_lp; 3285 3286 for_each_pipe(display, pipe) { 3287 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) { 3288 dirty |= WM_DIRTY_PIPE(pipe); 3289 /* Must disable LP1+ watermarks too */ 3290 dirty |= WM_DIRTY_LP_ALL; 3291 } 3292 } 3293 3294 if (old->enable_fbc_wm != new->enable_fbc_wm) { 3295 dirty |= WM_DIRTY_FBC; 3296 /* Must disable LP1+ watermarks too */ 3297 dirty |= WM_DIRTY_LP_ALL; 3298 } 3299 3300 if (old->partitioning != new->partitioning) { 3301 dirty |= WM_DIRTY_DDB; 3302 /* Must disable LP1+ watermarks too */ 3303 dirty |= WM_DIRTY_LP_ALL; 3304 } 3305 3306 /* LP1+ watermarks already deemed dirty, no need to continue */ 3307 if (dirty & WM_DIRTY_LP_ALL) 3308 return dirty; 3309 3310 /* Find the lowest numbered LP1+ watermark in need of an update... */ 3311 for (wm_lp = 1; wm_lp <= 3; wm_lp++) { 3312 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] || 3313 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1]) 3314 break; 3315 } 3316 3317 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */ 3318 for (; wm_lp <= 3; wm_lp++) 3319 dirty |= WM_DIRTY_LP(wm_lp); 3320 3321 return dirty; 3322 } 3323 3324 static bool _ilk_disable_lp_wm(struct intel_display *display, 3325 unsigned int dirty) 3326 { 3327 struct ilk_wm_values *previous = &display->wm.hw; 3328 bool changed = false; 3329 3330 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) { 3331 previous->wm_lp[2] &= ~WM_LP_ENABLE; 3332 intel_de_write(display, WM3_LP_ILK, previous->wm_lp[2]); 3333 changed = true; 3334 } 3335 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) { 3336 previous->wm_lp[1] &= ~WM_LP_ENABLE; 3337 intel_de_write(display, WM2_LP_ILK, previous->wm_lp[1]); 3338 changed = true; 3339 } 3340 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) { 3341 previous->wm_lp[0] &= ~WM_LP_ENABLE; 3342 intel_de_write(display, WM1_LP_ILK, previous->wm_lp[0]); 3343 changed = true; 3344 } 3345 3346 /* 3347 * Don't touch WM_LP_SPRITE_ENABLE here. 3348 * Doing so could cause underruns. 3349 */ 3350 3351 return changed; 3352 } 3353 3354 /* 3355 * The spec says we shouldn't write when we don't need, because every write 3356 * causes WMs to be re-evaluated, expending some power. 3357 */ 3358 static void ilk_write_wm_values(struct intel_display *display, 3359 struct ilk_wm_values *results) 3360 { 3361 struct ilk_wm_values *previous = &display->wm.hw; 3362 unsigned int dirty; 3363 3364 dirty = ilk_compute_wm_dirty(display, previous, results); 3365 if (!dirty) 3366 return; 3367 3368 _ilk_disable_lp_wm(display, dirty); 3369 3370 if (dirty & WM_DIRTY_PIPE(PIPE_A)) 3371 intel_de_write(display, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]); 3372 if (dirty & WM_DIRTY_PIPE(PIPE_B)) 3373 intel_de_write(display, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]); 3374 if (dirty & WM_DIRTY_PIPE(PIPE_C)) 3375 intel_de_write(display, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]); 3376 3377 if (dirty & WM_DIRTY_DDB) { 3378 if (display->platform.haswell || display->platform.broadwell) 3379 intel_de_rmw(display, WM_MISC, WM_MISC_DATA_PARTITION_5_6, 3380 results->partitioning == INTEL_DDB_PART_1_2 ? 0 : 3381 WM_MISC_DATA_PARTITION_5_6); 3382 else 3383 intel_de_rmw(display, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6, 3384 results->partitioning == INTEL_DDB_PART_1_2 ? 0 : 3385 DISP_DATA_PARTITION_5_6); 3386 } 3387 3388 if (dirty & WM_DIRTY_FBC) 3389 intel_de_rmw(display, DISP_ARB_CTL, DISP_FBC_WM_DIS, 3390 results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS); 3391 3392 if (dirty & WM_DIRTY_LP(1) && 3393 previous->wm_lp_spr[0] != results->wm_lp_spr[0]) 3394 intel_de_write(display, WM1S_LP_ILK, results->wm_lp_spr[0]); 3395 3396 if (DISPLAY_VER(display) >= 7) { 3397 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1]) 3398 intel_de_write(display, WM2S_LP_IVB, results->wm_lp_spr[1]); 3399 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2]) 3400 intel_de_write(display, WM3S_LP_IVB, results->wm_lp_spr[2]); 3401 } 3402 3403 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0]) 3404 intel_de_write(display, WM1_LP_ILK, results->wm_lp[0]); 3405 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1]) 3406 intel_de_write(display, WM2_LP_ILK, results->wm_lp[1]); 3407 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2]) 3408 intel_de_write(display, WM3_LP_ILK, results->wm_lp[2]); 3409 3410 display->wm.hw = *results; 3411 } 3412 3413 bool ilk_disable_cxsr(struct intel_display *display) 3414 { 3415 return _ilk_disable_lp_wm(display, WM_DIRTY_LP_ALL); 3416 } 3417 3418 static void ilk_compute_wm_config(struct intel_display *display, 3419 struct intel_wm_config *config) 3420 { 3421 struct intel_crtc *crtc; 3422 3423 /* Compute the currently _active_ config */ 3424 for_each_intel_crtc(display->drm, crtc) { 3425 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk; 3426 3427 if (!wm->pipe_enabled) 3428 continue; 3429 3430 config->sprites_enabled |= wm->sprites_enabled; 3431 config->sprites_scaled |= wm->sprites_scaled; 3432 config->num_pipes_active++; 3433 } 3434 } 3435 3436 static void ilk_program_watermarks(struct intel_display *display) 3437 { 3438 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; 3439 struct ilk_wm_maximums max; 3440 struct intel_wm_config config = {}; 3441 struct ilk_wm_values results = {}; 3442 enum intel_ddb_partitioning partitioning; 3443 3444 ilk_compute_wm_config(display, &config); 3445 3446 ilk_compute_wm_maximums(display, 1, &config, INTEL_DDB_PART_1_2, &max); 3447 ilk_wm_merge(display, &config, &max, &lp_wm_1_2); 3448 3449 /* 5/6 split only in single pipe config on IVB+ */ 3450 if (DISPLAY_VER(display) >= 7 && 3451 config.num_pipes_active == 1 && config.sprites_enabled) { 3452 ilk_compute_wm_maximums(display, 1, &config, INTEL_DDB_PART_5_6, &max); 3453 ilk_wm_merge(display, &config, &max, &lp_wm_5_6); 3454 3455 best_lp_wm = ilk_find_best_result(display, &lp_wm_1_2, &lp_wm_5_6); 3456 } else { 3457 best_lp_wm = &lp_wm_1_2; 3458 } 3459 3460 partitioning = (best_lp_wm == &lp_wm_1_2) ? 3461 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6; 3462 3463 ilk_compute_wm_results(display, best_lp_wm, partitioning, &results); 3464 3465 ilk_write_wm_values(display, &results); 3466 } 3467 3468 static void ilk_initial_watermarks(struct intel_atomic_state *state, 3469 struct intel_crtc *crtc) 3470 { 3471 struct intel_display *display = to_intel_display(crtc); 3472 const struct intel_crtc_state *crtc_state = 3473 intel_atomic_get_new_crtc_state(state, crtc); 3474 3475 mutex_lock(&display->wm.wm_mutex); 3476 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate; 3477 ilk_program_watermarks(display); 3478 mutex_unlock(&display->wm.wm_mutex); 3479 } 3480 3481 static void ilk_optimize_watermarks(struct intel_atomic_state *state, 3482 struct intel_crtc *crtc) 3483 { 3484 struct intel_display *display = to_intel_display(crtc); 3485 const struct intel_crtc_state *crtc_state = 3486 intel_atomic_get_new_crtc_state(state, crtc); 3487 3488 if (!crtc_state->wm.need_postvbl_update) 3489 return; 3490 3491 mutex_lock(&display->wm.wm_mutex); 3492 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal; 3493 ilk_program_watermarks(display); 3494 mutex_unlock(&display->wm.wm_mutex); 3495 } 3496 3497 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc) 3498 { 3499 struct intel_display *display = to_intel_display(crtc); 3500 struct ilk_wm_values *hw = &display->wm.hw; 3501 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); 3502 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal; 3503 enum pipe pipe = crtc->pipe; 3504 3505 hw->wm_pipe[pipe] = intel_de_read(display, WM0_PIPE_ILK(pipe)); 3506 3507 memset(active, 0, sizeof(*active)); 3508 3509 active->pipe_enabled = crtc->active; 3510 3511 if (active->pipe_enabled) { 3512 u32 tmp = hw->wm_pipe[pipe]; 3513 3514 /* 3515 * For active pipes LP0 watermark is marked as 3516 * enabled, and LP1+ watermaks as disabled since 3517 * we can't really reverse compute them in case 3518 * multiple pipes are active. 3519 */ 3520 active->wm[0].enable = true; 3521 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp); 3522 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp); 3523 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp); 3524 } else { 3525 int level; 3526 3527 /* 3528 * For inactive pipes, all watermark levels 3529 * should be marked as enabled but zeroed, 3530 * which is what we'd compute them to. 3531 */ 3532 for (level = 0; level < display->wm.num_levels; level++) 3533 active->wm[level].enable = true; 3534 } 3535 3536 crtc->wm.active.ilk = *active; 3537 } 3538 3539 static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state) 3540 { 3541 struct drm_plane *plane; 3542 struct intel_crtc *crtc; 3543 3544 for_each_intel_crtc(state->dev, crtc) { 3545 struct intel_crtc_state *crtc_state; 3546 3547 crtc_state = intel_atomic_get_crtc_state(state, crtc); 3548 if (IS_ERR(crtc_state)) 3549 return PTR_ERR(crtc_state); 3550 3551 if (crtc_state->hw.active) { 3552 /* 3553 * Preserve the inherited flag to avoid 3554 * taking the full modeset path. 3555 */ 3556 crtc_state->inherited = true; 3557 } 3558 } 3559 3560 drm_for_each_plane(plane, state->dev) { 3561 struct drm_plane_state *plane_state; 3562 3563 plane_state = drm_atomic_get_plane_state(state, plane); 3564 if (IS_ERR(plane_state)) 3565 return PTR_ERR(plane_state); 3566 } 3567 3568 return 0; 3569 } 3570 3571 /* 3572 * Calculate what we think the watermarks should be for the state we've read 3573 * out of the hardware and then immediately program those watermarks so that 3574 * we ensure the hardware settings match our internal state. 3575 * 3576 * We can calculate what we think WM's should be by creating a duplicate of the 3577 * current state (which was constructed during hardware readout) and running it 3578 * through the atomic check code to calculate new watermark values in the 3579 * state object. 3580 */ 3581 void ilk_wm_sanitize(struct intel_display *display) 3582 { 3583 struct drm_atomic_state *state; 3584 struct intel_atomic_state *intel_state; 3585 struct intel_crtc *crtc; 3586 struct intel_crtc_state *crtc_state; 3587 struct drm_modeset_acquire_ctx ctx; 3588 int ret; 3589 int i; 3590 3591 /* Only supported on platforms that use atomic watermark design */ 3592 if (!display->funcs.wm->optimize_watermarks) 3593 return; 3594 3595 if (drm_WARN_ON(display->drm, DISPLAY_VER(display) >= 9)) 3596 return; 3597 3598 state = drm_atomic_state_alloc(display->drm); 3599 if (drm_WARN_ON(display->drm, !state)) 3600 return; 3601 3602 intel_state = to_intel_atomic_state(state); 3603 3604 drm_modeset_acquire_init(&ctx, 0); 3605 3606 state->acquire_ctx = &ctx; 3607 to_intel_atomic_state(state)->internal = true; 3608 3609 retry: 3610 /* 3611 * Hardware readout is the only time we don't want to calculate 3612 * intermediate watermarks (since we don't trust the current 3613 * watermarks). 3614 */ 3615 if (!HAS_GMCH(display)) 3616 intel_state->skip_intermediate_wm = true; 3617 3618 ret = ilk_sanitize_watermarks_add_affected(state); 3619 if (ret) 3620 goto fail; 3621 3622 ret = intel_atomic_check(display->drm, state); 3623 if (ret) 3624 goto fail; 3625 3626 /* Write calculated watermark values back */ 3627 for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) { 3628 crtc_state->wm.need_postvbl_update = true; 3629 intel_optimize_watermarks(intel_state, crtc); 3630 3631 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm; 3632 } 3633 3634 fail: 3635 if (ret == -EDEADLK) { 3636 drm_atomic_state_clear(state); 3637 drm_modeset_backoff(&ctx); 3638 goto retry; 3639 } 3640 3641 /* 3642 * If we fail here, it means that the hardware appears to be 3643 * programmed in a way that shouldn't be possible, given our 3644 * understanding of watermark requirements. This might mean a 3645 * mistake in the hardware readout code or a mistake in the 3646 * watermark calculations for a given platform. Raise a WARN 3647 * so that this is noticeable. 3648 * 3649 * If this actually happens, we'll have to just leave the 3650 * BIOS-programmed watermarks untouched and hope for the best. 3651 */ 3652 drm_WARN(display->drm, ret, 3653 "Could not determine valid watermarks for inherited state\n"); 3654 3655 drm_atomic_state_put(state); 3656 3657 drm_modeset_drop_locks(&ctx); 3658 drm_modeset_acquire_fini(&ctx); 3659 } 3660 3661 #define _FW_WM(value, plane) \ 3662 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT) 3663 #define _FW_WM_VLV(value, plane) \ 3664 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT) 3665 3666 static void g4x_read_wm_values(struct intel_display *display, 3667 struct g4x_wm_values *wm) 3668 { 3669 u32 tmp; 3670 3671 tmp = intel_de_read(display, DSPFW1(display)); 3672 wm->sr.plane = _FW_WM(tmp, SR); 3673 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB); 3674 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB); 3675 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA); 3676 3677 tmp = intel_de_read(display, DSPFW2(display)); 3678 wm->fbc_en = tmp & DSPFW_FBC_SR_EN; 3679 wm->sr.fbc = _FW_WM(tmp, FBC_SR); 3680 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR); 3681 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB); 3682 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA); 3683 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA); 3684 3685 tmp = intel_de_read(display, DSPFW3(display)); 3686 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN; 3687 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); 3688 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR); 3689 wm->hpll.plane = _FW_WM(tmp, HPLL_SR); 3690 } 3691 3692 static void vlv_read_wm_values(struct intel_display *display, 3693 struct vlv_wm_values *wm) 3694 { 3695 enum pipe pipe; 3696 u32 tmp; 3697 3698 for_each_pipe(display, pipe) { 3699 tmp = intel_de_read(display, VLV_DDL(pipe)); 3700 3701 wm->ddl[pipe].plane[PLANE_PRIMARY] = 3702 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); 3703 wm->ddl[pipe].plane[PLANE_CURSOR] = 3704 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); 3705 wm->ddl[pipe].plane[PLANE_SPRITE0] = 3706 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); 3707 wm->ddl[pipe].plane[PLANE_SPRITE1] = 3708 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); 3709 } 3710 3711 tmp = intel_de_read(display, DSPFW1(display)); 3712 wm->sr.plane = _FW_WM(tmp, SR); 3713 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB); 3714 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB); 3715 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA); 3716 3717 tmp = intel_de_read(display, DSPFW2(display)); 3718 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB); 3719 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA); 3720 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA); 3721 3722 tmp = intel_de_read(display, DSPFW3(display)); 3723 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); 3724 3725 if (display->platform.cherryview) { 3726 tmp = intel_de_read(display, DSPFW7_CHV); 3727 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED); 3728 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC); 3729 3730 tmp = intel_de_read(display, DSPFW8_CHV); 3731 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF); 3732 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE); 3733 3734 tmp = intel_de_read(display, DSPFW9_CHV); 3735 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC); 3736 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC); 3737 3738 tmp = intel_de_read(display, DSPHOWM); 3739 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; 3740 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8; 3741 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8; 3742 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8; 3743 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8; 3744 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8; 3745 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8; 3746 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8; 3747 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8; 3748 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8; 3749 } else { 3750 tmp = intel_de_read(display, DSPFW7); 3751 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED); 3752 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC); 3753 3754 tmp = intel_de_read(display, DSPHOWM); 3755 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; 3756 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8; 3757 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8; 3758 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8; 3759 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8; 3760 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8; 3761 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8; 3762 } 3763 } 3764 3765 #undef _FW_WM 3766 #undef _FW_WM_VLV 3767 3768 static void g4x_wm_get_hw_state(struct intel_display *display) 3769 { 3770 struct g4x_wm_values *wm = &display->wm.g4x; 3771 struct intel_crtc *crtc; 3772 3773 g4x_read_wm_values(display, wm); 3774 3775 wm->cxsr = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN; 3776 3777 for_each_intel_crtc(display->drm, crtc) { 3778 struct intel_crtc_state *crtc_state = 3779 to_intel_crtc_state(crtc->base.state); 3780 struct g4x_wm_state *active = &crtc->wm.active.g4x; 3781 struct g4x_pipe_wm *raw; 3782 enum pipe pipe = crtc->pipe; 3783 enum plane_id plane_id; 3784 int level, max_level; 3785 3786 active->cxsr = wm->cxsr; 3787 active->hpll_en = wm->hpll_en; 3788 active->fbc_en = wm->fbc_en; 3789 3790 active->sr = wm->sr; 3791 active->hpll = wm->hpll; 3792 3793 for_each_plane_id_on_crtc(crtc, plane_id) { 3794 active->wm.plane[plane_id] = 3795 wm->pipe[pipe].plane[plane_id]; 3796 } 3797 3798 if (wm->cxsr && wm->hpll_en) 3799 max_level = G4X_WM_LEVEL_HPLL; 3800 else if (wm->cxsr) 3801 max_level = G4X_WM_LEVEL_SR; 3802 else 3803 max_level = G4X_WM_LEVEL_NORMAL; 3804 3805 level = G4X_WM_LEVEL_NORMAL; 3806 raw = &crtc_state->wm.g4x.raw[level]; 3807 for_each_plane_id_on_crtc(crtc, plane_id) 3808 raw->plane[plane_id] = active->wm.plane[plane_id]; 3809 3810 level = G4X_WM_LEVEL_SR; 3811 if (level > max_level) 3812 goto out; 3813 3814 raw = &crtc_state->wm.g4x.raw[level]; 3815 raw->plane[PLANE_PRIMARY] = active->sr.plane; 3816 raw->plane[PLANE_CURSOR] = active->sr.cursor; 3817 raw->plane[PLANE_SPRITE0] = 0; 3818 raw->fbc = active->sr.fbc; 3819 3820 level = G4X_WM_LEVEL_HPLL; 3821 if (level > max_level) 3822 goto out; 3823 3824 raw = &crtc_state->wm.g4x.raw[level]; 3825 raw->plane[PLANE_PRIMARY] = active->hpll.plane; 3826 raw->plane[PLANE_CURSOR] = active->hpll.cursor; 3827 raw->plane[PLANE_SPRITE0] = 0; 3828 raw->fbc = active->hpll.fbc; 3829 3830 level++; 3831 out: 3832 for_each_plane_id_on_crtc(crtc, plane_id) 3833 g4x_raw_plane_wm_set(crtc_state, level, 3834 plane_id, USHRT_MAX); 3835 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX); 3836 3837 g4x_invalidate_wms(crtc, active, level); 3838 3839 crtc_state->wm.g4x.optimal = *active; 3840 crtc_state->wm.g4x.intermediate = *active; 3841 3842 drm_dbg_kms(display->drm, 3843 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n", 3844 pipe_name(pipe), 3845 wm->pipe[pipe].plane[PLANE_PRIMARY], 3846 wm->pipe[pipe].plane[PLANE_CURSOR], 3847 wm->pipe[pipe].plane[PLANE_SPRITE0]); 3848 } 3849 3850 drm_dbg_kms(display->drm, 3851 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n", 3852 wm->sr.plane, wm->sr.cursor, wm->sr.fbc); 3853 drm_dbg_kms(display->drm, 3854 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n", 3855 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc); 3856 drm_dbg_kms(display->drm, "Initial SR=%s HPLL=%s FBC=%s\n", 3857 str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en), 3858 str_yes_no(wm->fbc_en)); 3859 } 3860 3861 static void g4x_wm_sanitize(struct intel_display *display) 3862 { 3863 struct intel_plane *plane; 3864 struct intel_crtc *crtc; 3865 3866 mutex_lock(&display->wm.wm_mutex); 3867 3868 for_each_intel_plane(display->drm, plane) { 3869 struct intel_crtc *crtc = 3870 intel_crtc_for_pipe(display, plane->pipe); 3871 struct intel_crtc_state *crtc_state = 3872 to_intel_crtc_state(crtc->base.state); 3873 struct intel_plane_state *plane_state = 3874 to_intel_plane_state(plane->base.state); 3875 enum plane_id plane_id = plane->id; 3876 int level; 3877 3878 if (plane_state->uapi.visible) 3879 continue; 3880 3881 for (level = 0; level < display->wm.num_levels; level++) { 3882 struct g4x_pipe_wm *raw = 3883 &crtc_state->wm.g4x.raw[level]; 3884 3885 raw->plane[plane_id] = 0; 3886 3887 if (plane_id == PLANE_PRIMARY) 3888 raw->fbc = 0; 3889 } 3890 } 3891 3892 for_each_intel_crtc(display->drm, crtc) { 3893 struct intel_crtc_state *crtc_state = 3894 to_intel_crtc_state(crtc->base.state); 3895 int ret; 3896 3897 ret = _g4x_compute_pipe_wm(crtc_state); 3898 drm_WARN_ON(display->drm, ret); 3899 3900 crtc_state->wm.g4x.intermediate = 3901 crtc_state->wm.g4x.optimal; 3902 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal; 3903 } 3904 3905 g4x_program_watermarks(display); 3906 3907 mutex_unlock(&display->wm.wm_mutex); 3908 } 3909 3910 static void vlv_wm_get_hw_state(struct intel_display *display) 3911 { 3912 struct vlv_wm_values *wm = &display->wm.vlv; 3913 struct intel_crtc *crtc; 3914 u32 val; 3915 int ret; 3916 3917 vlv_read_wm_values(display, wm); 3918 3919 wm->cxsr = intel_de_read(display, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; 3920 wm->level = VLV_WM_LEVEL_PM2; 3921 3922 if (display->platform.cherryview) { 3923 vlv_punit_get(display->drm); 3924 3925 val = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM); 3926 if (val & DSP_MAXFIFO_PM5_ENABLE) 3927 wm->level = VLV_WM_LEVEL_PM5; 3928 3929 /* 3930 * If DDR DVFS is disabled in the BIOS, Punit 3931 * will never ack the request. So if that happens 3932 * assume we don't have to enable/disable DDR DVFS 3933 * dynamically. To test that just set the REQ_ACK 3934 * bit to poke the Punit, but don't change the 3935 * HIGH/LOW bits so that we don't actually change 3936 * the current state. 3937 */ 3938 val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2); 3939 val |= FORCE_DDR_FREQ_REQ_ACK; 3940 vlv_punit_write(display->drm, PUNIT_REG_DDR_SETUP2, val); 3941 3942 ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2), 3943 (val & FORCE_DDR_FREQ_REQ_ACK) == 0, 3944 500, 3000, false); 3945 if (ret) { 3946 drm_dbg_kms(display->drm, 3947 "Punit not acking DDR DVFS request, " 3948 "assuming DDR DVFS is disabled\n"); 3949 display->wm.num_levels = VLV_WM_LEVEL_PM5 + 1; 3950 } else { 3951 val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2); 3952 if ((val & FORCE_DDR_HIGH_FREQ) == 0) 3953 wm->level = VLV_WM_LEVEL_DDR_DVFS; 3954 } 3955 3956 vlv_punit_put(display->drm); 3957 } 3958 3959 for_each_intel_crtc(display->drm, crtc) { 3960 struct intel_crtc_state *crtc_state = 3961 to_intel_crtc_state(crtc->base.state); 3962 struct vlv_wm_state *active = &crtc->wm.active.vlv; 3963 const struct vlv_fifo_state *fifo_state = 3964 &crtc_state->wm.vlv.fifo_state; 3965 enum pipe pipe = crtc->pipe; 3966 enum plane_id plane_id; 3967 int level; 3968 3969 vlv_get_fifo_size(crtc_state); 3970 3971 active->num_levels = wm->level + 1; 3972 active->cxsr = wm->cxsr; 3973 3974 for (level = 0; level < active->num_levels; level++) { 3975 struct g4x_pipe_wm *raw = 3976 &crtc_state->wm.vlv.raw[level]; 3977 3978 active->sr[level].plane = wm->sr.plane; 3979 active->sr[level].cursor = wm->sr.cursor; 3980 3981 for_each_plane_id_on_crtc(crtc, plane_id) { 3982 active->wm[level].plane[plane_id] = 3983 wm->pipe[pipe].plane[plane_id]; 3984 3985 raw->plane[plane_id] = 3986 vlv_invert_wm_value(active->wm[level].plane[plane_id], 3987 fifo_state->plane[plane_id]); 3988 } 3989 } 3990 3991 for_each_plane_id_on_crtc(crtc, plane_id) 3992 vlv_raw_plane_wm_set(crtc_state, level, 3993 plane_id, USHRT_MAX); 3994 vlv_invalidate_wms(crtc, active, level); 3995 3996 crtc_state->wm.vlv.optimal = *active; 3997 crtc_state->wm.vlv.intermediate = *active; 3998 3999 drm_dbg_kms(display->drm, 4000 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n", 4001 pipe_name(pipe), 4002 wm->pipe[pipe].plane[PLANE_PRIMARY], 4003 wm->pipe[pipe].plane[PLANE_CURSOR], 4004 wm->pipe[pipe].plane[PLANE_SPRITE0], 4005 wm->pipe[pipe].plane[PLANE_SPRITE1]); 4006 } 4007 4008 drm_dbg_kms(display->drm, 4009 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n", 4010 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr); 4011 } 4012 4013 static void vlv_wm_sanitize(struct intel_display *display) 4014 { 4015 struct intel_plane *plane; 4016 struct intel_crtc *crtc; 4017 4018 mutex_lock(&display->wm.wm_mutex); 4019 4020 for_each_intel_plane(display->drm, plane) { 4021 struct intel_crtc *crtc = 4022 intel_crtc_for_pipe(display, plane->pipe); 4023 struct intel_crtc_state *crtc_state = 4024 to_intel_crtc_state(crtc->base.state); 4025 struct intel_plane_state *plane_state = 4026 to_intel_plane_state(plane->base.state); 4027 enum plane_id plane_id = plane->id; 4028 int level; 4029 4030 if (plane_state->uapi.visible) 4031 continue; 4032 4033 for (level = 0; level < display->wm.num_levels; level++) { 4034 struct g4x_pipe_wm *raw = 4035 &crtc_state->wm.vlv.raw[level]; 4036 4037 raw->plane[plane_id] = 0; 4038 } 4039 } 4040 4041 for_each_intel_crtc(display->drm, crtc) { 4042 struct intel_crtc_state *crtc_state = 4043 to_intel_crtc_state(crtc->base.state); 4044 int ret; 4045 4046 ret = _vlv_compute_pipe_wm(crtc_state); 4047 drm_WARN_ON(display->drm, ret); 4048 4049 crtc_state->wm.vlv.intermediate = 4050 crtc_state->wm.vlv.optimal; 4051 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal; 4052 } 4053 4054 vlv_program_watermarks(display); 4055 4056 mutex_unlock(&display->wm.wm_mutex); 4057 } 4058 4059 /* 4060 * FIXME should probably kill this and improve 4061 * the real watermark readout/sanitation instead 4062 */ 4063 static void ilk_init_lp_watermarks(struct intel_display *display) 4064 { 4065 intel_de_rmw(display, WM3_LP_ILK, WM_LP_ENABLE, 0); 4066 intel_de_rmw(display, WM2_LP_ILK, WM_LP_ENABLE, 0); 4067 intel_de_rmw(display, WM1_LP_ILK, WM_LP_ENABLE, 0); 4068 4069 /* 4070 * Don't touch WM_LP_SPRITE_ENABLE here. 4071 * Doing so could cause underruns. 4072 */ 4073 } 4074 4075 static void ilk_wm_get_hw_state(struct intel_display *display) 4076 { 4077 struct ilk_wm_values *hw = &display->wm.hw; 4078 struct intel_crtc *crtc; 4079 4080 ilk_init_lp_watermarks(display); 4081 4082 for_each_intel_crtc(display->drm, crtc) 4083 ilk_pipe_wm_get_hw_state(crtc); 4084 4085 hw->wm_lp[0] = intel_de_read(display, WM1_LP_ILK); 4086 hw->wm_lp[1] = intel_de_read(display, WM2_LP_ILK); 4087 hw->wm_lp[2] = intel_de_read(display, WM3_LP_ILK); 4088 4089 hw->wm_lp_spr[0] = intel_de_read(display, WM1S_LP_ILK); 4090 if (DISPLAY_VER(display) >= 7) { 4091 hw->wm_lp_spr[1] = intel_de_read(display, WM2S_LP_IVB); 4092 hw->wm_lp_spr[2] = intel_de_read(display, WM3S_LP_IVB); 4093 } 4094 4095 if (display->platform.haswell || display->platform.broadwell) 4096 hw->partitioning = (intel_de_read(display, WM_MISC) & 4097 WM_MISC_DATA_PARTITION_5_6) ? 4098 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; 4099 else if (display->platform.ivybridge) 4100 hw->partitioning = (intel_de_read(display, DISP_ARB_CTL2) & 4101 DISP_DATA_PARTITION_5_6) ? 4102 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; 4103 4104 hw->enable_fbc_wm = 4105 !(intel_de_read(display, DISP_ARB_CTL) & DISP_FBC_WM_DIS); 4106 } 4107 4108 static const struct intel_wm_funcs ilk_wm_funcs = { 4109 .compute_watermarks = ilk_compute_watermarks, 4110 .initial_watermarks = ilk_initial_watermarks, 4111 .optimize_watermarks = ilk_optimize_watermarks, 4112 .get_hw_state = ilk_wm_get_hw_state, 4113 }; 4114 4115 static const struct intel_wm_funcs vlv_wm_funcs = { 4116 .compute_watermarks = vlv_compute_watermarks, 4117 .initial_watermarks = vlv_initial_watermarks, 4118 .optimize_watermarks = vlv_optimize_watermarks, 4119 .atomic_update_watermarks = vlv_atomic_update_fifo, 4120 .get_hw_state = vlv_wm_get_hw_state, 4121 .sanitize = vlv_wm_sanitize, 4122 }; 4123 4124 static const struct intel_wm_funcs g4x_wm_funcs = { 4125 .compute_watermarks = g4x_compute_watermarks, 4126 .initial_watermarks = g4x_initial_watermarks, 4127 .optimize_watermarks = g4x_optimize_watermarks, 4128 .get_hw_state = g4x_wm_get_hw_state, 4129 .sanitize = g4x_wm_sanitize, 4130 }; 4131 4132 static const struct intel_wm_funcs pnv_wm_funcs = { 4133 .compute_watermarks = i9xx_compute_watermarks, 4134 .update_wm = pnv_update_wm, 4135 }; 4136 4137 static const struct intel_wm_funcs i965_wm_funcs = { 4138 .compute_watermarks = i9xx_compute_watermarks, 4139 .update_wm = i965_update_wm, 4140 }; 4141 4142 static const struct intel_wm_funcs i9xx_wm_funcs = { 4143 .compute_watermarks = i9xx_compute_watermarks, 4144 .update_wm = i9xx_update_wm, 4145 }; 4146 4147 static const struct intel_wm_funcs i845_wm_funcs = { 4148 .compute_watermarks = i9xx_compute_watermarks, 4149 .update_wm = i845_update_wm, 4150 }; 4151 4152 static const struct intel_wm_funcs nop_funcs = { 4153 }; 4154 4155 void i9xx_wm_init(struct intel_display *display) 4156 { 4157 /* For FIFO watermark updates */ 4158 if (HAS_PCH_SPLIT(display)) { 4159 ilk_setup_wm_latency(display); 4160 display->funcs.wm = &ilk_wm_funcs; 4161 } else if (display->platform.valleyview || display->platform.cherryview) { 4162 vlv_setup_wm_latency(display); 4163 display->funcs.wm = &vlv_wm_funcs; 4164 } else if (display->platform.g4x) { 4165 g4x_setup_wm_latency(display); 4166 display->funcs.wm = &g4x_wm_funcs; 4167 } else if (display->platform.pineview) { 4168 if (!pnv_get_cxsr_latency(display)) { 4169 drm_info(display->drm, "Unknown FSB/MEM, disabling CxSR\n"); 4170 /* Disable CxSR and never update its watermark again */ 4171 intel_set_memory_cxsr(display, false); 4172 display->funcs.wm = &nop_funcs; 4173 } else { 4174 display->funcs.wm = &pnv_wm_funcs; 4175 } 4176 } else if (DISPLAY_VER(display) == 4) { 4177 display->funcs.wm = &i965_wm_funcs; 4178 } else if (DISPLAY_VER(display) == 3) { 4179 display->funcs.wm = &i9xx_wm_funcs; 4180 } else if (DISPLAY_VER(display) == 2) { 4181 if (INTEL_NUM_PIPES(display) == 1) 4182 display->funcs.wm = &i845_wm_funcs; 4183 else 4184 display->funcs.wm = &i9xx_wm_funcs; 4185 } else { 4186 drm_err(display->drm, 4187 "unexpected fall-through in %s\n", __func__); 4188 display->funcs.wm = &nop_funcs; 4189 } 4190 } 4191