1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include <linux/pci.h> 7 8 #include <drm/drm_print.h> 9 10 #include "i915_utils.h" 11 #include "intel_step.h" 12 #include "intel_crtc.h" 13 #include "intel_de.h" 14 #include "intel_display_core.h" 15 #include "intel_display_types.h" 16 #include "intel_flipq.h" 17 #include "intel_dmc.h" 18 #include "intel_dmc_regs.h" 19 #include "intel_dsb.h" 20 #include "intel_vblank.h" 21 #include "intel_vrr.h" 22 23 /** 24 * DOC: DMC Flip Queue 25 * 26 * A flip queue is a ring buffer implemented by the pipe DMC firmware. 27 * The driver inserts entries into the queues to be executed by the 28 * pipe DMC at a specified presentation timestamp (PTS). 29 * 30 * Each pipe DMC provides several queues: 31 * 32 * - 1 general queue (two DSB buffers executed per entry) 33 * - 3 plane queues (one DSB buffer executed per entry) 34 * - 1 fast queue (deprecated) 35 */ 36 37 #define for_each_flipq(flipq_id) \ 38 for ((flipq_id) = INTEL_FLIPQ_PLANE_1; (flipq_id) < MAX_INTEL_FLIPQ; (flipq_id)++) 39 40 static int intel_flipq_offset(enum intel_flipq_id flipq_id) 41 { 42 switch (flipq_id) { 43 case INTEL_FLIPQ_PLANE_1: 44 return 0x008; 45 case INTEL_FLIPQ_PLANE_2: 46 return 0x108; 47 case INTEL_FLIPQ_PLANE_3: 48 return 0x208; 49 case INTEL_FLIPQ_GENERAL: 50 return 0x308; 51 case INTEL_FLIPQ_FAST: 52 return 0x3c8; 53 default: 54 MISSING_CASE(flipq_id); 55 return 0; 56 } 57 } 58 59 static int intel_flipq_size_dw(enum intel_flipq_id flipq_id) 60 { 61 switch (flipq_id) { 62 case INTEL_FLIPQ_PLANE_1: 63 case INTEL_FLIPQ_PLANE_2: 64 case INTEL_FLIPQ_PLANE_3: 65 return 64; 66 case INTEL_FLIPQ_GENERAL: 67 case INTEL_FLIPQ_FAST: 68 return 48; 69 default: 70 MISSING_CASE(flipq_id); 71 return 1; 72 } 73 } 74 75 static int intel_flipq_elem_size_dw(enum intel_flipq_id flipq_id) 76 { 77 switch (flipq_id) { 78 case INTEL_FLIPQ_PLANE_1: 79 case INTEL_FLIPQ_PLANE_2: 80 case INTEL_FLIPQ_PLANE_3: 81 return 4; 82 case INTEL_FLIPQ_GENERAL: 83 case INTEL_FLIPQ_FAST: 84 return 6; 85 default: 86 MISSING_CASE(flipq_id); 87 return 1; 88 } 89 } 90 91 static int intel_flipq_size_entries(enum intel_flipq_id flipq_id) 92 { 93 return intel_flipq_size_dw(flipq_id) / intel_flipq_elem_size_dw(flipq_id); 94 } 95 96 static void intel_flipq_crtc_init(struct intel_crtc *crtc) 97 { 98 struct intel_display *display = to_intel_display(crtc); 99 enum intel_flipq_id flipq_id; 100 101 for_each_flipq(flipq_id) { 102 struct intel_flipq *flipq = &crtc->flipq[flipq_id]; 103 104 flipq->start_mmioaddr = intel_pipedmc_start_mmioaddr(crtc) + intel_flipq_offset(flipq_id); 105 flipq->flipq_id = flipq_id; 106 107 drm_dbg_kms(display->drm, "[CRTC:%d:%s] FQ %d: start 0x%x\n", 108 crtc->base.base.id, crtc->base.name, 109 flipq_id, flipq->start_mmioaddr); 110 } 111 } 112 113 bool intel_flipq_supported(struct intel_display *display) 114 { 115 if (!display->params.enable_flipq) 116 return false; 117 118 if (!display->dmc.dmc) 119 return false; 120 121 if (DISPLAY_VER(display) == 20) 122 return true; 123 124 /* DMC firmware expects VRR timing generator to be used */ 125 return DISPLAY_VER(display) >= 30 && intel_vrr_always_use_vrr_tg(display); 126 } 127 128 void intel_flipq_init(struct intel_display *display) 129 { 130 struct intel_crtc *crtc; 131 132 intel_dmc_wait_fw_load(display); 133 134 for_each_intel_crtc(display->drm, crtc) 135 intel_flipq_crtc_init(crtc); 136 } 137 138 static int cdclk_factor(struct intel_display *display) 139 { 140 if (DISPLAY_VER(display) >= 30) 141 return 120; 142 else 143 return 280; 144 } 145 146 int intel_flipq_exec_time_us(struct intel_display *display) 147 { 148 return intel_dsb_exec_time_us() + 149 DIV_ROUND_UP(display->cdclk.hw.cdclk * cdclk_factor(display), 540000) + 150 display->sagv.block_time_us; 151 } 152 153 static int intel_flipq_preempt_timeout_ms(struct intel_display *display) 154 { 155 return DIV_ROUND_UP(intel_flipq_exec_time_us(display), 1000); 156 } 157 158 static void intel_flipq_preempt(struct intel_crtc *crtc, bool preempt) 159 { 160 struct intel_display *display = to_intel_display(crtc); 161 162 intel_de_rmw(display, PIPEDMC_FQ_CTRL(crtc->pipe), 163 PIPEDMC_FQ_CTRL_PREEMPT, preempt ? PIPEDMC_FQ_CTRL_PREEMPT : 0); 164 165 if (preempt && 166 intel_de_wait_for_clear(display, 167 PIPEDMC_FQ_STATUS(crtc->pipe), 168 PIPEDMC_FQ_STATUS_BUSY, 169 intel_flipq_preempt_timeout_ms(display))) 170 drm_err(display->drm, "[CRTC:%d:%s] flip queue preempt timeout\n", 171 crtc->base.base.id, crtc->base.name); 172 } 173 174 static int intel_flipq_current_head(struct intel_crtc *crtc, enum intel_flipq_id flipq_id) 175 { 176 struct intel_display *display = to_intel_display(crtc); 177 178 return intel_de_read(display, PIPEDMC_FPQ_CHP(crtc->pipe, flipq_id)); 179 } 180 181 static void intel_flipq_write_tail(struct intel_crtc *crtc) 182 { 183 struct intel_display *display = to_intel_display(crtc); 184 185 intel_de_write(display, PIPEDMC_FPQ_ATOMIC_TP(crtc->pipe), 186 PIPEDMC_FPQ_PLANEQ_3_TP(crtc->flipq[INTEL_FLIPQ_PLANE_3].tail) | 187 PIPEDMC_FPQ_PLANEQ_2_TP(crtc->flipq[INTEL_FLIPQ_PLANE_2].tail) | 188 PIPEDMC_FPQ_PLANEQ_1_TP(crtc->flipq[INTEL_FLIPQ_PLANE_1].tail) | 189 PIPEDMC_FPQ_FASTQ_TP(crtc->flipq[INTEL_FLIPQ_FAST].tail) | 190 PIPEDMC_FPQ_GENERALQ_TP(crtc->flipq[INTEL_FLIPQ_GENERAL].tail)); 191 } 192 193 static void intel_flipq_sw_dmc_wake(struct intel_crtc *crtc) 194 { 195 struct intel_display *display = to_intel_display(crtc); 196 197 intel_de_write(display, PIPEDMC_FPQ_CTL1(crtc->pipe), PIPEDMC_SW_DMC_WAKE); 198 } 199 200 static int intel_flipq_exec_time_lines(const struct intel_crtc_state *crtc_state) 201 { 202 struct intel_display *display = to_intel_display(crtc_state); 203 204 return intel_usecs_to_scanlines(&crtc_state->hw.adjusted_mode, 205 intel_flipq_exec_time_us(display)); 206 } 207 208 void intel_flipq_dump(struct intel_crtc *crtc, 209 enum intel_flipq_id flipq_id) 210 { 211 struct intel_display *display = to_intel_display(crtc); 212 struct intel_flipq *flipq = &crtc->flipq[flipq_id]; 213 u32 tmp; 214 215 drm_dbg_kms(display->drm, 216 "[CRTC:%d:%s] FQ %d @ 0x%x: ", 217 crtc->base.base.id, crtc->base.name, flipq_id, 218 flipq->start_mmioaddr); 219 for (int i = 0 ; i < intel_flipq_size_dw(flipq_id); i++) { 220 printk(KERN_CONT " 0x%08x", 221 intel_de_read(display, PIPEDMC_FQ_RAM(flipq->start_mmioaddr, i))); 222 if (i % intel_flipq_elem_size_dw(flipq_id) == intel_flipq_elem_size_dw(flipq_id) - 1) 223 printk(KERN_CONT "\n"); 224 } 225 226 drm_dbg_kms(display->drm, 227 "[CRTC:%d:%s] FQ %d: chp=0x%x, hp=0x%x\n", 228 crtc->base.base.id, crtc->base.name, flipq_id, 229 intel_de_read(display, PIPEDMC_FPQ_CHP(crtc->pipe, flipq_id)), 230 intel_de_read(display, PIPEDMC_FPQ_HP(crtc->pipe, flipq_id))); 231 232 drm_dbg_kms(display->drm, 233 "[CRTC:%d:%s] FQ %d: current head %d\n", 234 crtc->base.base.id, crtc->base.name, flipq_id, 235 intel_flipq_current_head(crtc, flipq_id)); 236 237 drm_dbg_kms(display->drm, 238 "[CRTC:%d:%s] flip queue timestamp: 0x%x\n", 239 crtc->base.base.id, crtc->base.name, 240 intel_de_read(display, PIPEDMC_FPQ_TS(crtc->pipe))); 241 242 tmp = intel_de_read(display, PIPEDMC_FPQ_ATOMIC_TP(crtc->pipe)); 243 244 drm_dbg_kms(display->drm, 245 "[CRTC:%d:%s] flip queue atomic tails: P3 %d, P2 %d, P1 %d, G %d, F %d\n", 246 crtc->base.base.id, crtc->base.name, 247 REG_FIELD_GET(PIPEDMC_FPQ_PLANEQ_3_TP_MASK, tmp), 248 REG_FIELD_GET(PIPEDMC_FPQ_PLANEQ_2_TP_MASK, tmp), 249 REG_FIELD_GET(PIPEDMC_FPQ_PLANEQ_1_TP_MASK, tmp), 250 REG_FIELD_GET(PIPEDMC_FPQ_GENERALQ_TP_MASK, tmp), 251 REG_FIELD_GET(PIPEDMC_FPQ_FASTQ_TP_MASK, tmp)); 252 } 253 254 void intel_flipq_reset(struct intel_display *display, enum pipe pipe) 255 { 256 struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe); 257 enum intel_flipq_id flipq_id; 258 259 intel_de_write(display, PIPEDMC_FQ_CTRL(pipe), 0); 260 261 intel_de_write(display, PIPEDMC_SCANLINECMPLOWER(pipe), 0); 262 intel_de_write(display, PIPEDMC_SCANLINECMPUPPER(pipe), 0); 263 264 for_each_flipq(flipq_id) { 265 struct intel_flipq *flipq = &crtc->flipq[flipq_id]; 266 267 intel_de_write(display, PIPEDMC_FPQ_HP(pipe, flipq_id), 0); 268 intel_de_write(display, PIPEDMC_FPQ_CHP(pipe, flipq_id), 0); 269 270 flipq->tail = 0; 271 } 272 273 intel_de_write(display, PIPEDMC_FPQ_ATOMIC_TP(pipe), 0); 274 } 275 276 static enum pipedmc_event_id flipq_event_id(struct intel_display *display) 277 { 278 if (DISPLAY_VER(display) >= 30) 279 return PIPEDMC_EVENT_FULL_FQ_WAKE_TRIGGER; 280 else 281 return PIPEDMC_EVENT_SCANLINE_INRANGE_FQ_TRIGGER; 282 } 283 284 void intel_flipq_enable(const struct intel_crtc_state *crtc_state) 285 { 286 struct intel_display *display = to_intel_display(crtc_state); 287 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 288 /* FIXME what to do with VRR? */ 289 int scanline = intel_mode_vblank_start(&crtc_state->hw.adjusted_mode) - 290 intel_flipq_exec_time_lines(crtc_state); 291 292 if (DISPLAY_VER(display) >= 30) { 293 u32 start_mmioaddr = intel_pipedmc_start_mmioaddr(crtc); 294 295 /* undocumented magic DMC variables */ 296 intel_de_write(display, PTL_PIPEDMC_EXEC_TIME_LINES(start_mmioaddr), 297 intel_flipq_exec_time_lines(crtc_state)); 298 intel_de_write(display, PTL_PIPEDMC_END_OF_EXEC_GB(start_mmioaddr), 299 100); 300 } 301 302 intel_de_write(display, PIPEDMC_SCANLINECMPUPPER(crtc->pipe), 303 PIPEDMC_SCANLINE_UPPER(scanline)); 304 intel_de_write(display, PIPEDMC_SCANLINECMPLOWER(crtc->pipe), 305 PIPEDMC_SCANLINEINRANGECMP_EN | 306 PIPEDMC_SCANLINE_LOWER(scanline - 2)); 307 308 intel_pipedmc_enable_event(crtc, flipq_event_id(display)); 309 310 intel_de_write(display, PIPEDMC_FQ_CTRL(crtc->pipe), PIPEDMC_FQ_CTRL_ENABLE); 311 } 312 313 void intel_flipq_disable(const struct intel_crtc_state *crtc_state) 314 { 315 struct intel_display *display = to_intel_display(crtc_state); 316 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 317 318 intel_flipq_preempt(crtc, true); 319 320 intel_de_write(display, PIPEDMC_FQ_CTRL(crtc->pipe), 0); 321 322 intel_pipedmc_disable_event(crtc, flipq_event_id(display)); 323 324 intel_de_write(display, PIPEDMC_SCANLINECMPLOWER(crtc->pipe), 0); 325 intel_de_write(display, PIPEDMC_SCANLINECMPUPPER(crtc->pipe), 0); 326 } 327 328 static bool assert_flipq_has_room(struct intel_crtc *crtc, 329 enum intel_flipq_id flipq_id) 330 { 331 struct intel_display *display = to_intel_display(crtc); 332 struct intel_flipq *flipq = &crtc->flipq[flipq_id]; 333 int head, size = intel_flipq_size_entries(flipq_id); 334 335 head = intel_flipq_current_head(crtc, flipq_id); 336 337 return !drm_WARN(display->drm, 338 (flipq->tail + size - head) % size >= size - 1, 339 "[CRTC:%d:%s] FQ %d overflow (head %d, tail %d, size %d)\n", 340 crtc->base.base.id, crtc->base.name, flipq_id, 341 head, flipq->tail, size); 342 } 343 344 static void intel_flipq_write(struct intel_display *display, 345 struct intel_flipq *flipq, u32 data, int i) 346 { 347 intel_de_write(display, PIPEDMC_FQ_RAM(flipq->start_mmioaddr, flipq->tail * 348 intel_flipq_elem_size_dw(flipq->flipq_id) + i), data); 349 } 350 351 static void lnl_flipq_add(struct intel_display *display, 352 struct intel_flipq *flipq, 353 unsigned int pts, 354 enum intel_dsb_id dsb_id, 355 struct intel_dsb *dsb) 356 { 357 int i = 0; 358 359 switch (flipq->flipq_id) { 360 case INTEL_FLIPQ_GENERAL: 361 intel_flipq_write(display, flipq, pts, i++); 362 intel_flipq_write(display, flipq, intel_dsb_head(dsb), i++); 363 intel_flipq_write(display, flipq, LNL_FQ_INTERRUPT | 364 LNL_FQ_DSB_ID(dsb_id) | 365 LNL_FQ_DSB_SIZE(intel_dsb_size(dsb) / 64), i++); 366 intel_flipq_write(display, flipq, 0, i++); 367 intel_flipq_write(display, flipq, 0, i++); /* head for second DSB */ 368 intel_flipq_write(display, flipq, 0, i++); /* DSB engine + size for second DSB */ 369 break; 370 case INTEL_FLIPQ_PLANE_1: 371 case INTEL_FLIPQ_PLANE_2: 372 case INTEL_FLIPQ_PLANE_3: 373 intel_flipq_write(display, flipq, pts, i++); 374 intel_flipq_write(display, flipq, intel_dsb_head(dsb), i++); 375 intel_flipq_write(display, flipq, LNL_FQ_INTERRUPT | 376 LNL_FQ_DSB_ID(dsb_id) | 377 LNL_FQ_DSB_SIZE(intel_dsb_size(dsb) / 64), i++); 378 intel_flipq_write(display, flipq, 0, i++); 379 break; 380 default: 381 MISSING_CASE(flipq->flipq_id); 382 return; 383 } 384 } 385 386 static void ptl_flipq_add(struct intel_display *display, 387 struct intel_flipq *flipq, 388 unsigned int pts, 389 enum intel_dsb_id dsb_id, 390 struct intel_dsb *dsb) 391 { 392 int i = 0; 393 394 switch (flipq->flipq_id) { 395 case INTEL_FLIPQ_GENERAL: 396 intel_flipq_write(display, flipq, pts, i++); 397 intel_flipq_write(display, flipq, 0, i++); 398 intel_flipq_write(display, flipq, PTL_FQ_INTERRUPT | 399 PTL_FQ_DSB_ID(dsb_id) | 400 PTL_FQ_DSB_SIZE(intel_dsb_size(dsb) / 64), i++); 401 intel_flipq_write(display, flipq, intel_dsb_head(dsb), i++); 402 intel_flipq_write(display, flipq, 0, i++); /* DSB engine + size for second DSB */ 403 intel_flipq_write(display, flipq, 0, i++); /* head for second DSB */ 404 break; 405 case INTEL_FLIPQ_PLANE_1: 406 case INTEL_FLIPQ_PLANE_2: 407 case INTEL_FLIPQ_PLANE_3: 408 intel_flipq_write(display, flipq, pts, i++); 409 intel_flipq_write(display, flipq, 0, i++); 410 intel_flipq_write(display, flipq, PTL_FQ_INTERRUPT | 411 PTL_FQ_DSB_ID(dsb_id) | 412 PTL_FQ_DSB_SIZE(intel_dsb_size(dsb) / 64), i++); 413 intel_flipq_write(display, flipq, intel_dsb_head(dsb), i++); 414 break; 415 default: 416 MISSING_CASE(flipq->flipq_id); 417 return; 418 } 419 } 420 421 void intel_flipq_add(struct intel_crtc *crtc, 422 enum intel_flipq_id flipq_id, 423 unsigned int pts, 424 enum intel_dsb_id dsb_id, 425 struct intel_dsb *dsb) 426 { 427 struct intel_display *display = to_intel_display(crtc); 428 struct intel_flipq *flipq = &crtc->flipq[flipq_id]; 429 430 if (!assert_flipq_has_room(crtc, flipq_id)) 431 return; 432 433 pts += intel_de_read(display, PIPEDMC_FPQ_TS(crtc->pipe)); 434 435 intel_flipq_preempt(crtc, true); 436 437 if (DISPLAY_VER(display) >= 30) 438 ptl_flipq_add(display, flipq, pts, dsb_id, dsb); 439 else 440 lnl_flipq_add(display, flipq, pts, dsb_id, dsb); 441 442 flipq->tail = (flipq->tail + 1) % intel_flipq_size_entries(flipq->flipq_id); 443 intel_flipq_write_tail(crtc); 444 445 intel_flipq_preempt(crtc, false); 446 447 intel_flipq_sw_dmc_wake(crtc); 448 } 449 450 /* Wa_18034343758 */ 451 static bool need_dmc_halt_wa(struct intel_display *display) 452 { 453 return DISPLAY_VER(display) == 20 || 454 (display->platform.pantherlake && 455 IS_DISPLAY_STEP(display, STEP_A0, STEP_B0)); 456 } 457 458 void intel_flipq_wait_dmc_halt(struct intel_dsb *dsb, struct intel_crtc *crtc) 459 { 460 struct intel_display *display = to_intel_display(crtc); 461 462 if (need_dmc_halt_wa(display)) 463 intel_dsb_wait_usec(dsb, 2); 464 } 465 466 void intel_flipq_unhalt_dmc(struct intel_dsb *dsb, struct intel_crtc *crtc) 467 { 468 struct intel_display *display = to_intel_display(crtc); 469 470 if (need_dmc_halt_wa(display)) 471 intel_dsb_reg_write(dsb, PIPEDMC_CTL(crtc->pipe), 0); 472 } 473