1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 #include <linux/kernel.h> 6 #include <linux/pm_qos.h> 7 #include <linux/slab.h> 8 9 #include <drm/drm_atomic_helper.h> 10 #include <drm/drm_fourcc.h> 11 #include <drm/drm_plane.h> 12 #include <drm/drm_vblank_work.h> 13 14 #include "i915_vgpu.h" 15 #include "i9xx_plane.h" 16 #include "icl_dsi.h" 17 #include "intel_atomic.h" 18 #include "intel_atomic_plane.h" 19 #include "intel_color.h" 20 #include "intel_crtc.h" 21 #include "intel_cursor.h" 22 #include "intel_display_debugfs.h" 23 #include "intel_display_irq.h" 24 #include "intel_display_trace.h" 25 #include "intel_display_types.h" 26 #include "intel_drrs.h" 27 #include "intel_dsb.h" 28 #include "intel_dsi.h" 29 #include "intel_fifo_underrun.h" 30 #include "intel_pipe_crc.h" 31 #include "intel_psr.h" 32 #include "intel_sprite.h" 33 #include "intel_vblank.h" 34 #include "intel_vrr.h" 35 #include "skl_universal_plane.h" 36 37 static void assert_vblank_disabled(struct drm_crtc *crtc) 38 { 39 struct drm_i915_private *i915 = to_i915(crtc->dev); 40 41 if (I915_STATE_WARN(i915, drm_crtc_vblank_get(crtc) == 0, 42 "[CRTC:%d:%s] vblank assertion failure (expected off, current on)\n", 43 crtc->base.id, crtc->name)) 44 drm_crtc_vblank_put(crtc); 45 } 46 47 struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915) 48 { 49 return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0)); 50 } 51 52 struct intel_crtc *intel_crtc_for_pipe(struct drm_i915_private *i915, 53 enum pipe pipe) 54 { 55 struct intel_crtc *crtc; 56 57 for_each_intel_crtc(&i915->drm, crtc) { 58 if (crtc->pipe == pipe) 59 return crtc; 60 } 61 62 return NULL; 63 } 64 65 void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc) 66 { 67 drm_crtc_wait_one_vblank(&crtc->base); 68 } 69 70 void intel_wait_for_vblank_if_active(struct drm_i915_private *i915, 71 enum pipe pipe) 72 { 73 struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe); 74 75 if (crtc->active) 76 intel_crtc_wait_for_next_vblank(crtc); 77 } 78 79 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc) 80 { 81 struct drm_device *dev = crtc->base.dev; 82 struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)]; 83 84 if (!crtc->active) 85 return 0; 86 87 if (!vblank->max_vblank_count) 88 return (u32)drm_crtc_accurate_vblank_count(&crtc->base); 89 90 return crtc->base.funcs->get_vblank_counter(&crtc->base); 91 } 92 93 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state) 94 { 95 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); 96 97 /* 98 * From Gen 11, In case of dsi cmd mode, frame counter wouldnt 99 * have updated at the beginning of TE, if we want to use 100 * the hw counter, then we would find it updated in only 101 * the next TE, hence switching to sw counter. 102 */ 103 if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 | 104 I915_MODE_FLAG_DSI_USE_TE1)) 105 return 0; 106 107 /* 108 * On i965gm the hardware frame counter reads 109 * zero when the TV encoder is enabled :( 110 */ 111 if (IS_I965GM(dev_priv) && 112 (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT))) 113 return 0; 114 115 if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv)) 116 return 0xffffffff; /* full 32 bit counter */ 117 else if (DISPLAY_VER(dev_priv) >= 3) 118 return 0xffffff; /* only 24 bits of frame count */ 119 else 120 return 0; /* Gen2 doesn't have a hardware frame counter */ 121 } 122 123 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state) 124 { 125 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 126 127 assert_vblank_disabled(&crtc->base); 128 drm_crtc_set_max_vblank_count(&crtc->base, 129 intel_crtc_max_vblank_count(crtc_state)); 130 drm_crtc_vblank_on(&crtc->base); 131 132 /* 133 * Should really happen exactly when we enable the pipe 134 * but we want the frame counters in the trace, and that 135 * requires vblank support on some platforms/outputs. 136 */ 137 trace_intel_pipe_enable(crtc); 138 } 139 140 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state) 141 { 142 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 143 144 /* 145 * Should really happen exactly when we disable the pipe 146 * but we want the frame counters in the trace, and that 147 * requires vblank support on some platforms/outputs. 148 */ 149 trace_intel_pipe_disable(crtc); 150 151 drm_crtc_vblank_off(&crtc->base); 152 assert_vblank_disabled(&crtc->base); 153 } 154 155 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc) 156 { 157 struct intel_crtc_state *crtc_state; 158 159 crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL); 160 161 if (crtc_state) 162 intel_crtc_state_reset(crtc_state, crtc); 163 164 return crtc_state; 165 } 166 167 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state, 168 struct intel_crtc *crtc) 169 { 170 memset(crtc_state, 0, sizeof(*crtc_state)); 171 172 __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base); 173 174 crtc_state->cpu_transcoder = INVALID_TRANSCODER; 175 crtc_state->master_transcoder = INVALID_TRANSCODER; 176 crtc_state->hsw_workaround_pipe = INVALID_PIPE; 177 crtc_state->scaler_state.scaler_id = -1; 178 crtc_state->mst_master_transcoder = INVALID_TRANSCODER; 179 crtc_state->max_link_bpp_x16 = INT_MAX; 180 } 181 182 static struct intel_crtc *intel_crtc_alloc(void) 183 { 184 struct intel_crtc_state *crtc_state; 185 struct intel_crtc *crtc; 186 187 crtc = kzalloc(sizeof(*crtc), GFP_KERNEL); 188 if (!crtc) 189 return ERR_PTR(-ENOMEM); 190 191 crtc_state = intel_crtc_state_alloc(crtc); 192 if (!crtc_state) { 193 kfree(crtc); 194 return ERR_PTR(-ENOMEM); 195 } 196 197 crtc->base.state = &crtc_state->uapi; 198 crtc->config = crtc_state; 199 200 return crtc; 201 } 202 203 static void intel_crtc_free(struct intel_crtc *crtc) 204 { 205 intel_crtc_destroy_state(&crtc->base, crtc->base.state); 206 kfree(crtc); 207 } 208 209 static void intel_crtc_destroy(struct drm_crtc *_crtc) 210 { 211 struct intel_crtc *crtc = to_intel_crtc(_crtc); 212 213 cpu_latency_qos_remove_request(&crtc->vblank_pm_qos); 214 215 drm_crtc_cleanup(&crtc->base); 216 kfree(crtc); 217 } 218 219 static int intel_crtc_late_register(struct drm_crtc *crtc) 220 { 221 intel_crtc_debugfs_add(to_intel_crtc(crtc)); 222 return 0; 223 } 224 225 #define INTEL_CRTC_FUNCS \ 226 .set_config = drm_atomic_helper_set_config, \ 227 .destroy = intel_crtc_destroy, \ 228 .page_flip = drm_atomic_helper_page_flip, \ 229 .atomic_duplicate_state = intel_crtc_duplicate_state, \ 230 .atomic_destroy_state = intel_crtc_destroy_state, \ 231 .set_crc_source = intel_crtc_set_crc_source, \ 232 .verify_crc_source = intel_crtc_verify_crc_source, \ 233 .get_crc_sources = intel_crtc_get_crc_sources, \ 234 .late_register = intel_crtc_late_register 235 236 static const struct drm_crtc_funcs bdw_crtc_funcs = { 237 INTEL_CRTC_FUNCS, 238 239 .get_vblank_counter = g4x_get_vblank_counter, 240 .enable_vblank = bdw_enable_vblank, 241 .disable_vblank = bdw_disable_vblank, 242 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 243 }; 244 245 static const struct drm_crtc_funcs ilk_crtc_funcs = { 246 INTEL_CRTC_FUNCS, 247 248 .get_vblank_counter = g4x_get_vblank_counter, 249 .enable_vblank = ilk_enable_vblank, 250 .disable_vblank = ilk_disable_vblank, 251 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 252 }; 253 254 static const struct drm_crtc_funcs g4x_crtc_funcs = { 255 INTEL_CRTC_FUNCS, 256 257 .get_vblank_counter = g4x_get_vblank_counter, 258 .enable_vblank = i965_enable_vblank, 259 .disable_vblank = i965_disable_vblank, 260 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 261 }; 262 263 static const struct drm_crtc_funcs i965_crtc_funcs = { 264 INTEL_CRTC_FUNCS, 265 266 .get_vblank_counter = i915_get_vblank_counter, 267 .enable_vblank = i965_enable_vblank, 268 .disable_vblank = i965_disable_vblank, 269 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 270 }; 271 272 static const struct drm_crtc_funcs i915gm_crtc_funcs = { 273 INTEL_CRTC_FUNCS, 274 275 .get_vblank_counter = i915_get_vblank_counter, 276 .enable_vblank = i915gm_enable_vblank, 277 .disable_vblank = i915gm_disable_vblank, 278 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 279 }; 280 281 static const struct drm_crtc_funcs i915_crtc_funcs = { 282 INTEL_CRTC_FUNCS, 283 284 .get_vblank_counter = i915_get_vblank_counter, 285 .enable_vblank = i8xx_enable_vblank, 286 .disable_vblank = i8xx_disable_vblank, 287 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 288 }; 289 290 static const struct drm_crtc_funcs i8xx_crtc_funcs = { 291 INTEL_CRTC_FUNCS, 292 293 /* no hw vblank counter */ 294 .enable_vblank = i8xx_enable_vblank, 295 .disable_vblank = i8xx_disable_vblank, 296 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, 297 }; 298 299 int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe) 300 { 301 struct intel_plane *primary, *cursor; 302 const struct drm_crtc_funcs *funcs; 303 struct intel_crtc *crtc; 304 int sprite, ret; 305 306 crtc = intel_crtc_alloc(); 307 if (IS_ERR(crtc)) 308 return PTR_ERR(crtc); 309 310 crtc->pipe = pipe; 311 crtc->num_scalers = DISPLAY_RUNTIME_INFO(dev_priv)->num_scalers[pipe]; 312 313 if (DISPLAY_VER(dev_priv) >= 9) 314 primary = skl_universal_plane_create(dev_priv, pipe, 315 PLANE_PRIMARY); 316 else 317 primary = intel_primary_plane_create(dev_priv, pipe); 318 if (IS_ERR(primary)) { 319 ret = PTR_ERR(primary); 320 goto fail; 321 } 322 crtc->plane_ids_mask |= BIT(primary->id); 323 324 intel_init_fifo_underrun_reporting(dev_priv, crtc, false); 325 326 for_each_sprite(dev_priv, pipe, sprite) { 327 struct intel_plane *plane; 328 329 if (DISPLAY_VER(dev_priv) >= 9) 330 plane = skl_universal_plane_create(dev_priv, pipe, 331 PLANE_SPRITE0 + sprite); 332 else 333 plane = intel_sprite_plane_create(dev_priv, pipe, sprite); 334 if (IS_ERR(plane)) { 335 ret = PTR_ERR(plane); 336 goto fail; 337 } 338 crtc->plane_ids_mask |= BIT(plane->id); 339 } 340 341 cursor = intel_cursor_plane_create(dev_priv, pipe); 342 if (IS_ERR(cursor)) { 343 ret = PTR_ERR(cursor); 344 goto fail; 345 } 346 crtc->plane_ids_mask |= BIT(cursor->id); 347 348 if (HAS_GMCH(dev_priv)) { 349 if (IS_CHERRYVIEW(dev_priv) || 350 IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv)) 351 funcs = &g4x_crtc_funcs; 352 else if (DISPLAY_VER(dev_priv) == 4) 353 funcs = &i965_crtc_funcs; 354 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv)) 355 funcs = &i915gm_crtc_funcs; 356 else if (DISPLAY_VER(dev_priv) == 3) 357 funcs = &i915_crtc_funcs; 358 else 359 funcs = &i8xx_crtc_funcs; 360 } else { 361 if (DISPLAY_VER(dev_priv) >= 8) 362 funcs = &bdw_crtc_funcs; 363 else 364 funcs = &ilk_crtc_funcs; 365 } 366 367 ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base, 368 &primary->base, &cursor->base, 369 funcs, "pipe %c", pipe_name(pipe)); 370 if (ret) 371 goto fail; 372 373 if (DISPLAY_VER(dev_priv) >= 11) 374 drm_crtc_create_scaling_filter_property(&crtc->base, 375 BIT(DRM_SCALING_FILTER_DEFAULT) | 376 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR)); 377 378 intel_color_crtc_init(crtc); 379 intel_drrs_crtc_init(crtc); 380 intel_crtc_crc_init(crtc); 381 382 cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE); 383 384 drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe); 385 386 return 0; 387 388 fail: 389 intel_crtc_free(crtc); 390 391 return ret; 392 } 393 394 static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state) 395 { 396 return crtc_state->hw.active && 397 !intel_crtc_needs_modeset(crtc_state) && 398 !crtc_state->preload_luts && 399 intel_crtc_needs_color_update(crtc_state) && 400 !intel_color_uses_dsb(crtc_state); 401 } 402 403 static void intel_crtc_vblank_work(struct kthread_work *base) 404 { 405 struct drm_vblank_work *work = to_drm_vblank_work(base); 406 struct intel_crtc_state *crtc_state = 407 container_of(work, typeof(*crtc_state), vblank_work); 408 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 409 410 trace_intel_crtc_vblank_work_start(crtc); 411 412 intel_color_load_luts(crtc_state); 413 414 if (crtc_state->uapi.event) { 415 spin_lock_irq(&crtc->base.dev->event_lock); 416 drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event); 417 crtc_state->uapi.event = NULL; 418 spin_unlock_irq(&crtc->base.dev->event_lock); 419 } 420 421 trace_intel_crtc_vblank_work_end(crtc); 422 } 423 424 static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state) 425 { 426 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 427 428 drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base, 429 intel_crtc_vblank_work); 430 /* 431 * Interrupt latency is critical for getting the vblank 432 * work executed as early as possible during the vblank. 433 */ 434 cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0); 435 } 436 437 void intel_wait_for_vblank_workers(struct intel_atomic_state *state) 438 { 439 struct intel_crtc_state *crtc_state; 440 struct intel_crtc *crtc; 441 int i; 442 443 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { 444 if (!intel_crtc_needs_vblank_work(crtc_state)) 445 continue; 446 447 drm_vblank_work_flush(&crtc_state->vblank_work); 448 cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 449 PM_QOS_DEFAULT_VALUE); 450 } 451 } 452 453 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode, 454 int usecs) 455 { 456 /* paranoia */ 457 if (!adjusted_mode->crtc_htotal) 458 return 1; 459 460 return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock, 461 1000 * adjusted_mode->crtc_htotal); 462 } 463 464 static int intel_mode_vblank_start(const struct drm_display_mode *mode) 465 { 466 int vblank_start = mode->crtc_vblank_start; 467 468 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 469 vblank_start = DIV_ROUND_UP(vblank_start, 2); 470 471 return vblank_start; 472 } 473 474 static void intel_crtc_vblank_evade_scanlines(struct intel_atomic_state *state, 475 struct intel_crtc *crtc, 476 int *min, int *max, int *vblank_start) 477 { 478 const struct intel_crtc_state *old_crtc_state = 479 intel_atomic_get_old_crtc_state(state, crtc); 480 const struct intel_crtc_state *new_crtc_state = 481 intel_atomic_get_new_crtc_state(state, crtc); 482 const struct intel_crtc_state *crtc_state; 483 const struct drm_display_mode *adjusted_mode; 484 485 /* 486 * During fastsets/etc. the transcoder is still 487 * running with the old timings at this point. 488 * 489 * TODO: maybe just use the active timings here? 490 */ 491 if (intel_crtc_needs_modeset(new_crtc_state)) 492 crtc_state = new_crtc_state; 493 else 494 crtc_state = old_crtc_state; 495 496 adjusted_mode = &crtc_state->hw.adjusted_mode; 497 498 if (crtc->mode_flags & I915_MODE_FLAG_VRR) { 499 /* timing changes should happen with VRR disabled */ 500 drm_WARN_ON(state->base.dev, intel_crtc_needs_modeset(new_crtc_state) || 501 new_crtc_state->update_m_n || new_crtc_state->update_lrr); 502 503 if (intel_vrr_is_push_sent(crtc_state)) 504 *vblank_start = intel_vrr_vmin_vblank_start(crtc_state); 505 else 506 *vblank_start = intel_vrr_vmax_vblank_start(crtc_state); 507 } else { 508 *vblank_start = intel_mode_vblank_start(adjusted_mode); 509 } 510 511 /* FIXME needs to be calibrated sensibly */ 512 *min = *vblank_start - intel_usecs_to_scanlines(adjusted_mode, 513 VBLANK_EVASION_TIME_US); 514 *max = *vblank_start - 1; 515 516 /* 517 * M/N and TRANS_VTOTAL are double buffered on the transcoder's 518 * undelayed vblank, so with seamless M/N and LRR we must evade 519 * both vblanks. 520 * 521 * DSB execution waits for the transcoder's undelayed vblank, 522 * hence we must kick off the commit before that. 523 */ 524 if (new_crtc_state->dsb || new_crtc_state->update_m_n || new_crtc_state->update_lrr) 525 *min -= adjusted_mode->crtc_vblank_start - adjusted_mode->crtc_vdisplay; 526 } 527 528 /** 529 * intel_pipe_update_start() - start update of a set of display registers 530 * @state: the atomic state 531 * @crtc: the crtc 532 * 533 * Mark the start of an update to pipe registers that should be updated 534 * atomically regarding vblank. If the next vblank will happens within 535 * the next 100 us, this function waits until the vblank passes. 536 * 537 * After a successful call to this function, interrupts will be disabled 538 * until a subsequent call to intel_pipe_update_end(). That is done to 539 * avoid random delays. 540 */ 541 void intel_pipe_update_start(struct intel_atomic_state *state, 542 struct intel_crtc *crtc) 543 { 544 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 545 struct intel_crtc_state *new_crtc_state = 546 intel_atomic_get_new_crtc_state(state, crtc); 547 long timeout = msecs_to_jiffies_timeout(1); 548 int scanline, min, max, vblank_start; 549 wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base); 550 bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && 551 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI); 552 DEFINE_WAIT(wait); 553 554 intel_psr_lock(new_crtc_state); 555 556 if (new_crtc_state->do_async_flip) 557 return; 558 559 if (intel_crtc_needs_vblank_work(new_crtc_state)) 560 intel_crtc_vblank_work_init(new_crtc_state); 561 562 intel_crtc_vblank_evade_scanlines(state, crtc, &min, &max, &vblank_start); 563 if (min <= 0 || max <= 0) 564 goto irq_disable; 565 566 if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base))) 567 goto irq_disable; 568 569 /* 570 * Wait for psr to idle out after enabling the VBL interrupts 571 * VBL interrupts will start the PSR exit and prevent a PSR 572 * re-entry as well. 573 */ 574 intel_psr_wait_for_idle_locked(new_crtc_state); 575 576 local_irq_disable(); 577 578 crtc->debug.min_vbl = min; 579 crtc->debug.max_vbl = max; 580 trace_intel_pipe_update_start(crtc); 581 582 for (;;) { 583 /* 584 * prepare_to_wait() has a memory barrier, which guarantees 585 * other CPUs can see the task state update by the time we 586 * read the scanline. 587 */ 588 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); 589 590 scanline = intel_get_crtc_scanline(crtc); 591 if (scanline < min || scanline > max) 592 break; 593 594 if (!timeout) { 595 drm_err(&dev_priv->drm, 596 "Potential atomic update failure on pipe %c\n", 597 pipe_name(crtc->pipe)); 598 break; 599 } 600 601 local_irq_enable(); 602 603 timeout = schedule_timeout(timeout); 604 605 local_irq_disable(); 606 } 607 608 finish_wait(wq, &wait); 609 610 drm_crtc_vblank_put(&crtc->base); 611 612 /* 613 * On VLV/CHV DSI the scanline counter would appear to 614 * increment approx. 1/3 of a scanline before start of vblank. 615 * The registers still get latched at start of vblank however. 616 * This means we must not write any registers on the first 617 * line of vblank (since not the whole line is actually in 618 * vblank). And unfortunately we can't use the interrupt to 619 * wait here since it will fire too soon. We could use the 620 * frame start interrupt instead since it will fire after the 621 * critical scanline, but that would require more changes 622 * in the interrupt code. So for now we'll just do the nasty 623 * thing and poll for the bad scanline to pass us by. 624 * 625 * FIXME figure out if BXT+ DSI suffers from this as well 626 */ 627 while (need_vlv_dsi_wa && scanline == vblank_start) 628 scanline = intel_get_crtc_scanline(crtc); 629 630 crtc->debug.scanline_start = scanline; 631 crtc->debug.start_vbl_time = ktime_get(); 632 crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc); 633 634 trace_intel_pipe_update_vblank_evaded(crtc); 635 return; 636 637 irq_disable: 638 local_irq_disable(); 639 } 640 641 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE) 642 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) 643 { 644 u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time)); 645 unsigned int h; 646 647 h = ilog2(delta >> 9); 648 if (h >= ARRAY_SIZE(crtc->debug.vbl.times)) 649 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1; 650 crtc->debug.vbl.times[h]++; 651 652 crtc->debug.vbl.sum += delta; 653 if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min) 654 crtc->debug.vbl.min = delta; 655 if (delta > crtc->debug.vbl.max) 656 crtc->debug.vbl.max = delta; 657 658 if (delta > 1000 * VBLANK_EVASION_TIME_US) { 659 drm_dbg_kms(crtc->base.dev, 660 "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n", 661 pipe_name(crtc->pipe), 662 div_u64(delta, 1000), 663 VBLANK_EVASION_TIME_US); 664 crtc->debug.vbl.over++; 665 } 666 } 667 #else 668 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {} 669 #endif 670 671 /** 672 * intel_pipe_update_end() - end update of a set of display registers 673 * @state: the atomic state 674 * @crtc: the crtc 675 * 676 * Mark the end of an update started with intel_pipe_update_start(). This 677 * re-enables interrupts and verifies the update was actually completed 678 * before a vblank. 679 */ 680 void intel_pipe_update_end(struct intel_atomic_state *state, 681 struct intel_crtc *crtc) 682 { 683 struct intel_crtc_state *new_crtc_state = 684 intel_atomic_get_new_crtc_state(state, crtc); 685 enum pipe pipe = crtc->pipe; 686 int scanline_end = intel_get_crtc_scanline(crtc); 687 u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc); 688 ktime_t end_vbl_time = ktime_get(); 689 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 690 691 if (new_crtc_state->do_async_flip) 692 goto out; 693 694 trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end); 695 696 /* 697 * Incase of mipi dsi command mode, we need to set frame update 698 * request for every commit. 699 */ 700 if (DISPLAY_VER(dev_priv) >= 11 && 701 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI)) 702 icl_dsi_frame_update(new_crtc_state); 703 704 /* We're still in the vblank-evade critical section, this can't race. 705 * Would be slightly nice to just grab the vblank count and arm the 706 * event outside of the critical section - the spinlock might spin for a 707 * while ... */ 708 if (intel_crtc_needs_vblank_work(new_crtc_state)) { 709 drm_vblank_work_schedule(&new_crtc_state->vblank_work, 710 drm_crtc_accurate_vblank_count(&crtc->base) + 1, 711 false); 712 } else if (new_crtc_state->uapi.event) { 713 drm_WARN_ON(&dev_priv->drm, 714 drm_crtc_vblank_get(&crtc->base) != 0); 715 716 spin_lock(&crtc->base.dev->event_lock); 717 drm_crtc_arm_vblank_event(&crtc->base, 718 new_crtc_state->uapi.event); 719 spin_unlock(&crtc->base.dev->event_lock); 720 721 new_crtc_state->uapi.event = NULL; 722 } 723 724 /* 725 * Send VRR Push to terminate Vblank. If we are already in vblank 726 * this has to be done _after_ sampling the frame counter, as 727 * otherwise the push would immediately terminate the vblank and 728 * the sampled frame counter would correspond to the next frame 729 * instead of the current frame. 730 * 731 * There is a tiny race here (iff vblank evasion failed us) where 732 * we might sample the frame counter just before vmax vblank start 733 * but the push would be sent just after it. That would cause the 734 * push to affect the next frame instead of the current frame, 735 * which would cause the next frame to terminate already at vmin 736 * vblank start instead of vmax vblank start. 737 */ 738 intel_vrr_send_push(new_crtc_state); 739 740 local_irq_enable(); 741 742 if (intel_vgpu_active(dev_priv)) 743 goto out; 744 745 if (crtc->debug.start_vbl_count && 746 crtc->debug.start_vbl_count != end_vbl_count) { 747 drm_err(&dev_priv->drm, 748 "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n", 749 pipe_name(pipe), crtc->debug.start_vbl_count, 750 end_vbl_count, 751 ktime_us_delta(end_vbl_time, 752 crtc->debug.start_vbl_time), 753 crtc->debug.min_vbl, crtc->debug.max_vbl, 754 crtc->debug.scanline_start, scanline_end); 755 } 756 757 dbg_vblank_evade(crtc, end_vbl_time); 758 759 out: 760 intel_psr_unlock(new_crtc_state); 761 } 762