i915/display/intel_vblank.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2022-2023 Intel Corporation
 */

#include <linux/iopoll.h>

#include <drm/drm_print.h>
#include <drm/drm_vblank.h>

#include "i915_drv.h"
#include "intel_color.h"
#include "intel_crtc.h"
#include "intel_de.h"
#include "intel_display_jiffies.h"
#include "intel_display_regs.h"
#include "intel_display_types.h"
#include "intel_display_utils.h"
#include "intel_vblank.h"
#include "intel_vrr.h"

/*
 * This timing diagram depicts the video signal in and
 * around the vertical blanking period.
 *
 * Assumptions about the fictitious mode used in this example:
 *  vblank_start >= 3
 *  vsync_start = vblank_start + 1
 *  vsync_end = vblank_start + 2
 *  vtotal = vblank_start + 3
 *
 *           start of vblank:
 *           latch double buffered registers
 *           increment frame counter (ctg+)
 *           generate start of vblank interrupt (gen4+)
 *           |
 *           |          frame start:
 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 *           |          may be shifted forward 1-3 extra lines via TRANSCONF
 *           |          |
 *           |          |  start of vsync:
 *           |          |  generate vsync interrupt
 *           |          |  |
 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 * ----va---> <-----------------vb--------------------> <--------va-------------
 *       |          |       <----vs----->                     |
 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 *       |          |                                         |
 *       last visible pixel                                   first visible pixel
 *                  |                                         increment frame counter (gen3/4)
 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 *
 * x  = horizontal active
 * _  = horizontal blanking
 * hs = horizontal sync
 * va = vertical active
 * vb = vertical blanking
 * vs = vertical sync
 * vbs = vblank_start (number)
 *
 * Summary:
 * - most events happen at the start of horizontal sync
 * - frame start happens at the start of horizontal blank, 1-4 lines
 *   (depending on TRANSCONF settings) after the start of vblank
 * - gen3/4 pixel and frame counter are synchronized with the start
 *   of horizontal active on the first line of vertical active
 */

/*
 * Called from drm generic code, passed a 'crtc', which we use as a pipe index.
 */
u32 i915_get_vblank_counter(struct drm_crtc *crtc)
{
	struct intel_display *display = to_intel_display(crtc->dev);
	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
	const struct drm_display_mode *mode = &vblank->hwmode;
	enum pipe pipe = to_intel_crtc(crtc)->pipe;
	u32 pixel, vbl_start, hsync_start, htotal;
	u64 frame;

	/*
	 * On i965gm TV output the frame counter only works up to
	 * the point when we enable the TV encoder. After that the
	 * frame counter ceases to work and reads zero. We need a
	 * vblank wait before enabling the TV encoder and so we
	 * have to enable vblank interrupts while the frame counter
	 * is still in a working state. However the core vblank code
	 * does not like us returning non-zero frame counter values
	 * when we've told it that we don't have a working frame
	 * counter. Thus we must stop non-zero values leaking out.
	 */
	if (!vblank->max_vblank_count)
		return 0;

	htotal = mode->crtc_htotal;
	hsync_start = mode->crtc_hsync_start;
	vbl_start = intel_mode_vblank_start(mode);

	/* Convert to pixel count */
	vbl_start *= htotal;

	/* Start of vblank event occurs at start of hsync */
	vbl_start -= htotal - hsync_start;

	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	frame = intel_de_read64_2x32(display, PIPEFRAMEPIXEL(display, pipe),
				     PIPEFRAME(display, pipe));

	pixel = frame & PIPE_PIXEL_MASK;
	frame = (frame >> PIPE_FRAME_LOW_SHIFT) & 0xffffff;

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
	return (frame + (pixel >= vbl_start)) & 0xffffff;
}

u32 g4x_get_vblank_counter(struct drm_crtc *crtc)
{
	struct intel_display *display = to_intel_display(crtc->dev);
	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
	enum pipe pipe = to_intel_crtc(crtc)->pipe;

	if (!vblank->max_vblank_count)
		return 0;

	return intel_de_read(display, PIPE_FRMCOUNT_G4X(display, pipe));
}

static u32 intel_crtc_scanlines_since_frame_timestamp(struct intel_crtc *crtc)
{
	struct intel_display *display = to_intel_display(crtc);
	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(&crtc->base);
	const struct drm_display_mode *mode = &vblank->hwmode;
	u32 htotal = mode->crtc_htotal;
	u32 clock = mode->crtc_clock;
	u32 scan_prev_time, scan_curr_time, scan_post_time;

	/*
	 * To avoid the race condition where we might cross into the
	 * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR
	 * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR
	 * during the same frame.
	 */
	do {
		/*
		 * This field provides read back of the display
		 * pipe frame time stamp. The time stamp value
		 * is sampled at every start of vertical blank.
		 */
		scan_prev_time = intel_de_read_fw(display,
						  PIPE_FRMTMSTMP(crtc->pipe));

		/*
		 * The TIMESTAMP_CTR register has the current
		 * time stamp value.
		 */
		scan_curr_time = intel_de_read_fw(display, IVB_TIMESTAMP_CTR);

		scan_post_time = intel_de_read_fw(display,
						  PIPE_FRMTMSTMP(crtc->pipe));
	} while (scan_post_time != scan_prev_time);

	return div_u64(mul_u32_u32(scan_curr_time - scan_prev_time,
				   clock), 1000 * htotal);
}

/*
 * On certain encoders on certain platforms, pipe
 * scanline register will not work to get the scanline,
 * since the timings are driven from the PORT or issues
 * with scanline register updates.
 * This function will use Framestamp and current
 * timestamp registers to calculate the scanline.
 */
static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc)
{
	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(&crtc->base);
	const struct drm_display_mode *mode = &vblank->hwmode;
	u32 vblank_start = mode->crtc_vblank_start;
	u32 vtotal = mode->crtc_vtotal;
	u32 scanline;

	scanline = intel_crtc_scanlines_since_frame_timestamp(crtc);
	scanline = min(scanline, vtotal - 1);
	scanline = (scanline + vblank_start) % vtotal;

	return scanline;
}

int intel_crtc_scanline_offset(const struct intel_crtc_state *crtc_state)
{
	struct intel_display *display = to_intel_display(crtc_state);

	/*
	 * The scanline counter increments at the leading edge of hsync.
	 *
	 * On most platforms it starts counting from vtotal-1 on the
	 * first active line. That means the scanline counter value is
	 * always one less than what we would expect. Ie. just after
	 * start of vblank, which also occurs at start of hsync (on the
	 * last active line), the scanline counter will read vblank_start-1.
	 *
	 * On gen2 the scanline counter starts counting from 1 instead
	 * of vtotal-1, so we have to subtract one.
	 *
	 * On HSW+ the behaviour of the scanline counter depends on the output
	 * type. For DP ports it behaves like most other platforms, but on HDMI
	 * there's an extra 1 line difference. So we need to add two instead of
	 * one to the value.
	 *
	 * On VLV/CHV DSI the scanline counter would appear to increment
	 * approx. 1/3 of a scanline before start of vblank. Unfortunately
	 * that means we can't tell whether we're in vblank or not while
	 * we're on that particular line. We must still set scanline_offset
	 * to 1 so that the vblank timestamps come out correct when we query
	 * the scanline counter from within the vblank interrupt handler.
	 * However if queried just before the start of vblank we'll get an
	 * answer that's slightly in the future.
	 */
	if (DISPLAY_VER(display) >= 20 || display->platform.battlemage)
		return 1;
	else if (DISPLAY_VER(display) >= 9 ||
		 display->platform.broadwell || display->platform.haswell)
		return intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ? 2 : 1;
	else if (DISPLAY_VER(display) >= 3)
		return 1;
	else
		return -1;
}

/*
 * intel_de_read_fw(), only for fast reads of display block, no need for
 * forcewake etc.
 */
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct intel_display *display = to_intel_display(crtc);
	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(&crtc->base);
	const struct drm_display_mode *mode = &vblank->hwmode;
	enum pipe pipe = crtc->pipe;
	int position, vtotal;

	if (!crtc->active)
		return 0;

	if (crtc->mode_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP)
		return __intel_get_crtc_scanline_from_timestamp(crtc);

	vtotal = intel_mode_vtotal(mode);

	position = intel_de_read_fw(display, PIPEDSL(display, pipe)) & PIPEDSL_LINE_MASK;

	/*
	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
	 * read it just before the start of vblank.  So try it again
	 * so we don't accidentally end up spanning a vblank frame
	 * increment, causing the pipe_update_end() code to squak at us.
	 *
	 * The nature of this problem means we can't simply check the ISR
	 * bit and return the vblank start value; nor can we use the scanline
	 * debug register in the transcoder as it appears to have the same
	 * problem.  We may need to extend this to include other platforms,
	 * but so far testing only shows the problem on HSW.
	 */
	if (HAS_DDI(display) && !position) {
		int i, temp;

		for (i = 0; i < 100; i++) {
			udelay(1);
			temp = intel_de_read_fw(display,
						PIPEDSL(display, pipe)) & PIPEDSL_LINE_MASK;
			if (temp != position) {
				position = temp;
				break;
			}
		}
	}

	/*
	 * See update_scanline_offset() for the details on the
	 * scanline_offset adjustment.
	 */
	return (position + vtotal + crtc->scanline_offset) % vtotal;
}

/*
 * The uncore version of the spin lock functions is used to decide
 * whether we need to lock the uncore lock or not.  This is only
 * needed in i915, not in Xe.
 *
 * This lock in i915 is needed because some old platforms (at least
 * IVB and possibly HSW as well), which are not supported in Xe, need
 * all register accesses to the same cacheline to be serialized,
 * otherwise they may hang.
 */
#ifdef I915
static void intel_vblank_section_enter(struct intel_display *display)
	__acquires(i915->uncore.lock)
{
	struct drm_i915_private *i915 = to_i915(display->drm);
	spin_lock(&i915->uncore.lock);
}

static void intel_vblank_section_exit(struct intel_display *display)
	__releases(i915->uncore.lock)
{
	struct drm_i915_private *i915 = to_i915(display->drm);
	spin_unlock(&i915->uncore.lock);
}
#else
static void intel_vblank_section_enter(struct intel_display *display)
{
}

static void intel_vblank_section_exit(struct intel_display *display)
{
}
#endif

static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc,
				     bool in_vblank_irq,
				     int *vpos, int *hpos,
				     ktime_t *stime, ktime_t *etime,
				     const struct drm_display_mode *mode)
{
	struct intel_display *display = to_intel_display(_crtc->dev);
	struct intel_crtc *crtc = to_intel_crtc(_crtc);
	enum pipe pipe = crtc->pipe;
	int position;
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
	unsigned long irqflags;
	bool use_scanline_counter = DISPLAY_VER(display) >= 5 ||
		display->platform.g4x || DISPLAY_VER(display) == 2 ||
		crtc->mode_flags & I915_MODE_FLAG_USE_SCANLINE_COUNTER;

	if (drm_WARN_ON(display->drm, !mode->crtc_clock)) {
		drm_dbg(display->drm,
			"trying to get scanoutpos for disabled pipe %c\n",
			pipe_name(pipe));
		return false;
	}

	htotal = mode->crtc_htotal;
	hsync_start = mode->crtc_hsync_start;
	vtotal = intel_mode_vtotal(mode);
	vbl_start = intel_mode_vblank_start(mode);
	vbl_end = intel_mode_vblank_end(mode);

	/*
	 * Enter vblank critical section, as we will do multiple
	 * timing critical raw register reads, potentially with
	 * preemption disabled, so the following code must not block.
	 */
	local_irq_save(irqflags);
	intel_vblank_section_enter(display);

	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

	if (crtc->mode_flags & I915_MODE_FLAG_VRR) {
		int scanlines = intel_crtc_scanlines_since_frame_timestamp(crtc);

		position = __intel_get_crtc_scanline(crtc);

		/*
		 * Already exiting vblank? If so, shift our position
		 * so it looks like we're already approaching the full
		 * vblank end. This should make the generated timestamp
		 * more or less match when the active portion will start.
		 */
		if (position >= vbl_start && scanlines < position)
			position = min(crtc->vmax_vblank_start + scanlines, vtotal - 1);
	} else if (use_scanline_counter) {
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
		position = __intel_get_crtc_scanline(crtc);
	} else {
		/*
		 * Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
		position = (intel_de_read_fw(display, PIPEFRAMEPIXEL(display, pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;

		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;

		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		position = min(position, vtotal - 1);

		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
	}

	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	intel_vblank_section_exit(display);
	local_irq_restore(irqflags);

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;

	if (use_scanline_counter) {
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}

	return true;
}

bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error,
				     ktime_t *vblank_time, bool in_vblank_irq)
{
	return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
		crtc, max_error, vblank_time, in_vblank_irq,
		i915_get_crtc_scanoutpos);
}

int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct intel_display *display = to_intel_display(crtc);
	unsigned long irqflags;
	int position;

	local_irq_save(irqflags);
	intel_vblank_section_enter(display);

	position = __intel_get_crtc_scanline(crtc);

	intel_vblank_section_exit(display);
	local_irq_restore(irqflags);

	return position;
}

static bool pipe_scanline_is_moving(struct intel_display *display,
				    enum pipe pipe)
{
	i915_reg_t reg = PIPEDSL(display, pipe);
	u32 line1, line2;

	line1 = intel_de_read(display, reg) & PIPEDSL_LINE_MASK;
	msleep(5);
	line2 = intel_de_read(display, reg) & PIPEDSL_LINE_MASK;

	return line1 != line2;
}

static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
{
	struct intel_display *display = to_intel_display(crtc);
	enum pipe pipe = crtc->pipe;
	bool is_moving;
	int ret;

	/* Wait for the display line to settle/start moving */
	ret = poll_timeout_us(is_moving = pipe_scanline_is_moving(display, pipe),
			      is_moving == state,
			      500, 100 * 1000, false);
	if (ret)
		drm_err(display->drm,
			"pipe %c scanline %s wait timed out\n",
			pipe_name(pipe), str_on_off(state));
}

void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
{
	wait_for_pipe_scanline_moving(crtc, false);
}

void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
{
	wait_for_pipe_scanline_moving(crtc, true);
}

static void intel_crtc_active_timings(struct drm_display_mode *mode,
				      int *vmax_vblank_start,
				      const struct intel_crtc_state *crtc_state,
				      bool vrr_enable)
{
	drm_mode_init(mode, &crtc_state->hw.adjusted_mode);
	*vmax_vblank_start = 0;

	if (!vrr_enable)
		return;

	mode->crtc_vtotal = intel_vrr_vmax_vtotal(crtc_state);
	mode->crtc_vblank_end = intel_vrr_vmax_vtotal(crtc_state);
	mode->crtc_vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
	*vmax_vblank_start = intel_vrr_vmax_vblank_start(crtc_state);
}

void intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state,
				      bool vrr_enable)
{
	struct intel_display *display = to_intel_display(crtc_state);
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
	u8 mode_flags = crtc_state->mode_flags;
	struct drm_display_mode adjusted_mode;
	int vmax_vblank_start = 0;
	unsigned long irqflags;

	intel_crtc_active_timings(&adjusted_mode, &vmax_vblank_start,
				  crtc_state, vrr_enable);

	if (vrr_enable)
		drm_WARN_ON(display->drm, (mode_flags & I915_MODE_FLAG_VRR) == 0);
	else
		mode_flags &= ~I915_MODE_FLAG_VRR;

	/*
	 * Belts and suspenders locking to guarantee everyone sees 100%
	 * consistent state during fastset seamless refresh rate changes.
	 *
	 * vblank_time_lock takes care of all drm_vblank.c stuff, and
	 * uncore.lock takes care of __intel_get_crtc_scanline() which
	 * may get called elsewhere as well.
	 *
	 * TODO maybe just protect everything (including
	 * __intel_get_crtc_scanline()) with vblank_time_lock?
	 * Need to audit everything to make sure it's safe.
	 */
	spin_lock_irqsave(&display->drm->vblank_time_lock, irqflags);
	intel_vblank_section_enter(display);

	drm_calc_timestamping_constants(&crtc->base, &adjusted_mode);

	crtc->vmax_vblank_start = vmax_vblank_start;

	crtc->mode_flags = mode_flags;

	crtc->scanline_offset = intel_crtc_scanline_offset(crtc_state);
	intel_vblank_section_exit(display);
	spin_unlock_irqrestore(&display->drm->vblank_time_lock, irqflags);
}

int intel_mode_vdisplay(const struct drm_display_mode *mode)
{
	int vdisplay = mode->crtc_vdisplay;

	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vdisplay = DIV_ROUND_UP(vdisplay, 2);

	return vdisplay;
}

int intel_mode_vblank_start(const struct drm_display_mode *mode)
{
	int vblank_start = mode->crtc_vblank_start;

	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vblank_start = DIV_ROUND_UP(vblank_start, 2);

	return vblank_start;
}

int intel_mode_vblank_end(const struct drm_display_mode *mode)
{
	int vblank_end = mode->crtc_vblank_end;

	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vblank_end /= 2;

	return vblank_end;
}

int intel_mode_vtotal(const struct drm_display_mode *mode)
{
	int vtotal = mode->crtc_vtotal;

	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

	return vtotal;
}

int intel_mode_vblank_delay(const struct drm_display_mode *mode)
{
	return intel_mode_vblank_start(mode) - intel_mode_vdisplay(mode);
}

static const struct intel_crtc_state *
pre_commit_crtc_state(const struct intel_crtc_state *old_crtc_state,
		      const struct intel_crtc_state *new_crtc_state)
{
	/*
	 * During fastsets/etc. the transcoder is still
	 * running with the old timings at this point.
	 */
	if (intel_crtc_needs_modeset(new_crtc_state))
		return new_crtc_state;
	else
		return old_crtc_state;
}

const struct intel_crtc_state *
intel_pre_commit_crtc_state(struct intel_atomic_state *state,
			    struct intel_crtc *crtc)
{
	const struct intel_crtc_state *old_crtc_state =
		intel_atomic_get_old_crtc_state(state, crtc);
	const struct intel_crtc_state *new_crtc_state =
		intel_atomic_get_new_crtc_state(state, crtc);

	return pre_commit_crtc_state(old_crtc_state, new_crtc_state);
}

void intel_vblank_evade_init(const struct intel_crtc_state *old_crtc_state,
			     const struct intel_crtc_state *new_crtc_state,
			     struct intel_vblank_evade_ctx *evade)
{
	struct intel_display *display = to_intel_display(new_crtc_state);
	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
	const struct intel_crtc_state *crtc_state;
	const struct drm_display_mode *adjusted_mode;
	int vblank_delay;

	evade->crtc = crtc;

	evade->need_vlv_dsi_wa = (display->platform.valleyview ||
				  display->platform.cherryview) &&
		intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI);

	/* TODO: maybe just use the active timings here? */
	crtc_state = pre_commit_crtc_state(old_crtc_state, new_crtc_state);

	adjusted_mode = &crtc_state->hw.adjusted_mode;

	if (crtc->mode_flags & I915_MODE_FLAG_VRR) {
		/* timing changes should happen with VRR disabled */
		drm_WARN_ON(crtc->base.dev, intel_crtc_needs_modeset(new_crtc_state) ||
			    new_crtc_state->update_m_n || new_crtc_state->update_lrr);

		if (intel_vrr_is_push_sent(crtc_state))
			evade->vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
		else
			evade->vblank_start = intel_vrr_vmax_vblank_start(crtc_state);

		vblank_delay = crtc_state->set_context_latency;
	} else {
		evade->vblank_start = intel_mode_vblank_start(adjusted_mode);

		vblank_delay = intel_mode_vblank_delay(adjusted_mode);
	}

	/* FIXME needs to be calibrated sensibly */
	evade->min = evade->vblank_start - intel_usecs_to_scanlines(adjusted_mode,
								    VBLANK_EVASION_TIME_US);
	evade->max = evade->vblank_start - 1;

	/*
	 * M/N and TRANS_VTOTAL are double buffered on the transcoder's
	 * undelayed vblank, so with seamless M/N and LRR we must evade
	 * both vblanks.
	 *
	 * DSB execution waits for the transcoder's undelayed vblank,
	 * hence we must kick off the commit before that.
	 */
	if (intel_color_uses_dsb(new_crtc_state) ||
	    new_crtc_state->update_m_n || new_crtc_state->update_lrr)
		evade->min -= vblank_delay;
}

/* must be called with vblank interrupt already enabled! */
int intel_vblank_evade(struct intel_vblank_evade_ctx *evade)
{
	struct intel_crtc *crtc = evade->crtc;
	struct intel_display *display = to_intel_display(crtc);
	long timeout = msecs_to_jiffies_timeout(1);
	wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
	DEFINE_WAIT(wait);
	int scanline;

	if (evade->min <= 0 || evade->max <= 0)
		return 0;

	for (;;) {
		/*
		 * prepare_to_wait() has a memory barrier, which guarantees
		 * other CPUs can see the task state update by the time we
		 * read the scanline.
		 */
		prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);

		scanline = intel_get_crtc_scanline(crtc);
		if (scanline < evade->min || scanline > evade->max)
			break;

		if (!timeout) {
			drm_dbg_kms(display->drm,
				    "Potential atomic update failure on pipe %c\n",
				    pipe_name(crtc->pipe));
			break;
		}

		local_irq_enable();

		timeout = schedule_timeout(timeout);

		local_irq_disable();
	}

	finish_wait(wq, &wait);

	/*
	 * On VLV/CHV DSI the scanline counter would appear to
	 * increment approx. 1/3 of a scanline before start of vblank.
	 * The registers still get latched at start of vblank however.
	 * This means we must not write any registers on the first
	 * line of vblank (since not the whole line is actually in
	 * vblank). And unfortunately we can't use the interrupt to
	 * wait here since it will fire too soon. We could use the
	 * frame start interrupt instead since it will fire after the
	 * critical scanline, but that would require more changes
	 * in the interrupt code. So for now we'll just do the nasty
	 * thing and poll for the bad scanline to pass us by.
	 *
	 * FIXME figure out if BXT+ DSI suffers from this as well
	 */
	while (evade->need_vlv_dsi_wa && scanline == evade->vblank_start)
		scanline = intel_get_crtc_scanline(crtc);

	return scanline;
}

int intel_crtc_vblank_length(const struct intel_crtc_state *crtc_state)
{
	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;

	if (crtc_state->vrr.enable)
		return crtc_state->vrr.guardband;
	else
		return adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start;
}