i915/display/intel_bw.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2019 Intel Corporation
 */

#include <drm/drm_atomic_state_helper.h>
#include <drm/drm_print.h>
#include <drm/intel/intel_pcode_regs.h>

#include "intel_bw.h"
#include "intel_crtc.h"
#include "intel_de.h"
#include "intel_display_core.h"
#include "intel_display_regs.h"
#include "intel_display_types.h"
#include "intel_display_utils.h"
#include "intel_display_wa.h"
#include "intel_dram.h"
#include "intel_mchbar.h"
#include "intel_parent.h"
#include "skl_watermark.h"

struct intel_bw_state {
	struct intel_global_state base;

	/*
	 * Contains a bit mask, used to determine, whether correspondent
	 * pipe allows SAGV or not.
	 */
	u8 pipe_sagv_reject;

	/*
	 * From MTL onwards, to lock a QGV point, punit expects the peak BW of
	 * the selected QGV point as the parameter in multiples of 100MB/s
	 */
	u16 qgv_point_peakbw;

	/*
	 * Current QGV points mask, which restricts
	 * some particular SAGV states, not to confuse
	 * with pipe_sagv_mask.
	 */
	u16 qgv_points_mask;

	unsigned int data_rate[I915_MAX_PIPES];
	u8 num_active_planes[I915_MAX_PIPES];
};

/* Parameters for Qclk Geyserville (QGV) */
struct intel_qgv_point {
	u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
};

#define DEPROGBWPCLIMIT		60

#define PEAK_BW_THRESHOLD	20000

struct intel_psf_gv_point {
	u8 clk; /* clock in multiples of 16.6666 MHz */
};

struct intel_qgv_info {
	struct intel_qgv_point points[I915_NUM_QGV_POINTS];
	struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
	u8 num_qgv_points;
	u8 num_psf_points;
	u8 t_bl;
	u8 max_numchannels;
	u8 channel_width;
	u8 deinterleave;
};

static int dclk_freq_mhz(int ratio)
{
	/* multiple of 16.666 MHz (100/6) */
	return DIV_ROUND_CLOSEST(ratio * 100, 6);
}

static int dg1_mchbar_read_qgv_point_info(struct intel_display *display,
					  struct intel_qgv_point *sp,
					  int point)
{
	u32 dclk_ratio;
	u32 val;

	val = intel_mchbar_read(display, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
	dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
	if (val & DG1_QCLK_REFERENCE)
		dclk_ratio *= 6; /* 6 * 16.666 MHz = 100 MHz */
	else
		dclk_ratio *= 8; /* 8 * 16.666 MHz = 133 MHz */

	val = intel_mchbar_read(display, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
	if (val & DG1_GEAR_TYPE)
		dclk_ratio *= 2;

	sp->dclk = dclk_freq_mhz(dclk_ratio);
	if (sp->dclk == 0)
		return -EINVAL;

	val = intel_mchbar_read(display, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
	sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
	sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);

	val = intel_mchbar_read(display, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
	sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
	sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);

	sp->t_rc = sp->t_rp + sp->t_ras;

	return 0;
}

static int icl_pcode_read_qgv_point_info(struct intel_display *display,
					 struct intel_qgv_point *sp,
					 int point)
{
	u32 val = 0, val2 = 0;
	int ret;

	ret = intel_parent_pcode_read(display, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
				      ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
				      &val, &val2);
	if (ret)
		return ret;

	sp->dclk = dclk_freq_mhz(val & 0xffff);
	sp->t_rp = (val & 0xff0000) >> 16;
	sp->t_rcd = (val & 0xff000000) >> 24;

	sp->t_rdpre = val2 & 0xff;
	sp->t_ras = (val2 & 0xff00) >> 8;

	sp->t_rc = sp->t_rp + sp->t_ras;

	return 0;
}

static int adls_pcode_read_psf_gv_point_info(struct intel_display *display,
					     struct intel_psf_gv_point *points)
{
	u32 val = 0;
	int ret;
	int i;

	ret = intel_parent_pcode_read(display, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
				      ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
	if (ret)
		return ret;

	for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
		points[i].clk = val & 0xff;
		val >>= 8;
	}

	return 0;
}

static u16 icl_qgv_points_mask(struct intel_display *display)
{
	unsigned int num_psf_gv_points = display->bw.num_psf_gv_points;
	unsigned int num_qgv_points = display->bw.num_qgv_points;
	u16 qgv_points = 0, psf_points = 0;

	/*
	 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
	 * it with failure if we try masking any unadvertised points.
	 * So need to operate only with those returned from PCode.
	 */
	if (num_qgv_points > 0)
		qgv_points = GENMASK(num_qgv_points - 1, 0);

	if (num_psf_gv_points > 0)
		psf_points = GENMASK(num_psf_gv_points - 1, 0);

	return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
}

static bool is_sagv_enabled(struct intel_display *display, u16 points_mask)
{
	return !is_power_of_2(~points_mask & icl_qgv_points_mask(display) &
			      ICL_PCODE_REQ_QGV_PT_MASK);
}

static int icl_pcode_restrict_qgv_points(struct intel_display *display,
					 u32 points_mask)
{
	int ret;

	if (HAS_PMDEMAND(display))
		return 0;

	/* bspec says to keep retrying for at least 1 ms */
	ret = intel_parent_pcode_request(display, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
					 points_mask,
					 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
					 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
					 1);

	if (ret < 0) {
		drm_err(display->drm,
			"Failed to disable qgv points (0x%x) points: 0x%x\n",
			ret, points_mask);
		return ret;
	}

	display->sagv.status = is_sagv_enabled(display, points_mask) ?
		I915_SAGV_ENABLED : I915_SAGV_DISABLED;

	return 0;
}

static int mtl_read_qgv_point_info(struct intel_display *display,
				   struct intel_qgv_point *sp, int point)
{
	u32 val, val2;

	val = intel_de_read(display, MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
	val2 = intel_de_read(display, MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));

	sp->dclk = dclk_freq_mhz(REG_FIELD_GET(MTL_DCLK_MASK, val));
	sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
	sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val);

	sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2);
	sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2);

	sp->t_rc = sp->t_rp + sp->t_ras;

	return 0;
}

static int
intel_read_qgv_point_info(struct intel_display *display,
			  struct intel_qgv_point *sp,
			  int point)
{
	if (DISPLAY_VER(display) >= 14)
		return mtl_read_qgv_point_info(display, sp, point);
	else if (display->platform.dg1)
		return dg1_mchbar_read_qgv_point_info(display, sp, point);
	else
		return icl_pcode_read_qgv_point_info(display, sp, point);
}

static bool is_y_tile(struct intel_display *display)
{
	/* assume Y tile may be used if supported */
	return !HAS_4TILE(display);
}

static int icl_init_qgv_info(struct intel_display *display,
			     const struct dram_info *dram_info,
			     struct intel_qgv_info *qi)
{
	qi->num_qgv_points = dram_info->num_qgv_points;
	qi->num_psf_points = dram_info->num_psf_gv_points;

	if (DISPLAY_VER(display) >= 14) {
		switch (dram_info->type) {
		case INTEL_DRAM_DDR4:
			qi->t_bl = 4;
			qi->max_numchannels = 2;
			qi->channel_width = 64;
			qi->deinterleave = 2;
			break;
		case INTEL_DRAM_DDR5:
			qi->t_bl = 8;
			qi->max_numchannels = 4;
			qi->channel_width = 32;
			qi->deinterleave = 2;
			break;
		case INTEL_DRAM_LPDDR4:
		case INTEL_DRAM_LPDDR5:
			qi->t_bl = 16;
			/*
			 * Wa_16030862157
			 * Xe3p supports a fully-populated 16-channel LPDDR
			 * config (4 memory controllers x 4 channels); earlier
			 * D14+ platforms top out at 8.
			 */
			qi->max_numchannels =
				intel_display_wa(display, INTEL_DISPLAY_WA_16030862157) ? 16 : 8;
			qi->channel_width = 16;
			qi->deinterleave = 4;
			break;
		case INTEL_DRAM_GDDR:
		case INTEL_DRAM_GDDR_ECC:
			qi->channel_width = 32;
			break;
		default:
			MISSING_CASE(dram_info->type);
			return -EINVAL;
		}
	} else if (DISPLAY_VER(display) >= 12) {
		switch (dram_info->type) {
		case INTEL_DRAM_DDR4:
			qi->t_bl = is_y_tile(display) ? 8 : 4;
			qi->max_numchannels = 2;
			qi->channel_width = 64;
			qi->deinterleave = is_y_tile(display) ? 1 : 2;
			break;
		case INTEL_DRAM_DDR5:
			qi->t_bl = is_y_tile(display) ? 16 : 8;
			qi->max_numchannels = 4;
			qi->channel_width = 32;
			qi->deinterleave = is_y_tile(display) ? 1 : 2;
			break;
		case INTEL_DRAM_LPDDR4:
			if (display->platform.rocketlake) {
				qi->t_bl = 8;
				qi->max_numchannels = 4;
				qi->channel_width = 32;
				qi->deinterleave = 2;
				break;
			}
			fallthrough;
		case INTEL_DRAM_LPDDR5:
			qi->t_bl = 16;
			qi->max_numchannels = 8;
			qi->channel_width = 16;
			qi->deinterleave = is_y_tile(display) ? 2 : 4;
			break;
		default:
			qi->t_bl = 16;
			qi->max_numchannels = 1;
			break;
		}
	} else if (DISPLAY_VER(display) == 11) {
		qi->t_bl = dram_info->type == INTEL_DRAM_DDR4 ? 4 : 8;
		qi->max_numchannels = 1;
	}

	return 0;
}

static int icl_get_qgv_points(struct intel_display *display,
			      const struct dram_info *dram_info,
			      struct intel_qgv_info *qi)
{
	int i, ret;

	if (icl_init_qgv_info(display, dram_info, qi))
		return -EINVAL;

	if (drm_WARN_ON(display->drm,
			qi->num_qgv_points > ARRAY_SIZE(qi->points)))
		qi->num_qgv_points = ARRAY_SIZE(qi->points);

	for (i = 0; i < qi->num_qgv_points; i++) {
		struct intel_qgv_point *sp = &qi->points[i];

		ret = intel_read_qgv_point_info(display, sp, i);
		if (ret) {
			drm_dbg_kms(display->drm, "Could not read QGV %d info\n", i);
			return ret;
		}

		drm_dbg_kms(display->drm,
			    "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
			    i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
			    sp->t_rcd, sp->t_rc);
	}

	if (qi->num_psf_points > 0) {
		ret = adls_pcode_read_psf_gv_point_info(display, qi->psf_points);
		if (ret) {
			drm_err(display->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
			qi->num_psf_points = 0;
		}

		for (i = 0; i < qi->num_psf_points; i++)
			drm_dbg_kms(display->drm,
				    "PSF GV %d: CLK=%d\n",
				    i, qi->psf_points[i].clk);
	}

	return 0;
}

static int adl_calc_psf_bw(int clk)
{
	/*
	 * clk is multiples of 16.666MHz (100/6)
	 * According to BSpec PSF GV bandwidth is
	 * calculated as BW = 64 * clk * 16.666Mhz
	 */
	return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
}

static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
{
	u16 dclk = 0;
	int i;

	for (i = 0; i < qi->num_qgv_points; i++)
		dclk = max(dclk, qi->points[i].dclk);

	return dclk;
}

/*
 * Bandwidth parameters that are tied to the SoC (as opposed to struct
 * intel_display_bw_params).
 */
struct intel_soc_bw_params {
	u8 deprogbwlimit;
	u8 derating;
};

static const struct intel_soc_bw_params icl_bw_params = {
	.deprogbwlimit = 25,
	.derating = 10,
};

static const struct intel_soc_bw_params tgl_bw_params = {
	.deprogbwlimit = 34,
	.derating = 10,
};

static const struct intel_soc_bw_params rkl_bw_params = {
	.deprogbwlimit = 20,
	.derating = 10,
};

static const struct intel_soc_bw_params adl_s_bw_params = {
	.deprogbwlimit = 38,
	.derating = 10,
};

static const struct intel_soc_bw_params adl_p_bw_params = {
	.deprogbwlimit = 38,
	.derating = 20,
};

static const struct intel_soc_bw_params bmg_bw_params = {
	.deprogbwlimit = 53,
	.derating = 30,
};

static const struct intel_soc_bw_params bmg_ecc_bw_params = {
	.deprogbwlimit = 53,
	.derating = 45,
};

static const struct intel_soc_bw_params ptl_bw_params = {
	.deprogbwlimit = 65,
	.derating = 10,
};

static const struct intel_soc_bw_params wcl_bw_params = {
	.deprogbwlimit = 22,
	.derating = 10,
};

static const struct intel_soc_bw_params *get_soc_bw_params(struct intel_display *display,
							   const struct dram_info *dram_info)
{
	if (display->platform.icelake ||
	    display->platform.jasperlake ||
	    display->platform.elkhartlake)
		return &icl_bw_params;
	else if (display->platform.tigerlake ||
		 display->platform.dg1)
		return &tgl_bw_params;
	else if (display->platform.rocketlake)
		return &rkl_bw_params;
	else if (display->platform.alderlake_s ||
		 display->platform.meteorlake ||
		 display->platform.lunarlake)
		return &adl_s_bw_params;
	else if (display->platform.alderlake_p)
		return &adl_p_bw_params;
	else if (display->platform.battlemage &&
		 dram_info->type == INTEL_DRAM_GDDR_ECC)
		return &bmg_ecc_bw_params;
	else if (display->platform.battlemage)
		return &bmg_bw_params;
	else if (display->platform.pantherlake_wildcatlake)
		return &wcl_bw_params;
	else if (display->platform.pantherlake ||
		 display->platform.novalake)
		return &ptl_bw_params;

	return NULL;
}

/*
 * Bandwidth parameters that are tied to the display IP (as opposed to struct
 * intel_soc_bw_params).
 */
struct intel_display_bw_params {
	u16 displayrtids;
	u8 deburst;
};

static const struct intel_display_bw_params gen11_bw_params = {
	.deburst = 8,
	.displayrtids = 128,
};

static const struct intel_display_bw_params gen12_bw_params = {
	.deburst = 16,
	.displayrtids = 256,
};

static const struct intel_display_bw_params xelpdp_bw_params = {
	.deburst = 32,
	.displayrtids = 256,
};

static const struct intel_display_bw_params *get_display_bw_params(struct intel_display *display)
{
	if (DISPLAY_VER(display) >= 14) {
		return &xelpdp_bw_params;
	} else if (DISPLAY_VER(display) >= 12) {
		/*
		 * RKL's SoC was based on ICL and the display, even though being
		 * gen12, had changes to the memory interface to match gen11's,
		 * consequently inheriting gen11's display-specific bandwidth
		 * parameters.
		 */
		if (display->platform.rocketlake)
			return &gen11_bw_params;
		else
			return &gen12_bw_params;
	} else if (DISPLAY_VER(display) == 11) {
		return &gen11_bw_params;
	}

	return NULL;
}

static void update_sagv_status(struct intel_display *display, int qgv_points)
{
	/*
	 * In case if SAGV is disabled in BIOS, we always get 1
	 * SAGV point, but we can't send PCode commands to restrict it
	 * as it will fail and pointless anyway.
	 */
	if (qgv_points == 1)
		display->sagv.status = I915_SAGV_NOT_CONTROLLED;
	else
		display->sagv.status = I915_SAGV_ENABLED;
}

static int icl_get_bw_info(struct intel_display *display,
			   const struct dram_info *dram_info,
			   const struct intel_soc_bw_params *soc_bw_params,
			   const struct intel_display_bw_params *display_bw_params)
{
	struct intel_qgv_info qi = {};
	int num_channels = max_t(u8, 1, dram_info->num_channels);
	int ipqdepth, ipqdepthpch = 16;
	int dclk_max;
	int maxdebw;
	int num_groups = ARRAY_SIZE(display->bw.max);
	int i, ret;

	ret = icl_get_qgv_points(display, dram_info, &qi);
	if (ret) {
		drm_dbg_kms(display->drm,
			    "Failed to get memory subsystem information, ignoring bandwidth limits");
		return ret;
	}

	dclk_max = icl_sagv_max_dclk(&qi);
	maxdebw = min(soc_bw_params->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
	ipqdepth = min(ipqdepthpch, display_bw_params->displayrtids / num_channels);
	qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile(display) ? 4 : 2);

	display->bw.num_qgv_points = qi.num_qgv_points;
	display->bw.num_psf_gv_points = qi.num_psf_points;

	for (i = 0; i < num_groups; i++) {
		struct intel_bw_info *bi = &display->bw.max[i];
		int clpchgroup;
		int j;

		clpchgroup = (display_bw_params->deburst * qi.deinterleave / num_channels) << i;
		bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;

		for (j = 0; j < qi.num_qgv_points; j++) {
			const struct intel_qgv_point *sp = &qi.points[j];
			int ct, bw;

			/*
			 * Max row cycle time
			 *
			 * FIXME what is the logic behind the
			 * assumed burst length?
			 */
			ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
				   (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
			bw = sp->dclk * clpchgroup * 32 * num_channels / ct;

			bi->deratedbw[j] = min(maxdebw,
					       bw * (100 - soc_bw_params->derating) / 100);

			drm_dbg_kms(display->drm,
				    "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
				    i, j, bi->num_planes, bi->deratedbw[j]);
		}
	}

	return 0;
}

static int tgl_peakbw(int num_channels, int channel_width, int dclk)
{
	return num_channels * (channel_width / 8) * dclk;
}

static void xe3_add_peakbw_threshold(struct intel_display *display)
{
	u8 qgv_points = display->bw.num_qgv_points;

	if (!HAS_PEAK_BW_THRESHOLD(display))
		return;

	if (qgv_points >= I915_NUM_QGV_POINTS) {
		drm_dbg_kms(display->drm, "QGV points maxed out; skipping peak bandwidth threshold.\n");
		return;
	}

	if (qgv_points <= 1)
		return;

	display->bw.num_qgv_points++;

	display->bw.peakbw[qgv_points] = PEAK_BW_THRESHOLD;

	for (int i = 0; i < ARRAY_SIZE(display->bw.max); i++)
		display->bw.max[i].deratedbw[qgv_points] = PEAK_BW_THRESHOLD;

	drm_dbg_kms(display->drm, "An extra QGV point %d added for Peak bw threshold of %d\n",
		    qgv_points, PEAK_BW_THRESHOLD);
}

static int tgl_get_bw_info(struct intel_display *display,
			   const struct dram_info *dram_info,
			   const struct intel_soc_bw_params *soc_bw_params,
			   const struct intel_display_bw_params *display_bw_params)
{
	struct intel_qgv_info qi = {};
	int num_channels = max_t(u8, 1, dram_info->num_channels);
	int ipqdepth, ipqdepthpch = 16;
	int maxdebw, peakbw;
	int clperchgroup;
	int num_groups = ARRAY_SIZE(display->bw.max);
	int i, ret;

	ret = icl_get_qgv_points(display, dram_info, &qi);
	if (ret) {
		drm_dbg_kms(display->drm,
			    "Failed to get memory subsystem information, ignoring bandwidth limits");
		return ret;
	}

	if (DISPLAY_VER(display) < 14 &&
	    (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5))
		num_channels *= 2;

	if (num_channels < qi.max_numchannels && DISPLAY_VER(display) >= 12)
		qi.deinterleave = max(qi.deinterleave / 2, 1);

	if (DISPLAY_VER(display) >= 12 && num_channels > qi.max_numchannels)
		drm_warn(display->drm, "Number of channels exceeds max number of channels.");
	if (qi.max_numchannels != 0)
		num_channels = min_t(u8, num_channels, qi.max_numchannels);

	peakbw = tgl_peakbw(num_channels, qi.channel_width, icl_sagv_max_dclk(&qi));
	maxdebw = min(soc_bw_params->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 100);

	ipqdepth = min(ipqdepthpch, display_bw_params->displayrtids / num_channels);
	/*
	 * Wa_16030862157
	 * clperchgroup = 4kpagespermempage * clperchperblock,
	 * clperchperblock = max(8 / num_channels, 1) * interleave
	 *
	 * The 8 / num_channels truncating divide collapses to 0 for
	 * >8-channel configs (16-channel: 8 / 16 = 0); the max(..., 1) floor
	 * keeps clperchperblock >= 1 there while preserving the literal
	 * truncating divide for <=8-channel configs.
	 */
	clperchgroup = 4 * max(8 / num_channels, 1) * qi.deinterleave;

	display->bw.num_qgv_points = qi.num_qgv_points;
	display->bw.num_psf_gv_points = qi.num_psf_points;

	display->bw.max[0].num_planes = U8_MAX;

	for (i = 0; i < num_groups; i++) {
		struct intel_bw_info *bi = &display->bw.max[i];
		int clpchgroup;
		int j;

		clpchgroup = (display_bw_params->deburst * qi.deinterleave / num_channels) << i;

		if (i < num_groups - 1) {
			struct intel_bw_info *bi_next = &display->bw.max[i + 1];

			if (clpchgroup < clperchgroup)
				bi_next->num_planes = (ipqdepth - clpchgroup) / clpchgroup;
			else
				bi_next->num_planes = 0;
		}

		for (j = 0; j < qi.num_qgv_points; j++) {
			const struct intel_qgv_point *sp = &qi.points[j];
			int ct, bw;

			/*
			 * Max row cycle time
			 *
			 * FIXME what is the logic behind the
			 * assumed burst length?
			 */
			ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
				   (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
			bw = sp->dclk * clpchgroup * 32 * num_channels / ct;

			bi->deratedbw[j] = min(maxdebw,
					       bw * (100 - soc_bw_params->derating) / 100);

			drm_dbg_kms(display->drm,
				    "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
				    i, j, bi->num_planes, bi->deratedbw[j]);
		}
	}

	for (i = 0; i < qi.num_qgv_points; i++) {
		const struct intel_qgv_point *sp = &qi.points[i];

		display->bw.peakbw[i] = tgl_peakbw(num_channels, qi.channel_width, sp->dclk);

		drm_dbg_kms(display->drm, "QGV %d: peakbw=%u\n", i, display->bw.peakbw[i]);
	}

	/* For xe3 cases add an extra qgv point for Peak bw threshold */
	xe3_add_peakbw_threshold(display);

	for (i = 0; i < qi.num_psf_points; i++) {
		const struct intel_psf_gv_point *sp = &qi.psf_points[i];

		display->bw.psf_bw[i] = adl_calc_psf_bw(sp->clk);

		drm_dbg_kms(display->drm, "PSF GV %d: bw=%u\n", i, display->bw.psf_bw[i]);
	}

	return 0;
}

static void dg2_get_bw_info(struct intel_display *display)
{
	int i;

	display->bw.num_qgv_points = 1;

	display->bw.max[0].num_planes = U8_MAX;
	display->bw.max[0].deratedbw[0] = display->platform.dg2_g11 ? 38000 : 50000;

	drm_dbg_kms(display->drm,
		    "QGV 0: deratedbw=%u\n",
		    display->bw.max[0].deratedbw[0]);

	/* Bandwidth does not depend on # of planes; set all groups the same */
	for (i = 1; i < ARRAY_SIZE(display->bw.max); i++)
		display->bw.max[i] = display->bw.max[0];
}

static int xe2_hpd_get_bw_info(struct intel_display *display,
			       const struct dram_info *dram_info,
			       const struct intel_soc_bw_params *soc_bw_params)
{
	struct intel_qgv_info qi = {};
	int num_channels = dram_info->num_channels;
	int peakbw, maxdebw;
	int ret, i;

	ret = icl_get_qgv_points(display, dram_info, &qi);
	if (ret) {
		drm_dbg_kms(display->drm,
			    "Failed to get memory subsystem information, ignoring bandwidth limits");
		return ret;
	}

	peakbw = tgl_peakbw(num_channels, qi.channel_width, icl_sagv_max_dclk(&qi));
	maxdebw = min(soc_bw_params->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 100);

	display->bw.num_qgv_points = qi.num_qgv_points;

	display->bw.max[0].num_planes = U8_MAX;

	for (i = 0; i < qi.num_qgv_points; i++) {
		const struct intel_qgv_point *sp = &qi.points[i];
		int bw = tgl_peakbw(num_channels, qi.channel_width, sp->dclk);

		display->bw.max[0].deratedbw[i] =
			min(maxdebw, (100 - soc_bw_params->derating) * bw / 100);

		display->bw.peakbw[i] = bw;

		drm_dbg_kms(display->drm, "QGV %d: deratedbw=%u peakbw=%u\n",
			    i, display->bw.max[0].deratedbw[i], display->bw.peakbw[i]);
	}

	/* Bandwidth does not depend on # of planes; set all groups the same */
	for (i = 1; i < ARRAY_SIZE(display->bw.max); i++)
		display->bw.max[i] = display->bw.max[0];

	/*
	 * Xe2_HPD should always have exactly two QGV points representing
	 * battery and plugged-in operation.
	 */
	drm_WARN_ON(display->drm, qi.num_qgv_points != 2);

	return 0;
}

static unsigned int icl_max_bw_index(struct intel_display *display,
				     int num_planes, int qgv_point)
{
	int i;

	if (qgv_point >= display->bw.num_qgv_points)
		return UINT_MAX;

	/*
	 * Let's return max bw for 0 planes
	 */
	num_planes = max(1, num_planes);

	for (i = 0; i < ARRAY_SIZE(display->bw.max); i++) {
		const struct intel_bw_info *bi =
			&display->bw.max[i];

		if (num_planes >= bi->num_planes)
			return i;
	}

	return UINT_MAX;
}

static unsigned int tgl_max_bw_index(struct intel_display *display,
				     int num_planes, int qgv_point)
{
	int i;

	if (qgv_point >= display->bw.num_qgv_points)
		return UINT_MAX;

	for (i = ARRAY_SIZE(display->bw.max) - 1; i >= 0; i--) {
		const struct intel_bw_info *bi =
			&display->bw.max[i];

		if (num_planes <= bi->num_planes)
			return i;
	}

	return UINT_MAX;
}

static unsigned int adl_psf_bw(struct intel_display *display,
			       int psf_gv_point)
{
	return display->bw.psf_bw[psf_gv_point];
}

static unsigned int icl_qgv_bw(struct intel_display *display,
			       int num_active_planes, int qgv_point)
{
	unsigned int idx;

	if (DISPLAY_VER(display) >= 12)
		idx = tgl_max_bw_index(display, num_active_planes, qgv_point);
	else
		idx = icl_max_bw_index(display, num_active_planes, qgv_point);

	if (idx >= ARRAY_SIZE(display->bw.max))
		return 0;

	return display->bw.max[idx].deratedbw[qgv_point];
}

void intel_bw_init_hw(struct intel_display *display)
{
	const struct dram_info *dram_info;
	const struct intel_soc_bw_params *soc_bw_params;
	const struct intel_display_bw_params *display_bw_params;

	if (!HAS_DISPLAY(display))
		return;

	dram_info = intel_dram_info(display);
	soc_bw_params = get_soc_bw_params(display, dram_info);
	display_bw_params = get_display_bw_params(display);

	/*
	 * Starting with Xe3p_LPD, the hardware tells us whether memory has ECC
	 * enabled that would impact display bandwidth.  However, so far there
	 * are no instructions in Bspec on how to handle that case.  Let's
	 * complain if we ever find such a scenario.
	 */
	if (DISPLAY_VER(display) >= 35)
		drm_WARN_ON(display->drm, dram_info->ecc_impacting_de_bw);

	if (DISPLAY_VERx100(display) >= 1401 && display->platform.dgfx) {
		xe2_hpd_get_bw_info(display, dram_info, soc_bw_params);
	} else if (display->platform.dg2) {
		dg2_get_bw_info(display);
	} else if (DISPLAY_VER(display) >= 12) {
		tgl_get_bw_info(display, dram_info, soc_bw_params, display_bw_params);
	} else if (DISPLAY_VER(display) == 11) {
		icl_get_bw_info(display, dram_info, soc_bw_params, display_bw_params);
	}

	update_sagv_status(display, display->bw.num_qgv_points);
}

static unsigned int intel_bw_num_active_planes(struct intel_display *display,
					       const struct intel_bw_state *bw_state)
{
	unsigned int num_active_planes = 0;
	enum pipe pipe;

	for_each_pipe(display, pipe)
		num_active_planes += bw_state->num_active_planes[pipe];

	return num_active_planes;
}

static unsigned int intel_bw_data_rate(struct intel_display *display,
				       const struct intel_bw_state *bw_state)
{
	unsigned int data_rate = 0;
	enum pipe pipe;

	for_each_pipe(display, pipe)
		data_rate += bw_state->data_rate[pipe];

	if (DISPLAY_VER(display) >= 13 && intel_display_vtd_active(display))
		data_rate = DIV_ROUND_UP(data_rate * 105, 100);

	return data_rate;
}

struct intel_bw_state *to_intel_bw_state(struct intel_global_state *obj_state)
{
	return container_of(obj_state, struct intel_bw_state, base);
}

struct intel_bw_state *
intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
{
	struct intel_display *display = to_intel_display(state);
	struct intel_global_state *bw_state;

	bw_state = intel_atomic_get_old_global_obj_state(state, &display->bw.obj);

	return to_intel_bw_state(bw_state);
}

struct intel_bw_state *
intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
{
	struct intel_display *display = to_intel_display(state);
	struct intel_global_state *bw_state;

	bw_state = intel_atomic_get_new_global_obj_state(state, &display->bw.obj);

	return to_intel_bw_state(bw_state);
}

struct intel_bw_state *
intel_atomic_get_bw_state(struct intel_atomic_state *state)
{
	struct intel_display *display = to_intel_display(state);
	struct intel_global_state *bw_state;

	bw_state = intel_atomic_get_global_obj_state(state, &display->bw.obj);
	if (IS_ERR(bw_state))
		return ERR_CAST(bw_state);

	return to_intel_bw_state(bw_state);
}

static unsigned int icl_max_bw_qgv_point_mask(struct intel_display *display,
					      int num_active_planes)
{
	unsigned int num_qgv_points = display->bw.num_qgv_points;
	unsigned int max_bw_point = 0;
	unsigned int max_bw = 0;
	int i;

	for (i = 0; i < num_qgv_points; i++) {
		unsigned int max_data_rate =
			icl_qgv_bw(display, num_active_planes, i);

		/*
		 * We need to know which qgv point gives us
		 * maximum bandwidth in order to disable SAGV
		 * if we find that we exceed SAGV block time
		 * with watermarks. By that moment we already
		 * have those, as it is calculated earlier in
		 * intel_atomic_check,
		 */
		if (max_data_rate > max_bw) {
			max_bw_point = BIT(i);
			max_bw = max_data_rate;
		}
	}

	return max_bw_point;
}

static u16 icl_prepare_qgv_points_mask(struct intel_display *display,
				       unsigned int qgv_points,
				       unsigned int psf_points)
{
	return ~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
		 ADLS_PCODE_REQ_PSF_PT(psf_points)) & icl_qgv_points_mask(display);
}

static unsigned int icl_max_bw_psf_gv_point_mask(struct intel_display *display)
{
	unsigned int num_psf_gv_points = display->bw.num_psf_gv_points;
	unsigned int max_bw_point_mask = 0;
	unsigned int max_bw = 0;
	int i;

	for (i = 0; i < num_psf_gv_points; i++) {
		unsigned int max_data_rate = adl_psf_bw(display, i);

		if (max_data_rate > max_bw) {
			max_bw_point_mask = BIT(i);
			max_bw = max_data_rate;
		} else if (max_data_rate == max_bw) {
			max_bw_point_mask |= BIT(i);
		}
	}

	return max_bw_point_mask;
}

static void icl_force_disable_sagv(struct intel_display *display,
				   struct intel_bw_state *bw_state)
{
	unsigned int qgv_points = icl_max_bw_qgv_point_mask(display, 0);
	unsigned int psf_points = icl_max_bw_psf_gv_point_mask(display);

	bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(display,
								qgv_points,
								psf_points);

	drm_dbg_kms(display->drm, "Forcing SAGV disable: mask 0x%x\n",
		    bw_state->qgv_points_mask);

	icl_pcode_restrict_qgv_points(display, bw_state->qgv_points_mask);
}

void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
{
	struct intel_display *display = to_intel_display(state);
	const struct intel_bw_state *old_bw_state =
		intel_atomic_get_old_bw_state(state);
	const struct intel_bw_state *new_bw_state =
		intel_atomic_get_new_bw_state(state);
	u16 old_mask, new_mask;

	if (!new_bw_state)
		return;

	old_mask = old_bw_state->qgv_points_mask;
	new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;

	if (old_mask == new_mask)
		return;

	WARN_ON(!new_bw_state->base.changed);

	drm_dbg_kms(display->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
		    old_mask, new_mask);

	/*
	 * Restrict required qgv points before updating the configuration.
	 * According to BSpec we can't mask and unmask qgv points at the same
	 * time. Also masking should be done before updating the configuration
	 * and unmasking afterwards.
	 */
	icl_pcode_restrict_qgv_points(display, new_mask);
}

void icl_sagv_post_plane_update(struct intel_atomic_state *state)
{
	struct intel_display *display = to_intel_display(state);
	const struct intel_bw_state *old_bw_state =
		intel_atomic_get_old_bw_state(state);
	const struct intel_bw_state *new_bw_state =
		intel_atomic_get_new_bw_state(state);
	u16 old_mask, new_mask;

	if (!new_bw_state)
		return;

	old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
	new_mask = new_bw_state->qgv_points_mask;

	if (old_mask == new_mask)
		return;

	WARN_ON(!new_bw_state->base.changed);

	drm_dbg_kms(display->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
		    old_mask, new_mask);

	/*
	 * Allow required qgv points after updating the configuration.
	 * According to BSpec we can't mask and unmask qgv points at the same
	 * time. Also masking should be done before updating the configuration
	 * and unmasking afterwards.
	 */
	icl_pcode_restrict_qgv_points(display, new_mask);
}

static int mtl_find_qgv_points(struct intel_display *display,
			       unsigned int data_rate,
			       unsigned int num_active_planes,
			       struct intel_bw_state *new_bw_state)
{
	unsigned int best_rate = UINT_MAX;
	unsigned int num_qgv_points = display->bw.num_qgv_points;
	unsigned int qgv_peak_bw  = 0;
	int i;
	int ret;

	ret = intel_atomic_lock_global_state(&new_bw_state->base);
	if (ret)
		return ret;

	/*
	 * If SAGV cannot be enabled, disable the pcode SAGV by passing all 1's
	 * for qgv peak bw in PM Demand request. So assign UINT_MAX if SAGV is
	 * not enabled. PM Demand code will clamp the value for the register
	 */
	if (!intel_bw_can_enable_sagv(display, new_bw_state)) {
		new_bw_state->qgv_point_peakbw = U16_MAX;
		drm_dbg_kms(display->drm, "No SAGV, use UINT_MAX as peak bw.");
		return 0;
	}

	/*
	 * Find the best QGV point by comparing the data_rate with max data rate
	 * offered per plane group
	 */
	for (i = 0; i < num_qgv_points; i++) {
		unsigned int max_data_rate =
			icl_qgv_bw(display, num_active_planes, i);

		if (max_data_rate < data_rate)
			continue;

		if (max_data_rate < best_rate) {
			best_rate = max_data_rate;
			qgv_peak_bw = display->bw.peakbw[i];
		}

		drm_dbg_kms(display->drm, "QGV point %d: max bw %d required %d qgv_peak_bw: %d\n",
			    i, max_data_rate, data_rate, qgv_peak_bw);
	}

	drm_dbg_kms(display->drm, "Matching peaks QGV bw: %d for required data rate: %d\n",
		    qgv_peak_bw, data_rate);

	/*
	 * The display configuration cannot be supported if no QGV point
	 * satisfying the required data rate is found
	 */
	if (qgv_peak_bw == 0) {
		drm_dbg_kms(display->drm, "No QGV points for bw %d for display configuration(%d active planes).\n",
			    data_rate, num_active_planes);
		return -EINVAL;
	}

	/* MTL PM DEMAND expects QGV BW parameter in multiples of 100 mbps */
	new_bw_state->qgv_point_peakbw = qgv_peak_bw / 100;

	return 0;
}

static int icl_find_qgv_points(struct intel_display *display,
			       unsigned int data_rate,
			       unsigned int num_active_planes,
			       const struct intel_bw_state *old_bw_state,
			       struct intel_bw_state *new_bw_state)
{
	unsigned int num_psf_gv_points = display->bw.num_psf_gv_points;
	unsigned int num_qgv_points = display->bw.num_qgv_points;
	u16 psf_points = 0;
	u16 qgv_points = 0;
	int i;
	int ret;

	ret = intel_atomic_lock_global_state(&new_bw_state->base);
	if (ret)
		return ret;

	for (i = 0; i < num_qgv_points; i++) {
		unsigned int max_data_rate = icl_qgv_bw(display,
							num_active_planes, i);
		if (max_data_rate >= data_rate)
			qgv_points |= BIT(i);

		drm_dbg_kms(display->drm, "QGV point %d: max bw %d required %d\n",
			    i, max_data_rate, data_rate);
	}

	for (i = 0; i < num_psf_gv_points; i++) {
		unsigned int max_data_rate = adl_psf_bw(display, i);

		if (max_data_rate >= data_rate)
			psf_points |= BIT(i);

		drm_dbg_kms(display->drm, "PSF GV point %d: max bw %d"
			    " required %d\n",
			    i, max_data_rate, data_rate);
	}

	/*
	 * BSpec states that we always should have at least one allowed point
	 * left, so if we couldn't - simply reject the configuration for obvious
	 * reasons.
	 */
	if (qgv_points == 0) {
		drm_dbg_kms(display->drm, "No QGV points provide sufficient memory"
			    " bandwidth %d for display configuration(%d active planes).\n",
			    data_rate, num_active_planes);
		return -EINVAL;
	}

	if (num_psf_gv_points > 0 && psf_points == 0) {
		drm_dbg_kms(display->drm, "No PSF GV points provide sufficient memory"
			    " bandwidth %d for display configuration(%d active planes).\n",
			    data_rate, num_active_planes);
		return -EINVAL;
	}

	/*
	 * Leave only single point with highest bandwidth, if
	 * we can't enable SAGV due to the increased memory latency it may
	 * cause.
	 */
	if (!intel_bw_can_enable_sagv(display, new_bw_state)) {
		qgv_points = icl_max_bw_qgv_point_mask(display, num_active_planes);
		drm_dbg_kms(display->drm, "No SAGV, using single QGV point mask 0x%x\n",
			    qgv_points);
	}

	/*
	 * We store the ones which need to be masked as that is what PCode
	 * actually accepts as a parameter.
	 */
	new_bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(display,
								    qgv_points,
								    psf_points);
	/*
	 * If the actual mask had changed we need to make sure that
	 * the commits are serialized(in case this is a nomodeset, nonblocking)
	 */
	if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
		ret = intel_atomic_serialize_global_state(&new_bw_state->base);
		if (ret)
			return ret;
	}

	return 0;
}

static int intel_bw_check_qgv_points(struct intel_display *display,
				     const struct intel_bw_state *old_bw_state,
				     struct intel_bw_state *new_bw_state)
{
	unsigned int data_rate = intel_bw_data_rate(display, new_bw_state);
	unsigned int num_active_planes =
			intel_bw_num_active_planes(display, new_bw_state);

	data_rate = DIV_ROUND_UP(data_rate, 1000);

	if (HAS_PMDEMAND(display))
		return mtl_find_qgv_points(display, data_rate, num_active_planes,
					   new_bw_state);
	else
		return icl_find_qgv_points(display, data_rate, num_active_planes,
					   old_bw_state, new_bw_state);
}

static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
{
	struct intel_display *display = to_intel_display(state);
	const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
	struct intel_crtc *crtc;

	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state) {
		unsigned int old_data_rate =
			intel_crtc_bw_data_rate(old_crtc_state);
		unsigned int new_data_rate =
			intel_crtc_bw_data_rate(new_crtc_state);
		unsigned int old_active_planes =
			intel_crtc_bw_num_active_planes(old_crtc_state);
		unsigned int new_active_planes =
			intel_crtc_bw_num_active_planes(new_crtc_state);
		struct intel_bw_state *new_bw_state;

		/*
		 * Avoid locking the bw state when
		 * nothing significant has changed.
		 */
		if (old_data_rate == new_data_rate &&
		    old_active_planes == new_active_planes)
			continue;

		new_bw_state = intel_atomic_get_bw_state(state);
		if (IS_ERR(new_bw_state))
			return PTR_ERR(new_bw_state);

		new_bw_state->data_rate[crtc->pipe] = new_data_rate;
		new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;

		*changed = true;

		drm_dbg_kms(display->drm,
			    "[CRTC:%d:%s] data rate %u num active planes %u\n",
			    crtc->base.base.id, crtc->base.name,
			    new_bw_state->data_rate[crtc->pipe],
			    new_bw_state->num_active_planes[crtc->pipe]);
	}

	return 0;
}

static int intel_bw_check_sagv_mask(struct intel_atomic_state *state)
{
	struct intel_display *display = to_intel_display(state);
	const struct intel_crtc_state *old_crtc_state;
	const struct intel_crtc_state *new_crtc_state;
	const struct intel_bw_state *old_bw_state = NULL;
	struct intel_bw_state *new_bw_state = NULL;
	struct intel_crtc *crtc;
	int ret;

	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state) {
		if (intel_crtc_can_enable_sagv(old_crtc_state) ==
		    intel_crtc_can_enable_sagv(new_crtc_state))
			continue;

		new_bw_state = intel_atomic_get_bw_state(state);
		if (IS_ERR(new_bw_state))
			return PTR_ERR(new_bw_state);

		old_bw_state = intel_atomic_get_old_bw_state(state);

		if (intel_crtc_can_enable_sagv(new_crtc_state))
			new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
		else
			new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
	}

	if (!new_bw_state)
		return 0;

	if (intel_bw_can_enable_sagv(display, new_bw_state) !=
	    intel_bw_can_enable_sagv(display, old_bw_state)) {
		ret = intel_atomic_serialize_global_state(&new_bw_state->base);
		if (ret)
			return ret;
	} else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
		ret = intel_atomic_lock_global_state(&new_bw_state->base);
		if (ret)
			return ret;
	}

	return 0;
}

int intel_bw_atomic_check(struct intel_atomic_state *state)
{
	struct intel_display *display = to_intel_display(state);
	bool changed = false;
	struct intel_bw_state *new_bw_state;
	const struct intel_bw_state *old_bw_state;
	int ret;

	if (DISPLAY_VER(display) < 9)
		return 0;

	ret = intel_bw_check_sagv_mask(state);
	if (ret)
		return ret;

	/* FIXME earlier gens need some checks too */
	if (DISPLAY_VER(display) < 11)
		return 0;

	ret = intel_bw_check_data_rate(state, &changed);
	if (ret)
		return ret;

	old_bw_state = intel_atomic_get_old_bw_state(state);
	new_bw_state = intel_atomic_get_new_bw_state(state);

	if (new_bw_state &&
	    intel_bw_can_enable_sagv(display, old_bw_state) !=
	    intel_bw_can_enable_sagv(display, new_bw_state))
		changed = true;

	/*
	 * If none of our inputs (data rates, number of active
	 * planes, SAGV yes/no) changed then nothing to do here.
	 */
	if (!changed)
		return 0;

	ret = intel_bw_check_qgv_points(display, old_bw_state, new_bw_state);
	if (ret)
		return ret;

	return 0;
}

static void intel_bw_crtc_update(struct intel_bw_state *bw_state,
				 const struct intel_crtc_state *crtc_state)
{
	struct intel_display *display = to_intel_display(crtc_state);
	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);

	bw_state->data_rate[crtc->pipe] =
		intel_crtc_bw_data_rate(crtc_state);
	bw_state->num_active_planes[crtc->pipe] =
		intel_crtc_bw_num_active_planes(crtc_state);

	drm_dbg_kms(display->drm, "pipe %c data rate %u num active planes %u\n",
		    pipe_name(crtc->pipe),
		    bw_state->data_rate[crtc->pipe],
		    bw_state->num_active_planes[crtc->pipe]);
}

void intel_bw_update_hw_state(struct intel_display *display)
{
	struct intel_bw_state *bw_state =
		to_intel_bw_state(display->bw.obj.state);
	struct intel_crtc *crtc;

	if (DISPLAY_VER(display) < 9)
		return;

	bw_state->pipe_sagv_reject = 0;

	for_each_intel_crtc(display, crtc) {
		const struct intel_crtc_state *crtc_state =
			to_intel_crtc_state(crtc->base.state);
		enum pipe pipe = crtc->pipe;

		if (DISPLAY_VER(display) >= 11)
			intel_bw_crtc_update(bw_state, crtc_state);

		/* initially SAGV has been forced off */
		bw_state->pipe_sagv_reject |= BIT(pipe);
	}
}

void intel_bw_crtc_disable_noatomic(struct intel_crtc *crtc)
{
	struct intel_display *display = to_intel_display(crtc);
	struct intel_bw_state *bw_state =
		to_intel_bw_state(display->bw.obj.state);
	enum pipe pipe = crtc->pipe;

	if (DISPLAY_VER(display) < 9)
		return;

	bw_state->data_rate[pipe] = 0;
	bw_state->num_active_planes[pipe] = 0;
}

static struct intel_global_state *
intel_bw_duplicate_state(struct intel_global_obj *obj)
{
	struct intel_bw_state *state;

	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
	if (!state)
		return NULL;

	return &state->base;
}

static void intel_bw_destroy_state(struct intel_global_obj *obj,
				   struct intel_global_state *state)
{
	kfree(state);
}

static const struct intel_global_state_funcs intel_bw_funcs = {
	.atomic_duplicate_state = intel_bw_duplicate_state,
	.atomic_destroy_state = intel_bw_destroy_state,
};

int intel_bw_init(struct intel_display *display)
{
	struct intel_bw_state *state;

	state = kzalloc_obj(*state);
	if (!state)
		return -ENOMEM;

	intel_atomic_global_obj_init(display, &display->bw.obj,
				     &state->base, &intel_bw_funcs);

	/*
	 * Limit this only if we have SAGV. And for Display version 14 onwards
	 * sagv is handled though pmdemand requests
	 */
	if (intel_has_sagv(display) && IS_DISPLAY_VER(display, 11, 13))
		icl_force_disable_sagv(display, state);

	return 0;
}

bool intel_bw_pmdemand_needs_update(struct intel_atomic_state *state)
{
	const struct intel_bw_state *new_bw_state, *old_bw_state;

	new_bw_state = intel_atomic_get_new_bw_state(state);
	old_bw_state = intel_atomic_get_old_bw_state(state);

	if (new_bw_state &&
	    new_bw_state->qgv_point_peakbw != old_bw_state->qgv_point_peakbw)
		return true;

	return false;
}

bool intel_bw_can_enable_sagv(struct intel_display *display,
			      const struct intel_bw_state *bw_state)
{
	return bw_state->pipe_sagv_reject == 0;
}

int intel_bw_qgv_point_peakbw(const struct intel_bw_state *bw_state)
{
	return bw_state->qgv_point_peakbw;
}