xref: /linux/drivers/pmdomain/mediatek/mtk-mfg-pmdomain.c (revision 84318277d6334c6981ab326d4acc87c6a6ddc9b8)

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Driver for MediaTek MFlexGraphics Devices
 *
 * Copyright (C) 2025, Collabora Ltd.
 */

#include <linux/completion.h>
#include <linux/clk.h>
#include <linux/clk-provider.h>
#include <linux/container_of.h>
#include <linux/iopoll.h>
#include <linux/mailbox_client.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_platform.h>
#include <linux/of_reserved_mem.h>
#include <linux/overflow.h>
#include <linux/platform_device.h>
#include <linux/pm_domain.h>
#include <linux/pm_opp.h>
#include <linux/regulator/consumer.h>
#include <linux/units.h>

#define GPR_LP_STATE		0x0028
#define   EB_ON_SUSPEND		0x0
#define   EB_ON_RESUME		0x1
#define GPR_IPI_MAGIC		0x34

#define RPC_PWR_CON		0x0504
#define   PWR_ACK_M		GENMASK(31, 30)
#define RPC_DUMMY_REG_2		0x0658
#define RPC_GHPM_CFG0_CON	0x0800
#define   GHPM_ENABLE_M		BIT(0)
#define   GHPM_ON_SEQ_M		BIT(2)
#define RPC_GHPM_RO0_CON	0x09A4
#define   GHPM_STATE_M		GENMASK(7, 0)
#define   GHPM_PWR_STATE_M	BIT(16)

#define GF_REG_MAGIC			0x0000
#define GF_REG_GPU_OPP_IDX		0x0004
#define GF_REG_STK_OPP_IDX		0x0008
#define GF_REG_GPU_OPP_NUM		0x000c
#define GF_REG_STK_OPP_NUM		0x0010
#define GF_REG_GPU_OPP_SNUM		0x0014
#define GF_REG_STK_OPP_SNUM		0x0018
#define GF_REG_POWER_COUNT		0x001c
#define GF_REG_BUCK_COUNT		0x0020
#define GF_REG_MTCMOS_COUNT		0x0024
#define GF_REG_CG_COUNT			0x0028 /* CG = Clock Gate? */
#define GF_REG_ACTIVE_COUNT		0x002C
#define GF_REG_TEMP_RAW			0x0030
#define GF_REG_TEMP_NORM_GPU		0x0034
#define GF_REG_TEMP_HIGH_GPU		0x0038
#define GF_REG_TEMP_NORM_STK		0x003C
#define GF_REG_TEMP_HIGH_STK		0x0040
#define GF_REG_FREQ_CUR_GPU		0x0044
#define GF_REG_FREQ_CUR_STK		0x0048
#define GF_REG_FREQ_OUT_GPU		0x004C /* Guess: actual achieved freq */
#define GF_REG_FREQ_OUT_STK		0x0050 /* Guess: actual achieved freq */
#define GF_REG_FREQ_METER_GPU		0x0054 /* Seems unused, always 0 */
#define GF_REG_FREQ_METER_STK		0x0058 /* Seems unused, always 0 */
#define GF_REG_VOLT_CUR_GPU		0x005C /* in tens of microvolts */
#define GF_REG_VOLT_CUR_STK		0x0060 /* in tens of microvolts */
#define GF_REG_VOLT_CUR_GPU_SRAM	0x0064
#define GF_REG_VOLT_CUR_STK_SRAM	0x0068
#define GF_REG_VOLT_CUR_GPU_REG		0x006C /* Seems unused, always 0 */
#define GF_REG_VOLT_CUR_STK_REG		0x0070 /* Seems unused, always 0 */
#define GF_REG_VOLT_CUR_GPU_REG_SRAM	0x0074
#define GF_REG_VOLT_CUR_STK_REG_SRAM	0x0078
#define GF_REG_PWR_CUR_GPU		0x007C /* in milliwatts */
#define GF_REG_PWR_CUR_STK		0x0080 /* in milliwatts */
#define GF_REG_PWR_MAX_GPU		0x0084 /* in milliwatts */
#define GF_REG_PWR_MAX_STK		0x0088 /* in milliwatts */
#define GF_REG_PWR_MIN_GPU		0x008C /* in milliwatts */
#define GF_REG_PWR_MIN_STK		0x0090 /* in milliwatts */
#define GF_REG_LEAKAGE_RT_GPU		0x0094 /* Unknown */
#define GF_REG_LEAKAGE_RT_STK		0x0098 /* Unknown */
#define GF_REG_LEAKAGE_RT_SRAM		0x009C /* Unknown */
#define GF_REG_LEAKAGE_HT_GPU		0x00A0 /* Unknown */
#define GF_REG_LEAKAGE_HT_STK		0x00A4 /* Unknown */
#define GF_REG_LEAKAGE_HT_SRAM		0x00A8 /* Unknown */
#define GF_REG_VOLT_DAC_LOW_GPU		0x00AC /* Seems unused, always 0 */
#define GF_REG_VOLT_DAC_LOW_STK		0x00B0 /* Seems unused, always 0 */
#define GF_REG_OPP_CUR_CEIL		0x00B4
#define GF_REG_OPP_CUR_FLOOR		0x00B8
#define GF_REG_OPP_CUR_LIMITER_CEIL	0x00BC
#define GF_REG_OPP_CUR_LIMITER_FLOOR	0x00C0
#define GF_REG_OPP_PRIORITY_CEIL	0x00C4
#define GF_REG_OPP_PRIORITY_FLOOR	0x00C8
#define GF_REG_PWR_CTL			0x00CC
#define GF_REG_ACTIVE_SLEEP_CTL		0x00D0
#define GF_REG_DVFS_STATE		0x00D4
#define GF_REG_SHADER_PRESENT		0x00D8
#define GF_REG_ASENSOR_ENABLE		0x00DC
#define GF_REG_AGING_LOAD		0x00E0
#define GF_REG_AGING_MARGIN		0x00E4
#define GF_REG_AVS_ENABLE		0x00E8
#define GF_REG_AVS_MARGIN		0x00EC
#define GF_REG_CHIP_TYPE		0x00F0
#define GF_REG_SB_VERSION		0x00F4
#define GF_REG_PTP_VERSION		0x00F8
#define GF_REG_DBG_VERSION		0x00FC
#define GF_REG_KDBG_VERSION		0x0100
#define GF_REG_GPM1_MODE		0x0104
#define GF_REG_GPM3_MODE		0x0108
#define GF_REG_DFD_MODE			0x010C
#define GF_REG_DUAL_BUCK		0x0110
#define GF_REG_SEGMENT_ID		0x0114
#define GF_REG_POWER_TIME_H		0x0118
#define GF_REG_POWER_TIME_L		0x011C
#define GF_REG_PWR_STATUS		0x0120
#define GF_REG_STRESS_TEST		0x0124
#define GF_REG_TEST_MODE		0x0128
#define GF_REG_IPS_MODE			0x012C
#define GF_REG_TEMP_COMP_MODE		0x0130
#define GF_REG_HT_TEMP_COMP_MODE	0x0134
#define GF_REG_PWR_TRACKER_MODE		0x0138
#define GF_REG_OPP_TABLE_GPU		0x0314
#define GF_REG_OPP_TABLE_STK		0x09A4
#define GF_REG_OPP_TABLE_GPU_S		0x1034
#define GF_REG_OPP_TABLE_STK_S		0x16c4
#define GF_REG_LIMIT_TABLE		0x1d54
#define GF_REG_GPM3_TABLE		0x223C

#define MFG_MT8196_E2_ID		0x101
#define GPUEB_SLEEP_MAGIC		0x55667788UL
#define GPUEB_MEM_MAGIC			0xBABADADAUL

#define GPUEB_TIMEOUT_US		10000UL
#define GPUEB_POLL_US			50

#define MAX_OPP_NUM			70

#define GPUEB_MBOX_MAX_RX_SIZE		32 /* in bytes */

/*
 * This enum is part of the ABI of the GPUEB firmware. Don't change the
 * numbering, as you would wreak havoc.
 */
enum mtk_mfg_ipi_cmd {
	CMD_INIT_SHARED_MEM		= 0,
	CMD_GET_FREQ_BY_IDX		= 1,
	CMD_GET_POWER_BY_IDX		= 2,
	CMD_GET_OPPIDX_BY_FREQ		= 3,
	CMD_GET_LEAKAGE_POWER		= 4,
	CMD_SET_LIMIT			= 5,
	CMD_POWER_CONTROL		= 6,
	CMD_ACTIVE_SLEEP_CONTROL	= 7,
	CMD_COMMIT			= 8,
	CMD_DUAL_COMMIT			= 9,
	CMD_PDCA_CONFIG			= 10,
	CMD_UPDATE_DEBUG_OPP_INFO	= 11,
	CMD_SWITCH_LIMIT		= 12,
	CMD_FIX_TARGET_OPPIDX		= 13,
	CMD_FIX_DUAL_TARGET_OPPIDX	= 14,
	CMD_FIX_CUSTOM_FREQ_VOLT	= 15,
	CMD_FIX_DUAL_CUSTOM_FREQ_VOLT	= 16,
	CMD_SET_MFGSYS_CONFIG		= 17,
	CMD_MSSV_COMMIT			= 18,
	CMD_NUM				= 19,
};

/*
 * This struct is part of the ABI of the GPUEB firmware. Changing it, or
 * reordering fields in it, will break things, so don't do it. Thank you.
 */
struct __packed mtk_mfg_ipi_msg {
	__le32 magic;
	__le32 cmd;
	__le32 target;
	/*
	 * Downstream relies on the compiler to implicitly add the following
	 * padding, as it declares the struct as non-packed.
	 */
	__le32 reserved;
	union {
		s32 __bitwise oppidx;
		s32 __bitwise return_value;
		__le32 freq;
		__le32 volt;
		__le32 power;
		__le32 power_state;
		__le32 mode;
		__le32 value;
		struct {
			__le64 base;
			__le32 size;
		} shared_mem;
		struct {
			__le32 freq;
			__le32 volt;
		} custom;
		struct {
			__le32 limiter;
			s32 __bitwise ceiling_info;
			s32 __bitwise floor_info;
		} set_limit;
		struct {
			__le32 target;
			__le32 val;
		} mfg_cfg;
		struct {
			__le32 target;
			__le32 val;
		} mssv;
		struct {
			s32 __bitwise gpu_oppidx;
			s32 __bitwise stack_oppidx;
		} dual_commit;
		struct {
			__le32 fgpu;
			__le32 vgpu;
			__le32 fstack;
			__le32 vstack;
		} dual_custom;
	} u;
};

struct __packed mtk_mfg_ipi_sleep_msg {
	__le32 event;
	__le32 state;
	__le32 magic;
};

/**
 * struct mtk_mfg_opp_entry - OPP table entry from firmware
 * @freq_khz: The operating point's frequency in kilohertz
 * @voltage_core: The operating point's core voltage in tens of microvolts
 * @voltage_sram: The operating point's SRAM voltage in tens of microvolts
 * @posdiv: exponent of base 2 for PLL frequency divisor used for this OPP
 * @voltage_margin: Number of tens of microvolts the voltage can be undershot
 * @power_mw: estimate of power usage at this operating point, in milliwatts
 *
 * This struct is part of the ABI with the EB firmware. Do not change it.
 */
struct __packed mtk_mfg_opp_entry {
	__le32 freq_khz;
	__le32 voltage_core;
	__le32 voltage_sram;
	__le32 posdiv;
	__le32 voltage_margin;
	__le32 power_mw;
};

struct mtk_mfg_mbox {
	struct mbox_client cl;
	struct completion rx_done;
	struct mtk_mfg *mfg;
	struct mbox_chan *ch;
	void *rx_data;
};

struct mtk_mfg {
	struct generic_pm_domain pd;
	struct platform_device *pdev;
	struct clk *clk_eb;
	struct clk_bulk_data *gpu_clks;
	struct clk_hw clk_core_hw;
	struct clk_hw clk_stack_hw;
	struct regulator_bulk_data *gpu_regs;
	void __iomem *rpc;
	void __iomem *gpr;
	void __iomem *shared_mem;
	phys_addr_t shared_mem_phys;
	unsigned int shared_mem_size;
	u16 ghpm_en_reg;
	u32 ipi_magic;
	unsigned short num_gpu_opps;
	unsigned short num_stack_opps;
	struct dev_pm_opp_data *gpu_opps;
	struct dev_pm_opp_data *stack_opps;
	struct mtk_mfg_mbox *gf_mbox;
	struct mtk_mfg_mbox *slp_mbox;
	const struct mtk_mfg_variant *variant;
};

struct mtk_mfg_variant {
	const char *const *clk_names;
	unsigned int num_clks;
	const char *const *regulator_names;
	unsigned int num_regulators;
	/** @turbo_below: opp indices below this value are considered turbo */
	unsigned int turbo_below;
	int (*init)(struct mtk_mfg *mfg);
};

static inline struct mtk_mfg *mtk_mfg_from_genpd(struct generic_pm_domain *pd)
{
	return container_of(pd, struct mtk_mfg, pd);
}

static inline void mtk_mfg_update_reg_bits(void __iomem *addr, u32 mask, u32 val)
{
	writel((readl(addr) & ~mask) | (val & mask), addr);
}

static inline bool mtk_mfg_is_powered_on(struct mtk_mfg *mfg)
{
	return (readl(mfg->rpc + RPC_PWR_CON) & PWR_ACK_M) == PWR_ACK_M;
}

static unsigned long mtk_mfg_recalc_rate_gpu(struct clk_hw *hw,
					     unsigned long parent_rate)
{
	struct mtk_mfg *mfg = container_of(hw, struct mtk_mfg, clk_core_hw);

	return readl(mfg->shared_mem + GF_REG_FREQ_OUT_GPU) * HZ_PER_KHZ;
}

static int mtk_mfg_determine_rate(struct clk_hw *hw,
				  struct clk_rate_request *req)
{
	/*
	 * The determine_rate callback needs to be implemented to avoid returning
	 * the current clock frequency, rather than something even remotely
	 * close to the frequency that was asked for.
	 *
	 * Instead of writing considerable amounts of possibly slow code just to
	 * somehow figure out which of the three PLLs to round for, or even to
	 * do a search through one of two OPP tables in order to find the closest
	 * OPP of a frequency, just return the rate as-is. This avoids devfreq
	 * "rounding" a request for the lowest frequency to the possibly very
	 * high current frequency, breaking the powersave governor in the process.
	 */

	return 0;
}

static unsigned long mtk_mfg_recalc_rate_stack(struct clk_hw *hw,
					       unsigned long parent_rate)
{
	struct mtk_mfg *mfg = container_of(hw, struct mtk_mfg, clk_stack_hw);

	return readl(mfg->shared_mem + GF_REG_FREQ_OUT_STK) * HZ_PER_KHZ;
}

static const struct clk_ops mtk_mfg_clk_gpu_ops = {
	.recalc_rate = mtk_mfg_recalc_rate_gpu,
	.determine_rate = mtk_mfg_determine_rate,
};

static const struct clk_ops mtk_mfg_clk_stack_ops = {
	.recalc_rate = mtk_mfg_recalc_rate_stack,
	.determine_rate = mtk_mfg_determine_rate,
};

static const struct clk_init_data mtk_mfg_clk_gpu_init = {
	.name = "gpu-core",
	.ops = &mtk_mfg_clk_gpu_ops,
	.flags = CLK_GET_RATE_NOCACHE,
};

static const struct clk_init_data mtk_mfg_clk_stack_init = {
	.name = "gpu-stack",
	.ops = &mtk_mfg_clk_stack_ops,
	.flags = CLK_GET_RATE_NOCACHE,
};

static int mtk_mfg_eb_on(struct mtk_mfg *mfg)
{
	struct device *dev = &mfg->pdev->dev;
	u32 val;
	int ret;

	/*
	 * If MFG is already on from e.g. the bootloader, skip doing the
	 * power-on sequence, as it wouldn't work without powering it off first.
	 */
	if (mtk_mfg_is_powered_on(mfg))
		return 0;

	ret = readl_poll_timeout(mfg->rpc + RPC_GHPM_RO0_CON, val,
				 !(val & (GHPM_PWR_STATE_M | GHPM_STATE_M)),
				 GPUEB_POLL_US, GPUEB_TIMEOUT_US);
	if (ret) {
		dev_err(dev, "timed out waiting for EB to power on\n");
		return ret;
	}

	mtk_mfg_update_reg_bits(mfg->rpc + mfg->ghpm_en_reg, GHPM_ENABLE_M,
				GHPM_ENABLE_M);

	mtk_mfg_update_reg_bits(mfg->rpc + RPC_GHPM_CFG0_CON, GHPM_ON_SEQ_M, 0);
	mtk_mfg_update_reg_bits(mfg->rpc + RPC_GHPM_CFG0_CON, GHPM_ON_SEQ_M,
				GHPM_ON_SEQ_M);

	mtk_mfg_update_reg_bits(mfg->rpc + mfg->ghpm_en_reg, GHPM_ENABLE_M, 0);


	ret = readl_poll_timeout(mfg->rpc + RPC_PWR_CON, val,
				 (val & PWR_ACK_M) == PWR_ACK_M,
				 GPUEB_POLL_US, GPUEB_TIMEOUT_US);
	if (ret) {
		dev_err(dev, "timed out waiting for EB power ack, val = 0x%X\n",
			val);
		return ret;
	}

	ret = readl_poll_timeout(mfg->gpr + GPR_LP_STATE, val,
				 (val == EB_ON_RESUME),
				 GPUEB_POLL_US, GPUEB_TIMEOUT_US);
	if (ret) {
		dev_err(dev, "timed out waiting for EB to resume, status = 0x%X\n", val);
		return ret;
	}

	return 0;
}

static int mtk_mfg_eb_off(struct mtk_mfg *mfg)
{
	struct device *dev = &mfg->pdev->dev;
	struct mtk_mfg_ipi_sleep_msg msg = {
		.event = 0,
		.state = 0,
		.magic = GPUEB_SLEEP_MAGIC
	};
	u32 val;
	int ret;

	ret = mbox_send_message(mfg->slp_mbox->ch, &msg);
	if (ret < 0) {
		dev_err(dev, "Cannot send sleep command: %pe\n", ERR_PTR(ret));
		return ret;
	}

	ret = readl_poll_timeout(mfg->rpc + RPC_PWR_CON, val,
				 !(val & PWR_ACK_M), GPUEB_POLL_US,
				 GPUEB_TIMEOUT_US);

	if (ret) {
		dev_err(dev, "Timed out waiting for EB to power off, val=0x%08X\n", val);
		return ret;
	}

	return 0;
}

/**
 * mtk_mfg_send_ipi - synchronously send an IPI message on the gpufreq channel
 * @mfg: pointer to this driver instance's private &struct mtk_mfg
 * @msg: pointer to a message to send; will have magic filled and response assigned
 *
 * Send an IPI message on the gpufreq channel, and wait for a response. Once a
 * response is received, assign a pointer to the response buffer (valid until
 * next response is received) to @msg.
 *
 * Returns 0 on success, negative errno on failure.
 */
static int mtk_mfg_send_ipi(struct mtk_mfg *mfg, struct mtk_mfg_ipi_msg *msg)
{
	struct device *dev = &mfg->pdev->dev;
	unsigned long wait;
	int ret;

	msg->magic = mfg->ipi_magic;

	ret = mbox_send_message(mfg->gf_mbox->ch, msg);
	if (ret < 0) {
		dev_err(dev, "Cannot send GPUFreq IPI command: %pe\n", ERR_PTR(ret));
		return ret;
	}

	wait = wait_for_completion_timeout(&mfg->gf_mbox->rx_done, msecs_to_jiffies(500));
	if (!wait)
		return -ETIMEDOUT;

	msg = mfg->gf_mbox->rx_data;

	if (msg->u.return_value < 0) {
		dev_err(dev, "IPI return: %d\n", msg->u.return_value);
		return -EPROTO;
	}

	return 0;
}

static int mtk_mfg_init_shared_mem(struct mtk_mfg *mfg)
{
	struct device *dev = &mfg->pdev->dev;
	struct mtk_mfg_ipi_msg msg = {};
	int ret;

	dev_dbg(dev, "clearing GPUEB shared memory, 0x%X bytes\n", mfg->shared_mem_size);
	memset_io(mfg->shared_mem, 0, mfg->shared_mem_size);

	msg.cmd = CMD_INIT_SHARED_MEM;
	msg.u.shared_mem.base = mfg->shared_mem_phys;
	msg.u.shared_mem.size = mfg->shared_mem_size;

	ret = mtk_mfg_send_ipi(mfg, &msg);
	if (ret)
		return ret;

	if (readl(mfg->shared_mem + GF_REG_MAGIC) != GPUEB_MEM_MAGIC) {
		dev_err(dev, "EB did not initialise shared memory correctly\n");
		return -EIO;
	}

	return 0;
}

static int mtk_mfg_power_control(struct mtk_mfg *mfg, bool enabled)
{
	struct mtk_mfg_ipi_msg msg = {};

	msg.cmd = CMD_POWER_CONTROL;
	msg.u.power_state = enabled ? 1 : 0;

	return mtk_mfg_send_ipi(mfg, &msg);
}

static int mtk_mfg_set_oppidx(struct mtk_mfg *mfg, unsigned int opp_idx)
{
	struct mtk_mfg_ipi_msg msg = {};
	int ret;

	if (opp_idx >= mfg->num_gpu_opps)
		return -EINVAL;

	msg.cmd = CMD_FIX_DUAL_TARGET_OPPIDX;
	msg.u.dual_commit.gpu_oppidx = opp_idx;
	msg.u.dual_commit.stack_oppidx = opp_idx;

	ret = mtk_mfg_send_ipi(mfg, &msg);
	if (ret) {
		dev_err(&mfg->pdev->dev, "Failed to set OPP %u: %pe\n",
			opp_idx, ERR_PTR(ret));
		return ret;
	}

	return 0;
}

static int mtk_mfg_read_opp_tables(struct mtk_mfg *mfg)
{
	struct device *dev = &mfg->pdev->dev;
	struct mtk_mfg_opp_entry e = {};
	unsigned int i;

	mfg->num_gpu_opps = readl(mfg->shared_mem + GF_REG_GPU_OPP_NUM);
	mfg->num_stack_opps = readl(mfg->shared_mem + GF_REG_STK_OPP_NUM);

	if (mfg->num_gpu_opps > MAX_OPP_NUM || mfg->num_gpu_opps == 0) {
		dev_err(dev, "GPU OPP count (%u) out of range %u >= count > 0\n",
			mfg->num_gpu_opps, MAX_OPP_NUM);
		return -EINVAL;
	}

	if (mfg->num_stack_opps && mfg->num_stack_opps > MAX_OPP_NUM) {
		dev_err(dev, "Stack OPP count (%u) out of range %u >= count >= 0\n",
			mfg->num_stack_opps, MAX_OPP_NUM);
		return -EINVAL;
	}

	mfg->gpu_opps = devm_kcalloc(dev, mfg->num_gpu_opps,
				     sizeof(struct dev_pm_opp_data), GFP_KERNEL);
	if (!mfg->gpu_opps)
		return -ENOMEM;

	if (mfg->num_stack_opps) {
		mfg->stack_opps = devm_kcalloc(dev, mfg->num_stack_opps,
					       sizeof(struct dev_pm_opp_data), GFP_KERNEL);
		if (!mfg->stack_opps)
			return -ENOMEM;
	}

	for (i = 0; i < mfg->num_gpu_opps; i++) {
		memcpy_fromio(&e, mfg->shared_mem + GF_REG_OPP_TABLE_GPU + i * sizeof(e),
			      sizeof(e));
		if (mem_is_zero(&e, sizeof(e))) {
			dev_err(dev, "ran into an empty GPU OPP at index %u\n",
				i);
			return -EINVAL;
		}
		mfg->gpu_opps[i].freq = e.freq_khz * HZ_PER_KHZ;
		mfg->gpu_opps[i].u_volt = e.voltage_core * 10;
		mfg->gpu_opps[i].level = i;
		if (i < mfg->variant->turbo_below)
			mfg->gpu_opps[i].turbo = true;
	}

	for (i = 0; i < mfg->num_stack_opps; i++) {
		memcpy_fromio(&e, mfg->shared_mem + GF_REG_OPP_TABLE_STK + i * sizeof(e),
			      sizeof(e));
		if (mem_is_zero(&e, sizeof(e))) {
			dev_err(dev, "ran into an empty Stack OPP at index %u\n",
				i);
			return -EINVAL;
		}
		mfg->stack_opps[i].freq = e.freq_khz * HZ_PER_KHZ;
		mfg->stack_opps[i].u_volt = e.voltage_core * 10;
		mfg->stack_opps[i].level = i;
		if (i < mfg->variant->turbo_below)
			mfg->stack_opps[i].turbo = true;
	}

	return 0;
}

static const char *const mtk_mfg_mt8196_clk_names[] = {
	"core",
	"stack0",
	"stack1",
};

static const char *const mtk_mfg_mt8196_regulators[] = {
	"core",
	"stack",
	"sram",
};

static int mtk_mfg_mt8196_init(struct mtk_mfg *mfg)
{
	void __iomem *e2_base;

	e2_base = devm_platform_ioremap_resource_byname(mfg->pdev, "hw-revision");
	if (IS_ERR(e2_base))
		return dev_err_probe(&mfg->pdev->dev, PTR_ERR(e2_base),
				     "Couldn't get hw-revision register\n");

	clk_prepare_enable(mfg->clk_eb);

	if (readl(e2_base) == MFG_MT8196_E2_ID)
		mfg->ghpm_en_reg = RPC_DUMMY_REG_2;
	else
		mfg->ghpm_en_reg = RPC_GHPM_CFG0_CON;

	clk_disable_unprepare(mfg->clk_eb);

	return 0;
}

static const struct mtk_mfg_variant mtk_mfg_mt8196_variant = {
	.clk_names = mtk_mfg_mt8196_clk_names,
	.num_clks = ARRAY_SIZE(mtk_mfg_mt8196_clk_names),
	.regulator_names = mtk_mfg_mt8196_regulators,
	.num_regulators = ARRAY_SIZE(mtk_mfg_mt8196_regulators),
	.turbo_below = 7,
	.init = mtk_mfg_mt8196_init,
};

static void mtk_mfg_mbox_rx_callback(struct mbox_client *cl, void *mssg)
{
	struct mtk_mfg_mbox *mb = container_of(cl, struct mtk_mfg_mbox, cl);

	if (mb->rx_data)
		mb->rx_data = memcpy(mb->rx_data, mssg, GPUEB_MBOX_MAX_RX_SIZE);
	complete(&mb->rx_done);
}

static int mtk_mfg_attach_dev(struct generic_pm_domain *pd, struct device *dev)
{
	struct mtk_mfg *mfg = mtk_mfg_from_genpd(pd);
	struct dev_pm_opp_data *so = mfg->stack_opps;
	struct dev_pm_opp_data *go = mfg->gpu_opps;
	struct dev_pm_opp_data *prev_o;
	struct dev_pm_opp_data *o;
	int i, ret;

	for (i = mfg->num_gpu_opps - 1; i >= 0; i--) {
		/*
		 * Adding the lower of the two OPPs avoids gaps of indices in
		 * situations where the GPU OPPs are duplicated a couple of
		 * times when only the Stack OPP is being lowered at that index.
		 */
		if (i >= mfg->num_stack_opps || go[i].freq < so[i].freq)
			o = &go[i];
		else
			o = &so[i];

		/*
		 * Skip indices where both GPU and Stack OPPs are equal. Nominally,
		 * OPP core shouldn't care about dupes, but not doing so will cause
		 * dev_pm_opp_find_freq_ceil_indexed to -ERANGE later down the line.
		 */
		if (prev_o && prev_o->freq == o->freq)
			continue;

		ret = dev_pm_opp_add_dynamic(dev, o);
		if (ret) {
			dev_err(dev, "Failed to add OPP level %u from PD %s: %pe\n",
				o->level, pd->name, ERR_PTR(ret));
			dev_pm_opp_remove_all_dynamic(dev);
			return ret;
		}
		prev_o = o;
	}

	return 0;
}

static void mtk_mfg_detach_dev(struct generic_pm_domain *pd, struct device *dev)
{
	dev_pm_opp_remove_all_dynamic(dev);
}

static int mtk_mfg_set_performance(struct generic_pm_domain *pd,
				   unsigned int state)
{
	struct mtk_mfg *mfg = mtk_mfg_from_genpd(pd);

	/*
	 * pmdomain core intentionally sets a performance state before turning
	 * a domain on, and after turning it off. For the GPUEB however, it's
	 * only possible to act on performance requests when the GPUEB is
	 * powered on. To do this, return cleanly without taking action, and
	 * defer setting what pmdomain core set in mtk_mfg_power_on.
	 */
	if (mfg->pd.status != GENPD_STATE_ON)
		return 0;

	return mtk_mfg_set_oppidx(mfg, state);
}

static int mtk_mfg_power_on(struct generic_pm_domain *pd)
{
	struct mtk_mfg *mfg = mtk_mfg_from_genpd(pd);
	int ret;

	ret = regulator_bulk_enable(mfg->variant->num_regulators,
				    mfg->gpu_regs);
	if (ret)
		return ret;

	ret = clk_prepare_enable(mfg->clk_eb);
	if (ret)
		goto err_disable_regulators;

	ret = clk_bulk_prepare_enable(mfg->variant->num_clks, mfg->gpu_clks);
	if (ret)
		goto err_disable_eb_clk;

	ret = mtk_mfg_eb_on(mfg);
	if (ret)
		goto err_disable_clks;

	mfg->ipi_magic = readl(mfg->gpr + GPR_IPI_MAGIC);

	ret = mtk_mfg_power_control(mfg, true);
	if (ret)
		goto err_eb_off;

	/* Don't try to set a OPP in probe before OPPs have been read from EB */
	if (mfg->gpu_opps) {
		/* The aforementioned deferred setting of pmdomain's state */
		ret = mtk_mfg_set_oppidx(mfg, pd->performance_state);
		if (ret)
			dev_warn(&mfg->pdev->dev, "Failed to set oppidx in %s\n", __func__);
	}

	return 0;

err_eb_off:
	mtk_mfg_eb_off(mfg);
err_disable_clks:
	clk_bulk_disable_unprepare(mfg->variant->num_clks, mfg->gpu_clks);
err_disable_eb_clk:
	clk_disable_unprepare(mfg->clk_eb);
err_disable_regulators:
	regulator_bulk_disable(mfg->variant->num_regulators, mfg->gpu_regs);

	return ret;
}

static int mtk_mfg_power_off(struct generic_pm_domain *pd)
{
	struct mtk_mfg *mfg = mtk_mfg_from_genpd(pd);
	struct device *dev = &mfg->pdev->dev;
	int ret;

	ret = mtk_mfg_power_control(mfg, false);
	if (ret) {
		dev_err(dev, "power_control failed: %pe\n", ERR_PTR(ret));
		return ret;
	}

	ret = mtk_mfg_eb_off(mfg);
	if (ret) {
		dev_err(dev, "eb_off failed: %pe\n", ERR_PTR(ret));
		return ret;
	}

	clk_bulk_disable_unprepare(mfg->variant->num_clks, mfg->gpu_clks);
	clk_disable_unprepare(mfg->clk_eb);
	ret = regulator_bulk_disable(mfg->variant->num_regulators, mfg->gpu_regs);
	if (ret) {
		dev_err(dev, "Disabling regulators failed: %pe\n", ERR_PTR(ret));
		return ret;
	}

	return 0;
}

static int mtk_mfg_init_mbox(struct mtk_mfg *mfg)
{
	struct device *dev = &mfg->pdev->dev;
	struct mtk_mfg_mbox *gf;
	struct mtk_mfg_mbox *slp;

	gf = devm_kzalloc(dev, sizeof(*gf), GFP_KERNEL);
	if (!gf)
		return -ENOMEM;

	gf->rx_data = devm_kzalloc(dev, GPUEB_MBOX_MAX_RX_SIZE, GFP_KERNEL);
	if (!gf->rx_data)
		return -ENOMEM;

	gf->mfg = mfg;
	init_completion(&gf->rx_done);
	gf->cl.dev = dev;
	gf->cl.rx_callback = mtk_mfg_mbox_rx_callback;
	gf->cl.tx_tout = GPUEB_TIMEOUT_US / USEC_PER_MSEC;
	gf->ch = mbox_request_channel_byname(&gf->cl, "gpufreq");
	if (IS_ERR(gf->ch))
		return PTR_ERR(gf->ch);

	mfg->gf_mbox = gf;

	slp = devm_kzalloc(dev, sizeof(*slp), GFP_KERNEL);
	if (!slp)
		return -ENOMEM;

	slp->mfg = mfg;
	init_completion(&slp->rx_done);
	slp->cl.dev = dev;
	slp->cl.tx_tout = GPUEB_TIMEOUT_US / USEC_PER_MSEC;
	slp->cl.tx_block = true;
	slp->ch = mbox_request_channel_byname(&slp->cl, "sleep");
	if (IS_ERR(slp->ch)) {
		mbox_free_channel(gf->ch);
		return PTR_ERR(slp->ch);
	}

	mfg->slp_mbox = slp;

	return 0;
}

static int mtk_mfg_init_clk_provider(struct mtk_mfg *mfg)
{
	struct device *dev = &mfg->pdev->dev;
	struct clk_hw_onecell_data *clk_data;
	int ret;

	clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, 2), GFP_KERNEL);
	if (!clk_data)
		return -ENOMEM;

	clk_data->num = 2;

	mfg->clk_core_hw.init = &mtk_mfg_clk_gpu_init;
	mfg->clk_stack_hw.init = &mtk_mfg_clk_stack_init;

	ret = devm_clk_hw_register(dev, &mfg->clk_core_hw);
	if (ret)
		return dev_err_probe(dev, ret, "Couldn't register GPU core clock\n");

	ret = devm_clk_hw_register(dev, &mfg->clk_stack_hw);
	if (ret)
		return dev_err_probe(dev, ret, "Couldn't register GPU stack clock\n");

	clk_data->hws[0] = &mfg->clk_core_hw;
	clk_data->hws[1] = &mfg->clk_stack_hw;

	ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data);
	if (ret)
		return dev_err_probe(dev, ret, "Couldn't register clock provider\n");

	return 0;
}

static int mtk_mfg_probe(struct platform_device *pdev)
{
	struct mtk_mfg *mfg;
	struct device *dev = &pdev->dev;
	const struct mtk_mfg_variant *data = of_device_get_match_data(dev);
	struct resource res;
	int ret, i;

	mfg = devm_kzalloc(dev, sizeof(*mfg), GFP_KERNEL);
	if (!mfg)
		return -ENOMEM;

	mfg->pdev = pdev;
	mfg->variant = data;

	dev_set_drvdata(dev, mfg);

	mfg->gpr = devm_platform_ioremap_resource(pdev, 0);
	if (IS_ERR(mfg->gpr))
		return dev_err_probe(dev, PTR_ERR(mfg->gpr),
				     "Couldn't retrieve GPR MMIO registers\n");

	mfg->rpc = devm_platform_ioremap_resource(pdev, 1);
	if (IS_ERR(mfg->rpc))
		return dev_err_probe(dev, PTR_ERR(mfg->rpc),
				     "Couldn't retrieve RPC MMIO registers\n");

	mfg->clk_eb = devm_clk_get(dev, "eb");
	if (IS_ERR(mfg->clk_eb))
		return dev_err_probe(dev, PTR_ERR(mfg->clk_eb),
				     "Couldn't get 'eb' clock\n");

	mfg->gpu_clks = devm_kcalloc(dev, data->num_clks, sizeof(*mfg->gpu_clks),
				     GFP_KERNEL);
	if (!mfg->gpu_clks)
		return -ENOMEM;

	for (i = 0; i < data->num_clks; i++)
		mfg->gpu_clks[i].id = data->clk_names[i];

	ret = devm_clk_bulk_get(dev, data->num_clks, mfg->gpu_clks);
	if (ret)
		return dev_err_probe(dev, ret, "Couldn't get GPU clocks\n");

	mfg->gpu_regs = devm_kcalloc(dev, data->num_regulators,
				     sizeof(*mfg->gpu_regs), GFP_KERNEL);
	if (!mfg->gpu_regs)
		return -ENOMEM;

	for (i = 0; i < data->num_regulators; i++)
		mfg->gpu_regs[i].supply = data->regulator_names[i];

	ret = devm_regulator_bulk_get(dev, data->num_regulators, mfg->gpu_regs);
	if (ret)
		return dev_err_probe(dev, ret, "Couldn't get GPU regulators\n");

	ret = of_reserved_mem_region_to_resource(dev->of_node, 0, &res);
	if (ret)
		return dev_err_probe(dev, ret, "Couldn't get GPUEB shared memory\n");

	mfg->shared_mem = devm_ioremap(dev, res.start, resource_size(&res));
	if (!mfg->shared_mem)
		return dev_err_probe(dev, -ENOMEM, "Can't ioremap GPUEB shared memory\n");
	mfg->shared_mem_size = resource_size(&res);
	mfg->shared_mem_phys = res.start;

	if (data->init) {
		ret = data->init(mfg);
		if (ret)
			return dev_err_probe(dev, ret, "Variant init failed\n");
	}

	mfg->pd.name = dev_name(dev);
	mfg->pd.attach_dev = mtk_mfg_attach_dev;
	mfg->pd.detach_dev = mtk_mfg_detach_dev;
	mfg->pd.power_off = mtk_mfg_power_off;
	mfg->pd.power_on = mtk_mfg_power_on;
	mfg->pd.set_performance_state = mtk_mfg_set_performance;
	mfg->pd.flags = GENPD_FLAG_OPP_TABLE_FW;

	ret = pm_genpd_init(&mfg->pd, NULL, false);
	if (ret)
		return dev_err_probe(dev, ret, "Failed to initialise power domain\n");

	ret = mtk_mfg_init_mbox(mfg);
	if (ret) {
		dev_err_probe(dev, ret, "Couldn't initialise mailbox\n");
		goto err_remove_genpd;
	}

	ret = mtk_mfg_power_on(&mfg->pd);
	if (ret) {
		dev_err_probe(dev, ret, "Failed to power on MFG\n");
		goto err_free_mbox;
	}

	ret = mtk_mfg_init_shared_mem(mfg);
	if (ret) {
		dev_err_probe(dev, ret, "Couldn't initialize EB shared memory\n");
		goto err_power_off;
	}

	ret = mtk_mfg_read_opp_tables(mfg);
	if (ret) {
		dev_err_probe(dev, ret, "Error reading OPP tables from EB\n");
		goto err_power_off;
	}

	ret = mtk_mfg_init_clk_provider(mfg);
	if (ret)
		goto err_power_off;

	ret = of_genpd_add_provider_simple(dev->of_node, &mfg->pd);
	if (ret) {
		dev_err_probe(dev, ret, "Failed to add pmdomain provider\n");
		goto err_power_off;
	}

	return 0;

err_power_off:
	mtk_mfg_power_off(&mfg->pd);
err_free_mbox:
	mbox_free_channel(mfg->slp_mbox->ch);
	mfg->slp_mbox->ch = NULL;
	mbox_free_channel(mfg->gf_mbox->ch);
	mfg->gf_mbox->ch = NULL;
err_remove_genpd:
	pm_genpd_remove(&mfg->pd);

	return ret;
}

static const struct of_device_id mtk_mfg_of_match[] = {
	{ .compatible = "mediatek,mt8196-gpufreq", .data = &mtk_mfg_mt8196_variant },
	{}
};
MODULE_DEVICE_TABLE(of, mtk_mfg_of_match);

static void mtk_mfg_remove(struct platform_device *pdev)
{
	struct mtk_mfg *mfg = dev_get_drvdata(&pdev->dev);

	if (mtk_mfg_is_powered_on(mfg))
		mtk_mfg_power_off(&mfg->pd);

	of_genpd_del_provider(pdev->dev.of_node);
	pm_genpd_remove(&mfg->pd);

	mbox_free_channel(mfg->gf_mbox->ch);
	mfg->gf_mbox->ch = NULL;

	mbox_free_channel(mfg->slp_mbox->ch);
	mfg->slp_mbox->ch = NULL;
}

static struct platform_driver mtk_mfg_driver = {
	.driver = {
		.name = "mtk-mfg-pmdomain",
		.of_match_table = mtk_mfg_of_match,
		.suppress_bind_attrs = true,
	},
	.probe = mtk_mfg_probe,
	.remove = mtk_mfg_remove,
};
module_platform_driver(mtk_mfg_driver);

MODULE_AUTHOR("Nicolas Frattaroli <nicolas.frattaroli@collabora.com>");
MODULE_DESCRIPTION("MediaTek MFlexGraphics Power Domain Driver");
MODULE_LICENSE("GPL");