1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Driver for MediaTek MFlexGraphics Devices
4 *
5 * Copyright (C) 2025, Collabora Ltd.
6 */
7
8 #include <linux/completion.h>
9 #include <linux/clk.h>
10 #include <linux/clk-provider.h>
11 #include <linux/container_of.h>
12 #include <linux/iopoll.h>
13 #include <linux/nvmem-provider.h>
14 #include <linux/mailbox_client.h>
15 #include <linux/module.h>
16 #include <linux/of.h>
17 #include <linux/of_address.h>
18 #include <linux/of_platform.h>
19 #include <linux/of_reserved_mem.h>
20 #include <linux/overflow.h>
21 #include <linux/platform_device.h>
22 #include <linux/pm_domain.h>
23 #include <linux/pm_opp.h>
24 #include <linux/regulator/consumer.h>
25 #include <linux/units.h>
26
27 #define GPR_LP_STATE 0x0028
28 #define EB_ON_SUSPEND 0x0
29 #define EB_ON_RESUME 0x1
30 #define GPR_IPI_MAGIC 0x34
31
32 #define RPC_PWR_CON 0x0504
33 #define PWR_ACK_M GENMASK(31, 30)
34 #define RPC_DUMMY_REG_2 0x0658
35 #define RPC_GHPM_CFG0_CON 0x0800
36 #define GHPM_ENABLE_M BIT(0)
37 #define GHPM_ON_SEQ_M BIT(2)
38 #define RPC_GHPM_RO0_CON 0x09A4
39 #define GHPM_STATE_M GENMASK(7, 0)
40 #define GHPM_PWR_STATE_M BIT(16)
41
42 #define GF_REG_MAGIC 0x0000
43 #define GF_REG_GPU_OPP_IDX 0x0004
44 #define GF_REG_STK_OPP_IDX 0x0008
45 #define GF_REG_GPU_OPP_NUM 0x000c
46 #define GF_REG_STK_OPP_NUM 0x0010
47 #define GF_REG_GPU_OPP_SNUM 0x0014
48 #define GF_REG_STK_OPP_SNUM 0x0018
49 #define GF_REG_POWER_COUNT 0x001c
50 #define GF_REG_BUCK_COUNT 0x0020
51 #define GF_REG_MTCMOS_COUNT 0x0024
52 #define GF_REG_CG_COUNT 0x0028 /* CG = Clock Gate? */
53 #define GF_REG_ACTIVE_COUNT 0x002C
54 #define GF_REG_TEMP_RAW 0x0030
55 #define GF_REG_TEMP_NORM_GPU 0x0034
56 #define GF_REG_TEMP_HIGH_GPU 0x0038
57 #define GF_REG_TEMP_NORM_STK 0x003C
58 #define GF_REG_TEMP_HIGH_STK 0x0040
59 #define GF_REG_FREQ_CUR_GPU 0x0044
60 #define GF_REG_FREQ_CUR_STK 0x0048
61 #define GF_REG_FREQ_OUT_GPU 0x004C /* Guess: actual achieved freq */
62 #define GF_REG_FREQ_OUT_STK 0x0050 /* Guess: actual achieved freq */
63 #define GF_REG_FREQ_METER_GPU 0x0054 /* Seems unused, always 0 */
64 #define GF_REG_FREQ_METER_STK 0x0058 /* Seems unused, always 0 */
65 #define GF_REG_VOLT_CUR_GPU 0x005C /* in tens of microvolts */
66 #define GF_REG_VOLT_CUR_STK 0x0060 /* in tens of microvolts */
67 #define GF_REG_VOLT_CUR_GPU_SRAM 0x0064
68 #define GF_REG_VOLT_CUR_STK_SRAM 0x0068
69 #define GF_REG_VOLT_CUR_GPU_REG 0x006C /* Seems unused, always 0 */
70 #define GF_REG_VOLT_CUR_STK_REG 0x0070 /* Seems unused, always 0 */
71 #define GF_REG_VOLT_CUR_GPU_REG_SRAM 0x0074
72 #define GF_REG_VOLT_CUR_STK_REG_SRAM 0x0078
73 #define GF_REG_PWR_CUR_GPU 0x007C /* in milliwatts */
74 #define GF_REG_PWR_CUR_STK 0x0080 /* in milliwatts */
75 #define GF_REG_PWR_MAX_GPU 0x0084 /* in milliwatts */
76 #define GF_REG_PWR_MAX_STK 0x0088 /* in milliwatts */
77 #define GF_REG_PWR_MIN_GPU 0x008C /* in milliwatts */
78 #define GF_REG_PWR_MIN_STK 0x0090 /* in milliwatts */
79 #define GF_REG_LEAKAGE_RT_GPU 0x0094 /* Unknown */
80 #define GF_REG_LEAKAGE_RT_STK 0x0098 /* Unknown */
81 #define GF_REG_LEAKAGE_RT_SRAM 0x009C /* Unknown */
82 #define GF_REG_LEAKAGE_HT_GPU 0x00A0 /* Unknown */
83 #define GF_REG_LEAKAGE_HT_STK 0x00A4 /* Unknown */
84 #define GF_REG_LEAKAGE_HT_SRAM 0x00A8 /* Unknown */
85 #define GF_REG_VOLT_DAC_LOW_GPU 0x00AC /* Seems unused, always 0 */
86 #define GF_REG_VOLT_DAC_LOW_STK 0x00B0 /* Seems unused, always 0 */
87 #define GF_REG_OPP_CUR_CEIL 0x00B4
88 #define GF_REG_OPP_CUR_FLOOR 0x00B8
89 #define GF_REG_OPP_CUR_LIMITER_CEIL 0x00BC
90 #define GF_REG_OPP_CUR_LIMITER_FLOOR 0x00C0
91 #define GF_REG_OPP_PRIORITY_CEIL 0x00C4
92 #define GF_REG_OPP_PRIORITY_FLOOR 0x00C8
93 #define GF_REG_PWR_CTL 0x00CC
94 #define GF_REG_ACTIVE_SLEEP_CTL 0x00D0
95 #define GF_REG_DVFS_STATE 0x00D4
96 #define GF_REG_SHADER_PRESENT 0x00D8
97 #define GF_REG_ASENSOR_ENABLE 0x00DC
98 #define GF_REG_AGING_LOAD 0x00E0
99 #define GF_REG_AGING_MARGIN 0x00E4
100 #define GF_REG_AVS_ENABLE 0x00E8
101 #define GF_REG_AVS_MARGIN 0x00EC
102 #define GF_REG_CHIP_TYPE 0x00F0
103 #define GF_REG_SB_VERSION 0x00F4
104 #define GF_REG_PTP_VERSION 0x00F8
105 #define GF_REG_DBG_VERSION 0x00FC
106 #define GF_REG_KDBG_VERSION 0x0100
107 #define GF_REG_GPM1_MODE 0x0104
108 #define GF_REG_GPM3_MODE 0x0108
109 #define GF_REG_DFD_MODE 0x010C
110 #define GF_REG_DUAL_BUCK 0x0110
111 #define GF_REG_SEGMENT_ID 0x0114
112 #define GF_REG_POWER_TIME_H 0x0118
113 #define GF_REG_POWER_TIME_L 0x011C
114 #define GF_REG_PWR_STATUS 0x0120
115 #define GF_REG_STRESS_TEST 0x0124
116 #define GF_REG_TEST_MODE 0x0128
117 #define GF_REG_IPS_MODE 0x012C
118 #define GF_REG_TEMP_COMP_MODE 0x0130
119 #define GF_REG_HT_TEMP_COMP_MODE 0x0134
120 #define GF_REG_PWR_TRACKER_MODE 0x0138
121 #define GF_REG_OPP_TABLE_GPU 0x0314
122 #define GF_REG_OPP_TABLE_STK 0x09A4
123 #define GF_REG_OPP_TABLE_GPU_S 0x1034
124 #define GF_REG_OPP_TABLE_STK_S 0x16c4
125 #define GF_REG_LIMIT_TABLE 0x1d54
126 #define GF_REG_GPM3_TABLE 0x223C
127
128 #define MFG_MT8196_E2_ID 0x101
129 #define GPUEB_SLEEP_MAGIC 0x55667788UL
130 #define GPUEB_MEM_MAGIC 0xBABADADAUL
131
132 #define GPUEB_TIMEOUT_US 10000UL
133 #define GPUEB_POLL_US 50
134
135 #define MAX_OPP_NUM 70
136
137 #define GPUEB_MBOX_MAX_RX_SIZE 32 /* in bytes */
138
139 /*
140 * This enum is part of the ABI of the GPUEB firmware. Don't change the
141 * numbering, as you would wreak havoc.
142 */
143 enum mtk_mfg_ipi_cmd {
144 CMD_INIT_SHARED_MEM = 0,
145 CMD_GET_FREQ_BY_IDX = 1,
146 CMD_GET_POWER_BY_IDX = 2,
147 CMD_GET_OPPIDX_BY_FREQ = 3,
148 CMD_GET_LEAKAGE_POWER = 4,
149 CMD_SET_LIMIT = 5,
150 CMD_POWER_CONTROL = 6,
151 CMD_ACTIVE_SLEEP_CONTROL = 7,
152 CMD_COMMIT = 8,
153 CMD_DUAL_COMMIT = 9,
154 CMD_PDCA_CONFIG = 10,
155 CMD_UPDATE_DEBUG_OPP_INFO = 11,
156 CMD_SWITCH_LIMIT = 12,
157 CMD_FIX_TARGET_OPPIDX = 13,
158 CMD_FIX_DUAL_TARGET_OPPIDX = 14,
159 CMD_FIX_CUSTOM_FREQ_VOLT = 15,
160 CMD_FIX_DUAL_CUSTOM_FREQ_VOLT = 16,
161 CMD_SET_MFGSYS_CONFIG = 17,
162 CMD_MSSV_COMMIT = 18,
163 CMD_NUM = 19,
164 };
165
166 /*
167 * This struct is part of the ABI of the GPUEB firmware. Changing it, or
168 * reordering fields in it, will break things, so don't do it. Thank you.
169 */
170 struct __packed mtk_mfg_ipi_msg {
171 __le32 magic;
172 __le32 cmd;
173 __le32 target;
174 /*
175 * Downstream relies on the compiler to implicitly add the following
176 * padding, as it declares the struct as non-packed.
177 */
178 __le32 reserved;
179 union {
180 s32 __bitwise oppidx;
181 s32 __bitwise return_value;
182 __le32 freq;
183 __le32 volt;
184 __le32 power;
185 __le32 power_state;
186 __le32 mode;
187 __le32 value;
188 struct {
189 __le64 base;
190 __le32 size;
191 } shared_mem;
192 struct {
193 __le32 freq;
194 __le32 volt;
195 } custom;
196 struct {
197 __le32 limiter;
198 s32 __bitwise ceiling_info;
199 s32 __bitwise floor_info;
200 } set_limit;
201 struct {
202 __le32 target;
203 __le32 val;
204 } mfg_cfg;
205 struct {
206 __le32 target;
207 __le32 val;
208 } mssv;
209 struct {
210 s32 __bitwise gpu_oppidx;
211 s32 __bitwise stack_oppidx;
212 } dual_commit;
213 struct {
214 __le32 fgpu;
215 __le32 vgpu;
216 __le32 fstack;
217 __le32 vstack;
218 } dual_custom;
219 } u;
220 };
221
222 struct __packed mtk_mfg_ipi_sleep_msg {
223 __le32 event;
224 __le32 state;
225 __le32 magic;
226 };
227
228 /**
229 * struct mtk_mfg_opp_entry - OPP table entry from firmware
230 * @freq_khz: The operating point's frequency in kilohertz
231 * @voltage_core: The operating point's core voltage in tens of microvolts
232 * @voltage_sram: The operating point's SRAM voltage in tens of microvolts
233 * @posdiv: exponent of base 2 for PLL frequency divisor used for this OPP
234 * @voltage_margin: Number of tens of microvolts the voltage can be undershot
235 * @power_mw: estimate of power usage at this operating point, in milliwatts
236 *
237 * This struct is part of the ABI with the EB firmware. Do not change it.
238 */
239 struct mtk_mfg_opp_entry {
240 __le32 freq_khz;
241 __le32 voltage_core;
242 __le32 voltage_sram;
243 __le32 posdiv;
244 __le32 voltage_margin;
245 __le32 power_mw;
246 } __packed;
247
248 struct mtk_mfg_mbox {
249 struct mbox_client cl;
250 struct completion rx_done;
251 struct mtk_mfg *mfg;
252 struct mbox_chan *ch;
253 void *rx_data;
254 };
255
256 struct mtk_mfg {
257 struct generic_pm_domain pd;
258 struct platform_device *pdev;
259 struct clk *clk_eb;
260 struct clk_bulk_data *gpu_clks;
261 struct clk_hw clk_core_hw;
262 struct clk_hw clk_stack_hw;
263 struct regulator_bulk_data *gpu_regs;
264 void __iomem *rpc;
265 void __iomem *gpr;
266 void __iomem *shared_mem;
267 phys_addr_t shared_mem_phys;
268 unsigned int shared_mem_size;
269 u16 ghpm_en_reg;
270 u32 ipi_magic;
271 unsigned short num_gpu_opps;
272 unsigned short num_stack_opps;
273 struct dev_pm_opp_data *gpu_opps;
274 struct dev_pm_opp_data *stack_opps;
275 struct mtk_mfg_mbox *gf_mbox;
276 struct mtk_mfg_mbox *slp_mbox;
277 const struct mtk_mfg_variant *variant;
278 };
279
280 struct mtk_mfg_variant {
281 const char *const *clk_names;
282 unsigned int num_clks;
283 const char *const *regulator_names;
284 unsigned int num_regulators;
285 /** @turbo_below: opp indices below this value are considered turbo */
286 unsigned int turbo_below;
287 int (*init)(struct mtk_mfg *mfg);
288 };
289
mtk_mfg_from_genpd(struct generic_pm_domain * pd)290 static inline struct mtk_mfg *mtk_mfg_from_genpd(struct generic_pm_domain *pd)
291 {
292 return container_of(pd, struct mtk_mfg, pd);
293 }
294
mtk_mfg_update_reg_bits(void __iomem * addr,u32 mask,u32 val)295 static inline void mtk_mfg_update_reg_bits(void __iomem *addr, u32 mask, u32 val)
296 {
297 writel((readl(addr) & ~mask) | (val & mask), addr);
298 }
299
mtk_mfg_is_powered_on(struct mtk_mfg * mfg)300 static inline bool mtk_mfg_is_powered_on(struct mtk_mfg *mfg)
301 {
302 return (readl(mfg->rpc + RPC_PWR_CON) & PWR_ACK_M) == PWR_ACK_M;
303 }
304
mtk_mfg_recalc_rate_gpu(struct clk_hw * hw,unsigned long parent_rate)305 static unsigned long mtk_mfg_recalc_rate_gpu(struct clk_hw *hw,
306 unsigned long parent_rate)
307 {
308 struct mtk_mfg *mfg = container_of(hw, struct mtk_mfg, clk_core_hw);
309
310 return readl(mfg->shared_mem + GF_REG_FREQ_OUT_GPU) * HZ_PER_KHZ;
311 }
312
mtk_mfg_recalc_rate_stack(struct clk_hw * hw,unsigned long parent_rate)313 static unsigned long mtk_mfg_recalc_rate_stack(struct clk_hw *hw,
314 unsigned long parent_rate)
315 {
316 struct mtk_mfg *mfg = container_of(hw, struct mtk_mfg, clk_stack_hw);
317
318 return readl(mfg->shared_mem + GF_REG_FREQ_OUT_STK) * HZ_PER_KHZ;
319 }
320
321 static const struct clk_ops mtk_mfg_clk_gpu_ops = {
322 .recalc_rate = mtk_mfg_recalc_rate_gpu,
323 .determine_rate = clk_determine_rate_noop,
324 };
325
326 static const struct clk_ops mtk_mfg_clk_stack_ops = {
327 .recalc_rate = mtk_mfg_recalc_rate_stack,
328 .determine_rate = clk_determine_rate_noop,
329 };
330
331 static const struct clk_init_data mtk_mfg_clk_gpu_init = {
332 .name = "gpu-core",
333 .ops = &mtk_mfg_clk_gpu_ops,
334 .flags = CLK_GET_RATE_NOCACHE,
335 };
336
337 static const struct clk_init_data mtk_mfg_clk_stack_init = {
338 .name = "gpu-stack",
339 .ops = &mtk_mfg_clk_stack_ops,
340 .flags = CLK_GET_RATE_NOCACHE,
341 };
342
mtk_mfg_eb_on(struct mtk_mfg * mfg)343 static int mtk_mfg_eb_on(struct mtk_mfg *mfg)
344 {
345 struct device *dev = &mfg->pdev->dev;
346 u32 val;
347 int ret;
348
349 /*
350 * If MFG is already on from e.g. the bootloader, skip doing the
351 * power-on sequence, as it wouldn't work without powering it off first.
352 */
353 if (mtk_mfg_is_powered_on(mfg))
354 return 0;
355
356 ret = readl_poll_timeout(mfg->rpc + RPC_GHPM_RO0_CON, val,
357 !(val & (GHPM_PWR_STATE_M | GHPM_STATE_M)),
358 GPUEB_POLL_US, GPUEB_TIMEOUT_US);
359 if (ret) {
360 dev_err(dev, "timed out waiting for EB to power on\n");
361 return ret;
362 }
363
364 mtk_mfg_update_reg_bits(mfg->rpc + mfg->ghpm_en_reg, GHPM_ENABLE_M,
365 GHPM_ENABLE_M);
366
367 mtk_mfg_update_reg_bits(mfg->rpc + RPC_GHPM_CFG0_CON, GHPM_ON_SEQ_M, 0);
368 mtk_mfg_update_reg_bits(mfg->rpc + RPC_GHPM_CFG0_CON, GHPM_ON_SEQ_M,
369 GHPM_ON_SEQ_M);
370
371 mtk_mfg_update_reg_bits(mfg->rpc + mfg->ghpm_en_reg, GHPM_ENABLE_M, 0);
372
373
374 ret = readl_poll_timeout(mfg->rpc + RPC_PWR_CON, val,
375 (val & PWR_ACK_M) == PWR_ACK_M,
376 GPUEB_POLL_US, GPUEB_TIMEOUT_US);
377 if (ret) {
378 dev_err(dev, "timed out waiting for EB power ack, val = 0x%X\n",
379 val);
380 return ret;
381 }
382
383 ret = readl_poll_timeout(mfg->gpr + GPR_LP_STATE, val,
384 (val == EB_ON_RESUME),
385 GPUEB_POLL_US, GPUEB_TIMEOUT_US);
386 if (ret) {
387 dev_err(dev, "timed out waiting for EB to resume, status = 0x%X\n", val);
388 return ret;
389 }
390
391 return 0;
392 }
393
mtk_mfg_eb_off(struct mtk_mfg * mfg)394 static int mtk_mfg_eb_off(struct mtk_mfg *mfg)
395 {
396 struct device *dev = &mfg->pdev->dev;
397 struct mtk_mfg_ipi_sleep_msg msg = {
398 .event = 0,
399 .state = 0,
400 .magic = GPUEB_SLEEP_MAGIC
401 };
402 u32 val;
403 int ret;
404
405 ret = mbox_send_message(mfg->slp_mbox->ch, &msg);
406 if (ret < 0) {
407 dev_err(dev, "Cannot send sleep command: %pe\n", ERR_PTR(ret));
408 return ret;
409 }
410
411 ret = readl_poll_timeout(mfg->rpc + RPC_PWR_CON, val,
412 !(val & PWR_ACK_M), GPUEB_POLL_US,
413 GPUEB_TIMEOUT_US);
414
415 if (ret) {
416 dev_err(dev, "Timed out waiting for EB to power off, val=0x%08X\n", val);
417 return ret;
418 }
419
420 return 0;
421 }
422
423 /**
424 * mtk_mfg_send_ipi - synchronously send an IPI message on the gpufreq channel
425 * @mfg: pointer to this driver instance's private &struct mtk_mfg
426 * @msg: pointer to a message to send; will have magic filled and response assigned
427 *
428 * Send an IPI message on the gpufreq channel, and wait for a response. Once a
429 * response is received, assign a pointer to the response buffer (valid until
430 * next response is received) to @msg.
431 *
432 * Returns 0 on success, negative errno on failure.
433 */
mtk_mfg_send_ipi(struct mtk_mfg * mfg,struct mtk_mfg_ipi_msg * msg)434 static int mtk_mfg_send_ipi(struct mtk_mfg *mfg, struct mtk_mfg_ipi_msg *msg)
435 {
436 struct device *dev = &mfg->pdev->dev;
437 unsigned long wait;
438 int ret;
439
440 msg->magic = mfg->ipi_magic;
441
442 ret = mbox_send_message(mfg->gf_mbox->ch, msg);
443 if (ret < 0) {
444 dev_err(dev, "Cannot send GPUFreq IPI command: %pe\n", ERR_PTR(ret));
445 return ret;
446 }
447
448 wait = wait_for_completion_timeout(&mfg->gf_mbox->rx_done, msecs_to_jiffies(500));
449 if (!wait)
450 return -ETIMEDOUT;
451
452 msg = mfg->gf_mbox->rx_data;
453
454 if (msg->u.return_value < 0) {
455 dev_err(dev, "IPI return: %d\n", msg->u.return_value);
456 return -EPROTO;
457 }
458
459 return 0;
460 }
461
mtk_mfg_init_shared_mem(struct mtk_mfg * mfg)462 static int mtk_mfg_init_shared_mem(struct mtk_mfg *mfg)
463 {
464 struct device *dev = &mfg->pdev->dev;
465 struct mtk_mfg_ipi_msg msg = {};
466 int ret;
467
468 dev_dbg(dev, "clearing GPUEB shared memory, 0x%X bytes\n", mfg->shared_mem_size);
469 memset_io(mfg->shared_mem, 0, mfg->shared_mem_size);
470
471 msg.cmd = CMD_INIT_SHARED_MEM;
472 msg.u.shared_mem.base = mfg->shared_mem_phys;
473 msg.u.shared_mem.size = mfg->shared_mem_size;
474
475 ret = mtk_mfg_send_ipi(mfg, &msg);
476 if (ret)
477 return ret;
478
479 if (readl(mfg->shared_mem + GF_REG_MAGIC) != GPUEB_MEM_MAGIC) {
480 dev_err(dev, "EB did not initialise shared memory correctly\n");
481 return -EIO;
482 }
483
484 return 0;
485 }
486
mtk_mfg_power_control(struct mtk_mfg * mfg,bool enabled)487 static int mtk_mfg_power_control(struct mtk_mfg *mfg, bool enabled)
488 {
489 struct mtk_mfg_ipi_msg msg = {};
490
491 msg.cmd = CMD_POWER_CONTROL;
492 msg.u.power_state = enabled ? 1 : 0;
493
494 return mtk_mfg_send_ipi(mfg, &msg);
495 }
496
mtk_mfg_set_oppidx(struct mtk_mfg * mfg,unsigned int opp_idx)497 static int mtk_mfg_set_oppidx(struct mtk_mfg *mfg, unsigned int opp_idx)
498 {
499 struct mtk_mfg_ipi_msg msg = {};
500 int ret;
501
502 if (opp_idx >= mfg->num_gpu_opps)
503 return -EINVAL;
504
505 msg.cmd = CMD_FIX_DUAL_TARGET_OPPIDX;
506 msg.u.dual_commit.gpu_oppidx = opp_idx;
507 msg.u.dual_commit.stack_oppidx = opp_idx;
508
509 ret = mtk_mfg_send_ipi(mfg, &msg);
510 if (ret) {
511 dev_err(&mfg->pdev->dev, "Failed to set OPP %u: %pe\n",
512 opp_idx, ERR_PTR(ret));
513 return ret;
514 }
515
516 return 0;
517 }
518
mtk_mfg_read_opp_tables(struct mtk_mfg * mfg)519 static int mtk_mfg_read_opp_tables(struct mtk_mfg *mfg)
520 {
521 struct device *dev = &mfg->pdev->dev;
522 struct mtk_mfg_opp_entry e = {};
523 unsigned int i;
524
525 mfg->num_gpu_opps = readl(mfg->shared_mem + GF_REG_GPU_OPP_NUM);
526 mfg->num_stack_opps = readl(mfg->shared_mem + GF_REG_STK_OPP_NUM);
527
528 if (mfg->num_gpu_opps > MAX_OPP_NUM || mfg->num_gpu_opps == 0) {
529 dev_err(dev, "GPU OPP count (%u) out of range %u >= count > 0\n",
530 mfg->num_gpu_opps, MAX_OPP_NUM);
531 return -EINVAL;
532 }
533
534 if (mfg->num_stack_opps && mfg->num_stack_opps > MAX_OPP_NUM) {
535 dev_err(dev, "Stack OPP count (%u) out of range %u >= count >= 0\n",
536 mfg->num_stack_opps, MAX_OPP_NUM);
537 return -EINVAL;
538 }
539
540 mfg->gpu_opps = devm_kcalloc(dev, mfg->num_gpu_opps,
541 sizeof(struct dev_pm_opp_data), GFP_KERNEL);
542 if (!mfg->gpu_opps)
543 return -ENOMEM;
544
545 if (mfg->num_stack_opps) {
546 mfg->stack_opps = devm_kcalloc(dev, mfg->num_stack_opps,
547 sizeof(struct dev_pm_opp_data), GFP_KERNEL);
548 if (!mfg->stack_opps)
549 return -ENOMEM;
550 }
551
552 for (i = 0; i < mfg->num_gpu_opps; i++) {
553 memcpy_fromio(&e, mfg->shared_mem + GF_REG_OPP_TABLE_GPU + i * sizeof(e),
554 sizeof(e));
555 if (mem_is_zero(&e, sizeof(e))) {
556 dev_err(dev, "ran into an empty GPU OPP at index %u\n",
557 i);
558 return -EINVAL;
559 }
560 mfg->gpu_opps[i].freq = e.freq_khz * HZ_PER_KHZ;
561 mfg->gpu_opps[i].u_volt = e.voltage_core * 10;
562 mfg->gpu_opps[i].level = i;
563 if (i < mfg->variant->turbo_below)
564 mfg->gpu_opps[i].turbo = true;
565 }
566
567 for (i = 0; i < mfg->num_stack_opps; i++) {
568 memcpy_fromio(&e, mfg->shared_mem + GF_REG_OPP_TABLE_STK + i * sizeof(e),
569 sizeof(e));
570 if (mem_is_zero(&e, sizeof(e))) {
571 dev_err(dev, "ran into an empty Stack OPP at index %u\n",
572 i);
573 return -EINVAL;
574 }
575 mfg->stack_opps[i].freq = e.freq_khz * HZ_PER_KHZ;
576 mfg->stack_opps[i].u_volt = e.voltage_core * 10;
577 mfg->stack_opps[i].level = i;
578 if (i < mfg->variant->turbo_below)
579 mfg->stack_opps[i].turbo = true;
580 }
581
582 return 0;
583 }
584
585 static const char *const mtk_mfg_mt8196_clk_names[] = {
586 "core",
587 "stack0",
588 "stack1",
589 };
590
591 static const char *const mtk_mfg_mt8196_regulators[] = {
592 "core",
593 "stack",
594 "sram",
595 };
596
mtk_mfg_mt8196_init(struct mtk_mfg * mfg)597 static int mtk_mfg_mt8196_init(struct mtk_mfg *mfg)
598 {
599 void __iomem *e2_base;
600
601 e2_base = devm_platform_ioremap_resource_byname(mfg->pdev, "hw-revision");
602 if (IS_ERR(e2_base))
603 return dev_err_probe(&mfg->pdev->dev, PTR_ERR(e2_base),
604 "Couldn't get hw-revision register\n");
605
606 clk_prepare_enable(mfg->clk_eb);
607
608 if (readl(e2_base) == MFG_MT8196_E2_ID)
609 mfg->ghpm_en_reg = RPC_DUMMY_REG_2;
610 else
611 mfg->ghpm_en_reg = RPC_GHPM_CFG0_CON;
612
613 clk_disable_unprepare(mfg->clk_eb);
614
615 return 0;
616 }
617
618 static const struct mtk_mfg_variant mtk_mfg_mt8196_variant = {
619 .clk_names = mtk_mfg_mt8196_clk_names,
620 .num_clks = ARRAY_SIZE(mtk_mfg_mt8196_clk_names),
621 .regulator_names = mtk_mfg_mt8196_regulators,
622 .num_regulators = ARRAY_SIZE(mtk_mfg_mt8196_regulators),
623 .turbo_below = 7,
624 .init = mtk_mfg_mt8196_init,
625 };
626
mtk_mfg_mbox_rx_callback(struct mbox_client * cl,void * mssg)627 static void mtk_mfg_mbox_rx_callback(struct mbox_client *cl, void *mssg)
628 {
629 struct mtk_mfg_mbox *mb = container_of(cl, struct mtk_mfg_mbox, cl);
630
631 if (mb->rx_data)
632 mb->rx_data = memcpy(mb->rx_data, mssg, GPUEB_MBOX_MAX_RX_SIZE);
633 complete(&mb->rx_done);
634 }
635
mtk_mfg_attach_dev(struct generic_pm_domain * pd,struct device * dev)636 static int mtk_mfg_attach_dev(struct generic_pm_domain *pd, struct device *dev)
637 {
638 struct mtk_mfg *mfg = mtk_mfg_from_genpd(pd);
639 struct dev_pm_opp_data *so = mfg->stack_opps;
640 struct dev_pm_opp_data *go = mfg->gpu_opps;
641 struct dev_pm_opp_data *prev_o = NULL;
642 struct dev_pm_opp_data *o;
643 int i, ret;
644
645 for (i = mfg->num_gpu_opps - 1; i >= 0; i--) {
646 /*
647 * Adding the lower of the two OPPs avoids gaps of indices in
648 * situations where the GPU OPPs are duplicated a couple of
649 * times when only the Stack OPP is being lowered at that index.
650 */
651 if (i >= mfg->num_stack_opps || go[i].freq < so[i].freq)
652 o = &go[i];
653 else
654 o = &so[i];
655
656 /*
657 * Skip indices where both GPU and Stack OPPs are equal. Nominally,
658 * OPP core shouldn't care about dupes, but not doing so will cause
659 * dev_pm_opp_find_freq_ceil_indexed to -ERANGE later down the line.
660 */
661 if (prev_o && prev_o->freq == o->freq)
662 continue;
663
664 ret = dev_pm_opp_add_dynamic(dev, o);
665 if (ret) {
666 dev_err(dev, "Failed to add OPP level %u from PD %s: %pe\n",
667 o->level, pd->name, ERR_PTR(ret));
668 dev_pm_opp_remove_all_dynamic(dev);
669 return ret;
670 }
671 prev_o = o;
672 }
673
674 return 0;
675 }
676
mtk_mfg_detach_dev(struct generic_pm_domain * pd,struct device * dev)677 static void mtk_mfg_detach_dev(struct generic_pm_domain *pd, struct device *dev)
678 {
679 dev_pm_opp_remove_all_dynamic(dev);
680 }
681
mtk_mfg_set_performance(struct generic_pm_domain * pd,unsigned int state)682 static int mtk_mfg_set_performance(struct generic_pm_domain *pd,
683 unsigned int state)
684 {
685 struct mtk_mfg *mfg = mtk_mfg_from_genpd(pd);
686
687 /*
688 * pmdomain core intentionally sets a performance state before turning
689 * a domain on, and after turning it off. For the GPUEB however, it's
690 * only possible to act on performance requests when the GPUEB is
691 * powered on. To do this, return cleanly without taking action, and
692 * defer setting what pmdomain core set in mtk_mfg_power_on.
693 */
694 if (mfg->pd.status != GENPD_STATE_ON)
695 return 0;
696
697 return mtk_mfg_set_oppidx(mfg, state);
698 }
699
mtk_mfg_power_on(struct generic_pm_domain * pd)700 static int mtk_mfg_power_on(struct generic_pm_domain *pd)
701 {
702 struct mtk_mfg *mfg = mtk_mfg_from_genpd(pd);
703 int ret;
704
705 ret = regulator_bulk_enable(mfg->variant->num_regulators,
706 mfg->gpu_regs);
707 if (ret)
708 return ret;
709
710 ret = clk_prepare_enable(mfg->clk_eb);
711 if (ret)
712 goto err_disable_regulators;
713
714 ret = clk_bulk_prepare_enable(mfg->variant->num_clks, mfg->gpu_clks);
715 if (ret)
716 goto err_disable_eb_clk;
717
718 ret = mtk_mfg_eb_on(mfg);
719 if (ret)
720 goto err_disable_clks;
721
722 mfg->ipi_magic = readl(mfg->gpr + GPR_IPI_MAGIC);
723
724 ret = mtk_mfg_power_control(mfg, true);
725 if (ret)
726 goto err_eb_off;
727
728 /* Don't try to set a OPP in probe before OPPs have been read from EB */
729 if (mfg->gpu_opps) {
730 /* The aforementioned deferred setting of pmdomain's state */
731 ret = mtk_mfg_set_oppidx(mfg, pd->performance_state);
732 if (ret)
733 dev_warn(&mfg->pdev->dev, "Failed to set oppidx in %s\n", __func__);
734 }
735
736 return 0;
737
738 err_eb_off:
739 mtk_mfg_eb_off(mfg);
740 err_disable_clks:
741 clk_bulk_disable_unprepare(mfg->variant->num_clks, mfg->gpu_clks);
742 err_disable_eb_clk:
743 clk_disable_unprepare(mfg->clk_eb);
744 err_disable_regulators:
745 regulator_bulk_disable(mfg->variant->num_regulators, mfg->gpu_regs);
746
747 return ret;
748 }
749
mtk_mfg_power_off(struct generic_pm_domain * pd)750 static int mtk_mfg_power_off(struct generic_pm_domain *pd)
751 {
752 struct mtk_mfg *mfg = mtk_mfg_from_genpd(pd);
753 struct device *dev = &mfg->pdev->dev;
754 int ret;
755
756 ret = mtk_mfg_power_control(mfg, false);
757 if (ret) {
758 dev_err(dev, "power_control failed: %pe\n", ERR_PTR(ret));
759 return ret;
760 }
761
762 ret = mtk_mfg_eb_off(mfg);
763 if (ret) {
764 dev_err(dev, "eb_off failed: %pe\n", ERR_PTR(ret));
765 return ret;
766 }
767
768 clk_bulk_disable_unprepare(mfg->variant->num_clks, mfg->gpu_clks);
769 clk_disable_unprepare(mfg->clk_eb);
770 ret = regulator_bulk_disable(mfg->variant->num_regulators, mfg->gpu_regs);
771 if (ret) {
772 dev_err(dev, "Disabling regulators failed: %pe\n", ERR_PTR(ret));
773 return ret;
774 }
775
776 return 0;
777 }
778
mtk_mfg_init_mbox(struct mtk_mfg * mfg)779 static int mtk_mfg_init_mbox(struct mtk_mfg *mfg)
780 {
781 struct device *dev = &mfg->pdev->dev;
782 struct mtk_mfg_mbox *gf;
783 struct mtk_mfg_mbox *slp;
784
785 gf = devm_kzalloc(dev, sizeof(*gf), GFP_KERNEL);
786 if (!gf)
787 return -ENOMEM;
788
789 gf->rx_data = devm_kzalloc(dev, GPUEB_MBOX_MAX_RX_SIZE, GFP_KERNEL);
790 if (!gf->rx_data)
791 return -ENOMEM;
792
793 gf->mfg = mfg;
794 init_completion(&gf->rx_done);
795 gf->cl.dev = dev;
796 gf->cl.rx_callback = mtk_mfg_mbox_rx_callback;
797 gf->cl.tx_tout = GPUEB_TIMEOUT_US / USEC_PER_MSEC;
798 gf->ch = mbox_request_channel_byname(&gf->cl, "gpufreq");
799 if (IS_ERR(gf->ch))
800 return PTR_ERR(gf->ch);
801
802 mfg->gf_mbox = gf;
803
804 slp = devm_kzalloc(dev, sizeof(*slp), GFP_KERNEL);
805 if (!slp)
806 return -ENOMEM;
807
808 slp->mfg = mfg;
809 init_completion(&slp->rx_done);
810 slp->cl.dev = dev;
811 slp->cl.tx_tout = GPUEB_TIMEOUT_US / USEC_PER_MSEC;
812 slp->cl.tx_block = true;
813 slp->ch = mbox_request_channel_byname(&slp->cl, "sleep");
814 if (IS_ERR(slp->ch)) {
815 mbox_free_channel(gf->ch);
816 return PTR_ERR(slp->ch);
817 }
818
819 mfg->slp_mbox = slp;
820
821 return 0;
822 }
823
mtk_mfg_init_clk_provider(struct mtk_mfg * mfg)824 static int mtk_mfg_init_clk_provider(struct mtk_mfg *mfg)
825 {
826 struct device *dev = &mfg->pdev->dev;
827 struct clk_hw_onecell_data *clk_data;
828 int ret;
829
830 clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, 2), GFP_KERNEL);
831 if (!clk_data)
832 return -ENOMEM;
833
834 clk_data->num = 2;
835
836 mfg->clk_core_hw.init = &mtk_mfg_clk_gpu_init;
837 mfg->clk_stack_hw.init = &mtk_mfg_clk_stack_init;
838
839 ret = devm_clk_hw_register(dev, &mfg->clk_core_hw);
840 if (ret)
841 return dev_err_probe(dev, ret, "Couldn't register GPU core clock\n");
842
843 ret = devm_clk_hw_register(dev, &mfg->clk_stack_hw);
844 if (ret)
845 return dev_err_probe(dev, ret, "Couldn't register GPU stack clock\n");
846
847 clk_data->hws[0] = &mfg->clk_core_hw;
848 clk_data->hws[1] = &mfg->clk_stack_hw;
849
850 ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data);
851 if (ret)
852 return dev_err_probe(dev, ret, "Couldn't register clock provider\n");
853
854 return 0;
855 }
856
mtk_mfg_read_nvmem(void * priv,unsigned int offset,void * val,size_t bytes)857 static int mtk_mfg_read_nvmem(void *priv, unsigned int offset, void *val, size_t bytes)
858 {
859 struct mtk_mfg *mfg = priv;
860 u32 *buf = val;
861
862 if (bytes != 4)
863 return -EINVAL;
864
865 if (!mfg->shared_mem)
866 return -ENODEV;
867
868 if (offset + bytes >= mfg->shared_mem_size)
869 return -EINVAL;
870
871 *buf = readl(mfg->shared_mem + offset);
872
873 return 0;
874 }
875
mtk_mfg_init_nvmem_provider(struct mtk_mfg * mfg)876 static int mtk_mfg_init_nvmem_provider(struct mtk_mfg *mfg)
877 {
878 struct device *dev = &mfg->pdev->dev;
879 struct nvmem_cell_info cell = {};
880 struct nvmem_config config = {};
881 struct nvmem_device *nvdev;
882 int ret;
883
884 config.reg_read = mtk_mfg_read_nvmem;
885 config.dev = dev;
886 config.read_only = true;
887 config.priv = mfg;
888 config.size = 4;
889 config.word_size = 4;
890
891 nvdev = devm_nvmem_register(dev, &config);
892 if (IS_ERR(nvdev))
893 return dev_err_probe(dev, PTR_ERR(nvdev), "Couldn't register nvmem provider\n");
894
895 cell.name = "shader-present";
896 cell.offset = GF_REG_SHADER_PRESENT;
897 cell.bytes = 4;
898 cell.np = of_get_child_by_name(dev->of_node, cell.name);
899
900 ret = nvmem_add_one_cell(nvdev, &cell);
901 if (ret) {
902 of_node_put(cell.np);
903 return dev_err_probe(dev, ret, "Couldn't add cell %s\n", cell.name);
904 }
905
906 /* cell.np purposefully not put as nvmem_add_one_cell does not increase refcount */
907
908 return 0;
909 }
910
mtk_mfg_probe(struct platform_device * pdev)911 static int mtk_mfg_probe(struct platform_device *pdev)
912 {
913 struct mtk_mfg *mfg;
914 struct device *dev = &pdev->dev;
915 const struct mtk_mfg_variant *data = of_device_get_match_data(dev);
916 struct resource res;
917 int ret, i;
918
919 mfg = devm_kzalloc(dev, sizeof(*mfg), GFP_KERNEL);
920 if (!mfg)
921 return -ENOMEM;
922
923 mfg->pdev = pdev;
924 mfg->variant = data;
925
926 dev_set_drvdata(dev, mfg);
927
928 mfg->gpr = devm_platform_ioremap_resource(pdev, 0);
929 if (IS_ERR(mfg->gpr))
930 return dev_err_probe(dev, PTR_ERR(mfg->gpr),
931 "Couldn't retrieve GPR MMIO registers\n");
932
933 mfg->rpc = devm_platform_ioremap_resource(pdev, 1);
934 if (IS_ERR(mfg->rpc))
935 return dev_err_probe(dev, PTR_ERR(mfg->rpc),
936 "Couldn't retrieve RPC MMIO registers\n");
937
938 mfg->clk_eb = devm_clk_get(dev, "eb");
939 if (IS_ERR(mfg->clk_eb))
940 return dev_err_probe(dev, PTR_ERR(mfg->clk_eb),
941 "Couldn't get 'eb' clock\n");
942
943 mfg->gpu_clks = devm_kcalloc(dev, data->num_clks, sizeof(*mfg->gpu_clks),
944 GFP_KERNEL);
945 if (!mfg->gpu_clks)
946 return -ENOMEM;
947
948 for (i = 0; i < data->num_clks; i++)
949 mfg->gpu_clks[i].id = data->clk_names[i];
950
951 ret = devm_clk_bulk_get(dev, data->num_clks, mfg->gpu_clks);
952 if (ret)
953 return dev_err_probe(dev, ret, "Couldn't get GPU clocks\n");
954
955 mfg->gpu_regs = devm_kcalloc(dev, data->num_regulators,
956 sizeof(*mfg->gpu_regs), GFP_KERNEL);
957 if (!mfg->gpu_regs)
958 return -ENOMEM;
959
960 for (i = 0; i < data->num_regulators; i++)
961 mfg->gpu_regs[i].supply = data->regulator_names[i];
962
963 ret = devm_regulator_bulk_get(dev, data->num_regulators, mfg->gpu_regs);
964 if (ret)
965 return dev_err_probe(dev, ret, "Couldn't get GPU regulators\n");
966
967 ret = of_reserved_mem_region_to_resource(dev->of_node, 0, &res);
968 if (ret)
969 return dev_err_probe(dev, ret, "Couldn't get GPUEB shared memory\n");
970
971 mfg->shared_mem = devm_ioremap(dev, res.start, resource_size(&res));
972 if (!mfg->shared_mem)
973 return dev_err_probe(dev, -ENOMEM, "Can't ioremap GPUEB shared memory\n");
974 mfg->shared_mem_size = resource_size(&res);
975 mfg->shared_mem_phys = res.start;
976
977 if (data->init) {
978 ret = data->init(mfg);
979 if (ret)
980 return dev_err_probe(dev, ret, "Variant init failed\n");
981 }
982
983 mfg->pd.name = dev_name(dev);
984 mfg->pd.attach_dev = mtk_mfg_attach_dev;
985 mfg->pd.detach_dev = mtk_mfg_detach_dev;
986 mfg->pd.power_off = mtk_mfg_power_off;
987 mfg->pd.power_on = mtk_mfg_power_on;
988 mfg->pd.set_performance_state = mtk_mfg_set_performance;
989 mfg->pd.flags = GENPD_FLAG_OPP_TABLE_FW;
990
991 ret = pm_genpd_init(&mfg->pd, NULL, false);
992 if (ret)
993 return dev_err_probe(dev, ret, "Failed to initialise power domain\n");
994
995 ret = mtk_mfg_init_mbox(mfg);
996 if (ret) {
997 dev_err_probe(dev, ret, "Couldn't initialise mailbox\n");
998 goto err_remove_genpd;
999 }
1000
1001 ret = mtk_mfg_power_on(&mfg->pd);
1002 if (ret) {
1003 dev_err_probe(dev, ret, "Failed to power on MFG\n");
1004 goto err_free_mbox;
1005 }
1006
1007 ret = mtk_mfg_init_shared_mem(mfg);
1008 if (ret) {
1009 dev_err_probe(dev, ret, "Couldn't initialize EB shared memory\n");
1010 goto err_power_off;
1011 }
1012
1013 ret = mtk_mfg_read_opp_tables(mfg);
1014 if (ret) {
1015 dev_err_probe(dev, ret, "Error reading OPP tables from EB\n");
1016 goto err_power_off;
1017 }
1018
1019 ret = mtk_mfg_init_clk_provider(mfg);
1020 if (ret)
1021 goto err_power_off;
1022
1023 ret = mtk_mfg_init_nvmem_provider(mfg);
1024 if (ret)
1025 goto err_power_off;
1026
1027 ret = of_genpd_add_provider_simple(dev->of_node, &mfg->pd);
1028 if (ret) {
1029 dev_err_probe(dev, ret, "Failed to add pmdomain provider\n");
1030 goto err_power_off;
1031 }
1032
1033 return 0;
1034
1035 err_power_off:
1036 mtk_mfg_power_off(&mfg->pd);
1037 err_free_mbox:
1038 mbox_free_channel(mfg->slp_mbox->ch);
1039 mfg->slp_mbox->ch = NULL;
1040 mbox_free_channel(mfg->gf_mbox->ch);
1041 mfg->gf_mbox->ch = NULL;
1042 err_remove_genpd:
1043 pm_genpd_remove(&mfg->pd);
1044
1045 return ret;
1046 }
1047
1048 static const struct of_device_id mtk_mfg_of_match[] = {
1049 { .compatible = "mediatek,mt8196-gpufreq", .data = &mtk_mfg_mt8196_variant },
1050 {}
1051 };
1052 MODULE_DEVICE_TABLE(of, mtk_mfg_of_match);
1053
mtk_mfg_remove(struct platform_device * pdev)1054 static void mtk_mfg_remove(struct platform_device *pdev)
1055 {
1056 struct mtk_mfg *mfg = dev_get_drvdata(&pdev->dev);
1057
1058 if (mtk_mfg_is_powered_on(mfg))
1059 mtk_mfg_power_off(&mfg->pd);
1060
1061 of_genpd_del_provider(pdev->dev.of_node);
1062 pm_genpd_remove(&mfg->pd);
1063
1064 mbox_free_channel(mfg->gf_mbox->ch);
1065 mfg->gf_mbox->ch = NULL;
1066
1067 mbox_free_channel(mfg->slp_mbox->ch);
1068 mfg->slp_mbox->ch = NULL;
1069 }
1070
1071 static struct platform_driver mtk_mfg_driver = {
1072 .driver = {
1073 .name = "mtk-mfg-pmdomain",
1074 .of_match_table = mtk_mfg_of_match,
1075 .suppress_bind_attrs = true,
1076 },
1077 .probe = mtk_mfg_probe,
1078 .remove = mtk_mfg_remove,
1079 };
1080 module_platform_driver(mtk_mfg_driver);
1081
1082 MODULE_AUTHOR("Nicolas Frattaroli <nicolas.frattaroli@collabora.com>");
1083 MODULE_DESCRIPTION("MediaTek MFlexGraphics Power Domain Driver");
1084 MODULE_LICENSE("GPL");
1085