1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2019 NXP
4 */
5
6 #include <linux/module.h>
7 #include <linux/device.h>
8 #include <linux/platform_device.h>
9 #include <linux/devfreq.h>
10 #include <linux/pm_opp.h>
11 #include <linux/clk.h>
12 #include <linux/clk-provider.h>
13 #include <linux/arm-smccc.h>
14
15 #define IMX_SIP_DDR_DVFS 0xc2000004
16
17 /* Query available frequencies. */
18 #define IMX_SIP_DDR_DVFS_GET_FREQ_COUNT 0x10
19 #define IMX_SIP_DDR_DVFS_GET_FREQ_INFO 0x11
20
21 /*
22 * This should be in a 1:1 mapping with devicetree OPPs but
23 * firmware provides additional info.
24 */
25 struct imx8m_ddrc_freq {
26 unsigned long rate;
27 unsigned long smcarg;
28 int dram_core_parent_index;
29 int dram_alt_parent_index;
30 int dram_apb_parent_index;
31 };
32
33 /* Hardware limitation */
34 #define IMX8M_DDRC_MAX_FREQ_COUNT 4
35
36 /*
37 * i.MX8M DRAM Controller clocks have the following structure (abridged):
38 *
39 * +----------+ |\ +------+
40 * | dram_pll |-------|M| dram_core | |
41 * +----------+ |U|---------->| D |
42 * /--|X| | D |
43 * dram_alt_root | |/ | R |
44 * | | C |
45 * +---------+ | |
46 * |FIX DIV/4| | |
47 * +---------+ | |
48 * composite: | | |
49 * +----------+ | | |
50 * | dram_alt |----/ | |
51 * +----------+ | |
52 * | dram_apb |-------------------->| |
53 * +----------+ +------+
54 *
55 * The dram_pll is used for higher rates and dram_alt is used for lower rates.
56 *
57 * Frequency switching is implemented in TF-A (via SMC call) and can change the
58 * configuration of the clocks, including mux parents. The dram_alt and
59 * dram_apb clocks are "imx composite" and their parent can change too.
60 *
61 * We need to prepare/enable the new mux parents head of switching and update
62 * their information afterwards.
63 */
64 struct imx8m_ddrc {
65 struct devfreq_dev_profile profile;
66 struct devfreq *devfreq;
67
68 /* For frequency switching: */
69 struct clk *dram_core;
70 struct clk *dram_pll;
71 struct clk *dram_alt;
72 struct clk *dram_apb;
73
74 int freq_count;
75 struct imx8m_ddrc_freq freq_table[IMX8M_DDRC_MAX_FREQ_COUNT];
76 };
77
imx8m_ddrc_find_freq(struct imx8m_ddrc * priv,unsigned long rate)78 static struct imx8m_ddrc_freq *imx8m_ddrc_find_freq(struct imx8m_ddrc *priv,
79 unsigned long rate)
80 {
81 struct imx8m_ddrc_freq *freq;
82 int i;
83
84 /*
85 * Firmware reports values in MT/s, so we round-down from Hz
86 * Rounding is extra generous to ensure a match.
87 */
88 rate = DIV_ROUND_CLOSEST(rate, 250000);
89 for (i = 0; i < priv->freq_count; ++i) {
90 freq = &priv->freq_table[i];
91 if (freq->rate == rate ||
92 freq->rate + 1 == rate ||
93 freq->rate - 1 == rate)
94 return freq;
95 }
96
97 return NULL;
98 }
99
imx8m_ddrc_smc_set_freq(int target_freq)100 static void imx8m_ddrc_smc_set_freq(int target_freq)
101 {
102 struct arm_smccc_res res;
103 u32 online_cpus = 0;
104 int cpu;
105
106 local_irq_disable();
107
108 for_each_online_cpu(cpu)
109 online_cpus |= (1 << (cpu * 8));
110
111 /* change the ddr freqency */
112 arm_smccc_smc(IMX_SIP_DDR_DVFS, target_freq, online_cpus,
113 0, 0, 0, 0, 0, &res);
114
115 local_irq_enable();
116 }
117
clk_get_parent_by_index(struct clk * clk,int index)118 static struct clk *clk_get_parent_by_index(struct clk *clk, int index)
119 {
120 struct clk_hw *hw;
121
122 hw = clk_hw_get_parent_by_index(__clk_get_hw(clk), index);
123
124 return hw ? hw->clk : NULL;
125 }
126
imx8m_ddrc_set_freq(struct device * dev,struct imx8m_ddrc_freq * freq)127 static int imx8m_ddrc_set_freq(struct device *dev, struct imx8m_ddrc_freq *freq)
128 {
129 struct imx8m_ddrc *priv = dev_get_drvdata(dev);
130 struct clk *new_dram_core_parent;
131 struct clk *new_dram_alt_parent;
132 struct clk *new_dram_apb_parent;
133 int ret;
134
135 /*
136 * Fetch new parents
137 *
138 * new_dram_alt_parent and new_dram_apb_parent are optional but
139 * new_dram_core_parent is not.
140 */
141 new_dram_core_parent = clk_get_parent_by_index(
142 priv->dram_core, freq->dram_core_parent_index - 1);
143 if (!new_dram_core_parent) {
144 dev_err(dev, "failed to fetch new dram_core parent\n");
145 return -EINVAL;
146 }
147 if (freq->dram_alt_parent_index) {
148 new_dram_alt_parent = clk_get_parent_by_index(
149 priv->dram_alt,
150 freq->dram_alt_parent_index - 1);
151 if (!new_dram_alt_parent) {
152 dev_err(dev, "failed to fetch new dram_alt parent\n");
153 return -EINVAL;
154 }
155 } else
156 new_dram_alt_parent = NULL;
157
158 if (freq->dram_apb_parent_index) {
159 new_dram_apb_parent = clk_get_parent_by_index(
160 priv->dram_apb,
161 freq->dram_apb_parent_index - 1);
162 if (!new_dram_apb_parent) {
163 dev_err(dev, "failed to fetch new dram_apb parent\n");
164 return -EINVAL;
165 }
166 } else
167 new_dram_apb_parent = NULL;
168
169 /* increase reference counts and ensure clks are ON before switch */
170 ret = clk_prepare_enable(new_dram_core_parent);
171 if (ret) {
172 dev_err(dev, "failed to enable new dram_core parent: %d\n",
173 ret);
174 goto out;
175 }
176 ret = clk_prepare_enable(new_dram_alt_parent);
177 if (ret) {
178 dev_err(dev, "failed to enable new dram_alt parent: %d\n",
179 ret);
180 goto out_disable_core_parent;
181 }
182 ret = clk_prepare_enable(new_dram_apb_parent);
183 if (ret) {
184 dev_err(dev, "failed to enable new dram_apb parent: %d\n",
185 ret);
186 goto out_disable_alt_parent;
187 }
188
189 imx8m_ddrc_smc_set_freq(freq->smcarg);
190
191 /* update parents in clk tree after switch. */
192 ret = clk_set_parent(priv->dram_core, new_dram_core_parent);
193 if (ret)
194 dev_warn(dev, "failed to set dram_core parent: %d\n", ret);
195 if (new_dram_alt_parent) {
196 ret = clk_set_parent(priv->dram_alt, new_dram_alt_parent);
197 if (ret)
198 dev_warn(dev, "failed to set dram_alt parent: %d\n",
199 ret);
200 }
201 if (new_dram_apb_parent) {
202 ret = clk_set_parent(priv->dram_apb, new_dram_apb_parent);
203 if (ret)
204 dev_warn(dev, "failed to set dram_apb parent: %d\n",
205 ret);
206 }
207
208 /*
209 * Explicitly refresh dram PLL rate.
210 *
211 * Even if it's marked with CLK_GET_RATE_NOCACHE the rate will not be
212 * automatically refreshed when clk_get_rate is called on children.
213 */
214 clk_get_rate(priv->dram_pll);
215
216 /*
217 * clk_set_parent transfer the reference count from old parent.
218 * now we drop extra reference counts used during the switch
219 */
220 clk_disable_unprepare(new_dram_apb_parent);
221 out_disable_alt_parent:
222 clk_disable_unprepare(new_dram_alt_parent);
223 out_disable_core_parent:
224 clk_disable_unprepare(new_dram_core_parent);
225 out:
226 return ret;
227 }
228
imx8m_ddrc_target(struct device * dev,unsigned long * freq,u32 flags)229 static int imx8m_ddrc_target(struct device *dev, unsigned long *freq, u32 flags)
230 {
231 struct imx8m_ddrc *priv = dev_get_drvdata(dev);
232 struct imx8m_ddrc_freq *freq_info;
233 struct dev_pm_opp *new_opp;
234 unsigned long old_freq, new_freq;
235 int ret;
236
237 new_opp = devfreq_recommended_opp(dev, freq, flags);
238 if (IS_ERR(new_opp)) {
239 ret = PTR_ERR(new_opp);
240 dev_err(dev, "failed to get recommended opp: %d\n", ret);
241 return ret;
242 }
243 dev_pm_opp_put(new_opp);
244
245 old_freq = clk_get_rate(priv->dram_core);
246 if (*freq == old_freq)
247 return 0;
248
249 freq_info = imx8m_ddrc_find_freq(priv, *freq);
250 if (!freq_info)
251 return -EINVAL;
252
253 /*
254 * Read back the clk rate to verify switch was correct and so that
255 * we can report it on all error paths.
256 */
257 ret = imx8m_ddrc_set_freq(dev, freq_info);
258
259 new_freq = clk_get_rate(priv->dram_core);
260 if (ret)
261 dev_err(dev, "ddrc failed freq switch to %lu from %lu: error %d. now at %lu\n",
262 *freq, old_freq, ret, new_freq);
263 else if (*freq != new_freq)
264 dev_err(dev, "ddrc failed freq update to %lu from %lu, now at %lu\n",
265 *freq, old_freq, new_freq);
266 else
267 dev_dbg(dev, "ddrc freq set to %lu (was %lu)\n",
268 *freq, old_freq);
269
270 return ret;
271 }
272
imx8m_ddrc_get_cur_freq(struct device * dev,unsigned long * freq)273 static int imx8m_ddrc_get_cur_freq(struct device *dev, unsigned long *freq)
274 {
275 struct imx8m_ddrc *priv = dev_get_drvdata(dev);
276
277 *freq = clk_get_rate(priv->dram_core);
278
279 return 0;
280 }
281
imx8m_ddrc_init_freq_info(struct device * dev)282 static int imx8m_ddrc_init_freq_info(struct device *dev)
283 {
284 struct imx8m_ddrc *priv = dev_get_drvdata(dev);
285 struct arm_smccc_res res;
286 int index;
287
288 /* An error here means DDR DVFS API not supported by firmware */
289 arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_COUNT,
290 0, 0, 0, 0, 0, 0, &res);
291 priv->freq_count = res.a0;
292 if (priv->freq_count <= 0 ||
293 priv->freq_count > IMX8M_DDRC_MAX_FREQ_COUNT)
294 return -ENODEV;
295
296 for (index = 0; index < priv->freq_count; ++index) {
297 struct imx8m_ddrc_freq *freq = &priv->freq_table[index];
298
299 arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_INFO,
300 index, 0, 0, 0, 0, 0, &res);
301 /* Result should be strictly positive */
302 if ((long)res.a0 <= 0)
303 return -ENODEV;
304
305 freq->rate = res.a0;
306 freq->smcarg = index;
307 freq->dram_core_parent_index = res.a1;
308 freq->dram_alt_parent_index = res.a2;
309 freq->dram_apb_parent_index = res.a3;
310
311 /* dram_core has 2 options: dram_pll or dram_alt_root */
312 if (freq->dram_core_parent_index != 1 &&
313 freq->dram_core_parent_index != 2)
314 return -ENODEV;
315 /* dram_apb and dram_alt have exactly 8 possible parents */
316 if (freq->dram_alt_parent_index > 8 ||
317 freq->dram_apb_parent_index > 8)
318 return -ENODEV;
319 /* dram_core from alt requires explicit dram_alt parent */
320 if (freq->dram_core_parent_index == 2 &&
321 freq->dram_alt_parent_index == 0)
322 return -ENODEV;
323 }
324
325 return 0;
326 }
327
imx8m_ddrc_check_opps(struct device * dev)328 static int imx8m_ddrc_check_opps(struct device *dev)
329 {
330 struct imx8m_ddrc *priv = dev_get_drvdata(dev);
331 struct imx8m_ddrc_freq *freq_info;
332 struct dev_pm_opp *opp;
333 unsigned long freq;
334 int i, opp_count;
335
336 /* Enumerate DT OPPs and disable those not supported by firmware */
337 opp_count = dev_pm_opp_get_opp_count(dev);
338 if (opp_count < 0)
339 return opp_count;
340 for (i = 0, freq = 0; i < opp_count; ++i, ++freq) {
341 opp = dev_pm_opp_find_freq_ceil(dev, &freq);
342 if (IS_ERR(opp)) {
343 dev_err(dev, "Failed enumerating OPPs: %ld\n",
344 PTR_ERR(opp));
345 return PTR_ERR(opp);
346 }
347 dev_pm_opp_put(opp);
348
349 freq_info = imx8m_ddrc_find_freq(priv, freq);
350 if (!freq_info) {
351 dev_info(dev, "Disable unsupported OPP %luHz %luMT/s\n",
352 freq, DIV_ROUND_CLOSEST(freq, 250000));
353 dev_pm_opp_disable(dev, freq);
354 }
355 }
356
357 return 0;
358 }
359
imx8m_ddrc_exit(struct device * dev)360 static void imx8m_ddrc_exit(struct device *dev)
361 {
362 dev_pm_opp_of_remove_table(dev);
363 }
364
imx8m_ddrc_probe(struct platform_device * pdev)365 static int imx8m_ddrc_probe(struct platform_device *pdev)
366 {
367 struct device *dev = &pdev->dev;
368 struct imx8m_ddrc *priv;
369 const char *gov = DEVFREQ_GOV_USERSPACE;
370 int ret;
371
372 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
373 if (!priv)
374 return -ENOMEM;
375
376 platform_set_drvdata(pdev, priv);
377
378 ret = imx8m_ddrc_init_freq_info(dev);
379 if (ret) {
380 dev_err(dev, "failed to init firmware freq info: %d\n", ret);
381 return ret;
382 }
383
384 priv->dram_core = devm_clk_get(dev, "core");
385 if (IS_ERR(priv->dram_core)) {
386 ret = PTR_ERR(priv->dram_core);
387 dev_err(dev, "failed to fetch core clock: %d\n", ret);
388 return ret;
389 }
390 priv->dram_pll = devm_clk_get(dev, "pll");
391 if (IS_ERR(priv->dram_pll)) {
392 ret = PTR_ERR(priv->dram_pll);
393 dev_err(dev, "failed to fetch pll clock: %d\n", ret);
394 return ret;
395 }
396 priv->dram_alt = devm_clk_get(dev, "alt");
397 if (IS_ERR(priv->dram_alt)) {
398 ret = PTR_ERR(priv->dram_alt);
399 dev_err(dev, "failed to fetch alt clock: %d\n", ret);
400 return ret;
401 }
402 priv->dram_apb = devm_clk_get(dev, "apb");
403 if (IS_ERR(priv->dram_apb)) {
404 ret = PTR_ERR(priv->dram_apb);
405 dev_err(dev, "failed to fetch apb clock: %d\n", ret);
406 return ret;
407 }
408
409 ret = dev_pm_opp_of_add_table(dev);
410 if (ret < 0) {
411 dev_err(dev, "failed to get OPP table\n");
412 return ret;
413 }
414
415 ret = imx8m_ddrc_check_opps(dev);
416 if (ret < 0)
417 goto err;
418
419 priv->profile.target = imx8m_ddrc_target;
420 priv->profile.exit = imx8m_ddrc_exit;
421 priv->profile.get_cur_freq = imx8m_ddrc_get_cur_freq;
422 priv->profile.initial_freq = clk_get_rate(priv->dram_core);
423
424 priv->devfreq = devm_devfreq_add_device(dev, &priv->profile,
425 gov, NULL);
426 if (IS_ERR(priv->devfreq)) {
427 ret = PTR_ERR(priv->devfreq);
428 dev_err(dev, "failed to add devfreq device: %d\n", ret);
429 goto err;
430 }
431
432 return 0;
433
434 err:
435 dev_pm_opp_of_remove_table(dev);
436 return ret;
437 }
438
439 static const struct of_device_id imx8m_ddrc_of_match[] = {
440 { .compatible = "fsl,imx8m-ddrc", },
441 { /* sentinel */ },
442 };
443 MODULE_DEVICE_TABLE(of, imx8m_ddrc_of_match);
444
445 static struct platform_driver imx8m_ddrc_platdrv = {
446 .probe = imx8m_ddrc_probe,
447 .driver = {
448 .name = "imx8m-ddrc-devfreq",
449 .of_match_table = imx8m_ddrc_of_match,
450 },
451 };
452 module_platform_driver(imx8m_ddrc_platdrv);
453
454 MODULE_DESCRIPTION("i.MX8M DDR Controller frequency driver");
455 MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>");
456 MODULE_LICENSE("GPL v2");
457