1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2019 NXP 4 */ 5 6 #include <linux/module.h> 7 #include <linux/device.h> 8 #include <linux/platform_device.h> 9 #include <linux/devfreq.h> 10 #include <linux/pm_opp.h> 11 #include <linux/clk.h> 12 #include <linux/clk-provider.h> 13 #include <linux/arm-smccc.h> 14 15 #define IMX_SIP_DDR_DVFS 0xc2000004 16 17 /* Query available frequencies. */ 18 #define IMX_SIP_DDR_DVFS_GET_FREQ_COUNT 0x10 19 #define IMX_SIP_DDR_DVFS_GET_FREQ_INFO 0x11 20 21 /* 22 * This should be in a 1:1 mapping with devicetree OPPs but 23 * firmware provides additional info. 24 */ 25 struct imx8m_ddrc_freq { 26 unsigned long rate; 27 unsigned long smcarg; 28 int dram_core_parent_index; 29 int dram_alt_parent_index; 30 int dram_apb_parent_index; 31 }; 32 33 /* Hardware limitation */ 34 #define IMX8M_DDRC_MAX_FREQ_COUNT 4 35 36 /* 37 * i.MX8M DRAM Controller clocks have the following structure (abridged): 38 * 39 * +----------+ |\ +------+ 40 * | dram_pll |-------|M| dram_core | | 41 * +----------+ |U|---------->| D | 42 * /--|X| | D | 43 * dram_alt_root | |/ | R | 44 * | | C | 45 * +---------+ | | 46 * |FIX DIV/4| | | 47 * +---------+ | | 48 * composite: | | | 49 * +----------+ | | | 50 * | dram_alt |----/ | | 51 * +----------+ | | 52 * | dram_apb |-------------------->| | 53 * +----------+ +------+ 54 * 55 * The dram_pll is used for higher rates and dram_alt is used for lower rates. 56 * 57 * Frequency switching is implemented in TF-A (via SMC call) and can change the 58 * configuration of the clocks, including mux parents. The dram_alt and 59 * dram_apb clocks are "imx composite" and their parent can change too. 60 * 61 * We need to prepare/enable the new mux parents head of switching and update 62 * their information afterwards. 63 */ 64 struct imx8m_ddrc { 65 struct devfreq_dev_profile profile; 66 struct devfreq *devfreq; 67 68 /* For frequency switching: */ 69 struct clk *dram_core; 70 struct clk *dram_pll; 71 struct clk *dram_alt; 72 struct clk *dram_apb; 73 74 int freq_count; 75 struct imx8m_ddrc_freq freq_table[IMX8M_DDRC_MAX_FREQ_COUNT]; 76 }; 77 78 static struct imx8m_ddrc_freq *imx8m_ddrc_find_freq(struct imx8m_ddrc *priv, 79 unsigned long rate) 80 { 81 struct imx8m_ddrc_freq *freq; 82 int i; 83 84 /* 85 * Firmware reports values in MT/s, so we round-down from Hz 86 * Rounding is extra generous to ensure a match. 87 */ 88 rate = DIV_ROUND_CLOSEST(rate, 250000); 89 for (i = 0; i < priv->freq_count; ++i) { 90 freq = &priv->freq_table[i]; 91 if (freq->rate == rate || 92 freq->rate + 1 == rate || 93 freq->rate - 1 == rate) 94 return freq; 95 } 96 97 return NULL; 98 } 99 100 static void imx8m_ddrc_smc_set_freq(int target_freq) 101 { 102 struct arm_smccc_res res; 103 u32 online_cpus = 0; 104 int cpu; 105 106 local_irq_disable(); 107 108 for_each_online_cpu(cpu) 109 online_cpus |= (1 << (cpu * 8)); 110 111 /* change the ddr freqency */ 112 arm_smccc_smc(IMX_SIP_DDR_DVFS, target_freq, online_cpus, 113 0, 0, 0, 0, 0, &res); 114 115 local_irq_enable(); 116 } 117 118 static struct clk *clk_get_parent_by_index(struct clk *clk, int index) 119 { 120 struct clk_hw *hw; 121 122 hw = clk_hw_get_parent_by_index(__clk_get_hw(clk), index); 123 124 return hw ? hw->clk : NULL; 125 } 126 127 static int imx8m_ddrc_set_freq(struct device *dev, struct imx8m_ddrc_freq *freq) 128 { 129 struct imx8m_ddrc *priv = dev_get_drvdata(dev); 130 struct clk *new_dram_core_parent; 131 struct clk *new_dram_alt_parent; 132 struct clk *new_dram_apb_parent; 133 int ret; 134 135 /* 136 * Fetch new parents 137 * 138 * new_dram_alt_parent and new_dram_apb_parent are optional but 139 * new_dram_core_parent is not. 140 */ 141 new_dram_core_parent = clk_get_parent_by_index( 142 priv->dram_core, freq->dram_core_parent_index - 1); 143 if (!new_dram_core_parent) { 144 dev_err(dev, "failed to fetch new dram_core parent\n"); 145 return -EINVAL; 146 } 147 if (freq->dram_alt_parent_index) { 148 new_dram_alt_parent = clk_get_parent_by_index( 149 priv->dram_alt, 150 freq->dram_alt_parent_index - 1); 151 if (!new_dram_alt_parent) { 152 dev_err(dev, "failed to fetch new dram_alt parent\n"); 153 return -EINVAL; 154 } 155 } else 156 new_dram_alt_parent = NULL; 157 158 if (freq->dram_apb_parent_index) { 159 new_dram_apb_parent = clk_get_parent_by_index( 160 priv->dram_apb, 161 freq->dram_apb_parent_index - 1); 162 if (!new_dram_apb_parent) { 163 dev_err(dev, "failed to fetch new dram_apb parent\n"); 164 return -EINVAL; 165 } 166 } else 167 new_dram_apb_parent = NULL; 168 169 /* increase reference counts and ensure clks are ON before switch */ 170 ret = clk_prepare_enable(new_dram_core_parent); 171 if (ret) { 172 dev_err(dev, "failed to enable new dram_core parent: %d\n", 173 ret); 174 goto out; 175 } 176 ret = clk_prepare_enable(new_dram_alt_parent); 177 if (ret) { 178 dev_err(dev, "failed to enable new dram_alt parent: %d\n", 179 ret); 180 goto out_disable_core_parent; 181 } 182 ret = clk_prepare_enable(new_dram_apb_parent); 183 if (ret) { 184 dev_err(dev, "failed to enable new dram_apb parent: %d\n", 185 ret); 186 goto out_disable_alt_parent; 187 } 188 189 imx8m_ddrc_smc_set_freq(freq->smcarg); 190 191 /* update parents in clk tree after switch. */ 192 ret = clk_set_parent(priv->dram_core, new_dram_core_parent); 193 if (ret) 194 dev_warn(dev, "failed to set dram_core parent: %d\n", ret); 195 if (new_dram_alt_parent) { 196 ret = clk_set_parent(priv->dram_alt, new_dram_alt_parent); 197 if (ret) 198 dev_warn(dev, "failed to set dram_alt parent: %d\n", 199 ret); 200 } 201 if (new_dram_apb_parent) { 202 ret = clk_set_parent(priv->dram_apb, new_dram_apb_parent); 203 if (ret) 204 dev_warn(dev, "failed to set dram_apb parent: %d\n", 205 ret); 206 } 207 208 /* 209 * Explicitly refresh dram PLL rate. 210 * 211 * Even if it's marked with CLK_GET_RATE_NOCACHE the rate will not be 212 * automatically refreshed when clk_get_rate is called on children. 213 */ 214 clk_get_rate(priv->dram_pll); 215 216 /* 217 * clk_set_parent transfer the reference count from old parent. 218 * now we drop extra reference counts used during the switch 219 */ 220 clk_disable_unprepare(new_dram_apb_parent); 221 out_disable_alt_parent: 222 clk_disable_unprepare(new_dram_alt_parent); 223 out_disable_core_parent: 224 clk_disable_unprepare(new_dram_core_parent); 225 out: 226 return ret; 227 } 228 229 static int imx8m_ddrc_target(struct device *dev, unsigned long *freq, u32 flags) 230 { 231 struct imx8m_ddrc *priv = dev_get_drvdata(dev); 232 struct imx8m_ddrc_freq *freq_info; 233 struct dev_pm_opp *new_opp; 234 unsigned long old_freq, new_freq; 235 int ret; 236 237 new_opp = devfreq_recommended_opp(dev, freq, flags); 238 if (IS_ERR(new_opp)) { 239 ret = PTR_ERR(new_opp); 240 dev_err(dev, "failed to get recommended opp: %d\n", ret); 241 return ret; 242 } 243 dev_pm_opp_put(new_opp); 244 245 old_freq = clk_get_rate(priv->dram_core); 246 if (*freq == old_freq) 247 return 0; 248 249 freq_info = imx8m_ddrc_find_freq(priv, *freq); 250 if (!freq_info) 251 return -EINVAL; 252 253 /* 254 * Read back the clk rate to verify switch was correct and so that 255 * we can report it on all error paths. 256 */ 257 ret = imx8m_ddrc_set_freq(dev, freq_info); 258 259 new_freq = clk_get_rate(priv->dram_core); 260 if (ret) 261 dev_err(dev, "ddrc failed freq switch to %lu from %lu: error %d. now at %lu\n", 262 *freq, old_freq, ret, new_freq); 263 else if (*freq != new_freq) 264 dev_err(dev, "ddrc failed freq update to %lu from %lu, now at %lu\n", 265 *freq, old_freq, new_freq); 266 else 267 dev_dbg(dev, "ddrc freq set to %lu (was %lu)\n", 268 *freq, old_freq); 269 270 return ret; 271 } 272 273 static int imx8m_ddrc_get_cur_freq(struct device *dev, unsigned long *freq) 274 { 275 struct imx8m_ddrc *priv = dev_get_drvdata(dev); 276 277 *freq = clk_get_rate(priv->dram_core); 278 279 return 0; 280 } 281 282 static int imx8m_ddrc_init_freq_info(struct device *dev) 283 { 284 struct imx8m_ddrc *priv = dev_get_drvdata(dev); 285 struct arm_smccc_res res; 286 int index; 287 288 /* An error here means DDR DVFS API not supported by firmware */ 289 arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_COUNT, 290 0, 0, 0, 0, 0, 0, &res); 291 priv->freq_count = res.a0; 292 if (priv->freq_count <= 0 || 293 priv->freq_count > IMX8M_DDRC_MAX_FREQ_COUNT) 294 return -ENODEV; 295 296 for (index = 0; index < priv->freq_count; ++index) { 297 struct imx8m_ddrc_freq *freq = &priv->freq_table[index]; 298 299 arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_INFO, 300 index, 0, 0, 0, 0, 0, &res); 301 /* Result should be strictly positive */ 302 if ((long)res.a0 <= 0) 303 return -ENODEV; 304 305 freq->rate = res.a0; 306 freq->smcarg = index; 307 freq->dram_core_parent_index = res.a1; 308 freq->dram_alt_parent_index = res.a2; 309 freq->dram_apb_parent_index = res.a3; 310 311 /* dram_core has 2 options: dram_pll or dram_alt_root */ 312 if (freq->dram_core_parent_index != 1 && 313 freq->dram_core_parent_index != 2) 314 return -ENODEV; 315 /* dram_apb and dram_alt have exactly 8 possible parents */ 316 if (freq->dram_alt_parent_index > 8 || 317 freq->dram_apb_parent_index > 8) 318 return -ENODEV; 319 /* dram_core from alt requires explicit dram_alt parent */ 320 if (freq->dram_core_parent_index == 2 && 321 freq->dram_alt_parent_index == 0) 322 return -ENODEV; 323 } 324 325 return 0; 326 } 327 328 static int imx8m_ddrc_check_opps(struct device *dev) 329 { 330 struct imx8m_ddrc *priv = dev_get_drvdata(dev); 331 struct imx8m_ddrc_freq *freq_info; 332 struct dev_pm_opp *opp; 333 unsigned long freq; 334 int i, opp_count; 335 336 /* Enumerate DT OPPs and disable those not supported by firmware */ 337 opp_count = dev_pm_opp_get_opp_count(dev); 338 if (opp_count < 0) 339 return opp_count; 340 for (i = 0, freq = 0; i < opp_count; ++i, ++freq) { 341 opp = dev_pm_opp_find_freq_ceil(dev, &freq); 342 if (IS_ERR(opp)) { 343 dev_err(dev, "Failed enumerating OPPs: %ld\n", 344 PTR_ERR(opp)); 345 return PTR_ERR(opp); 346 } 347 dev_pm_opp_put(opp); 348 349 freq_info = imx8m_ddrc_find_freq(priv, freq); 350 if (!freq_info) { 351 dev_info(dev, "Disable unsupported OPP %luHz %luMT/s\n", 352 freq, DIV_ROUND_CLOSEST(freq, 250000)); 353 dev_pm_opp_disable(dev, freq); 354 } 355 } 356 357 return 0; 358 } 359 360 static void imx8m_ddrc_exit(struct device *dev) 361 { 362 dev_pm_opp_of_remove_table(dev); 363 } 364 365 static int imx8m_ddrc_probe(struct platform_device *pdev) 366 { 367 struct device *dev = &pdev->dev; 368 struct imx8m_ddrc *priv; 369 const char *gov = DEVFREQ_GOV_USERSPACE; 370 int ret; 371 372 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 373 if (!priv) 374 return -ENOMEM; 375 376 platform_set_drvdata(pdev, priv); 377 378 ret = imx8m_ddrc_init_freq_info(dev); 379 if (ret) { 380 dev_err(dev, "failed to init firmware freq info: %d\n", ret); 381 return ret; 382 } 383 384 priv->dram_core = devm_clk_get(dev, "core"); 385 if (IS_ERR(priv->dram_core)) { 386 ret = PTR_ERR(priv->dram_core); 387 dev_err(dev, "failed to fetch core clock: %d\n", ret); 388 return ret; 389 } 390 priv->dram_pll = devm_clk_get(dev, "pll"); 391 if (IS_ERR(priv->dram_pll)) { 392 ret = PTR_ERR(priv->dram_pll); 393 dev_err(dev, "failed to fetch pll clock: %d\n", ret); 394 return ret; 395 } 396 priv->dram_alt = devm_clk_get(dev, "alt"); 397 if (IS_ERR(priv->dram_alt)) { 398 ret = PTR_ERR(priv->dram_alt); 399 dev_err(dev, "failed to fetch alt clock: %d\n", ret); 400 return ret; 401 } 402 priv->dram_apb = devm_clk_get(dev, "apb"); 403 if (IS_ERR(priv->dram_apb)) { 404 ret = PTR_ERR(priv->dram_apb); 405 dev_err(dev, "failed to fetch apb clock: %d\n", ret); 406 return ret; 407 } 408 409 ret = dev_pm_opp_of_add_table(dev); 410 if (ret < 0) { 411 dev_err(dev, "failed to get OPP table\n"); 412 return ret; 413 } 414 415 ret = imx8m_ddrc_check_opps(dev); 416 if (ret < 0) 417 goto err; 418 419 priv->profile.target = imx8m_ddrc_target; 420 priv->profile.exit = imx8m_ddrc_exit; 421 priv->profile.get_cur_freq = imx8m_ddrc_get_cur_freq; 422 priv->profile.initial_freq = clk_get_rate(priv->dram_core); 423 424 priv->devfreq = devm_devfreq_add_device(dev, &priv->profile, 425 gov, NULL); 426 if (IS_ERR(priv->devfreq)) { 427 ret = PTR_ERR(priv->devfreq); 428 dev_err(dev, "failed to add devfreq device: %d\n", ret); 429 goto err; 430 } 431 432 return 0; 433 434 err: 435 dev_pm_opp_of_remove_table(dev); 436 return ret; 437 } 438 439 static const struct of_device_id imx8m_ddrc_of_match[] = { 440 { .compatible = "fsl,imx8m-ddrc", }, 441 { /* sentinel */ }, 442 }; 443 MODULE_DEVICE_TABLE(of, imx8m_ddrc_of_match); 444 445 static struct platform_driver imx8m_ddrc_platdrv = { 446 .probe = imx8m_ddrc_probe, 447 .driver = { 448 .name = "imx8m-ddrc-devfreq", 449 .of_match_table = imx8m_ddrc_of_match, 450 }, 451 }; 452 module_platform_driver(imx8m_ddrc_platdrv); 453 454 MODULE_DESCRIPTION("i.MX8M DDR Controller frequency driver"); 455 MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>"); 456 MODULE_LICENSE("GPL v2"); 457