1 // SPDX-License-Identifier: GPL-2.0-only 2 // 3 // Copyright (C) 2020-2021 Samuel Holland <samuel@sholland.org> 4 // 5 6 #include <linux/clk.h> 7 #include <linux/devfreq.h> 8 #include <linux/err.h> 9 #include <linux/io.h> 10 #include <linux/iopoll.h> 11 #include <linux/module.h> 12 #include <linux/of.h> 13 #include <linux/platform_device.h> 14 #include <linux/property.h> 15 16 #define MBUS_CR 0x0000 17 #define MBUS_CR_GET_DRAM_TYPE(x) (((x) >> 16) & 0x7) 18 #define MBUS_CR_DRAM_TYPE_DDR2 2 19 #define MBUS_CR_DRAM_TYPE_DDR3 3 20 #define MBUS_CR_DRAM_TYPE_DDR4 4 21 #define MBUS_CR_DRAM_TYPE_LPDDR2 6 22 #define MBUS_CR_DRAM_TYPE_LPDDR3 7 23 24 #define MBUS_TMR 0x000c 25 #define MBUS_TMR_PERIOD(x) ((x) - 1) 26 27 #define MBUS_PMU_CFG 0x009c 28 #define MBUS_PMU_CFG_PERIOD(x) (((x) - 1) << 16) 29 #define MBUS_PMU_CFG_UNIT (0x3 << 1) 30 #define MBUS_PMU_CFG_UNIT_B (0x0 << 1) 31 #define MBUS_PMU_CFG_UNIT_KB (0x1 << 1) 32 #define MBUS_PMU_CFG_UNIT_MB (0x2 << 1) 33 #define MBUS_PMU_CFG_ENABLE (0x1 << 0) 34 35 #define MBUS_PMU_BWCR(n) (0x00a0 + (0x04 * (n))) 36 37 #define MBUS_TOTAL_BWCR MBUS_PMU_BWCR(5) 38 #define MBUS_TOTAL_BWCR_H616 MBUS_PMU_BWCR(13) 39 40 #define MBUS_MDFSCR 0x0100 41 #define MBUS_MDFSCR_BUFFER_TIMING (0x1 << 15) 42 #define MBUS_MDFSCR_PAD_HOLD (0x1 << 13) 43 #define MBUS_MDFSCR_BYPASS (0x1 << 4) 44 #define MBUS_MDFSCR_MODE (0x1 << 1) 45 #define MBUS_MDFSCR_MODE_DFS (0x0 << 1) 46 #define MBUS_MDFSCR_MODE_CFS (0x1 << 1) 47 #define MBUS_MDFSCR_START (0x1 << 0) 48 49 #define MBUS_MDFSMRMR 0x0108 50 51 #define DRAM_PWRCTL 0x0004 52 #define DRAM_PWRCTL_SELFREF_EN (0x1 << 0) 53 54 #define DRAM_RFSHTMG 0x0090 55 #define DRAM_RFSHTMG_TREFI(x) ((x) << 16) 56 #define DRAM_RFSHTMG_TRFC(x) ((x) << 0) 57 58 #define DRAM_VTFCR 0x00b8 59 #define DRAM_VTFCR_VTF_ENABLE (0x3 << 8) 60 61 #define DRAM_ODTMAP 0x0120 62 63 #define DRAM_DX_MAX 4 64 65 #define DRAM_DXnGCR0(n) (0x0344 + 0x80 * (n)) 66 #define DRAM_DXnGCR0_DXODT (0x3 << 4) 67 #define DRAM_DXnGCR0_DXODT_DYNAMIC (0x0 << 4) 68 #define DRAM_DXnGCR0_DXODT_ENABLED (0x1 << 4) 69 #define DRAM_DXnGCR0_DXODT_DISABLED (0x2 << 4) 70 #define DRAM_DXnGCR0_DXEN (0x1 << 0) 71 72 struct sun8i_a33_mbus_variant { 73 u32 min_dram_divider; 74 u32 max_dram_divider; 75 u32 odt_freq_mhz; 76 }; 77 78 struct sun8i_a33_mbus { 79 const struct sun8i_a33_mbus_variant *variant; 80 void __iomem *reg_dram; 81 void __iomem *reg_mbus; 82 struct clk *clk_bus; 83 struct clk *clk_dram; 84 struct clk *clk_mbus; 85 struct devfreq *devfreq_dram; 86 struct devfreq_simple_ondemand_data gov_data; 87 struct devfreq_dev_profile profile; 88 u32 data_width; 89 u32 nominal_bw; 90 u32 odtmap; 91 u32 tREFI_ns; 92 u32 tRFC_ns; 93 unsigned long freq_table[]; 94 }; 95 96 /* 97 * The unit for this value is (MBUS clock cycles / MBUS_TMR_PERIOD). When 98 * MBUS_TMR_PERIOD is programmed to match the MBUS clock frequency in MHz, as 99 * it is during DRAM init and during probe, the resulting unit is microseconds. 100 */ 101 static int pmu_period = 50000; 102 module_param(pmu_period, int, 0644); 103 MODULE_PARM_DESC(pmu_period, "Bandwidth measurement period (microseconds)"); 104 105 static u32 sun8i_a33_mbus_get_peak_bw(struct sun8i_a33_mbus *priv) 106 { 107 /* Returns the peak transfer (in KiB) during any single PMU period. */ 108 return readl_relaxed(priv->reg_mbus + MBUS_TOTAL_BWCR); 109 } 110 111 static void sun8i_a33_mbus_restart_pmu_counters(struct sun8i_a33_mbus *priv) 112 { 113 u32 pmu_cfg = MBUS_PMU_CFG_PERIOD(pmu_period) | MBUS_PMU_CFG_UNIT_KB; 114 115 /* All PMU counters are cleared on a disable->enable transition. */ 116 writel_relaxed(pmu_cfg, 117 priv->reg_mbus + MBUS_PMU_CFG); 118 writel_relaxed(pmu_cfg | MBUS_PMU_CFG_ENABLE, 119 priv->reg_mbus + MBUS_PMU_CFG); 120 121 } 122 123 static void sun8i_a33_mbus_update_nominal_bw(struct sun8i_a33_mbus *priv, 124 u32 ddr_freq_mhz) 125 { 126 /* 127 * Nominal bandwidth (KiB per PMU period): 128 * 129 * DDR transfers microseconds KiB 130 * ------------- * ------------ * -------- 131 * microsecond PMU period transfer 132 */ 133 priv->nominal_bw = ddr_freq_mhz * pmu_period * priv->data_width / 1024; 134 } 135 136 static int sun8i_a33_mbus_set_dram_freq(struct sun8i_a33_mbus *priv, 137 unsigned long freq) 138 { 139 u32 ddr_freq_mhz = freq / USEC_PER_SEC; /* DDR */ 140 u32 dram_freq_mhz = ddr_freq_mhz / 2; /* SDR */ 141 u32 mctl_freq_mhz = dram_freq_mhz / 2; /* HDR */ 142 u32 dxodt, mdfscr, pwrctl, vtfcr; 143 u32 i, tREFI_32ck, tRFC_ck; 144 int ret; 145 146 /* The rate change is not effective until the MDFS process runs. */ 147 ret = clk_set_rate(priv->clk_dram, freq); 148 if (ret) 149 return ret; 150 151 /* Disable automatic self-refesh and VTF before starting MDFS. */ 152 pwrctl = readl_relaxed(priv->reg_dram + DRAM_PWRCTL) & 153 ~DRAM_PWRCTL_SELFREF_EN; 154 writel_relaxed(pwrctl, priv->reg_dram + DRAM_PWRCTL); 155 vtfcr = readl_relaxed(priv->reg_dram + DRAM_VTFCR); 156 writel_relaxed(vtfcr & ~DRAM_VTFCR_VTF_ENABLE, 157 priv->reg_dram + DRAM_VTFCR); 158 159 /* Set up MDFS and enable double buffering for timing registers. */ 160 mdfscr = MBUS_MDFSCR_MODE_DFS | 161 MBUS_MDFSCR_BYPASS | 162 MBUS_MDFSCR_PAD_HOLD | 163 MBUS_MDFSCR_BUFFER_TIMING; 164 writel(mdfscr, priv->reg_mbus + MBUS_MDFSCR); 165 166 /* Update the buffered copy of RFSHTMG. */ 167 tREFI_32ck = priv->tREFI_ns * mctl_freq_mhz / 1000 / 32; 168 tRFC_ck = DIV_ROUND_UP(priv->tRFC_ns * mctl_freq_mhz, 1000); 169 writel(DRAM_RFSHTMG_TREFI(tREFI_32ck) | DRAM_RFSHTMG_TRFC(tRFC_ck), 170 priv->reg_dram + DRAM_RFSHTMG); 171 172 /* Enable ODT if needed, or disable it to save power. */ 173 if (priv->odtmap && dram_freq_mhz > priv->variant->odt_freq_mhz) { 174 dxodt = DRAM_DXnGCR0_DXODT_DYNAMIC; 175 writel(priv->odtmap, priv->reg_dram + DRAM_ODTMAP); 176 } else { 177 dxodt = DRAM_DXnGCR0_DXODT_DISABLED; 178 writel(0, priv->reg_dram + DRAM_ODTMAP); 179 } 180 for (i = 0; i < DRAM_DX_MAX; ++i) { 181 void __iomem *reg = priv->reg_dram + DRAM_DXnGCR0(i); 182 183 writel((readl(reg) & ~DRAM_DXnGCR0_DXODT) | dxodt, reg); 184 } 185 186 dev_dbg(priv->devfreq_dram->dev.parent, 187 "Setting DRAM to %u MHz, tREFI=%u, tRFC=%u, ODT=%s\n", 188 dram_freq_mhz, tREFI_32ck, tRFC_ck, 189 dxodt == DRAM_DXnGCR0_DXODT_DYNAMIC ? "dynamic" : "disabled"); 190 191 /* Trigger hardware MDFS. */ 192 writel(mdfscr | MBUS_MDFSCR_START, priv->reg_mbus + MBUS_MDFSCR); 193 ret = readl_poll_timeout_atomic(priv->reg_mbus + MBUS_MDFSCR, mdfscr, 194 !(mdfscr & MBUS_MDFSCR_START), 10, 1000); 195 if (ret) 196 return ret; 197 198 /* Disable double buffering. */ 199 writel(0, priv->reg_mbus + MBUS_MDFSCR); 200 201 /* Restore VTF configuration. */ 202 writel_relaxed(vtfcr, priv->reg_dram + DRAM_VTFCR); 203 204 /* Enable automatic self-refresh at the lowest frequency only. */ 205 if (freq == priv->freq_table[0]) 206 pwrctl |= DRAM_PWRCTL_SELFREF_EN; 207 writel_relaxed(pwrctl, priv->reg_dram + DRAM_PWRCTL); 208 209 sun8i_a33_mbus_restart_pmu_counters(priv); 210 sun8i_a33_mbus_update_nominal_bw(priv, ddr_freq_mhz); 211 212 return 0; 213 } 214 215 static int sun8i_a33_mbus_set_dram_target(struct device *dev, 216 unsigned long *freq, u32 flags) 217 { 218 struct sun8i_a33_mbus *priv = dev_get_drvdata(dev); 219 struct devfreq *devfreq = priv->devfreq_dram; 220 struct dev_pm_opp *opp; 221 int ret; 222 223 opp = devfreq_recommended_opp(dev, freq, flags); 224 if (IS_ERR(opp)) 225 return PTR_ERR(opp); 226 227 dev_pm_opp_put(opp); 228 229 if (*freq == devfreq->previous_freq) 230 return 0; 231 232 ret = sun8i_a33_mbus_set_dram_freq(priv, *freq); 233 if (ret) { 234 dev_warn(dev, "failed to set DRAM frequency: %d\n", ret); 235 *freq = devfreq->previous_freq; 236 } 237 238 return ret; 239 } 240 241 static int sun8i_a33_mbus_get_dram_status(struct device *dev, 242 struct devfreq_dev_status *stat) 243 { 244 struct sun8i_a33_mbus *priv = dev_get_drvdata(dev); 245 246 stat->busy_time = sun8i_a33_mbus_get_peak_bw(priv); 247 stat->total_time = priv->nominal_bw; 248 stat->current_frequency = priv->devfreq_dram->previous_freq; 249 250 sun8i_a33_mbus_restart_pmu_counters(priv); 251 252 dev_dbg(dev, "Using %lu/%lu (%lu%%) at %lu MHz\n", 253 stat->busy_time, stat->total_time, 254 DIV_ROUND_CLOSEST(stat->busy_time * 100, stat->total_time), 255 stat->current_frequency / USEC_PER_SEC); 256 257 return 0; 258 } 259 260 static int sun8i_a33_mbus_hw_init(struct device *dev, 261 struct sun8i_a33_mbus *priv, 262 unsigned long ddr_freq) 263 { 264 u32 i, mbus_cr, mbus_freq_mhz; 265 266 /* Choose tREFI and tRFC to match the configured DRAM type. */ 267 mbus_cr = readl_relaxed(priv->reg_mbus + MBUS_CR); 268 switch (MBUS_CR_GET_DRAM_TYPE(mbus_cr)) { 269 case MBUS_CR_DRAM_TYPE_DDR2: 270 case MBUS_CR_DRAM_TYPE_DDR3: 271 case MBUS_CR_DRAM_TYPE_DDR4: 272 priv->tREFI_ns = 7800; 273 priv->tRFC_ns = 350; 274 break; 275 case MBUS_CR_DRAM_TYPE_LPDDR2: 276 case MBUS_CR_DRAM_TYPE_LPDDR3: 277 priv->tREFI_ns = 3900; 278 priv->tRFC_ns = 210; 279 break; 280 default: 281 return -EINVAL; 282 } 283 284 /* Save ODTMAP so it can be restored when raising the frequency. */ 285 priv->odtmap = readl_relaxed(priv->reg_dram + DRAM_ODTMAP); 286 287 /* Compute the DRAM data bus width by counting enabled DATx8 blocks. */ 288 for (i = 0; i < DRAM_DX_MAX; ++i) { 289 void __iomem *reg = priv->reg_dram + DRAM_DXnGCR0(i); 290 291 if (!(readl_relaxed(reg) & DRAM_DXnGCR0_DXEN)) 292 break; 293 } 294 priv->data_width = i; 295 296 dev_dbg(dev, "Detected %u-bit %sDDRx with%s ODT\n", 297 priv->data_width * 8, 298 MBUS_CR_GET_DRAM_TYPE(mbus_cr) > 4 ? "LP" : "", 299 priv->odtmap ? "" : "out"); 300 301 /* Program MBUS_TMR such that the PMU period unit is microseconds. */ 302 mbus_freq_mhz = clk_get_rate(priv->clk_mbus) / USEC_PER_SEC; 303 writel_relaxed(MBUS_TMR_PERIOD(mbus_freq_mhz), 304 priv->reg_mbus + MBUS_TMR); 305 306 /* "Master Ready Mask Register" bits must be set or MDFS will block. */ 307 writel_relaxed(0xffffffff, priv->reg_mbus + MBUS_MDFSMRMR); 308 309 sun8i_a33_mbus_restart_pmu_counters(priv); 310 sun8i_a33_mbus_update_nominal_bw(priv, ddr_freq / USEC_PER_SEC); 311 312 return 0; 313 } 314 315 static int __maybe_unused sun8i_a33_mbus_suspend(struct device *dev) 316 { 317 struct sun8i_a33_mbus *priv = dev_get_drvdata(dev); 318 319 clk_disable_unprepare(priv->clk_bus); 320 321 return 0; 322 } 323 324 static int __maybe_unused sun8i_a33_mbus_resume(struct device *dev) 325 { 326 struct sun8i_a33_mbus *priv = dev_get_drvdata(dev); 327 328 return clk_prepare_enable(priv->clk_bus); 329 } 330 331 static int sun8i_a33_mbus_probe(struct platform_device *pdev) 332 { 333 const struct sun8i_a33_mbus_variant *variant; 334 struct device *dev = &pdev->dev; 335 struct sun8i_a33_mbus *priv; 336 unsigned long base_freq; 337 unsigned int max_state; 338 const char *err; 339 int i, ret; 340 341 variant = device_get_match_data(dev); 342 if (!variant) 343 return -EINVAL; 344 345 max_state = variant->max_dram_divider - variant->min_dram_divider + 1; 346 347 priv = devm_kzalloc(dev, struct_size(priv, freq_table, max_state), GFP_KERNEL); 348 if (!priv) 349 return -ENOMEM; 350 351 platform_set_drvdata(pdev, priv); 352 353 priv->variant = variant; 354 355 priv->reg_dram = devm_platform_ioremap_resource_byname(pdev, "dram"); 356 if (IS_ERR(priv->reg_dram)) 357 return PTR_ERR(priv->reg_dram); 358 359 priv->reg_mbus = devm_platform_ioremap_resource_byname(pdev, "mbus"); 360 if (IS_ERR(priv->reg_mbus)) 361 return PTR_ERR(priv->reg_mbus); 362 363 priv->clk_bus = devm_clk_get(dev, "bus"); 364 if (IS_ERR(priv->clk_bus)) 365 return dev_err_probe(dev, PTR_ERR(priv->clk_bus), 366 "failed to get bus clock\n"); 367 368 priv->clk_dram = devm_clk_get(dev, "dram"); 369 if (IS_ERR(priv->clk_dram)) 370 return dev_err_probe(dev, PTR_ERR(priv->clk_dram), 371 "failed to get dram clock\n"); 372 373 priv->clk_mbus = devm_clk_get(dev, "mbus"); 374 if (IS_ERR(priv->clk_mbus)) 375 return dev_err_probe(dev, PTR_ERR(priv->clk_mbus), 376 "failed to get mbus clock\n"); 377 378 ret = clk_prepare_enable(priv->clk_bus); 379 if (ret) 380 return dev_err_probe(dev, ret, 381 "failed to enable bus clock\n"); 382 383 /* Lock the DRAM clock rate to keep priv->nominal_bw in sync. */ 384 ret = clk_rate_exclusive_get(priv->clk_dram); 385 if (ret) { 386 err = "failed to lock dram clock rate\n"; 387 goto err_disable_bus; 388 } 389 390 /* Lock the MBUS clock rate to keep MBUS_TMR_PERIOD in sync. */ 391 ret = clk_rate_exclusive_get(priv->clk_mbus); 392 if (ret) { 393 err = "failed to lock mbus clock rate\n"; 394 goto err_unlock_dram; 395 } 396 397 priv->gov_data.upthreshold = 10; 398 priv->gov_data.downdifferential = 5; 399 400 priv->profile.initial_freq = clk_get_rate(priv->clk_dram); 401 priv->profile.polling_ms = 1000; 402 priv->profile.target = sun8i_a33_mbus_set_dram_target; 403 priv->profile.get_dev_status = sun8i_a33_mbus_get_dram_status; 404 priv->profile.freq_table = priv->freq_table; 405 priv->profile.max_state = max_state; 406 407 ret = devm_pm_opp_set_clkname(dev, "dram"); 408 if (ret) { 409 err = "failed to add OPP table\n"; 410 goto err_unlock_mbus; 411 } 412 413 base_freq = clk_get_rate(clk_get_parent(priv->clk_dram)); 414 for (i = 0; i < max_state; ++i) { 415 unsigned int div = variant->max_dram_divider - i; 416 417 priv->freq_table[i] = base_freq / div; 418 419 ret = dev_pm_opp_add(dev, priv->freq_table[i], 0); 420 if (ret) { 421 err = "failed to add OPPs\n"; 422 goto err_remove_opps; 423 } 424 } 425 426 ret = sun8i_a33_mbus_hw_init(dev, priv, priv->profile.initial_freq); 427 if (ret) { 428 err = "failed to init hardware\n"; 429 goto err_remove_opps; 430 } 431 432 priv->devfreq_dram = devfreq_add_device(dev, &priv->profile, 433 DEVFREQ_GOV_SIMPLE_ONDEMAND, 434 &priv->gov_data); 435 if (IS_ERR(priv->devfreq_dram)) { 436 ret = PTR_ERR(priv->devfreq_dram); 437 err = "failed to add devfreq device\n"; 438 goto err_remove_opps; 439 } 440 441 /* 442 * This must be set manually after registering the devfreq device, 443 * because there is no way to select a dynamic OPP as the suspend OPP. 444 */ 445 priv->devfreq_dram->suspend_freq = priv->freq_table[0]; 446 447 return 0; 448 449 err_remove_opps: 450 dev_pm_opp_remove_all_dynamic(dev); 451 err_unlock_mbus: 452 clk_rate_exclusive_put(priv->clk_mbus); 453 err_unlock_dram: 454 clk_rate_exclusive_put(priv->clk_dram); 455 err_disable_bus: 456 clk_disable_unprepare(priv->clk_bus); 457 458 return dev_err_probe(dev, ret, err); 459 } 460 461 static void sun8i_a33_mbus_remove(struct platform_device *pdev) 462 { 463 struct sun8i_a33_mbus *priv = platform_get_drvdata(pdev); 464 unsigned long initial_freq = priv->profile.initial_freq; 465 struct device *dev = &pdev->dev; 466 int ret; 467 468 devfreq_remove_device(priv->devfreq_dram); 469 470 ret = sun8i_a33_mbus_set_dram_freq(priv, initial_freq); 471 if (ret) 472 dev_warn(dev, "failed to restore DRAM frequency: %d\n", ret); 473 474 dev_pm_opp_remove_all_dynamic(dev); 475 clk_rate_exclusive_put(priv->clk_mbus); 476 clk_rate_exclusive_put(priv->clk_dram); 477 clk_disable_unprepare(priv->clk_bus); 478 } 479 480 static const struct sun8i_a33_mbus_variant sun50i_a64_mbus = { 481 .min_dram_divider = 1, 482 .max_dram_divider = 4, 483 .odt_freq_mhz = 400, 484 }; 485 486 static const struct of_device_id sun8i_a33_mbus_of_match[] = { 487 { .compatible = "allwinner,sun50i-a64-mbus", .data = &sun50i_a64_mbus }, 488 { .compatible = "allwinner,sun50i-h5-mbus", .data = &sun50i_a64_mbus }, 489 { }, 490 }; 491 MODULE_DEVICE_TABLE(of, sun8i_a33_mbus_of_match); 492 493 static SIMPLE_DEV_PM_OPS(sun8i_a33_mbus_pm_ops, 494 sun8i_a33_mbus_suspend, sun8i_a33_mbus_resume); 495 496 static struct platform_driver sun8i_a33_mbus_driver = { 497 .probe = sun8i_a33_mbus_probe, 498 .remove_new = sun8i_a33_mbus_remove, 499 .driver = { 500 .name = "sun8i-a33-mbus", 501 .of_match_table = sun8i_a33_mbus_of_match, 502 .pm = pm_ptr(&sun8i_a33_mbus_pm_ops), 503 }, 504 }; 505 module_platform_driver(sun8i_a33_mbus_driver); 506 507 MODULE_AUTHOR("Samuel Holland <samuel@sholland.org>"); 508 MODULE_DESCRIPTION("Allwinner sun8i/sun50i MBUS DEVFREQ Driver"); 509 MODULE_LICENSE("GPL v2"); 510