1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (c) 2014, Fuzhou Rockchip Electronics Co., Ltd 4 */ 5 6 #include <linux/module.h> 7 #include <linux/platform_device.h> 8 #include <linux/clk.h> 9 #include <linux/hw_bitfield.h> 10 #include <linux/mmc/host.h> 11 #include <linux/of_address.h> 12 #include <linux/mmc/slot-gpio.h> 13 #include <linux/pm_runtime.h> 14 #include <linux/slab.h> 15 16 #include "dw_mmc.h" 17 #include "dw_mmc-pltfm.h" 18 19 #define RK3288_CLKGEN_DIV 2 20 #define SDMMC_TIMING_CON0 0x130 21 #define SDMMC_TIMING_CON1 0x134 22 #define SDMMC_MISC_CON 0x138 23 #define MEM_CLK_AUTOGATE_ENABLE BIT(5) 24 #define ROCKCHIP_MMC_DELAY_SEL BIT(10) 25 #define ROCKCHIP_MMC_DEGREE_MASK 0x3 26 #define ROCKCHIP_MMC_DEGREE_OFFSET 1 27 #define ROCKCHIP_MMC_DELAYNUM_OFFSET 2 28 #define ROCKCHIP_MMC_DELAYNUM_MASK (0xff << ROCKCHIP_MMC_DELAYNUM_OFFSET) 29 #define ROCKCHIP_MMC_DELAY_ELEMENT_PSEC 60 30 31 static const unsigned int freqs[] = { 100000, 200000, 300000, 400000 }; 32 33 struct dw_mci_rockchip_priv_data { 34 struct clk *drv_clk; 35 struct clk *sample_clk; 36 int default_sample_phase; 37 int num_phases; 38 bool internal_phase; 39 int sample_phase; 40 int drv_phase; 41 }; 42 43 /* 44 * Each fine delay is between 44ps-77ps. Assume each fine delay is 60ps to 45 * simplify calculations. So 45degs could be anywhere between 33deg and 57.8deg. 46 */ 47 static int rockchip_mmc_get_internal_phase(struct dw_mci *host, bool sample) 48 { 49 unsigned long rate = clk_get_rate(host->ciu_clk) / RK3288_CLKGEN_DIV; 50 u32 raw_value; 51 u16 degrees; 52 u32 delay_num = 0; 53 54 /* Constant signal, no measurable phase shift */ 55 if (!rate) 56 return 0; 57 58 if (sample) 59 raw_value = mci_readl(host, TIMING_CON1); 60 else 61 raw_value = mci_readl(host, TIMING_CON0); 62 63 raw_value >>= ROCKCHIP_MMC_DEGREE_OFFSET; 64 degrees = (raw_value & ROCKCHIP_MMC_DEGREE_MASK) * 90; 65 66 if (raw_value & ROCKCHIP_MMC_DELAY_SEL) { 67 /* degrees/delaynum * 1000000 */ 68 unsigned long factor = (ROCKCHIP_MMC_DELAY_ELEMENT_PSEC / 10) * 69 36 * (rate / 10000); 70 71 delay_num = (raw_value & ROCKCHIP_MMC_DELAYNUM_MASK); 72 delay_num >>= ROCKCHIP_MMC_DELAYNUM_OFFSET; 73 degrees += DIV_ROUND_CLOSEST(delay_num * factor, 1000000); 74 } 75 76 return degrees % 360; 77 } 78 79 static int rockchip_mmc_get_phase(struct dw_mci *host, bool sample) 80 { 81 struct dw_mci_rockchip_priv_data *priv = host->priv; 82 struct clk *clock = sample ? priv->sample_clk : priv->drv_clk; 83 84 if (priv->internal_phase) 85 return rockchip_mmc_get_internal_phase(host, sample); 86 else 87 return clk_get_phase(clock); 88 } 89 90 static int rockchip_mmc_set_internal_phase(struct dw_mci *host, bool sample, int degrees) 91 { 92 unsigned long rate = clk_get_rate(host->ciu_clk) / RK3288_CLKGEN_DIV; 93 u8 nineties, remainder; 94 u8 delay_num; 95 u32 raw_value; 96 u32 delay; 97 98 /* 99 * The below calculation is based on the output clock from 100 * MMC host to the card, which expects the phase clock inherits 101 * the clock rate from its parent, namely the output clock 102 * provider of MMC host. However, things may go wrong if 103 * (1) It is orphan. 104 * (2) It is assigned to the wrong parent. 105 * 106 * This check help debug the case (1), which seems to be the 107 * most likely problem we often face and which makes it difficult 108 * for people to debug unstable mmc tuning results. 109 */ 110 if (!rate) { 111 dev_err(host->dev, "%s: invalid clk rate\n", __func__); 112 return -EINVAL; 113 } 114 115 nineties = degrees / 90; 116 remainder = (degrees % 90); 117 118 /* 119 * Due to the inexact nature of the "fine" delay, we might 120 * actually go non-monotonic. We don't go _too_ monotonic 121 * though, so we should be OK. Here are options of how we may 122 * work: 123 * 124 * Ideally we end up with: 125 * 1.0, 2.0, ..., 69.0, 70.0, ..., 89.0, 90.0 126 * 127 * On one extreme (if delay is actually 44ps): 128 * .73, 1.5, ..., 50.6, 51.3, ..., 65.3, 90.0 129 * The other (if delay is actually 77ps): 130 * 1.3, 2.6, ..., 88.6. 89.8, ..., 114.0, 90 131 * 132 * It's possible we might make a delay that is up to 25 133 * degrees off from what we think we're making. That's OK 134 * though because we should be REALLY far from any bad range. 135 */ 136 137 /* 138 * Convert to delay; do a little extra work to make sure we 139 * don't overflow 32-bit / 64-bit numbers. 140 */ 141 delay = 10000000; /* PSECS_PER_SEC / 10000 / 10 */ 142 delay *= remainder; 143 delay = DIV_ROUND_CLOSEST(delay, 144 (rate / 1000) * 36 * 145 (ROCKCHIP_MMC_DELAY_ELEMENT_PSEC / 10)); 146 147 delay_num = (u8) min_t(u32, delay, 255); 148 149 raw_value = delay_num ? ROCKCHIP_MMC_DELAY_SEL : 0; 150 raw_value |= delay_num << ROCKCHIP_MMC_DELAYNUM_OFFSET; 151 raw_value |= nineties; 152 153 if (sample) 154 mci_writel(host, TIMING_CON1, 155 FIELD_PREP_WM16(GENMASK(11, 1), raw_value)); 156 else 157 mci_writel(host, TIMING_CON0, 158 FIELD_PREP_WM16(GENMASK(11, 1), raw_value)); 159 160 dev_dbg(host->dev, "set %s_phase(%d) delay_nums=%u actual_degrees=%d\n", 161 sample ? "sample" : "drv", degrees, delay_num, 162 rockchip_mmc_get_phase(host, sample) 163 ); 164 165 return 0; 166 } 167 168 static int rockchip_mmc_set_phase(struct dw_mci *host, bool sample, int degrees) 169 { 170 struct dw_mci_rockchip_priv_data *priv = host->priv; 171 struct clk *clock = sample ? priv->sample_clk : priv->drv_clk; 172 173 if (priv->internal_phase) 174 return rockchip_mmc_set_internal_phase(host, sample, degrees); 175 else 176 return clk_set_phase(clock, degrees); 177 } 178 179 static void dw_mci_rk3288_set_ios(struct dw_mci *host, struct mmc_ios *ios) 180 { 181 struct dw_mci_rockchip_priv_data *priv = host->priv; 182 int ret; 183 unsigned int cclkin; 184 u32 bus_hz; 185 186 if (ios->clock == 0) 187 return; 188 189 /* 190 * cclkin: source clock of mmc controller 191 * bus_hz: card interface clock generated by CLKGEN 192 * bus_hz = cclkin / RK3288_CLKGEN_DIV 193 * ios->clock = (div == 0) ? bus_hz : (bus_hz / (2 * div)) 194 * 195 * Note: div can only be 0 or 1, but div must be set to 1 for eMMC 196 * DDR52 8-bit mode. 197 */ 198 if (ios->bus_width == MMC_BUS_WIDTH_8 && 199 ios->timing == MMC_TIMING_MMC_DDR52) 200 cclkin = 2 * ios->clock * RK3288_CLKGEN_DIV; 201 else 202 cclkin = ios->clock * RK3288_CLKGEN_DIV; 203 204 ret = clk_set_rate(host->ciu_clk, cclkin); 205 if (ret) 206 dev_warn(host->dev, "failed to set rate %uHz err: %d\n", cclkin, ret); 207 208 bus_hz = clk_get_rate(host->ciu_clk) / RK3288_CLKGEN_DIV; 209 if (bus_hz != host->bus_hz) { 210 host->bus_hz = bus_hz; 211 /* force dw_mci_setup_bus() */ 212 host->current_speed = 0; 213 } 214 215 /* Make sure we use phases which we can enumerate with */ 216 if (!IS_ERR(priv->sample_clk) && ios->timing <= MMC_TIMING_SD_HS) 217 rockchip_mmc_set_phase(host, true, priv->default_sample_phase); 218 219 /* 220 * Set the drive phase offset based on speed mode to achieve hold times. 221 * 222 * NOTE: this is _not_ a value that is dynamically tuned and is also 223 * _not_ a value that will vary from board to board. It is a value 224 * that could vary between different SoC models if they had massively 225 * different output clock delays inside their dw_mmc IP block (delay_o), 226 * but since it's OK to overshoot a little we don't need to do complex 227 * calculations and can pick values that will just work for everyone. 228 * 229 * When picking values we'll stick with picking 0/90/180/270 since 230 * those can be made very accurately on all known Rockchip SoCs. 231 * 232 * Note that these values match values from the DesignWare Databook 233 * tables for the most part except for SDR12 and "ID mode". For those 234 * two modes the databook calculations assume a clock in of 50MHz. As 235 * seen above, we always use a clock in rate that is exactly the 236 * card's input clock (times RK3288_CLKGEN_DIV, but that gets divided 237 * back out before the controller sees it). 238 * 239 * From measurement of a single device, it appears that delay_o is 240 * about .5 ns. Since we try to leave a bit of margin, it's expected 241 * that numbers here will be fine even with much larger delay_o 242 * (the 1.4 ns assumed by the DesignWare Databook would result in the 243 * same results, for instance). 244 */ 245 if (!IS_ERR(priv->drv_clk)) { 246 int phase; 247 248 /* 249 * In almost all cases a 90 degree phase offset will provide 250 * sufficient hold times across all valid input clock rates 251 * assuming delay_o is not absurd for a given SoC. We'll use 252 * that as a default. 253 */ 254 phase = 90; 255 256 switch (ios->timing) { 257 case MMC_TIMING_MMC_DDR52: 258 /* 259 * Since clock in rate with MMC_DDR52 is doubled when 260 * bus width is 8 we need to double the phase offset 261 * to get the same timings. 262 */ 263 if (ios->bus_width == MMC_BUS_WIDTH_8) 264 phase = 180; 265 break; 266 case MMC_TIMING_UHS_SDR104: 267 case MMC_TIMING_MMC_HS200: 268 /* 269 * In the case of 150 MHz clock (typical max for 270 * Rockchip SoCs), 90 degree offset will add a delay 271 * of 1.67 ns. That will meet min hold time of .8 ns 272 * as long as clock output delay is < .87 ns. On 273 * SoCs measured this seems to be OK, but it doesn't 274 * hurt to give margin here, so we use 180. 275 */ 276 phase = 180; 277 break; 278 } 279 280 rockchip_mmc_set_phase(host, false, phase); 281 } 282 } 283 284 #define TUNING_ITERATION_TO_PHASE(i, num_phases) \ 285 (DIV_ROUND_UP((i) * 360, num_phases)) 286 287 static int dw_mci_rk3288_execute_tuning(struct dw_mci_slot *slot, u32 opcode) 288 { 289 struct dw_mci *host = slot->host; 290 struct dw_mci_rockchip_priv_data *priv = host->priv; 291 struct mmc_host *mmc = slot->mmc; 292 int ret = 0; 293 int i; 294 bool v, prev_v = 0, first_v; 295 struct range_t { 296 int start; 297 int end; /* inclusive */ 298 }; 299 struct range_t *ranges; 300 unsigned int range_count = 0; 301 int longest_range_len = -1; 302 int longest_range = -1; 303 int middle_phase; 304 int phase; 305 306 if (IS_ERR(priv->sample_clk)) { 307 dev_err(host->dev, "Tuning clock (sample_clk) not defined.\n"); 308 return -EIO; 309 } 310 311 ranges = kmalloc_objs(*ranges, priv->num_phases / 2 + 1); 312 if (!ranges) 313 return -ENOMEM; 314 315 /* Try each phase and extract good ranges */ 316 for (i = 0; i < priv->num_phases; ) { 317 rockchip_mmc_set_phase(host, true, 318 TUNING_ITERATION_TO_PHASE( 319 i, 320 priv->num_phases)); 321 322 v = !mmc_send_tuning(mmc, opcode, NULL); 323 324 if (i == 0) 325 first_v = v; 326 327 if ((!prev_v) && v) { 328 range_count++; 329 ranges[range_count-1].start = i; 330 } 331 if (v) { 332 ranges[range_count-1].end = i; 333 i++; 334 } else if (i == priv->num_phases - 1) { 335 /* No extra skipping rules if we're at the end */ 336 i++; 337 } else { 338 /* 339 * No need to check too close to an invalid 340 * one since testing bad phases is slow. Skip 341 * 20 degrees. 342 */ 343 i += DIV_ROUND_UP(20 * priv->num_phases, 360); 344 345 /* Always test the last one */ 346 if (i >= priv->num_phases) 347 i = priv->num_phases - 1; 348 } 349 350 prev_v = v; 351 } 352 353 if (range_count == 0) { 354 dev_warn(host->dev, "All phases bad!"); 355 ret = -EIO; 356 goto free; 357 } 358 359 /* wrap around case, merge the end points */ 360 if ((range_count > 1) && first_v && v) { 361 ranges[0].start = ranges[range_count-1].start; 362 range_count--; 363 } 364 365 if (ranges[0].start == 0 && ranges[0].end == priv->num_phases - 1) { 366 rockchip_mmc_set_phase(host, true, priv->default_sample_phase); 367 368 dev_info(host->dev, "All phases work, using default phase %d.", 369 priv->default_sample_phase); 370 goto free; 371 } 372 373 /* Find the longest range */ 374 for (i = 0; i < range_count; i++) { 375 int len = (ranges[i].end - ranges[i].start + 1); 376 377 if (len < 0) 378 len += priv->num_phases; 379 380 if (longest_range_len < len) { 381 longest_range_len = len; 382 longest_range = i; 383 } 384 385 dev_dbg(host->dev, "Good phase range %d-%d (%d len)\n", 386 TUNING_ITERATION_TO_PHASE(ranges[i].start, 387 priv->num_phases), 388 TUNING_ITERATION_TO_PHASE(ranges[i].end, 389 priv->num_phases), 390 len 391 ); 392 } 393 394 dev_dbg(host->dev, "Best phase range %d-%d (%d len)\n", 395 TUNING_ITERATION_TO_PHASE(ranges[longest_range].start, 396 priv->num_phases), 397 TUNING_ITERATION_TO_PHASE(ranges[longest_range].end, 398 priv->num_phases), 399 longest_range_len 400 ); 401 402 middle_phase = ranges[longest_range].start + longest_range_len / 2; 403 middle_phase %= priv->num_phases; 404 phase = TUNING_ITERATION_TO_PHASE(middle_phase, priv->num_phases); 405 dev_info(host->dev, "Successfully tuned phase to %d\n", phase); 406 407 rockchip_mmc_set_phase(host, true, phase); 408 409 free: 410 kfree(ranges); 411 return ret; 412 } 413 414 static int dw_mci_common_parse_dt(struct dw_mci *host) 415 { 416 struct device_node *np = host->dev->of_node; 417 struct dw_mci_rockchip_priv_data *priv; 418 419 priv = devm_kzalloc(host->dev, sizeof(*priv), GFP_KERNEL); 420 if (!priv) 421 return -ENOMEM; 422 423 if (of_property_read_u32(np, "rockchip,desired-num-phases", 424 &priv->num_phases)) 425 priv->num_phases = 360; 426 427 if (of_property_read_u32(np, "rockchip,default-sample-phase", 428 &priv->default_sample_phase)) 429 priv->default_sample_phase = 0; 430 431 host->priv = priv; 432 433 return 0; 434 } 435 436 static int dw_mci_rk3288_parse_dt(struct dw_mci *host) 437 { 438 struct dw_mci_rockchip_priv_data *priv; 439 int err; 440 441 err = dw_mci_common_parse_dt(host); 442 if (err) 443 return err; 444 445 priv = host->priv; 446 447 priv->drv_clk = devm_clk_get(host->dev, "ciu-drive"); 448 if (IS_ERR(priv->drv_clk)) 449 dev_dbg(host->dev, "ciu-drive not available\n"); 450 451 priv->sample_clk = devm_clk_get(host->dev, "ciu-sample"); 452 if (IS_ERR(priv->sample_clk)) 453 dev_dbg(host->dev, "ciu-sample not available\n"); 454 455 priv->internal_phase = false; 456 457 return 0; 458 } 459 460 static int dw_mci_rk3576_parse_dt(struct dw_mci *host) 461 { 462 struct dw_mci_rockchip_priv_data *priv; 463 int err = dw_mci_common_parse_dt(host); 464 if (err) 465 return err; 466 467 priv = host->priv; 468 469 priv->internal_phase = true; 470 471 return 0; 472 } 473 474 static int dw_mci_rockchip_init(struct dw_mci *host) 475 { 476 struct dw_mci_rockchip_priv_data *priv = host->priv; 477 int ret, i; 478 479 /* It is slot 8 on Rockchip SoCs */ 480 host->sdio_id0 = 8; 481 482 if (of_device_is_compatible(host->dev->of_node, "rockchip,rk3288-dw-mshc")) { 483 host->bus_hz /= RK3288_CLKGEN_DIV; 484 485 /* clock driver will fail if the clock is less than the lowest source clock 486 * divided by the internal clock divider. Test for the lowest available 487 * clock and set the minimum freq to clock / clock divider. 488 */ 489 490 for (i = 0; i < ARRAY_SIZE(freqs); i++) { 491 ret = clk_round_rate(host->ciu_clk, freqs[i] * RK3288_CLKGEN_DIV); 492 if (ret > 0) { 493 host->minimum_speed = ret / RK3288_CLKGEN_DIV; 494 break; 495 } 496 } 497 if (ret < 0) 498 dev_warn(host->dev, "no valid minimum freq: %d\n", ret); 499 } 500 501 if (priv->internal_phase) 502 mci_writel(host, MISC_CON, MEM_CLK_AUTOGATE_ENABLE); 503 504 return 0; 505 } 506 507 static const struct dw_mci_drv_data rk2928_drv_data = { 508 .init = dw_mci_rockchip_init, 509 }; 510 511 static const struct dw_mci_drv_data rk3288_drv_data = { 512 .common_caps = MMC_CAP_CMD23, 513 .set_ios = dw_mci_rk3288_set_ios, 514 .execute_tuning = dw_mci_rk3288_execute_tuning, 515 .parse_dt = dw_mci_rk3288_parse_dt, 516 .init = dw_mci_rockchip_init, 517 }; 518 519 static const struct dw_mci_drv_data rk3576_drv_data = { 520 .common_caps = MMC_CAP_CMD23, 521 .set_ios = dw_mci_rk3288_set_ios, 522 .execute_tuning = dw_mci_rk3288_execute_tuning, 523 .parse_dt = dw_mci_rk3576_parse_dt, 524 .init = dw_mci_rockchip_init, 525 }; 526 527 static const struct of_device_id dw_mci_rockchip_match[] = { 528 { .compatible = "rockchip,rk2928-dw-mshc", 529 .data = &rk2928_drv_data }, 530 { .compatible = "rockchip,rk3288-dw-mshc", 531 .data = &rk3288_drv_data }, 532 { .compatible = "rockchip,rk3576-dw-mshc", 533 .data = &rk3576_drv_data }, 534 {}, 535 }; 536 MODULE_DEVICE_TABLE(of, dw_mci_rockchip_match); 537 538 static int dw_mci_rockchip_probe(struct platform_device *pdev) 539 { 540 const struct dw_mci_drv_data *drv_data; 541 const struct of_device_id *match; 542 int ret; 543 544 if (!pdev->dev.of_node) 545 return -ENODEV; 546 547 match = of_match_node(dw_mci_rockchip_match, pdev->dev.of_node); 548 drv_data = match->data; 549 550 pm_runtime_get_noresume(&pdev->dev); 551 pm_runtime_set_active(&pdev->dev); 552 pm_runtime_enable(&pdev->dev); 553 pm_runtime_set_autosuspend_delay(&pdev->dev, 50); 554 pm_runtime_use_autosuspend(&pdev->dev); 555 556 ret = dw_mci_pltfm_register(pdev, drv_data); 557 if (ret) { 558 pm_runtime_disable(&pdev->dev); 559 pm_runtime_set_suspended(&pdev->dev); 560 pm_runtime_put_noidle(&pdev->dev); 561 return ret; 562 } 563 564 pm_runtime_put_autosuspend(&pdev->dev); 565 566 return 0; 567 } 568 569 static void dw_mci_rockchip_remove(struct platform_device *pdev) 570 { 571 pm_runtime_get_sync(&pdev->dev); 572 pm_runtime_disable(&pdev->dev); 573 pm_runtime_put_noidle(&pdev->dev); 574 575 dw_mci_pltfm_remove(pdev); 576 } 577 578 static int dw_mci_rockchip_runtime_suspend(struct device *dev) 579 { 580 struct platform_device *pdev = to_platform_device(dev); 581 struct dw_mci *host = platform_get_drvdata(pdev); 582 struct dw_mci_rockchip_priv_data *priv = host->priv; 583 584 if (priv->internal_phase) { 585 priv->sample_phase = rockchip_mmc_get_phase(host, true); 586 priv->drv_phase = rockchip_mmc_get_phase(host, false); 587 } 588 589 return dw_mci_runtime_suspend(dev); 590 } 591 592 static int dw_mci_rockchip_runtime_resume(struct device *dev) 593 { 594 struct platform_device *pdev = to_platform_device(dev); 595 struct dw_mci *host = platform_get_drvdata(pdev); 596 struct dw_mci_rockchip_priv_data *priv = host->priv; 597 int ret; 598 599 ret = dw_mci_runtime_resume(dev); 600 if (ret) 601 return ret; 602 603 if (priv->internal_phase) { 604 rockchip_mmc_set_phase(host, true, priv->sample_phase); 605 rockchip_mmc_set_phase(host, false, priv->drv_phase); 606 mci_writel(host, MISC_CON, MEM_CLK_AUTOGATE_ENABLE); 607 } 608 609 return ret; 610 } 611 612 static const struct dev_pm_ops dw_mci_rockchip_dev_pm_ops = { 613 SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 614 RUNTIME_PM_OPS(dw_mci_rockchip_runtime_suspend, dw_mci_rockchip_runtime_resume, NULL) 615 }; 616 617 static struct platform_driver dw_mci_rockchip_pltfm_driver = { 618 .probe = dw_mci_rockchip_probe, 619 .remove = dw_mci_rockchip_remove, 620 .driver = { 621 .name = "dwmmc_rockchip", 622 .probe_type = PROBE_PREFER_ASYNCHRONOUS, 623 .of_match_table = dw_mci_rockchip_match, 624 .pm = pm_ptr(&dw_mci_rockchip_dev_pm_ops), 625 }, 626 }; 627 628 module_platform_driver(dw_mci_rockchip_pltfm_driver); 629 630 MODULE_AUTHOR("Addy Ke <addy.ke@rock-chips.com>"); 631 MODULE_DESCRIPTION("Rockchip Specific DW-MSHC Driver Extension"); 632 MODULE_ALIAS("platform:dwmmc_rockchip"); 633 MODULE_LICENSE("GPL v2"); 634