1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Tegra20 External Memory Controller driver 4 * 5 * Author: Dmitry Osipenko <digetx@gmail.com> 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/clk.h> 10 #include <linux/clk/tegra.h> 11 #include <linux/debugfs.h> 12 #include <linux/devfreq.h> 13 #include <linux/err.h> 14 #include <linux/interconnect-provider.h> 15 #include <linux/interrupt.h> 16 #include <linux/io.h> 17 #include <linux/iopoll.h> 18 #include <linux/kernel.h> 19 #include <linux/module.h> 20 #include <linux/mutex.h> 21 #include <linux/of.h> 22 #include <linux/platform_device.h> 23 #include <linux/pm_opp.h> 24 #include <linux/slab.h> 25 #include <linux/sort.h> 26 #include <linux/types.h> 27 28 #include <soc/tegra/common.h> 29 #include <soc/tegra/fuse.h> 30 31 #include "../jedec_ddr.h" 32 #include "../of_memory.h" 33 34 #include "mc.h" 35 #include "tegra-emc-common.h" 36 37 #define EMC_INTSTATUS 0x000 38 #define EMC_INTMASK 0x004 39 #define EMC_DBG 0x008 40 #define EMC_ADR_CFG_0 0x010 41 #define EMC_TIMING_CONTROL 0x028 42 #define EMC_RC 0x02c 43 #define EMC_RFC 0x030 44 #define EMC_RAS 0x034 45 #define EMC_RP 0x038 46 #define EMC_R2W 0x03c 47 #define EMC_W2R 0x040 48 #define EMC_R2P 0x044 49 #define EMC_W2P 0x048 50 #define EMC_RD_RCD 0x04c 51 #define EMC_WR_RCD 0x050 52 #define EMC_RRD 0x054 53 #define EMC_REXT 0x058 54 #define EMC_WDV 0x05c 55 #define EMC_QUSE 0x060 56 #define EMC_QRST 0x064 57 #define EMC_QSAFE 0x068 58 #define EMC_RDV 0x06c 59 #define EMC_REFRESH 0x070 60 #define EMC_BURST_REFRESH_NUM 0x074 61 #define EMC_PDEX2WR 0x078 62 #define EMC_PDEX2RD 0x07c 63 #define EMC_PCHG2PDEN 0x080 64 #define EMC_ACT2PDEN 0x084 65 #define EMC_AR2PDEN 0x088 66 #define EMC_RW2PDEN 0x08c 67 #define EMC_TXSR 0x090 68 #define EMC_TCKE 0x094 69 #define EMC_TFAW 0x098 70 #define EMC_TRPAB 0x09c 71 #define EMC_TCLKSTABLE 0x0a0 72 #define EMC_TCLKSTOP 0x0a4 73 #define EMC_TREFBW 0x0a8 74 #define EMC_QUSE_EXTRA 0x0ac 75 #define EMC_ODT_WRITE 0x0b0 76 #define EMC_ODT_READ 0x0b4 77 #define EMC_MRR 0x0ec 78 #define EMC_FBIO_CFG5 0x104 79 #define EMC_FBIO_CFG6 0x114 80 #define EMC_STAT_CONTROL 0x160 81 #define EMC_STAT_LLMC_CONTROL 0x178 82 #define EMC_STAT_PWR_CLOCK_LIMIT 0x198 83 #define EMC_STAT_PWR_CLOCKS 0x19c 84 #define EMC_STAT_PWR_COUNT 0x1a0 85 #define EMC_AUTO_CAL_INTERVAL 0x2a8 86 #define EMC_CFG_2 0x2b8 87 #define EMC_CFG_DIG_DLL 0x2bc 88 #define EMC_DLL_XFORM_DQS 0x2c0 89 #define EMC_DLL_XFORM_QUSE 0x2c4 90 #define EMC_ZCAL_REF_CNT 0x2e0 91 #define EMC_ZCAL_WAIT_CNT 0x2e4 92 #define EMC_CFG_CLKTRIM_0 0x2d0 93 #define EMC_CFG_CLKTRIM_1 0x2d4 94 #define EMC_CFG_CLKTRIM_2 0x2d8 95 96 #define EMC_CLKCHANGE_REQ_ENABLE BIT(0) 97 #define EMC_CLKCHANGE_PD_ENABLE BIT(1) 98 #define EMC_CLKCHANGE_SR_ENABLE BIT(2) 99 100 #define EMC_TIMING_UPDATE BIT(0) 101 102 #define EMC_REFRESH_OVERFLOW_INT BIT(3) 103 #define EMC_CLKCHANGE_COMPLETE_INT BIT(4) 104 #define EMC_MRR_DIVLD_INT BIT(5) 105 106 #define EMC_DBG_READ_MUX_ASSEMBLY BIT(0) 107 #define EMC_DBG_WRITE_MUX_ACTIVE BIT(1) 108 #define EMC_DBG_FORCE_UPDATE BIT(2) 109 #define EMC_DBG_READ_DQM_CTRL BIT(9) 110 #define EMC_DBG_CFG_PRIORITY BIT(24) 111 112 #define EMC_FBIO_CFG5_DRAM_WIDTH_X16 BIT(4) 113 #define EMC_FBIO_CFG5_DRAM_TYPE GENMASK(1, 0) 114 115 #define EMC_MRR_DEV_SELECTN GENMASK(31, 30) 116 #define EMC_MRR_MRR_MA GENMASK(23, 16) 117 #define EMC_MRR_MRR_DATA GENMASK(15, 0) 118 119 #define EMC_ADR_CFG_0_EMEM_NUMDEV GENMASK(25, 24) 120 121 #define EMC_PWR_GATHER_CLEAR (1 << 8) 122 #define EMC_PWR_GATHER_DISABLE (2 << 8) 123 #define EMC_PWR_GATHER_ENABLE (3 << 8) 124 125 enum emc_dram_type { 126 DRAM_TYPE_RESERVED, 127 DRAM_TYPE_DDR1, 128 DRAM_TYPE_LPDDR2, 129 DRAM_TYPE_DDR2, 130 }; 131 132 static const u16 emc_timing_registers[] = { 133 EMC_RC, 134 EMC_RFC, 135 EMC_RAS, 136 EMC_RP, 137 EMC_R2W, 138 EMC_W2R, 139 EMC_R2P, 140 EMC_W2P, 141 EMC_RD_RCD, 142 EMC_WR_RCD, 143 EMC_RRD, 144 EMC_REXT, 145 EMC_WDV, 146 EMC_QUSE, 147 EMC_QRST, 148 EMC_QSAFE, 149 EMC_RDV, 150 EMC_REFRESH, 151 EMC_BURST_REFRESH_NUM, 152 EMC_PDEX2WR, 153 EMC_PDEX2RD, 154 EMC_PCHG2PDEN, 155 EMC_ACT2PDEN, 156 EMC_AR2PDEN, 157 EMC_RW2PDEN, 158 EMC_TXSR, 159 EMC_TCKE, 160 EMC_TFAW, 161 EMC_TRPAB, 162 EMC_TCLKSTABLE, 163 EMC_TCLKSTOP, 164 EMC_TREFBW, 165 EMC_QUSE_EXTRA, 166 EMC_FBIO_CFG6, 167 EMC_ODT_WRITE, 168 EMC_ODT_READ, 169 EMC_FBIO_CFG5, 170 EMC_CFG_DIG_DLL, 171 EMC_DLL_XFORM_DQS, 172 EMC_DLL_XFORM_QUSE, 173 EMC_ZCAL_REF_CNT, 174 EMC_ZCAL_WAIT_CNT, 175 EMC_AUTO_CAL_INTERVAL, 176 EMC_CFG_CLKTRIM_0, 177 EMC_CFG_CLKTRIM_1, 178 EMC_CFG_CLKTRIM_2, 179 }; 180 181 struct emc_timing { 182 unsigned long rate; 183 u32 data[ARRAY_SIZE(emc_timing_registers)]; 184 }; 185 186 struct tegra_emc { 187 struct device *dev; 188 struct tegra_mc *mc; 189 struct icc_provider provider; 190 struct notifier_block clk_nb; 191 struct clk *clk; 192 void __iomem *regs; 193 unsigned int dram_bus_width; 194 195 struct emc_timing *timings; 196 unsigned int num_timings; 197 198 struct { 199 struct dentry *root; 200 unsigned long min_rate; 201 unsigned long max_rate; 202 } debugfs; 203 204 struct tegra_emc_rate_requests reqs; 205 206 struct devfreq_simple_ondemand_data ondemand_data; 207 208 /* memory chip identity information */ 209 union lpddr2_basic_config4 basic_conf4; 210 unsigned int manufacturer_id; 211 unsigned int revision_id1; 212 unsigned int revision_id2; 213 214 bool mrr_error; 215 }; 216 217 static irqreturn_t tegra20_emc_isr(int irq, void *data) 218 { 219 struct tegra_emc *emc = data; 220 u32 intmask = EMC_REFRESH_OVERFLOW_INT; 221 u32 status; 222 223 status = readl_relaxed(emc->regs + EMC_INTSTATUS) & intmask; 224 if (!status) 225 return IRQ_NONE; 226 227 /* notify about HW problem */ 228 if (status & EMC_REFRESH_OVERFLOW_INT) 229 dev_err_ratelimited(emc->dev, 230 "refresh request overflow timeout\n"); 231 232 /* clear interrupts */ 233 writel_relaxed(status, emc->regs + EMC_INTSTATUS); 234 235 return IRQ_HANDLED; 236 } 237 238 static struct emc_timing *tegra20_emc_find_timing(struct tegra_emc *emc, 239 unsigned long rate) 240 { 241 struct emc_timing *timing = NULL; 242 unsigned int i; 243 244 for (i = 0; i < emc->num_timings; i++) { 245 if (emc->timings[i].rate >= rate) { 246 timing = &emc->timings[i]; 247 break; 248 } 249 } 250 251 if (!timing) { 252 dev_err(emc->dev, "no timing for rate %lu\n", rate); 253 return NULL; 254 } 255 256 return timing; 257 } 258 259 static int emc_prepare_timing_change(struct tegra_emc *emc, unsigned long rate) 260 { 261 struct emc_timing *timing = tegra20_emc_find_timing(emc, rate); 262 unsigned int i; 263 264 if (!timing) 265 return -EINVAL; 266 267 dev_dbg(emc->dev, "%s: using timing rate %lu for requested rate %lu\n", 268 __func__, timing->rate, rate); 269 270 /* program shadow registers */ 271 for (i = 0; i < ARRAY_SIZE(timing->data); i++) 272 writel_relaxed(timing->data[i], 273 emc->regs + emc_timing_registers[i]); 274 275 /* wait until programming has settled */ 276 readl_relaxed(emc->regs + emc_timing_registers[i - 1]); 277 278 return 0; 279 } 280 281 static int emc_complete_timing_change(struct tegra_emc *emc, bool flush) 282 { 283 int err; 284 u32 v; 285 286 dev_dbg(emc->dev, "%s: flush %d\n", __func__, flush); 287 288 if (flush) { 289 /* manually initiate memory timing update */ 290 writel_relaxed(EMC_TIMING_UPDATE, 291 emc->regs + EMC_TIMING_CONTROL); 292 return 0; 293 } 294 295 err = readl_relaxed_poll_timeout_atomic(emc->regs + EMC_INTSTATUS, v, 296 v & EMC_CLKCHANGE_COMPLETE_INT, 297 1, 100); 298 if (err) { 299 dev_err(emc->dev, "emc-car handshake timeout: %d\n", err); 300 return err; 301 } 302 303 return 0; 304 } 305 306 static int tegra20_emc_clk_change_notify(struct notifier_block *nb, 307 unsigned long msg, void *data) 308 { 309 struct tegra_emc *emc = container_of(nb, struct tegra_emc, clk_nb); 310 struct clk_notifier_data *cnd = data; 311 int err; 312 313 switch (msg) { 314 case PRE_RATE_CHANGE: 315 err = emc_prepare_timing_change(emc, cnd->new_rate); 316 break; 317 318 case ABORT_RATE_CHANGE: 319 err = emc_prepare_timing_change(emc, cnd->old_rate); 320 if (err) 321 break; 322 323 err = emc_complete_timing_change(emc, true); 324 break; 325 326 case POST_RATE_CHANGE: 327 err = emc_complete_timing_change(emc, false); 328 break; 329 330 default: 331 return NOTIFY_DONE; 332 } 333 334 return notifier_from_errno(err); 335 } 336 337 static int load_one_timing_from_dt(struct tegra_emc *emc, 338 struct emc_timing *timing, 339 struct device_node *node) 340 { 341 u32 rate; 342 int err; 343 344 if (!of_device_is_compatible(node, "nvidia,tegra20-emc-table")) { 345 dev_err(emc->dev, "incompatible DT node: %pOF\n", node); 346 return -EINVAL; 347 } 348 349 err = of_property_read_u32(node, "clock-frequency", &rate); 350 if (err) { 351 dev_err(emc->dev, "timing %pOF: failed to read rate: %d\n", 352 node, err); 353 return err; 354 } 355 356 err = of_property_read_u32_array(node, "nvidia,emc-registers", 357 timing->data, 358 ARRAY_SIZE(emc_timing_registers)); 359 if (err) { 360 dev_err(emc->dev, 361 "timing %pOF: failed to read emc timing data: %d\n", 362 node, err); 363 return err; 364 } 365 366 /* 367 * The EMC clock rate is twice the bus rate, and the bus rate is 368 * measured in kHz. 369 */ 370 timing->rate = rate * 2 * 1000; 371 372 dev_dbg(emc->dev, "%s: %pOF: EMC rate %lu\n", 373 __func__, node, timing->rate); 374 375 return 0; 376 } 377 378 static int cmp_timings(const void *_a, const void *_b) 379 { 380 const struct emc_timing *a = _a; 381 const struct emc_timing *b = _b; 382 383 if (a->rate < b->rate) 384 return -1; 385 386 if (a->rate > b->rate) 387 return 1; 388 389 return 0; 390 } 391 392 static int tegra20_emc_load_timings_from_dt(struct tegra_emc *emc, 393 struct device_node *node) 394 { 395 struct emc_timing *timing; 396 int child_count; 397 int err; 398 399 child_count = of_get_child_count(node); 400 if (!child_count) { 401 dev_err(emc->dev, "no memory timings in DT node: %pOF\n", node); 402 return -EINVAL; 403 } 404 405 emc->timings = devm_kcalloc(emc->dev, child_count, sizeof(*timing), 406 GFP_KERNEL); 407 if (!emc->timings) 408 return -ENOMEM; 409 410 timing = emc->timings; 411 412 for_each_child_of_node_scoped(node, child) { 413 if (of_node_name_eq(child, "lpddr2")) 414 continue; 415 416 err = load_one_timing_from_dt(emc, timing++, child); 417 if (err) 418 return err; 419 420 emc->num_timings++; 421 } 422 423 sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings, 424 NULL); 425 426 dev_info_once(emc->dev, 427 "got %u timings for RAM code %u (min %luMHz max %luMHz)\n", 428 emc->num_timings, 429 tegra_read_ram_code(), 430 emc->timings[0].rate / 1000000, 431 emc->timings[emc->num_timings - 1].rate / 1000000); 432 433 return 0; 434 } 435 436 static struct device_node * 437 tegra20_emc_find_node_by_ram_code(struct tegra_emc *emc) 438 { 439 struct device *dev = emc->dev; 440 struct device_node *np; 441 u32 value, ram_code; 442 int err; 443 444 if (emc->mrr_error) { 445 dev_warn(dev, "memory timings skipped due to MRR error\n"); 446 return NULL; 447 } 448 449 if (of_get_child_count(dev->of_node) == 0) { 450 dev_info_once(dev, "device-tree doesn't have memory timings\n"); 451 return NULL; 452 } 453 454 if (!of_property_read_bool(dev->of_node, "nvidia,use-ram-code")) 455 return of_node_get(dev->of_node); 456 457 ram_code = tegra_read_ram_code(); 458 459 for_each_child_of_node(dev->of_node, np) { 460 if (!of_node_name_eq(np, "emc-tables")) 461 continue; 462 err = of_property_read_u32(np, "nvidia,ram-code", &value); 463 if (err || value != ram_code) { 464 struct device_node *lpddr2_np; 465 bool cfg_mismatches = false; 466 467 lpddr2_np = of_get_child_by_name(np, "lpddr2"); 468 if (lpddr2_np) { 469 const struct lpddr2_info *info; 470 471 info = of_lpddr2_get_info(lpddr2_np, dev); 472 if (info) { 473 if (info->manufacturer_id >= 0 && 474 info->manufacturer_id != emc->manufacturer_id) 475 cfg_mismatches = true; 476 477 if (info->revision_id1 >= 0 && 478 info->revision_id1 != emc->revision_id1) 479 cfg_mismatches = true; 480 481 if (info->revision_id2 >= 0 && 482 info->revision_id2 != emc->revision_id2) 483 cfg_mismatches = true; 484 485 if (info->density != emc->basic_conf4.density) 486 cfg_mismatches = true; 487 488 if (info->io_width != emc->basic_conf4.io_width) 489 cfg_mismatches = true; 490 491 if (info->arch_type != emc->basic_conf4.arch_type) 492 cfg_mismatches = true; 493 } else { 494 dev_err(dev, "failed to parse %pOF\n", lpddr2_np); 495 cfg_mismatches = true; 496 } 497 498 of_node_put(lpddr2_np); 499 } else { 500 cfg_mismatches = true; 501 } 502 503 if (cfg_mismatches) { 504 continue; 505 } 506 } 507 508 return np; 509 } 510 511 dev_err(dev, "no memory timings for RAM code %u found in device tree\n", 512 ram_code); 513 514 return NULL; 515 } 516 517 static int emc_read_lpddr_mode_register(struct tegra_emc *emc, 518 unsigned int emem_dev, 519 unsigned int register_addr, 520 unsigned int *register_data) 521 { 522 u32 memory_dev = emem_dev ? 1 : 2; 523 u32 val, mr_mask = 0xff; 524 int err; 525 526 /* clear data-valid interrupt status */ 527 writel_relaxed(EMC_MRR_DIVLD_INT, emc->regs + EMC_INTSTATUS); 528 529 /* issue mode register read request */ 530 val = FIELD_PREP(EMC_MRR_DEV_SELECTN, memory_dev); 531 val |= FIELD_PREP(EMC_MRR_MRR_MA, register_addr); 532 533 writel_relaxed(val, emc->regs + EMC_MRR); 534 535 /* wait for the LPDDR2 data-valid interrupt */ 536 err = readl_relaxed_poll_timeout_atomic(emc->regs + EMC_INTSTATUS, val, 537 val & EMC_MRR_DIVLD_INT, 538 1, 100); 539 if (err) { 540 dev_err(emc->dev, "mode register %u read failed: %d\n", 541 register_addr, err); 542 emc->mrr_error = true; 543 return err; 544 } 545 546 /* read out mode register data */ 547 val = readl_relaxed(emc->regs + EMC_MRR); 548 *register_data = FIELD_GET(EMC_MRR_MRR_DATA, val) & mr_mask; 549 550 return 0; 551 } 552 553 static void emc_read_lpddr_sdram_info(struct tegra_emc *emc, 554 unsigned int emem_dev, 555 bool print_out) 556 { 557 /* these registers are standard for all LPDDR JEDEC memory chips */ 558 emc_read_lpddr_mode_register(emc, emem_dev, 5, &emc->manufacturer_id); 559 emc_read_lpddr_mode_register(emc, emem_dev, 6, &emc->revision_id1); 560 emc_read_lpddr_mode_register(emc, emem_dev, 7, &emc->revision_id2); 561 emc_read_lpddr_mode_register(emc, emem_dev, 8, &emc->basic_conf4.value); 562 563 if (!print_out) 564 return; 565 566 dev_info(emc->dev, "SDRAM[dev%u]: manufacturer: 0x%x (%s) rev1: 0x%x rev2: 0x%x prefetch: S%u density: %uMbit iowidth: %ubit\n", 567 emem_dev, emc->manufacturer_id, 568 lpddr2_jedec_manufacturer(emc->manufacturer_id), 569 emc->revision_id1, emc->revision_id2, 570 4 >> emc->basic_conf4.arch_type, 571 64 << emc->basic_conf4.density, 572 32 >> emc->basic_conf4.io_width); 573 } 574 575 static int emc_setup_hw(struct tegra_emc *emc) 576 { 577 u32 emc_cfg, emc_dbg, emc_fbio, emc_adr_cfg; 578 u32 intmask = EMC_REFRESH_OVERFLOW_INT; 579 static bool print_sdram_info_once; 580 enum emc_dram_type dram_type; 581 const char *dram_type_str; 582 unsigned int emem_numdev; 583 584 emc_cfg = readl_relaxed(emc->regs + EMC_CFG_2); 585 586 /* 587 * Depending on a memory type, DRAM should enter either self-refresh 588 * or power-down state on EMC clock change. 589 */ 590 if (!(emc_cfg & EMC_CLKCHANGE_PD_ENABLE) && 591 !(emc_cfg & EMC_CLKCHANGE_SR_ENABLE)) { 592 dev_err(emc->dev, 593 "bootloader didn't specify DRAM auto-suspend mode\n"); 594 return -EINVAL; 595 } 596 597 /* enable EMC and CAR to handshake on PLL divider/source changes */ 598 emc_cfg |= EMC_CLKCHANGE_REQ_ENABLE; 599 writel_relaxed(emc_cfg, emc->regs + EMC_CFG_2); 600 601 /* initialize interrupt */ 602 writel_relaxed(intmask, emc->regs + EMC_INTMASK); 603 writel_relaxed(intmask, emc->regs + EMC_INTSTATUS); 604 605 /* ensure that unwanted debug features are disabled */ 606 emc_dbg = readl_relaxed(emc->regs + EMC_DBG); 607 emc_dbg |= EMC_DBG_CFG_PRIORITY; 608 emc_dbg &= ~EMC_DBG_READ_MUX_ASSEMBLY; 609 emc_dbg &= ~EMC_DBG_WRITE_MUX_ACTIVE; 610 emc_dbg &= ~EMC_DBG_FORCE_UPDATE; 611 writel_relaxed(emc_dbg, emc->regs + EMC_DBG); 612 613 emc_fbio = readl_relaxed(emc->regs + EMC_FBIO_CFG5); 614 615 if (emc_fbio & EMC_FBIO_CFG5_DRAM_WIDTH_X16) 616 emc->dram_bus_width = 16; 617 else 618 emc->dram_bus_width = 32; 619 620 dram_type = FIELD_GET(EMC_FBIO_CFG5_DRAM_TYPE, emc_fbio); 621 622 switch (dram_type) { 623 case DRAM_TYPE_RESERVED: 624 dram_type_str = "INVALID"; 625 break; 626 case DRAM_TYPE_DDR1: 627 dram_type_str = "DDR1"; 628 break; 629 case DRAM_TYPE_LPDDR2: 630 dram_type_str = "LPDDR2"; 631 break; 632 case DRAM_TYPE_DDR2: 633 dram_type_str = "DDR2"; 634 break; 635 } 636 637 emc_adr_cfg = readl_relaxed(emc->regs + EMC_ADR_CFG_0); 638 emem_numdev = FIELD_GET(EMC_ADR_CFG_0_EMEM_NUMDEV, emc_adr_cfg) + 1; 639 640 dev_info_once(emc->dev, "%ubit DRAM bus, %u %s %s attached\n", 641 emc->dram_bus_width, emem_numdev, dram_type_str, 642 emem_numdev == 2 ? "devices" : "device"); 643 644 if (dram_type == DRAM_TYPE_LPDDR2) { 645 while (emem_numdev--) 646 emc_read_lpddr_sdram_info(emc, emem_numdev, 647 !print_sdram_info_once); 648 print_sdram_info_once = true; 649 } 650 651 return 0; 652 } 653 654 static long emc_round_rate(unsigned long rate, 655 unsigned long min_rate, 656 unsigned long max_rate, 657 void *arg) 658 { 659 struct emc_timing *timing = NULL; 660 struct tegra_emc *emc = arg; 661 unsigned int i; 662 663 if (!emc->num_timings) 664 return clk_get_rate(emc->clk); 665 666 min_rate = min(min_rate, emc->timings[emc->num_timings - 1].rate); 667 668 for (i = 0; i < emc->num_timings; i++) { 669 if (emc->timings[i].rate < rate && i != emc->num_timings - 1) 670 continue; 671 672 if (emc->timings[i].rate > max_rate) { 673 i = max(i, 1u) - 1; 674 675 if (emc->timings[i].rate < min_rate) 676 break; 677 } 678 679 if (emc->timings[i].rate < min_rate) 680 continue; 681 682 timing = &emc->timings[i]; 683 break; 684 } 685 686 if (!timing) { 687 dev_err(emc->dev, "no timing for rate %lu min %lu max %lu\n", 688 rate, min_rate, max_rate); 689 return -EINVAL; 690 } 691 692 return timing->rate; 693 } 694 695 /* 696 * debugfs interface 697 * 698 * The memory controller driver exposes some files in debugfs that can be used 699 * to control the EMC frequency. The top-level directory can be found here: 700 * 701 * /sys/kernel/debug/emc 702 * 703 * It contains the following files: 704 * 705 * - available_rates: This file contains a list of valid, space-separated 706 * EMC frequencies. 707 * 708 * - min_rate: Writing a value to this file sets the given frequency as the 709 * floor of the permitted range. If this is higher than the currently 710 * configured EMC frequency, this will cause the frequency to be 711 * increased so that it stays within the valid range. 712 * 713 * - max_rate: Similarily to the min_rate file, writing a value to this file 714 * sets the given frequency as the ceiling of the permitted range. If 715 * the value is lower than the currently configured EMC frequency, this 716 * will cause the frequency to be decreased so that it stays within the 717 * valid range. 718 */ 719 720 static bool tegra20_emc_validate_rate(struct tegra_emc *emc, unsigned long rate) 721 { 722 unsigned int i; 723 724 for (i = 0; i < emc->num_timings; i++) 725 if (rate == emc->timings[i].rate) 726 return true; 727 728 return false; 729 } 730 731 static int tegra20_emc_debug_available_rates_show(struct seq_file *s, void *data) 732 { 733 struct tegra_emc *emc = s->private; 734 const char *prefix = ""; 735 unsigned int i; 736 737 for (i = 0; i < emc->num_timings; i++) { 738 seq_printf(s, "%s%lu", prefix, emc->timings[i].rate); 739 prefix = " "; 740 } 741 742 seq_puts(s, "\n"); 743 744 return 0; 745 } 746 DEFINE_SHOW_ATTRIBUTE(tegra20_emc_debug_available_rates); 747 748 static int tegra20_emc_debug_min_rate_get(void *data, u64 *rate) 749 { 750 struct tegra_emc *emc = data; 751 752 *rate = emc->debugfs.min_rate; 753 754 return 0; 755 } 756 757 static int tegra20_emc_debug_min_rate_set(void *data, u64 rate) 758 { 759 struct tegra_emc *emc = data; 760 int err; 761 762 if (!tegra20_emc_validate_rate(emc, rate)) 763 return -EINVAL; 764 765 err = tegra_emc_set_min_rate(&emc->reqs, rate, TEGRA_EMC_RATE_DEBUG); 766 if (err < 0) 767 return err; 768 769 emc->debugfs.min_rate = rate; 770 771 return 0; 772 } 773 774 DEFINE_SIMPLE_ATTRIBUTE(tegra20_emc_debug_min_rate_fops, 775 tegra20_emc_debug_min_rate_get, 776 tegra20_emc_debug_min_rate_set, "%llu\n"); 777 778 static int tegra20_emc_debug_max_rate_get(void *data, u64 *rate) 779 { 780 struct tegra_emc *emc = data; 781 782 *rate = emc->debugfs.max_rate; 783 784 return 0; 785 } 786 787 static int tegra20_emc_debug_max_rate_set(void *data, u64 rate) 788 { 789 struct tegra_emc *emc = data; 790 int err; 791 792 if (!tegra20_emc_validate_rate(emc, rate)) 793 return -EINVAL; 794 795 err = tegra_emc_set_max_rate(&emc->reqs, rate, TEGRA_EMC_RATE_DEBUG); 796 if (err < 0) 797 return err; 798 799 emc->debugfs.max_rate = rate; 800 801 return 0; 802 } 803 804 DEFINE_SIMPLE_ATTRIBUTE(tegra20_emc_debug_max_rate_fops, 805 tegra20_emc_debug_max_rate_get, 806 tegra20_emc_debug_max_rate_set, "%llu\n"); 807 808 static void tegra20_emc_debugfs_init(struct tegra_emc *emc) 809 { 810 struct device *dev = emc->dev; 811 unsigned int i; 812 int err; 813 814 emc->debugfs.min_rate = ULONG_MAX; 815 emc->debugfs.max_rate = 0; 816 817 for (i = 0; i < emc->num_timings; i++) { 818 if (emc->timings[i].rate < emc->debugfs.min_rate) 819 emc->debugfs.min_rate = emc->timings[i].rate; 820 821 if (emc->timings[i].rate > emc->debugfs.max_rate) 822 emc->debugfs.max_rate = emc->timings[i].rate; 823 } 824 825 if (!emc->num_timings) { 826 emc->debugfs.min_rate = clk_get_rate(emc->clk); 827 emc->debugfs.max_rate = emc->debugfs.min_rate; 828 } 829 830 err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate, 831 emc->debugfs.max_rate); 832 if (err < 0) { 833 dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n", 834 emc->debugfs.min_rate, emc->debugfs.max_rate, 835 emc->clk); 836 } 837 838 emc->debugfs.root = debugfs_create_dir("emc", NULL); 839 840 debugfs_create_file("available_rates", 0444, emc->debugfs.root, 841 emc, &tegra20_emc_debug_available_rates_fops); 842 debugfs_create_file("min_rate", 0644, emc->debugfs.root, 843 emc, &tegra20_emc_debug_min_rate_fops); 844 debugfs_create_file("max_rate", 0644, emc->debugfs.root, 845 emc, &tegra20_emc_debug_max_rate_fops); 846 } 847 848 static inline struct tegra_emc * 849 to_tegra_emc_provider(struct icc_provider *provider) 850 { 851 return container_of(provider, struct tegra_emc, provider); 852 } 853 854 static struct icc_node_data * 855 emc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data) 856 { 857 struct icc_provider *provider = data; 858 struct icc_node_data *ndata; 859 struct icc_node *node; 860 861 /* External Memory is the only possible ICC route */ 862 list_for_each_entry(node, &provider->nodes, node_list) { 863 if (node->id != TEGRA_ICC_EMEM) 864 continue; 865 866 ndata = kzalloc_obj(*ndata); 867 if (!ndata) 868 return ERR_PTR(-ENOMEM); 869 870 /* 871 * SRC and DST nodes should have matching TAG in order to have 872 * it set by default for a requested path. 873 */ 874 ndata->tag = TEGRA_MC_ICC_TAG_ISO; 875 ndata->node = node; 876 877 return ndata; 878 } 879 880 return ERR_PTR(-EPROBE_DEFER); 881 } 882 883 static int emc_icc_set(struct icc_node *src, struct icc_node *dst) 884 { 885 struct tegra_emc *emc = to_tegra_emc_provider(dst->provider); 886 unsigned long long peak_bw = icc_units_to_bps(dst->peak_bw); 887 unsigned long long avg_bw = icc_units_to_bps(dst->avg_bw); 888 unsigned long long rate = max(avg_bw, peak_bw); 889 unsigned int dram_data_bus_width_bytes; 890 int err; 891 892 /* 893 * Tegra20 EMC runs on x2 clock rate of SDRAM bus because DDR data 894 * is sampled on both clock edges. This means that EMC clock rate 895 * equals to the peak data-rate. 896 */ 897 dram_data_bus_width_bytes = emc->dram_bus_width / 8; 898 do_div(rate, dram_data_bus_width_bytes); 899 rate = min_t(u64, rate, U32_MAX); 900 901 err = tegra_emc_set_min_rate(&emc->reqs, rate, TEGRA_EMC_RATE_ICC); 902 if (err) 903 return err; 904 905 return 0; 906 } 907 908 static int tegra20_emc_interconnect_init(struct tegra_emc *emc) 909 { 910 const struct tegra_mc_soc *soc; 911 struct icc_node *node; 912 int err; 913 914 emc->mc = devm_tegra_memory_controller_get(emc->dev); 915 if (IS_ERR(emc->mc)) 916 return PTR_ERR(emc->mc); 917 918 soc = emc->mc->soc; 919 920 emc->provider.dev = emc->dev; 921 emc->provider.set = emc_icc_set; 922 emc->provider.data = &emc->provider; 923 emc->provider.aggregate = soc->icc_ops->aggregate; 924 emc->provider.xlate_extended = emc_of_icc_xlate_extended; 925 926 icc_provider_init(&emc->provider); 927 928 /* create External Memory Controller node */ 929 node = icc_node_create(TEGRA_ICC_EMC); 930 if (IS_ERR(node)) 931 return PTR_ERR(node); 932 933 node->name = "External Memory Controller"; 934 icc_node_add(node, &emc->provider); 935 936 /* link External Memory Controller to External Memory (DRAM) */ 937 err = icc_link_create(node, TEGRA_ICC_EMEM); 938 if (err) 939 goto remove_nodes; 940 941 /* create External Memory node */ 942 node = icc_node_create(TEGRA_ICC_EMEM); 943 if (IS_ERR(node)) { 944 err = PTR_ERR(node); 945 goto remove_nodes; 946 } 947 948 node->name = "External Memory (DRAM)"; 949 icc_node_add(node, &emc->provider); 950 951 err = icc_provider_register(&emc->provider); 952 if (err) 953 goto remove_nodes; 954 955 return 0; 956 957 remove_nodes: 958 icc_nodes_remove(&emc->provider); 959 960 return dev_err_probe(emc->dev, err, "failed to initialize ICC\n"); 961 } 962 963 static void devm_tegra20_emc_unset_callback(void *data) 964 { 965 tegra20_clk_set_emc_round_callback(NULL, NULL); 966 } 967 968 static void devm_tegra20_emc_unreg_clk_notifier(void *data) 969 { 970 struct tegra_emc *emc = data; 971 972 clk_notifier_unregister(emc->clk, &emc->clk_nb); 973 } 974 975 static int tegra20_emc_init_clk(struct tegra_emc *emc) 976 { 977 int err; 978 979 tegra20_clk_set_emc_round_callback(emc_round_rate, emc); 980 981 err = devm_add_action_or_reset(emc->dev, devm_tegra20_emc_unset_callback, 982 NULL); 983 if (err) 984 return err; 985 986 emc->clk = devm_clk_get(emc->dev, NULL); 987 if (IS_ERR(emc->clk)) 988 return dev_err_probe(emc->dev, PTR_ERR(emc->clk), 989 "failed to get EMC clock\n"); 990 991 err = clk_notifier_register(emc->clk, &emc->clk_nb); 992 if (err) 993 return dev_err_probe(emc->dev, err, "failed to register clk notifier\n"); 994 995 err = devm_add_action_or_reset(emc->dev, 996 devm_tegra20_emc_unreg_clk_notifier, emc); 997 if (err) 998 return err; 999 1000 return 0; 1001 } 1002 1003 static int tegra20_emc_devfreq_target(struct device *dev, unsigned long *freq, 1004 u32 flags) 1005 { 1006 struct tegra_emc *emc = dev_get_drvdata(dev); 1007 struct dev_pm_opp *opp; 1008 unsigned long rate; 1009 1010 opp = devfreq_recommended_opp(dev, freq, flags); 1011 if (IS_ERR(opp)) { 1012 dev_err(dev, "failed to find opp for %lu Hz\n", *freq); 1013 return PTR_ERR(opp); 1014 } 1015 1016 rate = dev_pm_opp_get_freq(opp); 1017 dev_pm_opp_put(opp); 1018 1019 return tegra_emc_set_min_rate(&emc->reqs, rate, TEGRA_EMC_RATE_DEVFREQ); 1020 } 1021 1022 static int tegra20_emc_devfreq_get_dev_status(struct device *dev, 1023 struct devfreq_dev_status *stat) 1024 { 1025 struct tegra_emc *emc = dev_get_drvdata(dev); 1026 1027 /* freeze counters */ 1028 writel_relaxed(EMC_PWR_GATHER_DISABLE, emc->regs + EMC_STAT_CONTROL); 1029 1030 /* 1031 * busy_time: number of clocks EMC request was accepted 1032 * total_time: number of clocks PWR_GATHER control was set to ENABLE 1033 */ 1034 stat->busy_time = readl_relaxed(emc->regs + EMC_STAT_PWR_COUNT); 1035 stat->total_time = readl_relaxed(emc->regs + EMC_STAT_PWR_CLOCKS); 1036 stat->current_frequency = clk_get_rate(emc->clk); 1037 1038 /* clear counters and restart */ 1039 writel_relaxed(EMC_PWR_GATHER_CLEAR, emc->regs + EMC_STAT_CONTROL); 1040 writel_relaxed(EMC_PWR_GATHER_ENABLE, emc->regs + EMC_STAT_CONTROL); 1041 1042 return 0; 1043 } 1044 1045 static struct devfreq_dev_profile tegra20_emc_devfreq_profile = { 1046 .polling_ms = 30, 1047 .target = tegra20_emc_devfreq_target, 1048 .get_dev_status = tegra20_emc_devfreq_get_dev_status, 1049 }; 1050 1051 static int tegra20_emc_devfreq_init(struct tegra_emc *emc) 1052 { 1053 struct devfreq *devfreq; 1054 1055 /* 1056 * PWR_COUNT is 1/2 of PWR_CLOCKS at max, and thus, the up-threshold 1057 * should be less than 50. Secondly, multiple active memory clients 1058 * may cause over 20% of lost clock cycles due to stalls caused by 1059 * competing memory accesses. This means that threshold should be 1060 * set to a less than 30 in order to have a properly working governor. 1061 */ 1062 emc->ondemand_data.upthreshold = 20; 1063 1064 /* 1065 * Reset statistic gathers state, select global bandwidth for the 1066 * statistics collection mode and set clocks counter saturation 1067 * limit to maximum. 1068 */ 1069 writel_relaxed(0x00000000, emc->regs + EMC_STAT_CONTROL); 1070 writel_relaxed(0x00000000, emc->regs + EMC_STAT_LLMC_CONTROL); 1071 writel_relaxed(0xffffffff, emc->regs + EMC_STAT_PWR_CLOCK_LIMIT); 1072 1073 devfreq = devm_devfreq_add_device(emc->dev, &tegra20_emc_devfreq_profile, 1074 DEVFREQ_GOV_SIMPLE_ONDEMAND, 1075 &emc->ondemand_data); 1076 if (IS_ERR(devfreq)) 1077 return dev_err_probe(emc->dev, PTR_ERR(devfreq), 1078 "failed to initialize devfreq\n"); 1079 1080 return 0; 1081 } 1082 1083 static int tegra20_emc_probe(struct platform_device *pdev) 1084 { 1085 struct tegra_core_opp_params opp_params = {}; 1086 struct device_node *np; 1087 struct tegra_emc *emc; 1088 int irq, err; 1089 1090 irq = platform_get_irq(pdev, 0); 1091 if (irq < 0) 1092 return irq; 1093 1094 emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL); 1095 if (!emc) 1096 return -ENOMEM; 1097 1098 emc->clk_nb.notifier_call = tegra20_emc_clk_change_notify; 1099 emc->dev = &pdev->dev; 1100 1101 emc->regs = devm_platform_ioremap_resource(pdev, 0); 1102 if (IS_ERR(emc->regs)) 1103 return PTR_ERR(emc->regs); 1104 1105 err = emc_setup_hw(emc); 1106 if (err) 1107 return err; 1108 1109 np = tegra20_emc_find_node_by_ram_code(emc); 1110 if (np) { 1111 err = tegra20_emc_load_timings_from_dt(emc, np); 1112 of_node_put(np); 1113 if (err) 1114 return err; 1115 } 1116 1117 err = devm_request_irq(&pdev->dev, irq, tegra20_emc_isr, 0, 1118 dev_name(&pdev->dev), emc); 1119 if (err) { 1120 dev_err(&pdev->dev, "failed to request IRQ: %d\n", err); 1121 return err; 1122 } 1123 1124 err = tegra20_emc_init_clk(emc); 1125 if (err) 1126 return err; 1127 1128 opp_params.init_state = true; 1129 1130 err = devm_tegra_core_dev_init_opp_table(&pdev->dev, &opp_params); 1131 if (err) 1132 return err; 1133 1134 platform_set_drvdata(pdev, emc); 1135 tegra_emc_rate_requests_init(&emc->reqs, &pdev->dev); 1136 tegra20_emc_debugfs_init(emc); 1137 tegra20_emc_interconnect_init(emc); 1138 tegra20_emc_devfreq_init(emc); 1139 1140 /* 1141 * Don't allow the kernel module to be unloaded. Unloading adds some 1142 * extra complexity which doesn't really worth the effort in a case of 1143 * this driver. 1144 */ 1145 try_module_get(THIS_MODULE); 1146 1147 return 0; 1148 } 1149 1150 static const struct of_device_id tegra20_emc_of_match[] = { 1151 { .compatible = "nvidia,tegra20-emc", }, 1152 {}, 1153 }; 1154 MODULE_DEVICE_TABLE(of, tegra20_emc_of_match); 1155 1156 static struct platform_driver tegra20_emc_driver = { 1157 .probe = tegra20_emc_probe, 1158 .driver = { 1159 .name = "tegra20-emc", 1160 .of_match_table = tegra20_emc_of_match, 1161 .suppress_bind_attrs = true, 1162 .sync_state = icc_sync_state, 1163 }, 1164 }; 1165 module_platform_driver(tegra20_emc_driver); 1166 1167 MODULE_AUTHOR("Dmitry Osipenko <digetx@gmail.com>"); 1168 MODULE_DESCRIPTION("NVIDIA Tegra20 EMC driver"); 1169 MODULE_SOFTDEP("pre: governor_simpleondemand"); 1170 MODULE_LICENSE("GPL v2"); 1171