1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2014-2026 NVIDIA CORPORATION. All rights reserved. 4 */ 5 6 #include <linux/clk.h> 7 #include <linux/delay.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/export.h> 10 #include <linux/interrupt.h> 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/of.h> 14 #include <linux/of_platform.h> 15 #include <linux/platform_device.h> 16 #include <linux/slab.h> 17 #include <linux/sort.h> 18 #include <linux/tegra-icc.h> 19 20 #include <soc/tegra/fuse.h> 21 22 #include "mc.h" 23 24 static const struct of_device_id tegra_mc_of_match[] = { 25 #ifdef CONFIG_ARCH_TEGRA_2x_SOC 26 { .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc }, 27 #endif 28 #ifdef CONFIG_ARCH_TEGRA_3x_SOC 29 { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc }, 30 #endif 31 #ifdef CONFIG_ARCH_TEGRA_114_SOC 32 { .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc }, 33 #endif 34 #ifdef CONFIG_ARCH_TEGRA_124_SOC 35 { .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc }, 36 #endif 37 #ifdef CONFIG_ARCH_TEGRA_132_SOC 38 { .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc }, 39 #endif 40 #ifdef CONFIG_ARCH_TEGRA_210_SOC 41 { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc }, 42 #endif 43 #ifdef CONFIG_ARCH_TEGRA_186_SOC 44 { .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc }, 45 #endif 46 #ifdef CONFIG_ARCH_TEGRA_194_SOC 47 { .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc }, 48 #endif 49 #ifdef CONFIG_ARCH_TEGRA_234_SOC 50 { .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc }, 51 #endif 52 #ifdef CONFIG_ARCH_TEGRA_264_SOC 53 { .compatible = "nvidia,tegra264-mc", .data = &tegra264_mc_soc }, 54 #endif 55 { /* sentinel */ } 56 }; 57 MODULE_DEVICE_TABLE(of, tegra_mc_of_match); 58 59 const struct tegra_mc_regs tegra20_mc_regs = { 60 .cfg_channel_enable = 0xdf8, 61 .err_status = 0x08, 62 .err_add = 0x0c, 63 .err_add_hi = 0x11fc, 64 .err_vpr_status = 0x654, 65 .err_vpr_add = 0x658, 66 .err_sec_status = 0x67c, 67 .err_sec_add = 0x680, 68 .err_mts_status = 0x9b0, 69 .err_mts_add = 0x9b4, 70 .err_gen_co_status = 0xc00, 71 .err_gen_co_add = 0xc04, 72 .err_route_status = 0x9c0, 73 .err_route_add = 0x9c4, 74 }; 75 76 static void tegra_mc_devm_action_put_device(void *data) 77 { 78 struct tegra_mc *mc = data; 79 80 put_device(mc->dev); 81 } 82 83 /** 84 * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle 85 * @dev: device pointer for the consumer device 86 * 87 * This function will search for the Memory Controller node in a device-tree 88 * and retrieve the Memory Controller handle. 89 * 90 * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc. 91 */ 92 struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev) 93 { 94 struct platform_device *pdev; 95 struct device_node *np; 96 struct tegra_mc *mc; 97 int err; 98 99 np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0); 100 if (!np) 101 return ERR_PTR(-ENOENT); 102 103 pdev = of_find_device_by_node(np); 104 of_node_put(np); 105 if (!pdev) 106 return ERR_PTR(-ENODEV); 107 108 mc = platform_get_drvdata(pdev); 109 if (!mc) { 110 put_device(&pdev->dev); 111 return ERR_PTR(-EPROBE_DEFER); 112 } 113 114 err = devm_add_action_or_reset(dev, tegra_mc_devm_action_put_device, mc); 115 if (err) 116 return ERR_PTR(err); 117 118 return mc; 119 } 120 EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get); 121 122 int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev) 123 { 124 if (mc->soc->ops && mc->soc->ops->probe_device) 125 return mc->soc->ops->probe_device(mc, dev); 126 127 return 0; 128 } 129 EXPORT_SYMBOL_GPL(tegra_mc_probe_device); 130 131 int tegra_mc_get_carveout_info(struct tegra_mc *mc, unsigned int id, 132 phys_addr_t *base, u64 *size) 133 { 134 u32 offset; 135 136 if (id < 1 || id >= mc->soc->num_carveouts) 137 return -EINVAL; 138 139 if (id < 6) 140 offset = 0xc0c + 0x50 * (id - 1); 141 else 142 offset = 0x2004 + 0x50 * (id - 6); 143 144 *base = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x0); 145 #ifdef CONFIG_PHYS_ADDR_T_64BIT 146 *base |= (phys_addr_t)mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x4) << 32; 147 #endif 148 149 if (size) 150 *size = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x8) << 17; 151 152 return 0; 153 } 154 EXPORT_SYMBOL_GPL(tegra_mc_get_carveout_info); 155 156 static int tegra_mc_block_dma_common(struct tegra_mc *mc, 157 const struct tegra_mc_reset *rst) 158 { 159 unsigned long flags; 160 u32 value; 161 162 spin_lock_irqsave(&mc->lock, flags); 163 164 value = mc_readl(mc, rst->control) | BIT(rst->bit); 165 mc_writel(mc, value, rst->control); 166 167 spin_unlock_irqrestore(&mc->lock, flags); 168 169 return 0; 170 } 171 172 static bool tegra_mc_dma_idling_common(struct tegra_mc *mc, 173 const struct tegra_mc_reset *rst) 174 { 175 return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0; 176 } 177 178 static int tegra_mc_unblock_dma_common(struct tegra_mc *mc, 179 const struct tegra_mc_reset *rst) 180 { 181 unsigned long flags; 182 u32 value; 183 184 spin_lock_irqsave(&mc->lock, flags); 185 186 value = mc_readl(mc, rst->control) & ~BIT(rst->bit); 187 mc_writel(mc, value, rst->control); 188 189 spin_unlock_irqrestore(&mc->lock, flags); 190 191 return 0; 192 } 193 194 static int tegra_mc_reset_status_common(struct tegra_mc *mc, 195 const struct tegra_mc_reset *rst) 196 { 197 return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0; 198 } 199 200 const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = { 201 .block_dma = tegra_mc_block_dma_common, 202 .dma_idling = tegra_mc_dma_idling_common, 203 .unblock_dma = tegra_mc_unblock_dma_common, 204 .reset_status = tegra_mc_reset_status_common, 205 }; 206 207 static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev) 208 { 209 return container_of(rcdev, struct tegra_mc, reset); 210 } 211 212 static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc, 213 unsigned long id) 214 { 215 unsigned int i; 216 217 for (i = 0; i < mc->soc->num_resets; i++) 218 if (mc->soc->resets[i].id == id) 219 return &mc->soc->resets[i]; 220 221 return NULL; 222 } 223 224 static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev, 225 unsigned long id) 226 { 227 struct tegra_mc *mc = reset_to_mc(rcdev); 228 const struct tegra_mc_reset_ops *rst_ops; 229 const struct tegra_mc_reset *rst; 230 int retries = 500; 231 int err; 232 233 rst = tegra_mc_reset_find(mc, id); 234 if (!rst) 235 return -ENODEV; 236 237 rst_ops = mc->soc->reset_ops; 238 if (!rst_ops) 239 return -ENODEV; 240 241 /* DMA flushing will fail if reset is already asserted */ 242 if (rst_ops->reset_status) { 243 /* check whether reset is asserted */ 244 if (rst_ops->reset_status(mc, rst)) 245 return 0; 246 } 247 248 if (rst_ops->block_dma) { 249 /* block clients DMA requests */ 250 err = rst_ops->block_dma(mc, rst); 251 if (err) { 252 dev_err(mc->dev, "failed to block %s DMA: %d\n", 253 rst->name, err); 254 return err; 255 } 256 } 257 258 if (rst_ops->dma_idling) { 259 /* wait for completion of the outstanding DMA requests */ 260 while (!rst_ops->dma_idling(mc, rst)) { 261 if (!retries--) { 262 dev_err(mc->dev, "failed to flush %s DMA\n", 263 rst->name); 264 return -EBUSY; 265 } 266 267 usleep_range(10, 100); 268 } 269 } 270 271 if (rst_ops->hotreset_assert) { 272 /* clear clients DMA requests sitting before arbitration */ 273 err = rst_ops->hotreset_assert(mc, rst); 274 if (err) { 275 dev_err(mc->dev, "failed to hot reset %s: %d\n", 276 rst->name, err); 277 return err; 278 } 279 } 280 281 return 0; 282 } 283 284 static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev, 285 unsigned long id) 286 { 287 struct tegra_mc *mc = reset_to_mc(rcdev); 288 const struct tegra_mc_reset_ops *rst_ops; 289 const struct tegra_mc_reset *rst; 290 int err; 291 292 rst = tegra_mc_reset_find(mc, id); 293 if (!rst) 294 return -ENODEV; 295 296 rst_ops = mc->soc->reset_ops; 297 if (!rst_ops) 298 return -ENODEV; 299 300 if (rst_ops->hotreset_deassert) { 301 /* take out client from hot reset */ 302 err = rst_ops->hotreset_deassert(mc, rst); 303 if (err) { 304 dev_err(mc->dev, "failed to deassert hot reset %s: %d\n", 305 rst->name, err); 306 return err; 307 } 308 } 309 310 if (rst_ops->unblock_dma) { 311 /* allow new DMA requests to proceed to arbitration */ 312 err = rst_ops->unblock_dma(mc, rst); 313 if (err) { 314 dev_err(mc->dev, "failed to unblock %s DMA : %d\n", 315 rst->name, err); 316 return err; 317 } 318 } 319 320 return 0; 321 } 322 323 static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev, 324 unsigned long id) 325 { 326 struct tegra_mc *mc = reset_to_mc(rcdev); 327 const struct tegra_mc_reset_ops *rst_ops; 328 const struct tegra_mc_reset *rst; 329 330 rst = tegra_mc_reset_find(mc, id); 331 if (!rst) 332 return -ENODEV; 333 334 rst_ops = mc->soc->reset_ops; 335 if (!rst_ops) 336 return -ENODEV; 337 338 return rst_ops->reset_status(mc, rst); 339 } 340 341 static const struct reset_control_ops tegra_mc_reset_ops = { 342 .assert = tegra_mc_hotreset_assert, 343 .deassert = tegra_mc_hotreset_deassert, 344 .status = tegra_mc_hotreset_status, 345 }; 346 347 static int tegra_mc_reset_setup(struct tegra_mc *mc) 348 { 349 int err; 350 351 mc->reset.ops = &tegra_mc_reset_ops; 352 mc->reset.owner = THIS_MODULE; 353 mc->reset.of_node = mc->dev->of_node; 354 mc->reset.of_reset_n_cells = 1; 355 mc->reset.nr_resets = mc->soc->num_resets; 356 357 err = reset_controller_register(&mc->reset); 358 if (err < 0) 359 return err; 360 361 return 0; 362 } 363 364 int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate) 365 { 366 unsigned int i; 367 struct tegra_mc_timing *timing = NULL; 368 369 for (i = 0; i < mc->num_timings; i++) { 370 if (mc->timings[i].rate == rate) { 371 timing = &mc->timings[i]; 372 break; 373 } 374 } 375 376 if (!timing) { 377 dev_err(mc->dev, "no memory timing registered for rate %lu\n", 378 rate); 379 return -EINVAL; 380 } 381 382 for (i = 0; i < mc->soc->num_emem_regs; ++i) 383 mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]); 384 385 return 0; 386 } 387 EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration); 388 389 unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc) 390 { 391 u8 dram_count; 392 393 dram_count = mc_readl(mc, MC_EMEM_ADR_CFG); 394 dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV; 395 dram_count++; 396 397 return dram_count; 398 } 399 EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count); 400 401 const irq_handler_t tegra30_mc_irq_handlers[] = { 402 tegra30_mc_handle_irq 403 }; 404 405 #if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \ 406 defined(CONFIG_ARCH_TEGRA_114_SOC) || \ 407 defined(CONFIG_ARCH_TEGRA_124_SOC) || \ 408 defined(CONFIG_ARCH_TEGRA_132_SOC) || \ 409 defined(CONFIG_ARCH_TEGRA_210_SOC) 410 static void tegra_mc_setup_latency_allowance(struct tegra_mc *mc) 411 { 412 unsigned long long tick; 413 unsigned int i; 414 u32 value; 415 416 /* compute the number of MC clock cycles per tick */ 417 tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk); 418 do_div(tick, NSEC_PER_SEC); 419 420 value = mc_readl(mc, MC_EMEM_ARB_CFG); 421 value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK; 422 value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick); 423 mc_writel(mc, value, MC_EMEM_ARB_CFG); 424 425 /* write latency allowance defaults */ 426 for (i = 0; i < mc->soc->num_clients; i++) { 427 const struct tegra_mc_client *client = &mc->soc->clients[i]; 428 u32 value; 429 430 value = mc_readl(mc, client->regs.la.reg); 431 value &= ~(client->regs.la.mask << client->regs.la.shift); 432 value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift; 433 mc_writel(mc, value, client->regs.la.reg); 434 } 435 436 /* latch new values */ 437 mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL); 438 } 439 440 static int load_one_timing(struct tegra_mc *mc, 441 struct tegra_mc_timing *timing, 442 struct device_node *node) 443 { 444 int err; 445 u32 tmp; 446 447 err = of_property_read_u32(node, "clock-frequency", &tmp); 448 if (err) { 449 dev_err(mc->dev, 450 "timing %pOFn: failed to read rate\n", node); 451 return err; 452 } 453 454 timing->rate = tmp; 455 timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs, 456 sizeof(u32), GFP_KERNEL); 457 if (!timing->emem_data) 458 return -ENOMEM; 459 460 err = of_property_read_u32_array(node, "nvidia,emem-configuration", 461 timing->emem_data, 462 mc->soc->num_emem_regs); 463 if (err) { 464 dev_err(mc->dev, 465 "timing %pOFn: failed to read EMEM configuration\n", 466 node); 467 return err; 468 } 469 470 return 0; 471 } 472 473 static int load_timings(struct tegra_mc *mc, struct device_node *node) 474 { 475 struct tegra_mc_timing *timing; 476 int child_count = of_get_child_count(node); 477 int i = 0, err; 478 479 mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing), 480 GFP_KERNEL); 481 if (!mc->timings) 482 return -ENOMEM; 483 484 mc->num_timings = child_count; 485 486 for_each_child_of_node_scoped(node, child) { 487 timing = &mc->timings[i++]; 488 489 err = load_one_timing(mc, timing, child); 490 if (err) 491 return err; 492 } 493 494 return 0; 495 } 496 497 static int tegra_mc_setup_timings(struct tegra_mc *mc) 498 { 499 u32 ram_code, node_ram_code; 500 int err; 501 502 ram_code = tegra_read_ram_code(); 503 504 mc->num_timings = 0; 505 506 for_each_child_of_node_scoped(mc->dev->of_node, node) { 507 err = of_property_read_u32(node, "nvidia,ram-code", 508 &node_ram_code); 509 if (err || (node_ram_code != ram_code)) 510 continue; 511 512 err = load_timings(mc, node); 513 if (err) 514 return err; 515 break; 516 } 517 518 if (mc->num_timings == 0) 519 dev_warn(mc->dev, 520 "no memory timings for RAM code %u registered\n", 521 ram_code); 522 523 return 0; 524 } 525 526 int tegra30_mc_probe(struct tegra_mc *mc) 527 { 528 int err; 529 530 mc->clk = devm_clk_get_optional(mc->dev, "mc"); 531 if (IS_ERR(mc->clk)) 532 return dev_err_probe(mc->dev, PTR_ERR(mc->clk), 533 "failed to get MC clock\n"); 534 535 /* ensure that debug features are disabled */ 536 mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG); 537 538 tegra_mc_setup_latency_allowance(mc); 539 540 err = tegra_mc_setup_timings(mc); 541 if (err < 0) 542 return dev_err_probe(mc->dev, err, "failed to setup timings\n"); 543 544 return 0; 545 } 546 547 const struct tegra_mc_ops tegra30_mc_ops = { 548 .probe = tegra30_mc_probe, 549 }; 550 #endif 551 552 static int mc_global_intstatus_to_channel(const struct tegra_mc *mc, u32 status, 553 unsigned int *mc_channel) 554 { 555 if ((status & mc->soc->ch_intmask) == 0) 556 return -EINVAL; 557 558 *mc_channel = __ffs((status & mc->soc->ch_intmask) >> 559 mc->soc->global_intstatus_channel_shift); 560 561 return 0; 562 } 563 564 static u32 mc_channel_to_global_intstatus(const struct tegra_mc *mc, 565 unsigned int channel) 566 { 567 return BIT(channel) << mc->soc->global_intstatus_channel_shift; 568 } 569 570 irqreturn_t tegra30_mc_handle_irq(int irq, void *data) 571 { 572 struct tegra_mc *mc = data; 573 unsigned int bit, channel; 574 unsigned long status; 575 576 if (mc->soc->num_channels) { 577 u32 global_status; 578 int err; 579 580 global_status = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MC_GLOBAL_INTSTATUS); 581 err = mc_global_intstatus_to_channel(mc, global_status, &channel); 582 if (err < 0) { 583 dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n", 584 global_status); 585 return IRQ_NONE; 586 } 587 588 /* mask all interrupts to avoid flooding */ 589 status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmasks[0].mask; 590 } else { 591 status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmasks[0].mask; 592 } 593 594 if (!status) 595 return IRQ_NONE; 596 597 for_each_set_bit(bit, &status, 32) { 598 const char *error = tegra_mc_status_names[bit] ?: "unknown"; 599 const char *client = "unknown", *desc; 600 const char *direction, *secure; 601 u32 status_reg, addr_reg; 602 u32 intmask = BIT(bit); 603 phys_addr_t addr = 0; 604 #ifdef CONFIG_PHYS_ADDR_T_64BIT 605 u32 addr_hi_reg = 0; 606 #endif 607 unsigned int i; 608 char perm[7]; 609 u8 id, type; 610 u32 value; 611 612 switch (intmask) { 613 case MC_INT_DECERR_VPR: 614 status_reg = mc->soc->regs->err_vpr_status; 615 addr_reg = mc->soc->regs->err_vpr_add; 616 break; 617 618 case MC_INT_SECERR_SEC: 619 status_reg = mc->soc->regs->err_sec_status; 620 addr_reg = mc->soc->regs->err_sec_add; 621 break; 622 623 case MC_INT_DECERR_MTS: 624 status_reg = mc->soc->regs->err_mts_status; 625 addr_reg = mc->soc->regs->err_mts_add; 626 break; 627 628 case MC_INT_DECERR_GENERALIZED_CARVEOUT: 629 status_reg = mc->soc->regs->err_gen_co_status; 630 addr_reg = mc->soc->regs->err_gen_co_add; 631 break; 632 633 case MC_INT_DECERR_ROUTE_SANITY: 634 status_reg = mc->soc->regs->err_route_status; 635 addr_reg = mc->soc->regs->err_route_add; 636 break; 637 638 default: 639 status_reg = mc->soc->regs->err_status; 640 addr_reg = mc->soc->regs->err_add; 641 642 #ifdef CONFIG_PHYS_ADDR_T_64BIT 643 if (mc->soc->has_addr_hi_reg) 644 addr_hi_reg = mc->soc->regs->err_add_hi; 645 #endif 646 break; 647 } 648 649 if (mc->soc->num_channels) 650 value = mc_ch_readl(mc, channel, status_reg); 651 else 652 value = mc_readl(mc, status_reg); 653 654 #ifdef CONFIG_PHYS_ADDR_T_64BIT 655 if (mc->soc->num_address_bits > 32) { 656 if (addr_hi_reg) { 657 if (mc->soc->num_channels) 658 addr = mc_ch_readl(mc, channel, addr_hi_reg); 659 else 660 addr = mc_readl(mc, addr_hi_reg); 661 } else if (mc->soc->mc_addr_hi_mask) { 662 addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) & 663 mc->soc->mc_addr_hi_mask); 664 } else { 665 dev_err_ratelimited(mc->dev, "Unable to determine high address!"); 666 return IRQ_NONE; 667 } 668 addr <<= 32; 669 } 670 #endif 671 672 if (value & MC_ERR_STATUS_RW) 673 direction = "write"; 674 else 675 direction = "read"; 676 677 if (value & MC_ERR_STATUS_SECURITY) 678 secure = "secure "; 679 else 680 secure = ""; 681 682 id = value & mc->soc->client_id_mask; 683 684 for (i = 0; i < mc->soc->num_clients; i++) { 685 if (mc->soc->clients[i].id == id) { 686 client = mc->soc->clients[i].name; 687 break; 688 } 689 } 690 691 type = (value & mc->soc->mc_err_status_type_mask) >> 692 MC_ERR_STATUS_TYPE_SHIFT; 693 desc = tegra20_mc_error_names[type]; 694 695 switch (value & mc->soc->mc_err_status_type_mask) { 696 case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE: 697 perm[0] = ' '; 698 perm[1] = '['; 699 700 if (value & MC_ERR_STATUS_READABLE) 701 perm[2] = 'R'; 702 else 703 perm[2] = '-'; 704 705 if (value & MC_ERR_STATUS_WRITABLE) 706 perm[3] = 'W'; 707 else 708 perm[3] = '-'; 709 710 if (value & MC_ERR_STATUS_NONSECURE) 711 perm[4] = '-'; 712 else 713 perm[4] = 'S'; 714 715 perm[5] = ']'; 716 perm[6] = '\0'; 717 break; 718 719 default: 720 perm[0] = '\0'; 721 break; 722 } 723 724 if (mc->soc->num_channels) 725 value = mc_ch_readl(mc, channel, addr_reg); 726 else 727 value = mc_readl(mc, addr_reg); 728 addr |= value; 729 730 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n", 731 client, secure, direction, &addr, error, 732 desc, perm); 733 } 734 735 /* clear interrupts */ 736 if (mc->soc->num_channels) { 737 mc_ch_writel(mc, channel, status, MC_INTSTATUS); 738 mc_ch_writel(mc, MC_BROADCAST_CHANNEL, 739 mc_channel_to_global_intstatus(mc, channel), 740 MC_GLOBAL_INTSTATUS); 741 } else { 742 mc_writel(mc, status, MC_INTSTATUS); 743 } 744 745 return IRQ_HANDLED; 746 } 747 748 const char *const tegra_mc_status_names[32] = { 749 [ 1] = "External interrupt", 750 [ 6] = "EMEM address decode error", 751 [ 7] = "GART page fault", 752 [ 8] = "Security violation", 753 [ 9] = "EMEM arbitration error", 754 [10] = "Page fault", 755 [11] = "Invalid APB ASID update", 756 [12] = "VPR violation", 757 [13] = "Secure carveout violation", 758 [16] = "MTS carveout violation", 759 [17] = "Generalized carveout violation", 760 [20] = "Route Sanity error", 761 [21] = "GIC_MSI error", 762 }; 763 764 const char *const tegra20_mc_error_names[8] = { 765 [2] = "EMEM decode error", 766 [3] = "TrustZone violation", 767 [4] = "Carveout violation", 768 [6] = "SMMU translation error", 769 }; 770 771 struct icc_node *tegra_mc_icc_xlate(const struct of_phandle_args *spec, void *data) 772 { 773 struct tegra_mc *mc = icc_provider_to_tegra_mc(data); 774 struct icc_node *node; 775 776 list_for_each_entry(node, &mc->provider.nodes, node_list) { 777 if (node->id == spec->args[0]) 778 return node; 779 } 780 781 /* 782 * If a client driver calls devm_of_icc_get() before the MC driver 783 * is probed, then return EPROBE_DEFER to the client driver. 784 */ 785 return ERR_PTR(-EPROBE_DEFER); 786 } 787 788 static int tegra_mc_icc_get(struct icc_node *node, u32 *average, u32 *peak) 789 { 790 *average = 0; 791 *peak = 0; 792 793 return 0; 794 } 795 796 static int tegra_mc_icc_set(struct icc_node *src, struct icc_node *dst) 797 { 798 return 0; 799 } 800 801 const struct tegra_mc_icc_ops tegra_mc_icc_ops = { 802 .xlate = tegra_mc_icc_xlate, 803 .aggregate = icc_std_aggregate, 804 .get_bw = tegra_mc_icc_get, 805 .set = tegra_mc_icc_set, 806 }; 807 808 /* 809 * Memory Controller (MC) has few Memory Clients that are issuing memory 810 * bandwidth allocation requests to the MC interconnect provider. The MC 811 * provider aggregates the requests and then sends the aggregated request 812 * up to the External Memory Controller (EMC) interconnect provider which 813 * re-configures hardware interface to External Memory (EMEM) in accordance 814 * to the required bandwidth. Each MC interconnect node represents an 815 * individual Memory Client. 816 * 817 * Memory interconnect topology: 818 * 819 * +----+ 820 * +--------+ | | 821 * | TEXSRD +--->+ | 822 * +--------+ | | 823 * | | +-----+ +------+ 824 * ... | MC +--->+ EMC +--->+ EMEM | 825 * | | +-----+ +------+ 826 * +--------+ | | 827 * | DISP.. +--->+ | 828 * +--------+ | | 829 * +----+ 830 */ 831 static int tegra_mc_interconnect_setup(struct tegra_mc *mc) 832 { 833 struct icc_node *node; 834 unsigned int i; 835 int err; 836 837 /* older device-trees don't have interconnect properties */ 838 if (!device_property_present(mc->dev, "#interconnect-cells") || 839 !mc->soc->icc_ops) 840 return 0; 841 842 mc->provider.dev = mc->dev; 843 mc->provider.data = &mc->provider; 844 mc->provider.set = mc->soc->icc_ops->set; 845 mc->provider.aggregate = mc->soc->icc_ops->aggregate; 846 mc->provider.get_bw = mc->soc->icc_ops->get_bw; 847 mc->provider.xlate = mc->soc->icc_ops->xlate; 848 mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended; 849 850 icc_provider_init(&mc->provider); 851 852 /* create Memory Controller node */ 853 node = icc_node_create(TEGRA_ICC_MC); 854 if (IS_ERR(node)) 855 return PTR_ERR(node); 856 857 node->name = "Memory Controller"; 858 icc_node_add(node, &mc->provider); 859 860 /* link Memory Controller to External Memory Controller */ 861 err = icc_link_create(node, TEGRA_ICC_EMC); 862 if (err) 863 goto remove_nodes; 864 865 for (i = 0; i < mc->soc->num_clients; i++) { 866 /* create MC client node */ 867 node = icc_node_create(mc->soc->clients[i].id); 868 if (IS_ERR(node)) { 869 err = PTR_ERR(node); 870 goto remove_nodes; 871 } 872 873 node->name = mc->soc->clients[i].name; 874 icc_node_add(node, &mc->provider); 875 876 /* link Memory Client to Memory Controller */ 877 err = icc_link_create(node, TEGRA_ICC_MC); 878 if (err) 879 goto remove_nodes; 880 881 node->data = (struct tegra_mc_client *)&(mc->soc->clients[i]); 882 } 883 884 err = icc_provider_register(&mc->provider); 885 if (err) 886 goto remove_nodes; 887 888 return 0; 889 890 remove_nodes: 891 icc_nodes_remove(&mc->provider); 892 893 return err; 894 } 895 896 static void tegra_mc_num_channel_enabled(struct tegra_mc *mc) 897 { 898 unsigned int i; 899 u32 value; 900 901 value = mc_ch_readl(mc, 0, mc->soc->regs->cfg_channel_enable); 902 if (value <= 0) { 903 mc->num_channels = mc->soc->num_channels; 904 return; 905 } 906 907 for (i = 0; i < 32; i++) { 908 if (value & BIT(i)) 909 mc->num_channels++; 910 } 911 } 912 913 static int tegra_mc_probe(struct platform_device *pdev) 914 { 915 struct tegra_mc *mc; 916 u64 mask; 917 int err; 918 919 mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL); 920 if (!mc) 921 return -ENOMEM; 922 923 platform_set_drvdata(pdev, mc); 924 spin_lock_init(&mc->lock); 925 mc->soc = of_device_get_match_data(&pdev->dev); 926 mc->dev = &pdev->dev; 927 928 mask = DMA_BIT_MASK(mc->soc->num_address_bits); 929 930 err = dma_coerce_mask_and_coherent(&pdev->dev, mask); 931 if (err < 0) { 932 dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err); 933 return err; 934 } 935 936 /* length of MC tick in nanoseconds */ 937 mc->tick = 30; 938 939 mc->regs = devm_platform_ioremap_resource(pdev, 0); 940 if (IS_ERR(mc->regs)) 941 return PTR_ERR(mc->regs); 942 943 mc->debugfs.root = debugfs_create_dir("mc", NULL); 944 945 if (mc->soc->ops && mc->soc->ops->probe) { 946 err = mc->soc->ops->probe(mc); 947 if (err < 0) 948 return err; 949 } 950 951 tegra_mc_num_channel_enabled(mc); 952 953 if (mc->soc->handle_irq) { 954 unsigned int i; 955 956 WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n"); 957 958 for (i = 0; i < mc->soc->num_interrupts; i++) { 959 int irq; 960 961 irq = platform_get_irq(pdev, i); 962 if (irq < 0) 963 return irq; 964 965 err = devm_request_irq(&pdev->dev, irq, mc->soc->handle_irq[i], 0, 966 dev_name(&pdev->dev), mc); 967 if (err < 0) { 968 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", irq, err); 969 return err; 970 } 971 } 972 973 for (i = 0; i < mc->soc->num_intmasks; i++) { 974 if (mc->soc->num_channels) 975 mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmasks[i].mask, 976 mc->soc->intmasks[i].reg); 977 else 978 mc_writel(mc, mc->soc->intmasks[i].mask, mc->soc->intmasks[i].reg); 979 } 980 } 981 982 if (mc->soc->reset_ops) { 983 err = tegra_mc_reset_setup(mc); 984 if (err < 0) 985 dev_err(&pdev->dev, "failed to register reset controller: %d\n", err); 986 } 987 988 err = tegra_mc_interconnect_setup(mc); 989 if (err < 0) 990 dev_err(&pdev->dev, "failed to initialize interconnect: %d\n", 991 err); 992 993 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) { 994 mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc); 995 if (IS_ERR(mc->smmu)) { 996 dev_err(&pdev->dev, "failed to probe SMMU: %pe\n", mc->smmu); 997 mc->smmu = NULL; 998 } 999 } 1000 1001 return 0; 1002 } 1003 1004 static void tegra_mc_sync_state(struct device *dev) 1005 { 1006 struct tegra_mc *mc = dev_get_drvdata(dev); 1007 1008 /* check whether ICC provider is registered */ 1009 if (mc->provider.dev == dev) 1010 icc_sync_state(dev); 1011 } 1012 1013 static struct platform_driver tegra_mc_driver = { 1014 .driver = { 1015 .name = "tegra-mc", 1016 .of_match_table = tegra_mc_of_match, 1017 .suppress_bind_attrs = true, 1018 .sync_state = tegra_mc_sync_state, 1019 }, 1020 .prevent_deferred_probe = true, 1021 .probe = tegra_mc_probe, 1022 }; 1023 1024 static int tegra_mc_init(void) 1025 { 1026 return platform_driver_register(&tegra_mc_driver); 1027 } 1028 arch_initcall(tegra_mc_init); 1029 1030 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>"); 1031 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver"); 1032