1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2014-2025 NVIDIA CORPORATION. All rights reserved.
4 */
5
6 #include <linux/clk.h>
7 #include <linux/delay.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/export.h>
10 #include <linux/interrupt.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/of.h>
14 #include <linux/of_platform.h>
15 #include <linux/platform_device.h>
16 #include <linux/slab.h>
17 #include <linux/sort.h>
18 #include <linux/tegra-icc.h>
19
20 #include <soc/tegra/fuse.h>
21
22 #include "mc.h"
23
24 static const struct of_device_id tegra_mc_of_match[] = {
25 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
26 { .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
27 #endif
28 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
29 { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
30 #endif
31 #ifdef CONFIG_ARCH_TEGRA_114_SOC
32 { .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
33 #endif
34 #ifdef CONFIG_ARCH_TEGRA_124_SOC
35 { .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
36 #endif
37 #ifdef CONFIG_ARCH_TEGRA_132_SOC
38 { .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
39 #endif
40 #ifdef CONFIG_ARCH_TEGRA_210_SOC
41 { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
42 #endif
43 #ifdef CONFIG_ARCH_TEGRA_186_SOC
44 { .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc },
45 #endif
46 #ifdef CONFIG_ARCH_TEGRA_194_SOC
47 { .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc },
48 #endif
49 #ifdef CONFIG_ARCH_TEGRA_234_SOC
50 { .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc },
51 #endif
52 #ifdef CONFIG_ARCH_TEGRA_264_SOC
53 { .compatible = "nvidia,tegra264-mc", .data = &tegra264_mc_soc },
54 #endif
55 { /* sentinel */ }
56 };
57 MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
58
tegra_mc_devm_action_put_device(void * data)59 static void tegra_mc_devm_action_put_device(void *data)
60 {
61 struct tegra_mc *mc = data;
62
63 put_device(mc->dev);
64 }
65
66 /**
67 * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle
68 * @dev: device pointer for the consumer device
69 *
70 * This function will search for the Memory Controller node in a device-tree
71 * and retrieve the Memory Controller handle.
72 *
73 * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc.
74 */
devm_tegra_memory_controller_get(struct device * dev)75 struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev)
76 {
77 struct platform_device *pdev;
78 struct device_node *np;
79 struct tegra_mc *mc;
80 int err;
81
82 np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0);
83 if (!np)
84 return ERR_PTR(-ENOENT);
85
86 pdev = of_find_device_by_node(np);
87 of_node_put(np);
88 if (!pdev)
89 return ERR_PTR(-ENODEV);
90
91 mc = platform_get_drvdata(pdev);
92 if (!mc) {
93 put_device(&pdev->dev);
94 return ERR_PTR(-EPROBE_DEFER);
95 }
96
97 err = devm_add_action_or_reset(dev, tegra_mc_devm_action_put_device, mc);
98 if (err)
99 return ERR_PTR(err);
100
101 return mc;
102 }
103 EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get);
104
tegra_mc_probe_device(struct tegra_mc * mc,struct device * dev)105 int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev)
106 {
107 if (mc->soc->ops && mc->soc->ops->probe_device)
108 return mc->soc->ops->probe_device(mc, dev);
109
110 return 0;
111 }
112 EXPORT_SYMBOL_GPL(tegra_mc_probe_device);
113
tegra_mc_get_carveout_info(struct tegra_mc * mc,unsigned int id,phys_addr_t * base,u64 * size)114 int tegra_mc_get_carveout_info(struct tegra_mc *mc, unsigned int id,
115 phys_addr_t *base, u64 *size)
116 {
117 u32 offset;
118
119 if (id < 1 || id >= mc->soc->num_carveouts)
120 return -EINVAL;
121
122 if (id < 6)
123 offset = 0xc0c + 0x50 * (id - 1);
124 else
125 offset = 0x2004 + 0x50 * (id - 6);
126
127 *base = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x0);
128 #ifdef CONFIG_PHYS_ADDR_T_64BIT
129 *base |= (phys_addr_t)mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x4) << 32;
130 #endif
131
132 if (size)
133 *size = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x8) << 17;
134
135 return 0;
136 }
137 EXPORT_SYMBOL_GPL(tegra_mc_get_carveout_info);
138
tegra_mc_block_dma_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)139 static int tegra_mc_block_dma_common(struct tegra_mc *mc,
140 const struct tegra_mc_reset *rst)
141 {
142 unsigned long flags;
143 u32 value;
144
145 spin_lock_irqsave(&mc->lock, flags);
146
147 value = mc_readl(mc, rst->control) | BIT(rst->bit);
148 mc_writel(mc, value, rst->control);
149
150 spin_unlock_irqrestore(&mc->lock, flags);
151
152 return 0;
153 }
154
tegra_mc_dma_idling_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)155 static bool tegra_mc_dma_idling_common(struct tegra_mc *mc,
156 const struct tegra_mc_reset *rst)
157 {
158 return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
159 }
160
tegra_mc_unblock_dma_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)161 static int tegra_mc_unblock_dma_common(struct tegra_mc *mc,
162 const struct tegra_mc_reset *rst)
163 {
164 unsigned long flags;
165 u32 value;
166
167 spin_lock_irqsave(&mc->lock, flags);
168
169 value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
170 mc_writel(mc, value, rst->control);
171
172 spin_unlock_irqrestore(&mc->lock, flags);
173
174 return 0;
175 }
176
tegra_mc_reset_status_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)177 static int tegra_mc_reset_status_common(struct tegra_mc *mc,
178 const struct tegra_mc_reset *rst)
179 {
180 return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
181 }
182
183 const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = {
184 .block_dma = tegra_mc_block_dma_common,
185 .dma_idling = tegra_mc_dma_idling_common,
186 .unblock_dma = tegra_mc_unblock_dma_common,
187 .reset_status = tegra_mc_reset_status_common,
188 };
189
reset_to_mc(struct reset_controller_dev * rcdev)190 static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
191 {
192 return container_of(rcdev, struct tegra_mc, reset);
193 }
194
tegra_mc_reset_find(struct tegra_mc * mc,unsigned long id)195 static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
196 unsigned long id)
197 {
198 unsigned int i;
199
200 for (i = 0; i < mc->soc->num_resets; i++)
201 if (mc->soc->resets[i].id == id)
202 return &mc->soc->resets[i];
203
204 return NULL;
205 }
206
tegra_mc_hotreset_assert(struct reset_controller_dev * rcdev,unsigned long id)207 static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
208 unsigned long id)
209 {
210 struct tegra_mc *mc = reset_to_mc(rcdev);
211 const struct tegra_mc_reset_ops *rst_ops;
212 const struct tegra_mc_reset *rst;
213 int retries = 500;
214 int err;
215
216 rst = tegra_mc_reset_find(mc, id);
217 if (!rst)
218 return -ENODEV;
219
220 rst_ops = mc->soc->reset_ops;
221 if (!rst_ops)
222 return -ENODEV;
223
224 /* DMA flushing will fail if reset is already asserted */
225 if (rst_ops->reset_status) {
226 /* check whether reset is asserted */
227 if (rst_ops->reset_status(mc, rst))
228 return 0;
229 }
230
231 if (rst_ops->block_dma) {
232 /* block clients DMA requests */
233 err = rst_ops->block_dma(mc, rst);
234 if (err) {
235 dev_err(mc->dev, "failed to block %s DMA: %d\n",
236 rst->name, err);
237 return err;
238 }
239 }
240
241 if (rst_ops->dma_idling) {
242 /* wait for completion of the outstanding DMA requests */
243 while (!rst_ops->dma_idling(mc, rst)) {
244 if (!retries--) {
245 dev_err(mc->dev, "failed to flush %s DMA\n",
246 rst->name);
247 return -EBUSY;
248 }
249
250 usleep_range(10, 100);
251 }
252 }
253
254 if (rst_ops->hotreset_assert) {
255 /* clear clients DMA requests sitting before arbitration */
256 err = rst_ops->hotreset_assert(mc, rst);
257 if (err) {
258 dev_err(mc->dev, "failed to hot reset %s: %d\n",
259 rst->name, err);
260 return err;
261 }
262 }
263
264 return 0;
265 }
266
tegra_mc_hotreset_deassert(struct reset_controller_dev * rcdev,unsigned long id)267 static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
268 unsigned long id)
269 {
270 struct tegra_mc *mc = reset_to_mc(rcdev);
271 const struct tegra_mc_reset_ops *rst_ops;
272 const struct tegra_mc_reset *rst;
273 int err;
274
275 rst = tegra_mc_reset_find(mc, id);
276 if (!rst)
277 return -ENODEV;
278
279 rst_ops = mc->soc->reset_ops;
280 if (!rst_ops)
281 return -ENODEV;
282
283 if (rst_ops->hotreset_deassert) {
284 /* take out client from hot reset */
285 err = rst_ops->hotreset_deassert(mc, rst);
286 if (err) {
287 dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",
288 rst->name, err);
289 return err;
290 }
291 }
292
293 if (rst_ops->unblock_dma) {
294 /* allow new DMA requests to proceed to arbitration */
295 err = rst_ops->unblock_dma(mc, rst);
296 if (err) {
297 dev_err(mc->dev, "failed to unblock %s DMA : %d\n",
298 rst->name, err);
299 return err;
300 }
301 }
302
303 return 0;
304 }
305
tegra_mc_hotreset_status(struct reset_controller_dev * rcdev,unsigned long id)306 static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
307 unsigned long id)
308 {
309 struct tegra_mc *mc = reset_to_mc(rcdev);
310 const struct tegra_mc_reset_ops *rst_ops;
311 const struct tegra_mc_reset *rst;
312
313 rst = tegra_mc_reset_find(mc, id);
314 if (!rst)
315 return -ENODEV;
316
317 rst_ops = mc->soc->reset_ops;
318 if (!rst_ops)
319 return -ENODEV;
320
321 return rst_ops->reset_status(mc, rst);
322 }
323
324 static const struct reset_control_ops tegra_mc_reset_ops = {
325 .assert = tegra_mc_hotreset_assert,
326 .deassert = tegra_mc_hotreset_deassert,
327 .status = tegra_mc_hotreset_status,
328 };
329
tegra_mc_reset_setup(struct tegra_mc * mc)330 static int tegra_mc_reset_setup(struct tegra_mc *mc)
331 {
332 int err;
333
334 mc->reset.ops = &tegra_mc_reset_ops;
335 mc->reset.owner = THIS_MODULE;
336 mc->reset.of_node = mc->dev->of_node;
337 mc->reset.of_reset_n_cells = 1;
338 mc->reset.nr_resets = mc->soc->num_resets;
339
340 err = reset_controller_register(&mc->reset);
341 if (err < 0)
342 return err;
343
344 return 0;
345 }
346
tegra_mc_write_emem_configuration(struct tegra_mc * mc,unsigned long rate)347 int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
348 {
349 unsigned int i;
350 struct tegra_mc_timing *timing = NULL;
351
352 for (i = 0; i < mc->num_timings; i++) {
353 if (mc->timings[i].rate == rate) {
354 timing = &mc->timings[i];
355 break;
356 }
357 }
358
359 if (!timing) {
360 dev_err(mc->dev, "no memory timing registered for rate %lu\n",
361 rate);
362 return -EINVAL;
363 }
364
365 for (i = 0; i < mc->soc->num_emem_regs; ++i)
366 mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
367
368 return 0;
369 }
370 EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration);
371
tegra_mc_get_emem_device_count(struct tegra_mc * mc)372 unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
373 {
374 u8 dram_count;
375
376 dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
377 dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
378 dram_count++;
379
380 return dram_count;
381 }
382 EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count);
383
384 #if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \
385 defined(CONFIG_ARCH_TEGRA_114_SOC) || \
386 defined(CONFIG_ARCH_TEGRA_124_SOC) || \
387 defined(CONFIG_ARCH_TEGRA_132_SOC) || \
388 defined(CONFIG_ARCH_TEGRA_210_SOC)
tegra_mc_setup_latency_allowance(struct tegra_mc * mc)389 static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
390 {
391 unsigned long long tick;
392 unsigned int i;
393 u32 value;
394
395 /* compute the number of MC clock cycles per tick */
396 tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);
397 do_div(tick, NSEC_PER_SEC);
398
399 value = mc_readl(mc, MC_EMEM_ARB_CFG);
400 value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
401 value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
402 mc_writel(mc, value, MC_EMEM_ARB_CFG);
403
404 /* write latency allowance defaults */
405 for (i = 0; i < mc->soc->num_clients; i++) {
406 const struct tegra_mc_client *client = &mc->soc->clients[i];
407 u32 value;
408
409 value = mc_readl(mc, client->regs.la.reg);
410 value &= ~(client->regs.la.mask << client->regs.la.shift);
411 value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift;
412 mc_writel(mc, value, client->regs.la.reg);
413 }
414
415 /* latch new values */
416 mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
417
418 return 0;
419 }
420
load_one_timing(struct tegra_mc * mc,struct tegra_mc_timing * timing,struct device_node * node)421 static int load_one_timing(struct tegra_mc *mc,
422 struct tegra_mc_timing *timing,
423 struct device_node *node)
424 {
425 int err;
426 u32 tmp;
427
428 err = of_property_read_u32(node, "clock-frequency", &tmp);
429 if (err) {
430 dev_err(mc->dev,
431 "timing %pOFn: failed to read rate\n", node);
432 return err;
433 }
434
435 timing->rate = tmp;
436 timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
437 sizeof(u32), GFP_KERNEL);
438 if (!timing->emem_data)
439 return -ENOMEM;
440
441 err = of_property_read_u32_array(node, "nvidia,emem-configuration",
442 timing->emem_data,
443 mc->soc->num_emem_regs);
444 if (err) {
445 dev_err(mc->dev,
446 "timing %pOFn: failed to read EMEM configuration\n",
447 node);
448 return err;
449 }
450
451 return 0;
452 }
453
load_timings(struct tegra_mc * mc,struct device_node * node)454 static int load_timings(struct tegra_mc *mc, struct device_node *node)
455 {
456 struct tegra_mc_timing *timing;
457 int child_count = of_get_child_count(node);
458 int i = 0, err;
459
460 mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
461 GFP_KERNEL);
462 if (!mc->timings)
463 return -ENOMEM;
464
465 mc->num_timings = child_count;
466
467 for_each_child_of_node_scoped(node, child) {
468 timing = &mc->timings[i++];
469
470 err = load_one_timing(mc, timing, child);
471 if (err)
472 return err;
473 }
474
475 return 0;
476 }
477
tegra_mc_setup_timings(struct tegra_mc * mc)478 static int tegra_mc_setup_timings(struct tegra_mc *mc)
479 {
480 u32 ram_code, node_ram_code;
481 int err;
482
483 ram_code = tegra_read_ram_code();
484
485 mc->num_timings = 0;
486
487 for_each_child_of_node_scoped(mc->dev->of_node, node) {
488 err = of_property_read_u32(node, "nvidia,ram-code",
489 &node_ram_code);
490 if (err || (node_ram_code != ram_code))
491 continue;
492
493 err = load_timings(mc, node);
494 if (err)
495 return err;
496 break;
497 }
498
499 if (mc->num_timings == 0)
500 dev_warn(mc->dev,
501 "no memory timings for RAM code %u registered\n",
502 ram_code);
503
504 return 0;
505 }
506
tegra30_mc_probe(struct tegra_mc * mc)507 int tegra30_mc_probe(struct tegra_mc *mc)
508 {
509 int err;
510
511 mc->clk = devm_clk_get_optional(mc->dev, "mc");
512 if (IS_ERR(mc->clk)) {
513 dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk));
514 return PTR_ERR(mc->clk);
515 }
516
517 /* ensure that debug features are disabled */
518 mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG);
519
520 err = tegra_mc_setup_latency_allowance(mc);
521 if (err < 0) {
522 dev_err(mc->dev, "failed to setup latency allowance: %d\n", err);
523 return err;
524 }
525
526 err = tegra_mc_setup_timings(mc);
527 if (err < 0) {
528 dev_err(mc->dev, "failed to setup timings: %d\n", err);
529 return err;
530 }
531
532 return 0;
533 }
534
535 const struct tegra_mc_ops tegra30_mc_ops = {
536 .probe = tegra30_mc_probe,
537 .handle_irq = tegra30_mc_handle_irq,
538 };
539 #endif
540
mc_global_intstatus_to_channel(const struct tegra_mc * mc,u32 status,unsigned int * mc_channel)541 static int mc_global_intstatus_to_channel(const struct tegra_mc *mc, u32 status,
542 unsigned int *mc_channel)
543 {
544 if ((status & mc->soc->ch_intmask) == 0)
545 return -EINVAL;
546
547 *mc_channel = __ffs((status & mc->soc->ch_intmask) >>
548 mc->soc->global_intstatus_channel_shift);
549
550 return 0;
551 }
552
mc_channel_to_global_intstatus(const struct tegra_mc * mc,unsigned int channel)553 static u32 mc_channel_to_global_intstatus(const struct tegra_mc *mc,
554 unsigned int channel)
555 {
556 return BIT(channel) << mc->soc->global_intstatus_channel_shift;
557 }
558
tegra30_mc_handle_irq(int irq,void * data)559 irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
560 {
561 struct tegra_mc *mc = data;
562 unsigned int bit, channel;
563 unsigned long status;
564
565 if (mc->soc->num_channels) {
566 u32 global_status;
567 int err;
568
569 global_status = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MC_GLOBAL_INTSTATUS);
570 err = mc_global_intstatus_to_channel(mc, global_status, &channel);
571 if (err < 0) {
572 dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n",
573 global_status);
574 return IRQ_NONE;
575 }
576
577 /* mask all interrupts to avoid flooding */
578 status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmask;
579 } else {
580 status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
581 }
582
583 if (!status)
584 return IRQ_NONE;
585
586 for_each_set_bit(bit, &status, 32) {
587 const char *error = tegra_mc_status_names[bit] ?: "unknown";
588 const char *client = "unknown", *desc;
589 const char *direction, *secure;
590 u32 status_reg, addr_reg;
591 u32 intmask = BIT(bit);
592 phys_addr_t addr = 0;
593 #ifdef CONFIG_PHYS_ADDR_T_64BIT
594 u32 addr_hi_reg = 0;
595 #endif
596 unsigned int i;
597 char perm[7];
598 u8 id, type;
599 u32 value;
600
601 switch (intmask) {
602 case MC_INT_DECERR_VPR:
603 status_reg = MC_ERR_VPR_STATUS;
604 addr_reg = MC_ERR_VPR_ADR;
605 break;
606
607 case MC_INT_SECERR_SEC:
608 status_reg = MC_ERR_SEC_STATUS;
609 addr_reg = MC_ERR_SEC_ADR;
610 break;
611
612 case MC_INT_DECERR_MTS:
613 status_reg = MC_ERR_MTS_STATUS;
614 addr_reg = MC_ERR_MTS_ADR;
615 break;
616
617 case MC_INT_DECERR_GENERALIZED_CARVEOUT:
618 status_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS;
619 addr_reg = MC_ERR_GENERALIZED_CARVEOUT_ADR;
620 break;
621
622 case MC_INT_DECERR_ROUTE_SANITY:
623 status_reg = MC_ERR_ROUTE_SANITY_STATUS;
624 addr_reg = MC_ERR_ROUTE_SANITY_ADR;
625 break;
626
627 default:
628 status_reg = MC_ERR_STATUS;
629 addr_reg = MC_ERR_ADR;
630
631 #ifdef CONFIG_PHYS_ADDR_T_64BIT
632 if (mc->soc->has_addr_hi_reg)
633 addr_hi_reg = MC_ERR_ADR_HI;
634 #endif
635 break;
636 }
637
638 if (mc->soc->num_channels)
639 value = mc_ch_readl(mc, channel, status_reg);
640 else
641 value = mc_readl(mc, status_reg);
642
643 #ifdef CONFIG_PHYS_ADDR_T_64BIT
644 if (mc->soc->num_address_bits > 32) {
645 if (addr_hi_reg) {
646 if (mc->soc->num_channels)
647 addr = mc_ch_readl(mc, channel, addr_hi_reg);
648 else
649 addr = mc_readl(mc, addr_hi_reg);
650 } else {
651 addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
652 MC_ERR_STATUS_ADR_HI_MASK);
653 }
654 addr <<= 32;
655 }
656 #endif
657
658 if (value & MC_ERR_STATUS_RW)
659 direction = "write";
660 else
661 direction = "read";
662
663 if (value & MC_ERR_STATUS_SECURITY)
664 secure = "secure ";
665 else
666 secure = "";
667
668 id = value & mc->soc->client_id_mask;
669
670 for (i = 0; i < mc->soc->num_clients; i++) {
671 if (mc->soc->clients[i].id == id) {
672 client = mc->soc->clients[i].name;
673 break;
674 }
675 }
676
677 type = (value & MC_ERR_STATUS_TYPE_MASK) >>
678 MC_ERR_STATUS_TYPE_SHIFT;
679 desc = tegra_mc_error_names[type];
680
681 switch (value & MC_ERR_STATUS_TYPE_MASK) {
682 case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
683 perm[0] = ' ';
684 perm[1] = '[';
685
686 if (value & MC_ERR_STATUS_READABLE)
687 perm[2] = 'R';
688 else
689 perm[2] = '-';
690
691 if (value & MC_ERR_STATUS_WRITABLE)
692 perm[3] = 'W';
693 else
694 perm[3] = '-';
695
696 if (value & MC_ERR_STATUS_NONSECURE)
697 perm[4] = '-';
698 else
699 perm[4] = 'S';
700
701 perm[5] = ']';
702 perm[6] = '\0';
703 break;
704
705 default:
706 perm[0] = '\0';
707 break;
708 }
709
710 if (mc->soc->num_channels)
711 value = mc_ch_readl(mc, channel, addr_reg);
712 else
713 value = mc_readl(mc, addr_reg);
714 addr |= value;
715
716 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
717 client, secure, direction, &addr, error,
718 desc, perm);
719 }
720
721 /* clear interrupts */
722 if (mc->soc->num_channels) {
723 mc_ch_writel(mc, channel, status, MC_INTSTATUS);
724 mc_ch_writel(mc, MC_BROADCAST_CHANNEL,
725 mc_channel_to_global_intstatus(mc, channel),
726 MC_GLOBAL_INTSTATUS);
727 } else {
728 mc_writel(mc, status, MC_INTSTATUS);
729 }
730
731 return IRQ_HANDLED;
732 }
733
734 const char *const tegra_mc_status_names[32] = {
735 [ 1] = "External interrupt",
736 [ 6] = "EMEM address decode error",
737 [ 7] = "GART page fault",
738 [ 8] = "Security violation",
739 [ 9] = "EMEM arbitration error",
740 [10] = "Page fault",
741 [11] = "Invalid APB ASID update",
742 [12] = "VPR violation",
743 [13] = "Secure carveout violation",
744 [16] = "MTS carveout violation",
745 [17] = "Generalized carveout violation",
746 [20] = "Route Sanity error",
747 };
748
749 const char *const tegra_mc_error_names[8] = {
750 [2] = "EMEM decode error",
751 [3] = "TrustZone violation",
752 [4] = "Carveout violation",
753 [6] = "SMMU translation error",
754 };
755
tegra_mc_icc_xlate(const struct of_phandle_args * spec,void * data)756 struct icc_node *tegra_mc_icc_xlate(const struct of_phandle_args *spec, void *data)
757 {
758 struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
759 struct icc_node *node;
760
761 list_for_each_entry(node, &mc->provider.nodes, node_list) {
762 if (node->id == spec->args[0])
763 return node;
764 }
765
766 /*
767 * If a client driver calls devm_of_icc_get() before the MC driver
768 * is probed, then return EPROBE_DEFER to the client driver.
769 */
770 return ERR_PTR(-EPROBE_DEFER);
771 }
772
tegra_mc_icc_get(struct icc_node * node,u32 * average,u32 * peak)773 static int tegra_mc_icc_get(struct icc_node *node, u32 *average, u32 *peak)
774 {
775 *average = 0;
776 *peak = 0;
777
778 return 0;
779 }
780
tegra_mc_icc_set(struct icc_node * src,struct icc_node * dst)781 static int tegra_mc_icc_set(struct icc_node *src, struct icc_node *dst)
782 {
783 return 0;
784 }
785
786 const struct tegra_mc_icc_ops tegra_mc_icc_ops = {
787 .xlate = tegra_mc_icc_xlate,
788 .aggregate = icc_std_aggregate,
789 .get_bw = tegra_mc_icc_get,
790 .set = tegra_mc_icc_set,
791 };
792
793 /*
794 * Memory Controller (MC) has few Memory Clients that are issuing memory
795 * bandwidth allocation requests to the MC interconnect provider. The MC
796 * provider aggregates the requests and then sends the aggregated request
797 * up to the External Memory Controller (EMC) interconnect provider which
798 * re-configures hardware interface to External Memory (EMEM) in accordance
799 * to the required bandwidth. Each MC interconnect node represents an
800 * individual Memory Client.
801 *
802 * Memory interconnect topology:
803 *
804 * +----+
805 * +--------+ | |
806 * | TEXSRD +--->+ |
807 * +--------+ | |
808 * | | +-----+ +------+
809 * ... | MC +--->+ EMC +--->+ EMEM |
810 * | | +-----+ +------+
811 * +--------+ | |
812 * | DISP.. +--->+ |
813 * +--------+ | |
814 * +----+
815 */
tegra_mc_interconnect_setup(struct tegra_mc * mc)816 static int tegra_mc_interconnect_setup(struct tegra_mc *mc)
817 {
818 struct icc_node *node;
819 unsigned int i;
820 int err;
821
822 /* older device-trees don't have interconnect properties */
823 if (!device_property_present(mc->dev, "#interconnect-cells") ||
824 !mc->soc->icc_ops)
825 return 0;
826
827 mc->provider.dev = mc->dev;
828 mc->provider.data = &mc->provider;
829 mc->provider.set = mc->soc->icc_ops->set;
830 mc->provider.aggregate = mc->soc->icc_ops->aggregate;
831 mc->provider.get_bw = mc->soc->icc_ops->get_bw;
832 mc->provider.xlate = mc->soc->icc_ops->xlate;
833 mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended;
834
835 icc_provider_init(&mc->provider);
836
837 /* create Memory Controller node */
838 node = icc_node_create(TEGRA_ICC_MC);
839 if (IS_ERR(node))
840 return PTR_ERR(node);
841
842 node->name = "Memory Controller";
843 icc_node_add(node, &mc->provider);
844
845 /* link Memory Controller to External Memory Controller */
846 err = icc_link_create(node, TEGRA_ICC_EMC);
847 if (err)
848 goto remove_nodes;
849
850 for (i = 0; i < mc->soc->num_clients; i++) {
851 /* create MC client node */
852 node = icc_node_create(mc->soc->clients[i].id);
853 if (IS_ERR(node)) {
854 err = PTR_ERR(node);
855 goto remove_nodes;
856 }
857
858 node->name = mc->soc->clients[i].name;
859 icc_node_add(node, &mc->provider);
860
861 /* link Memory Client to Memory Controller */
862 err = icc_link_create(node, TEGRA_ICC_MC);
863 if (err)
864 goto remove_nodes;
865
866 node->data = (struct tegra_mc_client *)&(mc->soc->clients[i]);
867 }
868
869 err = icc_provider_register(&mc->provider);
870 if (err)
871 goto remove_nodes;
872
873 return 0;
874
875 remove_nodes:
876 icc_nodes_remove(&mc->provider);
877
878 return err;
879 }
880
tegra_mc_num_channel_enabled(struct tegra_mc * mc)881 static void tegra_mc_num_channel_enabled(struct tegra_mc *mc)
882 {
883 unsigned int i;
884 u32 value;
885
886 value = mc_ch_readl(mc, 0, MC_EMEM_ADR_CFG_CHANNEL_ENABLE);
887 if (value <= 0) {
888 mc->num_channels = mc->soc->num_channels;
889 return;
890 }
891
892 for (i = 0; i < 32; i++) {
893 if (value & BIT(i))
894 mc->num_channels++;
895 }
896 }
897
tegra_mc_probe(struct platform_device * pdev)898 static int tegra_mc_probe(struct platform_device *pdev)
899 {
900 struct tegra_mc *mc;
901 u64 mask;
902 int err;
903
904 mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
905 if (!mc)
906 return -ENOMEM;
907
908 platform_set_drvdata(pdev, mc);
909 spin_lock_init(&mc->lock);
910 mc->soc = of_device_get_match_data(&pdev->dev);
911 mc->dev = &pdev->dev;
912
913 mask = DMA_BIT_MASK(mc->soc->num_address_bits);
914
915 err = dma_coerce_mask_and_coherent(&pdev->dev, mask);
916 if (err < 0) {
917 dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);
918 return err;
919 }
920
921 /* length of MC tick in nanoseconds */
922 mc->tick = 30;
923
924 mc->regs = devm_platform_ioremap_resource(pdev, 0);
925 if (IS_ERR(mc->regs))
926 return PTR_ERR(mc->regs);
927
928 mc->debugfs.root = debugfs_create_dir("mc", NULL);
929
930 if (mc->soc->ops && mc->soc->ops->probe) {
931 err = mc->soc->ops->probe(mc);
932 if (err < 0)
933 return err;
934 }
935
936 tegra_mc_num_channel_enabled(mc);
937
938 if (mc->soc->ops && mc->soc->ops->handle_irq) {
939 mc->irq = platform_get_irq(pdev, 0);
940 if (mc->irq < 0)
941 return mc->irq;
942
943 WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
944
945 if (mc->soc->num_channels)
946 mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmask,
947 MC_INTMASK);
948 else
949 mc_writel(mc, mc->soc->intmask, MC_INTMASK);
950
951 err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0,
952 dev_name(&pdev->dev), mc);
953 if (err < 0) {
954 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
955 err);
956 return err;
957 }
958 }
959
960 if (mc->soc->reset_ops) {
961 err = tegra_mc_reset_setup(mc);
962 if (err < 0)
963 dev_err(&pdev->dev, "failed to register reset controller: %d\n", err);
964 }
965
966 err = tegra_mc_interconnect_setup(mc);
967 if (err < 0)
968 dev_err(&pdev->dev, "failed to initialize interconnect: %d\n",
969 err);
970
971 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
972 mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
973 if (IS_ERR(mc->smmu)) {
974 dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
975 PTR_ERR(mc->smmu));
976 mc->smmu = NULL;
977 }
978 }
979
980 return 0;
981 }
982
tegra_mc_sync_state(struct device * dev)983 static void tegra_mc_sync_state(struct device *dev)
984 {
985 struct tegra_mc *mc = dev_get_drvdata(dev);
986
987 /* check whether ICC provider is registered */
988 if (mc->provider.dev == dev)
989 icc_sync_state(dev);
990 }
991
992 static struct platform_driver tegra_mc_driver = {
993 .driver = {
994 .name = "tegra-mc",
995 .of_match_table = tegra_mc_of_match,
996 .suppress_bind_attrs = true,
997 .sync_state = tegra_mc_sync_state,
998 },
999 .prevent_deferred_probe = true,
1000 .probe = tegra_mc_probe,
1001 };
1002
tegra_mc_init(void)1003 static int tegra_mc_init(void)
1004 {
1005 return platform_driver_register(&tegra_mc_driver);
1006 }
1007 arch_initcall(tegra_mc_init);
1008
1009 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
1010 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
1011