1 // SPDX-License-Identifier: GPL-2.0-only
3  * Copyright (C) 2014-2025 NVIDIA CORPORATION.  All rights reserved.
8 #include <linux/dma-mapping.h>
18 #include <linux/tegra-icc.h>
20 #include <soc/tegra/fuse.h>
26 	{ .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
29 	{ .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
32 	{ .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
35 	{ .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
38 	{ .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
41 	{ .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
44 	{ .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc },
47 	{ .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc },
50 	{ .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc },
53 	{ .compatible = "nvidia,tegra264-mc", .data = &tegra264_mc_soc },
63 	put_device(mc->dev);  in tegra_mc_devm_action_put_device()
67  * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle
70  * This function will search for the Memory Controller node in a device-tree
71  * and retrieve the Memory Controller handle.
82 	np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0);  in devm_tegra_memory_controller_get()
84 		return ERR_PTR(-ENOENT);  in devm_tegra_memory_controller_get()
89 		return ERR_PTR(-ENODEV);  in devm_tegra_memory_controller_get()
93 		put_device(&pdev->dev);  in devm_tegra_memory_controller_get()
94 		return ERR_PTR(-EPROBE_DEFER);  in devm_tegra_memory_controller_get()
107 	if (mc->soc->ops && mc->soc->ops->probe_device)  in tegra_mc_probe_device()
108 		return mc->soc->ops->probe_device(mc, dev);  in tegra_mc_probe_device()
119 	if (id < 1 || id >= mc->soc->num_carveouts)  in tegra_mc_get_carveout_info()
120 		return -EINVAL;  in tegra_mc_get_carveout_info()
123 		offset = 0xc0c + 0x50 * (id - 1);  in tegra_mc_get_carveout_info()
125 		offset = 0x2004 + 0x50 * (id - 6);  in tegra_mc_get_carveout_info()
145 	spin_lock_irqsave(&mc->lock, flags);  in tegra_mc_block_dma_common()
147 	value = mc_readl(mc, rst->control) | BIT(rst->bit);  in tegra_mc_block_dma_common()
148 	mc_writel(mc, value, rst->control);  in tegra_mc_block_dma_common()
150 	spin_unlock_irqrestore(&mc->lock, flags);  in tegra_mc_block_dma_common()
158 	return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;  in tegra_mc_dma_idling_common()
167 	spin_lock_irqsave(&mc->lock, flags);  in tegra_mc_unblock_dma_common()
169 	value = mc_readl(mc, rst->control) & ~BIT(rst->bit);  in tegra_mc_unblock_dma_common()
170 	mc_writel(mc, value, rst->control);  in tegra_mc_unblock_dma_common()
172 	spin_unlock_irqrestore(&mc->lock, flags);  in tegra_mc_unblock_dma_common()
180 	return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;  in tegra_mc_reset_status_common()
200 	for (i = 0; i < mc->soc->num_resets; i++)  in tegra_mc_reset_find()
201 		if (mc->soc->resets[i].id == id)  in tegra_mc_reset_find()
202 			return &mc->soc->resets[i];  in tegra_mc_reset_find()
218 		return -ENODEV;  in tegra_mc_hotreset_assert()
220 	rst_ops = mc->soc->reset_ops;  in tegra_mc_hotreset_assert()
222 		return -ENODEV;  in tegra_mc_hotreset_assert()
225 	if (rst_ops->reset_status) {  in tegra_mc_hotreset_assert()
227 		if (rst_ops->reset_status(mc, rst))  in tegra_mc_hotreset_assert()
231 	if (rst_ops->block_dma) {  in tegra_mc_hotreset_assert()
233 		err = rst_ops->block_dma(mc, rst);  in tegra_mc_hotreset_assert()
235 			dev_err(mc->dev, "failed to block %s DMA: %d\n",  in tegra_mc_hotreset_assert()
236 				rst->name, err);  in tegra_mc_hotreset_assert()
241 	if (rst_ops->dma_idling) {  in tegra_mc_hotreset_assert()
243 		while (!rst_ops->dma_idling(mc, rst)) {  in tegra_mc_hotreset_assert()
244 			if (!retries--) {  in tegra_mc_hotreset_assert()
245 				dev_err(mc->dev, "failed to flush %s DMA\n",  in tegra_mc_hotreset_assert()
246 					rst->name);  in tegra_mc_hotreset_assert()
247 				return -EBUSY;  in tegra_mc_hotreset_assert()
254 	if (rst_ops->hotreset_assert) {  in tegra_mc_hotreset_assert()
256 		err = rst_ops->hotreset_assert(mc, rst);  in tegra_mc_hotreset_assert()
258 			dev_err(mc->dev, "failed to hot reset %s: %d\n",  in tegra_mc_hotreset_assert()
259 				rst->name, err);  in tegra_mc_hotreset_assert()
277 		return -ENODEV;  in tegra_mc_hotreset_deassert()
279 	rst_ops = mc->soc->reset_ops;  in tegra_mc_hotreset_deassert()
281 		return -ENODEV;  in tegra_mc_hotreset_deassert()
283 	if (rst_ops->hotreset_deassert) {  in tegra_mc_hotreset_deassert()
285 		err = rst_ops->hotreset_deassert(mc, rst);  in tegra_mc_hotreset_deassert()
287 			dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",  in tegra_mc_hotreset_deassert()
288 				rst->name, err);  in tegra_mc_hotreset_deassert()
293 	if (rst_ops->unblock_dma) {  in tegra_mc_hotreset_deassert()
295 		err = rst_ops->unblock_dma(mc, rst);  in tegra_mc_hotreset_deassert()
297 			dev_err(mc->dev, "failed to unblock %s DMA : %d\n",  in tegra_mc_hotreset_deassert()
298 				rst->name, err);  in tegra_mc_hotreset_deassert()
315 		return -ENODEV;  in tegra_mc_hotreset_status()
317 	rst_ops = mc->soc->reset_ops;  in tegra_mc_hotreset_status()
319 		return -ENODEV;  in tegra_mc_hotreset_status()
321 	return rst_ops->reset_status(mc, rst);  in tegra_mc_hotreset_status()
334 	mc->reset.ops = &tegra_mc_reset_ops;  in tegra_mc_reset_setup()
335 	mc->reset.owner = THIS_MODULE;  in tegra_mc_reset_setup()
336 	mc->reset.of_node = mc->dev->of_node;  in tegra_mc_reset_setup()
337 	mc->reset.of_reset_n_cells = 1;  in tegra_mc_reset_setup()
338 	mc->reset.nr_resets = mc->soc->num_resets;  in tegra_mc_reset_setup()
340 	err = reset_controller_register(&mc->reset);  in tegra_mc_reset_setup()
352 	for (i = 0; i < mc->num_timings; i++) {  in tegra_mc_write_emem_configuration()
353 		if (mc->timings[i].rate == rate) {  in tegra_mc_write_emem_configuration()
354 			timing = &mc->timings[i];  in tegra_mc_write_emem_configuration()
360 		dev_err(mc->dev, "no memory timing registered for rate %lu\n",  in tegra_mc_write_emem_configuration()
362 		return -EINVAL;  in tegra_mc_write_emem_configuration()
365 	for (i = 0; i < mc->soc->num_emem_regs; ++i)  in tegra_mc_write_emem_configuration()
366 		mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);  in tegra_mc_write_emem_configuration()
396 	tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);  in tegra_mc_setup_latency_allowance()
405 	for (i = 0; i < mc->soc->num_clients; i++) {  in tegra_mc_setup_latency_allowance()
406 		const struct tegra_mc_client *client = &mc->soc->clients[i];  in tegra_mc_setup_latency_allowance()
409 		value = mc_readl(mc, client->regs.la.reg);  in tegra_mc_setup_latency_allowance()
410 		value &= ~(client->regs.la.mask << client->regs.la.shift);  in tegra_mc_setup_latency_allowance()
411 		value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift;  in tegra_mc_setup_latency_allowance()
412 		mc_writel(mc, value, client->regs.la.reg);  in tegra_mc_setup_latency_allowance()
428 	err = of_property_read_u32(node, "clock-frequency", &tmp);  in load_one_timing()
430 		dev_err(mc->dev,  in load_one_timing()
435 	timing->rate = tmp;  in load_one_timing()
436 	timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,  in load_one_timing()
438 	if (!timing->emem_data)  in load_one_timing()
439 		return -ENOMEM;  in load_one_timing()
441 	err = of_property_read_u32_array(node, "nvidia,emem-configuration",  in load_one_timing()
442 					 timing->emem_data,  in load_one_timing()
443 					 mc->soc->num_emem_regs);  in load_one_timing()
445 		dev_err(mc->dev,  in load_one_timing()
460 	mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),  in load_timings()
462 	if (!mc->timings)  in load_timings()
463 		return -ENOMEM;  in load_timings()
465 	mc->num_timings = child_count;  in load_timings()
468 		timing = &mc->timings[i++];  in load_timings()
485 	mc->num_timings = 0;  in tegra_mc_setup_timings()
487 	for_each_child_of_node_scoped(mc->dev->of_node, node) {  in tegra_mc_setup_timings()
488 		err = of_property_read_u32(node, "nvidia,ram-code",  in tegra_mc_setup_timings()
499 	if (mc->num_timings == 0)  in tegra_mc_setup_timings()
500 		dev_warn(mc->dev,  in tegra_mc_setup_timings()
511 	mc->clk = devm_clk_get_optional(mc->dev, "mc");  in tegra30_mc_probe()
512 	if (IS_ERR(mc->clk)) {  in tegra30_mc_probe()
513 		dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk));  in tegra30_mc_probe()
514 		return PTR_ERR(mc->clk);  in tegra30_mc_probe()
522 		dev_err(mc->dev, "failed to setup latency allowance: %d\n", err);  in tegra30_mc_probe()
528 		dev_err(mc->dev, "failed to setup timings: %d\n", err);  in tegra30_mc_probe()
544 	if ((status & mc->soc->ch_intmask) == 0)  in mc_global_intstatus_to_channel()
545 		return -EINVAL;  in mc_global_intstatus_to_channel()
547 	*mc_channel = __ffs((status & mc->soc->ch_intmask) >>  in mc_global_intstatus_to_channel()
548 			    mc->soc->global_intstatus_channel_shift);  in mc_global_intstatus_to_channel()
556 	return BIT(channel) << mc->soc->global_intstatus_channel_shift;  in mc_channel_to_global_intstatus()
565 	if (mc->soc->num_channels) {  in tegra30_mc_handle_irq()
572 			dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n",  in tegra30_mc_handle_irq()
578 		status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmask;  in tegra30_mc_handle_irq()
580 		status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;  in tegra30_mc_handle_irq()
632 			if (mc->soc->has_addr_hi_reg)  in tegra30_mc_handle_irq()
638 		if (mc->soc->num_channels)  in tegra30_mc_handle_irq()
644 		if (mc->soc->num_address_bits > 32) {  in tegra30_mc_handle_irq()
646 				if (mc->soc->num_channels)  in tegra30_mc_handle_irq()
668 		id = value & mc->soc->client_id_mask;  in tegra30_mc_handle_irq()
670 		for (i = 0; i < mc->soc->num_clients; i++) {  in tegra30_mc_handle_irq()
671 			if (mc->soc->clients[i].id == id) {  in tegra30_mc_handle_irq()
672 				client = mc->soc->clients[i].name;  in tegra30_mc_handle_irq()
689 				perm[2] = '-';  in tegra30_mc_handle_irq()
694 				perm[3] = '-';  in tegra30_mc_handle_irq()
697 				perm[4] = '-';  in tegra30_mc_handle_irq()
710 		if (mc->soc->num_channels)  in tegra30_mc_handle_irq()
716 		dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",  in tegra30_mc_handle_irq()
722 	if (mc->soc->num_channels) {  in tegra30_mc_handle_irq()
761 	list_for_each_entry(node, &mc->provider.nodes, node_list) {  in tegra_mc_icc_xlate()
762 		if (node->id == spec->args[0])  in tegra_mc_icc_xlate()
770 	return ERR_PTR(-EPROBE_DEFER);  in tegra_mc_icc_xlate()
794  * Memory Controller (MC) has few Memory Clients that are issuing memory
797  * up to the External Memory Controller (EMC) interconnect provider which
798  * re-configures hardware interface to External Memory (EMEM) in accordance
804  *               +----+
805  * +--------+    |    |
806  * | TEXSRD +--->+    |
807  * +--------+    |    |
808  *               |    |    +-----+    +------+
809  *    ...        | MC +--->+ EMC +--->+ EMEM |
810  *               |    |    +-----+    +------+
811  * +--------+    |    |
812  * | DISP.. +--->+    |
813  * +--------+    |    |
814  *               +----+
822 	/* older device-trees don't have interconnect properties */  in tegra_mc_interconnect_setup()
823 	if (!device_property_present(mc->dev, "#interconnect-cells") ||  in tegra_mc_interconnect_setup()
824 	    !mc->soc->icc_ops)  in tegra_mc_interconnect_setup()
827 	mc->provider.dev = mc->dev;  in tegra_mc_interconnect_setup()
828 	mc->provider.data = &mc->provider;  in tegra_mc_interconnect_setup()
829 	mc->provider.set = mc->soc->icc_ops->set;  in tegra_mc_interconnect_setup()
830 	mc->provider.aggregate = mc->soc->icc_ops->aggregate;  in tegra_mc_interconnect_setup()
831 	mc->provider.get_bw = mc->soc->icc_ops->get_bw;  in tegra_mc_interconnect_setup()
832 	mc->provider.xlate = mc->soc->icc_ops->xlate;  in tegra_mc_interconnect_setup()
833 	mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended;  in tegra_mc_interconnect_setup()
835 	icc_provider_init(&mc->provider);  in tegra_mc_interconnect_setup()
837 	/* create Memory Controller node */  in tegra_mc_interconnect_setup()
842 	node->name = "Memory Controller";  in tegra_mc_interconnect_setup()
843 	icc_node_add(node, &mc->provider);  in tegra_mc_interconnect_setup()
845 	/* link Memory Controller to External Memory Controller */  in tegra_mc_interconnect_setup()
850 	for (i = 0; i < mc->soc->num_clients; i++) {  in tegra_mc_interconnect_setup()
852 		node = icc_node_create(mc->soc->clients[i].id);  in tegra_mc_interconnect_setup()
858 		node->name = mc->soc->clients[i].name;  in tegra_mc_interconnect_setup()
859 		icc_node_add(node, &mc->provider);  in tegra_mc_interconnect_setup()
861 		/* link Memory Client to Memory Controller */  in tegra_mc_interconnect_setup()
866 		node->data = (struct tegra_mc_client *)&(mc->soc->clients[i]);  in tegra_mc_interconnect_setup()
869 	err = icc_provider_register(&mc->provider);  in tegra_mc_interconnect_setup()
876 	icc_nodes_remove(&mc->provider);  in tegra_mc_interconnect_setup()
888 		mc->num_channels = mc->soc->num_channels;  in tegra_mc_num_channel_enabled()
894 			mc->num_channels++;  in tegra_mc_num_channel_enabled()
904 	mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);  in tegra_mc_probe()
906 		return -ENOMEM;  in tegra_mc_probe()
909 	spin_lock_init(&mc->lock);  in tegra_mc_probe()
910 	mc->soc = of_device_get_match_data(&pdev->dev);  in tegra_mc_probe()
911 	mc->dev = &pdev->dev;  in tegra_mc_probe()
913 	mask = DMA_BIT_MASK(mc->soc->num_address_bits);  in tegra_mc_probe()
915 	err = dma_coerce_mask_and_coherent(&pdev->dev, mask);  in tegra_mc_probe()
917 		dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);  in tegra_mc_probe()
922 	mc->tick = 30;  in tegra_mc_probe()
924 	mc->regs = devm_platform_ioremap_resource(pdev, 0);  in tegra_mc_probe()
925 	if (IS_ERR(mc->regs))  in tegra_mc_probe()
926 		return PTR_ERR(mc->regs);  in tegra_mc_probe()
928 	mc->debugfs.root = debugfs_create_dir("mc", NULL);  in tegra_mc_probe()
930 	if (mc->soc->ops && mc->soc->ops->probe) {  in tegra_mc_probe()
931 		err = mc->soc->ops->probe(mc);  in tegra_mc_probe()
938 	if (mc->soc->ops && mc->soc->ops->handle_irq) {  in tegra_mc_probe()
939 		mc->irq = platform_get_irq(pdev, 0);  in tegra_mc_probe()
940 		if (mc->irq < 0)  in tegra_mc_probe()
941 			return mc->irq;  in tegra_mc_probe()
943 		WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");  in tegra_mc_probe()
945 		if (mc->soc->num_channels)  in tegra_mc_probe()
946 			mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmask,  in tegra_mc_probe()
949 			mc_writel(mc, mc->soc->intmask, MC_INTMASK);  in tegra_mc_probe()
951 		err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0,  in tegra_mc_probe()
952 				       dev_name(&pdev->dev), mc);  in tegra_mc_probe()
954 			dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,  in tegra_mc_probe()
960 	if (mc->soc->reset_ops) {  in tegra_mc_probe()
963 			dev_err(&pdev->dev, "failed to register reset controller: %d\n", err);  in tegra_mc_probe()
968 		dev_err(&pdev->dev, "failed to initialize interconnect: %d\n",  in tegra_mc_probe()
971 	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {  in tegra_mc_probe()
972 		mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);  in tegra_mc_probe()
973 		if (IS_ERR(mc->smmu)) {  in tegra_mc_probe()
974 			dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",  in tegra_mc_probe()
975 				PTR_ERR(mc->smmu));  in tegra_mc_probe()
976 			mc->smmu = NULL;  in tegra_mc_probe()
988 	if (mc->provider.dev == dev)  in tegra_mc_sync_state()
994 		.name = "tegra-mc",
1010 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");