1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (c) 2015-2016 MediaTek Inc.
14 #include <linux/dma-mapping.h>
30 #include <dt-bindings/memory/mtk-memory-port.h>
31 #include <dt-bindings/memory/mt2701-larb-port.h>
35 #include <asm/dma-iommu.h>
38 #define arm_iommu_attach_device(...)	-ENODEV
86 #define MT2701_M4U_TF_LARB(TF)			(6 - (((TF) >> 13) & 0x7))
133 	return component_bind_all(dev, &data->larb_imu);  in mtk_iommu_v1_bind()
140 	component_unbind_all(dev, &data->larb_imu);  in mtk_iommu_v1_unbind()
157 	for (i = ARRAY_SIZE(mt2701_m4u_in_larb) - 1; i >= 0; i--)  in mt2701_m4u_to_larb()
166 	int larb = mt2701_m4u_to_larb(id);  in mt2701_m4u_to_port()  local
168 	return id - mt2701_m4u_in_larb[larb];  in mt2701_m4u_to_port()
174 			data->base + REG_MMU_INV_SEL);  in mtk_iommu_v1_tlb_flush_all()
175 	writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);  in mtk_iommu_v1_tlb_flush_all()
186 		data->base + REG_MMU_INV_SEL);  in mtk_iommu_v1_tlb_flush_range()
188 		data->base + REG_MMU_INVLD_START_A);  in mtk_iommu_v1_tlb_flush_range()
189 	writel_relaxed((iova + size - 1) & F_MMU_FAULT_VA_MSK,  in mtk_iommu_v1_tlb_flush_range()
190 		data->base + REG_MMU_INVLD_END_A);  in mtk_iommu_v1_tlb_flush_range()
191 	writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE);  in mtk_iommu_v1_tlb_flush_range()
193 	ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE,  in mtk_iommu_v1_tlb_flush_range()
196 		dev_warn(data->dev,  in mtk_iommu_v1_tlb_flush_range()
201 	writel_relaxed(0, data->base + REG_MMU_CPE_DONE);  in mtk_iommu_v1_tlb_flush_range()
207 	struct mtk_iommu_v1_domain *dom = data->m4u_dom;  in mtk_iommu_v1_isr()
212 	int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST);  in mtk_iommu_v1_isr()
213 	fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA);  in mtk_iommu_v1_isr()
216 	fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA);  in mtk_iommu_v1_isr()
217 	regval = readl_relaxed(data->base + REG_MMU_INT_ID);  in mtk_iommu_v1_isr()
225 	if (report_iommu_fault(&dom->domain, data->dev, fault_iova,  in mtk_iommu_v1_isr()
227 		dev_err_ratelimited(data->dev,  in mtk_iommu_v1_isr()
228 			"fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d\n",  in mtk_iommu_v1_isr()
233 	regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL);  in mtk_iommu_v1_isr()
235 	writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL);  in mtk_iommu_v1_isr()
250 	for (i = 0; i < fwspec->num_ids; ++i) {  in mtk_iommu_v1_config()
251 		larbid = mt2701_m4u_to_larb(fwspec->ids[i]);  in mtk_iommu_v1_config()
252 		portid = mt2701_m4u_to_port(fwspec->ids[i]);  in mtk_iommu_v1_config()
253 		larb_mmu = &data->larb_imu[larbid];  in mtk_iommu_v1_config()
255 		dev_dbg(dev, "%s iommu port: %d\n",  in mtk_iommu_v1_config()
259 			larb_mmu->mmu |= MTK_SMI_MMU_EN(portid);  in mtk_iommu_v1_config()
261 			larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid);  in mtk_iommu_v1_config()
267 	struct mtk_iommu_v1_domain *dom = data->m4u_dom;  in mtk_iommu_v1_domain_finalise()
269 	spin_lock_init(&dom->pgtlock);  in mtk_iommu_v1_domain_finalise()
271 	dom->pgt_va = dma_alloc_coherent(data->dev, M2701_IOMMU_PGT_SIZE,  in mtk_iommu_v1_domain_finalise()
272 					 &dom->pgt_pa, GFP_KERNEL);  in mtk_iommu_v1_domain_finalise()
273 	if (!dom->pgt_va)  in mtk_iommu_v1_domain_finalise()
274 		return -ENOMEM;  in mtk_iommu_v1_domain_finalise()
276 	writel(dom->pgt_pa, data->base + REG_MMU_PT_BASE_ADDR);  in mtk_iommu_v1_domain_finalise()
278 	dom->data = data;  in mtk_iommu_v1_domain_finalise()
291 	dom->domain.pgsize_bitmap = MT2701_IOMMU_PAGE_SIZE;  in mtk_iommu_v1_domain_alloc_paging()
293 	return &dom->domain;  in mtk_iommu_v1_domain_alloc_paging()
299 	struct mtk_iommu_v1_data *data = dom->data;  in mtk_iommu_v1_domain_free()
301 	dma_free_coherent(data->dev, M2701_IOMMU_PGT_SIZE,  in mtk_iommu_v1_domain_free()
302 			dom->pgt_va, dom->pgt_pa);  in mtk_iommu_v1_domain_free()
314 	mtk_mapping = data->mapping;  in mtk_iommu_v1_attach_device()
315 	if (mtk_mapping->domain != domain)  in mtk_iommu_v1_attach_device()
318 	if (!data->m4u_dom) {  in mtk_iommu_v1_attach_device()
319 		data->m4u_dom = dom;  in mtk_iommu_v1_attach_device()
322 			data->m4u_dom = NULL;  in mtk_iommu_v1_attach_device()
356 	u32 *pgt_base_iova = dom->pgt_va + (iova  >> MT2701_IOMMU_PAGE_SHIFT);  in mtk_iommu_v1_map()
359 	spin_lock_irqsave(&dom->pgtlock, flags);  in mtk_iommu_v1_map()
367 	spin_unlock_irqrestore(&dom->pgtlock, flags);  in mtk_iommu_v1_map()
370 	mtk_iommu_v1_tlb_flush_range(dom->data, iova, *mapped);  in mtk_iommu_v1_map()
372 	return i == pgcount ? 0 : -EEXIST;  in mtk_iommu_v1_map()
381 	u32 *pgt_base_iova = dom->pgt_va + (iova  >> MT2701_IOMMU_PAGE_SHIFT);  in mtk_iommu_v1_unmap()
384 	spin_lock_irqsave(&dom->pgtlock, flags);  in mtk_iommu_v1_unmap()
386 	spin_unlock_irqrestore(&dom->pgtlock, flags);  in mtk_iommu_v1_unmap()
388 	mtk_iommu_v1_tlb_flush_range(dom->data, iova, size);  in mtk_iommu_v1_unmap()
399 	spin_lock_irqsave(&dom->pgtlock, flags);  in mtk_iommu_v1_iova_to_phys()
400 	pa = *(dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT));  in mtk_iommu_v1_iova_to_phys()
401 	pa = pa & (~(MT2701_IOMMU_PAGE_SIZE - 1));  in mtk_iommu_v1_iova_to_phys()
402 	spin_unlock_irqrestore(&dom->pgtlock, flags);  in mtk_iommu_v1_iova_to_phys()
421 	if (args->args_count != 1) {  in mtk_iommu_v1_create_mapping()
422 		dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n",  in mtk_iommu_v1_create_mapping()
423 			args->args_count);  in mtk_iommu_v1_create_mapping()
424 		return -EINVAL;  in mtk_iommu_v1_create_mapping()
427 	ret = iommu_fwspec_init(dev, of_fwnode_handle(args->np));  in mtk_iommu_v1_create_mapping()
433 		m4updev = of_find_device_by_node(args->np);  in mtk_iommu_v1_create_mapping()
435 			return -EINVAL;  in mtk_iommu_v1_create_mapping()
440 	ret = iommu_fwspec_add_ids(dev, args->args, 1);  in mtk_iommu_v1_create_mapping()
445 	mtk_mapping = data->mapping;  in mtk_iommu_v1_create_mapping()
452 		data->mapping = mtk_mapping;  in mtk_iommu_v1_create_mapping()
467 	while (!of_parse_phandle_with_args(dev->of_node, "iommus",  in mtk_iommu_v1_probe_device()
468 					   "#iommu-cells",  in mtk_iommu_v1_probe_device()
476 		/* dev->iommu_fwspec might have changed */  in mtk_iommu_v1_probe_device()
482 		return ERR_PTR(-ENODEV);  in mtk_iommu_v1_probe_device()
486 	/* Link the consumer device with the smi-larb device(supplier) */  in mtk_iommu_v1_probe_device()
487 	larbid = mt2701_m4u_to_larb(fwspec->ids[0]);  in mtk_iommu_v1_probe_device()
489 		return ERR_PTR(-EINVAL);  in mtk_iommu_v1_probe_device()
491 	for (idx = 1; idx < fwspec->num_ids; idx++) {  in mtk_iommu_v1_probe_device()
492 		larbidx = mt2701_m4u_to_larb(fwspec->ids[idx]);  in mtk_iommu_v1_probe_device()
494 			dev_err(dev, "Can only use one larb. Fail@larb%d-%d.\n",  in mtk_iommu_v1_probe_device()
496 			return ERR_PTR(-EINVAL);  in mtk_iommu_v1_probe_device()
500 	larbdev = data->larb_imu[larbid].dev;  in mtk_iommu_v1_probe_device()
502 		return ERR_PTR(-EINVAL);  in mtk_iommu_v1_probe_device()
509 	return &data->iommu;  in mtk_iommu_v1_probe_device()
517 	err = arm_iommu_attach_device(dev, data->mapping);  in mtk_iommu_v1_probe_finalize()
519 		dev_err(dev, "Can't create IOMMU mapping - DMA-OPS will not work\n");  in mtk_iommu_v1_probe_finalize()
530 	larbid = mt2701_m4u_to_larb(fwspec->ids[0]);  in mtk_iommu_v1_release_device()
531 	larbdev = data->larb_imu[larbid].dev;  in mtk_iommu_v1_release_device()
540 	ret = clk_prepare_enable(data->bclk);  in mtk_iommu_v1_hw_init()
542 		dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret);  in mtk_iommu_v1_hw_init()
547 	writel_relaxed(regval, data->base + REG_MMU_CTRL_REG);  in mtk_iommu_v1_hw_init()
557 	writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL);  in mtk_iommu_v1_hw_init()
560 	writel_relaxed(data->protect_base,  in mtk_iommu_v1_hw_init()
561 			data->base + REG_MMU_IVRP_PADDR);  in mtk_iommu_v1_hw_init()
563 	writel_relaxed(F_MMU_DCM_ON, data->base + REG_MMU_DCM);  in mtk_iommu_v1_hw_init()
565 	if (devm_request_irq(data->dev, data->irq, mtk_iommu_v1_isr, 0,  in mtk_iommu_v1_hw_init()
566 			     dev_name(data->dev), (void *)data)) {  in mtk_iommu_v1_hw_init()
567 		writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR);  in mtk_iommu_v1_hw_init()
568 		clk_disable_unprepare(data->bclk);  in mtk_iommu_v1_hw_init()
569 		dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq);  in mtk_iommu_v1_hw_init()
570 		return -ENODEV;  in mtk_iommu_v1_hw_init()
594 	{ .compatible = "mediatek,mt2701-m4u", },
606 	struct device			*dev = &pdev->dev;  in mtk_iommu_v1_probe()
615 		return -ENOMEM;  in mtk_iommu_v1_probe()
617 	data->dev = dev;  in mtk_iommu_v1_probe()
623 		return -ENOMEM;  in mtk_iommu_v1_probe()
624 	data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN);  in mtk_iommu_v1_probe()
627 	data->base = devm_ioremap_resource(dev, res);  in mtk_iommu_v1_probe()
628 	if (IS_ERR(data->base))  in mtk_iommu_v1_probe()
629 		return PTR_ERR(data->base);  in mtk_iommu_v1_probe()
631 	data->irq = platform_get_irq(pdev, 0);  in mtk_iommu_v1_probe()
632 	if (data->irq < 0)  in mtk_iommu_v1_probe()
633 		return data->irq;  in mtk_iommu_v1_probe()
635 	data->bclk = devm_clk_get(dev, "bclk");  in mtk_iommu_v1_probe()
636 	if (IS_ERR(data->bclk))  in mtk_iommu_v1_probe()
637 		return PTR_ERR(data->bclk);  in mtk_iommu_v1_probe()
639 	larb_nr = of_count_phandle_with_args(dev->of_node,  in mtk_iommu_v1_probe()
648 		larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i);  in mtk_iommu_v1_probe()
650 			return -EINVAL;  in mtk_iommu_v1_probe()
660 			return -ENODEV;  in mtk_iommu_v1_probe()
662 		if (!plarbdev->dev.driver) {  in mtk_iommu_v1_probe()
664 			return -EPROBE_DEFER;  in mtk_iommu_v1_probe()
666 		data->larb_imu[i].dev = &plarbdev->dev;  in mtk_iommu_v1_probe()
678 	ret = iommu_device_sysfs_add(&data->iommu, &pdev->dev, NULL,  in mtk_iommu_v1_probe()
679 				     dev_name(&pdev->dev));  in mtk_iommu_v1_probe()
683 	ret = iommu_device_register(&data->iommu, &mtk_iommu_v1_ops, dev);  in mtk_iommu_v1_probe()
693 	iommu_device_unregister(&data->iommu);  in mtk_iommu_v1_probe()
695 	iommu_device_sysfs_remove(&data->iommu);  in mtk_iommu_v1_probe()
697 	clk_disable_unprepare(data->bclk);  in mtk_iommu_v1_probe()
705 	iommu_device_sysfs_remove(&data->iommu);  in mtk_iommu_v1_remove()
706 	iommu_device_unregister(&data->iommu);  in mtk_iommu_v1_remove()
708 	clk_disable_unprepare(data->bclk);  in mtk_iommu_v1_remove()
709 	devm_free_irq(&pdev->dev, data->irq, data);  in mtk_iommu_v1_remove()
710 	component_master_del(&pdev->dev, &mtk_iommu_v1_com_ops);  in mtk_iommu_v1_remove()
716 	struct mtk_iommu_v1_suspend_reg *reg = &data->reg;  in mtk_iommu_v1_suspend()
717 	void __iomem *base = data->base;  in mtk_iommu_v1_suspend()
719 	reg->standard_axi_mode = readl_relaxed(base +  in mtk_iommu_v1_suspend()
721 	reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM);  in mtk_iommu_v1_suspend()
722 	reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG);  in mtk_iommu_v1_suspend()
723 	reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL);  in mtk_iommu_v1_suspend()
730 	struct mtk_iommu_v1_suspend_reg *reg = &data->reg;  in mtk_iommu_v1_resume()
731 	void __iomem *base = data->base;  in mtk_iommu_v1_resume()
733 	writel_relaxed(data->m4u_dom->pgt_pa, base + REG_MMU_PT_BASE_ADDR);  in mtk_iommu_v1_resume()
734 	writel_relaxed(reg->standard_axi_mode,  in mtk_iommu_v1_resume()
736 	writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM);  in mtk_iommu_v1_resume()
737 	writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG);  in mtk_iommu_v1_resume()
738 	writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL);  in mtk_iommu_v1_resume()
739 	writel_relaxed(data->protect_base, base + REG_MMU_IVRP_PADDR);  in mtk_iommu_v1_resume()
751 		.name = "mtk-iommu-v1",