11802d0beSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 20df4fabeSYong Wu /* 30df4fabeSYong Wu * Copyright (c) 2015-2016 MediaTek Inc. 40df4fabeSYong Wu * Author: Yong Wu <yong.wu@mediatek.com> 50df4fabeSYong Wu */ 657c8a661SMike Rapoport #include <linux/memblock.h> 70df4fabeSYong Wu #include <linux/bug.h> 80df4fabeSYong Wu #include <linux/clk.h> 90df4fabeSYong Wu #include <linux/component.h> 100df4fabeSYong Wu #include <linux/device.h> 110df4fabeSYong Wu #include <linux/dma-iommu.h> 120df4fabeSYong Wu #include <linux/err.h> 130df4fabeSYong Wu #include <linux/interrupt.h> 140df4fabeSYong Wu #include <linux/io.h> 150df4fabeSYong Wu #include <linux/iommu.h> 160df4fabeSYong Wu #include <linux/iopoll.h> 170df4fabeSYong Wu #include <linux/list.h> 180df4fabeSYong Wu #include <linux/of_address.h> 190df4fabeSYong Wu #include <linux/of_iommu.h> 200df4fabeSYong Wu #include <linux/of_irq.h> 210df4fabeSYong Wu #include <linux/of_platform.h> 220df4fabeSYong Wu #include <linux/platform_device.h> 230df4fabeSYong Wu #include <linux/slab.h> 240df4fabeSYong Wu #include <linux/spinlock.h> 250df4fabeSYong Wu #include <asm/barrier.h> 260df4fabeSYong Wu #include <soc/mediatek/smi.h> 270df4fabeSYong Wu 289ca340c9SHonghui Zhang #include "mtk_iommu.h" 290df4fabeSYong Wu 300df4fabeSYong Wu #define REG_MMU_PT_BASE_ADDR 0x000 31907ba6a1SYong Wu #define MMU_PT_ADDR_MASK GENMASK(31, 7) 320df4fabeSYong Wu 330df4fabeSYong Wu #define REG_MMU_INVALIDATE 0x020 340df4fabeSYong Wu #define F_ALL_INVLD 0x2 350df4fabeSYong Wu #define F_MMU_INV_RANGE 0x1 360df4fabeSYong Wu 370df4fabeSYong Wu #define REG_MMU_INVLD_START_A 0x024 380df4fabeSYong Wu #define REG_MMU_INVLD_END_A 0x028 390df4fabeSYong Wu 400df4fabeSYong Wu #define REG_MMU_INV_SEL 0x038 410df4fabeSYong Wu #define F_INVLD_EN0 BIT(0) 420df4fabeSYong Wu #define F_INVLD_EN1 BIT(1) 430df4fabeSYong Wu 4475eed350SChao Hao #define REG_MMU_MISC_CTRL 0x048 45*4bb2bf4cSChao Hao #define F_MMU_IN_ORDER_WR_EN_MASK (BIT(1) | BIT(17)) 46*4bb2bf4cSChao Hao #define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19)) 47*4bb2bf4cSChao Hao 480df4fabeSYong Wu #define REG_MMU_DCM_DIS 0x050 490df4fabeSYong Wu 500df4fabeSYong Wu #define REG_MMU_CTRL_REG 0x110 51acb3c92aSYong Wu #define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4) 520df4fabeSYong Wu #define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4) 53acb3c92aSYong Wu #define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5) 540df4fabeSYong Wu 550df4fabeSYong Wu #define REG_MMU_IVRP_PADDR 0x114 5670ca608bSYong Wu 5730e2fccfSYong Wu #define REG_MMU_VLD_PA_RNG 0x118 5830e2fccfSYong Wu #define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA)) 590df4fabeSYong Wu 600df4fabeSYong Wu #define REG_MMU_INT_CONTROL0 0x120 610df4fabeSYong Wu #define F_L2_MULIT_HIT_EN BIT(0) 620df4fabeSYong Wu #define F_TABLE_WALK_FAULT_INT_EN BIT(1) 630df4fabeSYong Wu #define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2) 640df4fabeSYong Wu #define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3) 650df4fabeSYong Wu #define F_PREFETCH_FIFO_ERR_INT_EN BIT(5) 660df4fabeSYong Wu #define F_MISS_FIFO_ERR_INT_EN BIT(6) 670df4fabeSYong Wu #define F_INT_CLR_BIT BIT(12) 680df4fabeSYong Wu 690df4fabeSYong Wu #define REG_MMU_INT_MAIN_CONTROL 0x124 7015a01f4cSYong Wu /* mmu0 | mmu1 */ 7115a01f4cSYong Wu #define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7)) 7215a01f4cSYong Wu #define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8)) 7315a01f4cSYong Wu #define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9)) 7415a01f4cSYong Wu #define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10)) 7515a01f4cSYong Wu #define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11)) 7615a01f4cSYong Wu #define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12)) 7715a01f4cSYong Wu #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13)) 780df4fabeSYong Wu 790df4fabeSYong Wu #define REG_MMU_CPE_DONE 0x12C 800df4fabeSYong Wu 810df4fabeSYong Wu #define REG_MMU_FAULT_ST1 0x134 8215a01f4cSYong Wu #define F_REG_MMU0_FAULT_MASK GENMASK(6, 0) 8315a01f4cSYong Wu #define F_REG_MMU1_FAULT_MASK GENMASK(13, 7) 840df4fabeSYong Wu 8515a01f4cSYong Wu #define REG_MMU0_FAULT_VA 0x13c 860df4fabeSYong Wu #define F_MMU_FAULT_VA_WRITE_BIT BIT(1) 870df4fabeSYong Wu #define F_MMU_FAULT_VA_LAYER_BIT BIT(0) 880df4fabeSYong Wu 8915a01f4cSYong Wu #define REG_MMU0_INVLD_PA 0x140 9015a01f4cSYong Wu #define REG_MMU1_FAULT_VA 0x144 9115a01f4cSYong Wu #define REG_MMU1_INVLD_PA 0x148 9215a01f4cSYong Wu #define REG_MMU0_INT_ID 0x150 9315a01f4cSYong Wu #define REG_MMU1_INT_ID 0x154 9415a01f4cSYong Wu #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) 9515a01f4cSYong Wu #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) 960df4fabeSYong Wu 970df4fabeSYong Wu #define MTK_PROTECT_PA_ALIGN 128 980df4fabeSYong Wu 99a9467d95SYong Wu /* 100a9467d95SYong Wu * Get the local arbiter ID and the portid within the larb arbiter 101a9467d95SYong Wu * from mtk_m4u_id which is defined by MTK_M4U_ID. 102a9467d95SYong Wu */ 103e6dec923SYong Wu #define MTK_M4U_TO_LARB(id) (((id) >> 5) & 0xf) 104a9467d95SYong Wu #define MTK_M4U_TO_PORT(id) ((id) & 0x1f) 105a9467d95SYong Wu 1066b717796SChao Hao #define HAS_4GB_MODE BIT(0) 1076b717796SChao Hao /* HW will use the EMI clock if there isn't the "bclk". */ 1086b717796SChao Hao #define HAS_BCLK BIT(1) 1096b717796SChao Hao #define HAS_VLD_PA_RNG BIT(2) 1106b717796SChao Hao #define RESET_AXI BIT(3) 111*4bb2bf4cSChao Hao #define OUT_ORDER_WR_EN BIT(4) 1126b717796SChao Hao 1136b717796SChao Hao #define MTK_IOMMU_HAS_FLAG(pdata, _x) \ 1146b717796SChao Hao ((((pdata)->flags) & (_x)) == (_x)) 1156b717796SChao Hao 1160df4fabeSYong Wu struct mtk_iommu_domain { 1170df4fabeSYong Wu struct io_pgtable_cfg cfg; 1180df4fabeSYong Wu struct io_pgtable_ops *iop; 1190df4fabeSYong Wu 1200df4fabeSYong Wu struct iommu_domain domain; 1210df4fabeSYong Wu }; 1220df4fabeSYong Wu 123b65f5016SArvind Yadav static const struct iommu_ops mtk_iommu_ops; 1240df4fabeSYong Wu 12576ce6546SYong Wu /* 12676ce6546SYong Wu * In M4U 4GB mode, the physical address is remapped as below: 12776ce6546SYong Wu * 12876ce6546SYong Wu * CPU Physical address: 12976ce6546SYong Wu * ==================== 13076ce6546SYong Wu * 13176ce6546SYong Wu * 0 1G 2G 3G 4G 5G 13276ce6546SYong Wu * |---A---|---B---|---C---|---D---|---E---| 13376ce6546SYong Wu * +--I/O--+------------Memory-------------+ 13476ce6546SYong Wu * 13576ce6546SYong Wu * IOMMU output physical address: 13676ce6546SYong Wu * ============================= 13776ce6546SYong Wu * 13876ce6546SYong Wu * 4G 5G 6G 7G 8G 13976ce6546SYong Wu * |---E---|---B---|---C---|---D---| 14076ce6546SYong Wu * +------------Memory-------------+ 14176ce6546SYong Wu * 14276ce6546SYong Wu * The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the 14376ce6546SYong Wu * bit32 of the CPU physical address always is needed to set, and for Region 14476ce6546SYong Wu * 'E', the CPU physical address keep as is. 14576ce6546SYong Wu * Additionally, The iommu consumers always use the CPU phyiscal address. 14676ce6546SYong Wu */ 147b4dad40eSYong Wu #define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL 14876ce6546SYong Wu 1497c3a2ec0SYong Wu static LIST_HEAD(m4ulist); /* List all the M4U HWs */ 1507c3a2ec0SYong Wu 1517c3a2ec0SYong Wu #define for_each_m4u(data) list_for_each_entry(data, &m4ulist, list) 1527c3a2ec0SYong Wu 1537c3a2ec0SYong Wu /* 1547c3a2ec0SYong Wu * There may be 1 or 2 M4U HWs, But we always expect they are in the same domain 1557c3a2ec0SYong Wu * for the performance. 1567c3a2ec0SYong Wu * 1577c3a2ec0SYong Wu * Here always return the mtk_iommu_data of the first probed M4U where the 1587c3a2ec0SYong Wu * iommu domain information is recorded. 1597c3a2ec0SYong Wu */ 1607c3a2ec0SYong Wu static struct mtk_iommu_data *mtk_iommu_get_m4u_data(void) 1617c3a2ec0SYong Wu { 1627c3a2ec0SYong Wu struct mtk_iommu_data *data; 1637c3a2ec0SYong Wu 1647c3a2ec0SYong Wu for_each_m4u(data) 1657c3a2ec0SYong Wu return data; 1667c3a2ec0SYong Wu 1677c3a2ec0SYong Wu return NULL; 1687c3a2ec0SYong Wu } 1697c3a2ec0SYong Wu 1700df4fabeSYong Wu static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) 1710df4fabeSYong Wu { 1720df4fabeSYong Wu return container_of(dom, struct mtk_iommu_domain, domain); 1730df4fabeSYong Wu } 1740df4fabeSYong Wu 1750df4fabeSYong Wu static void mtk_iommu_tlb_flush_all(void *cookie) 1760df4fabeSYong Wu { 1770df4fabeSYong Wu struct mtk_iommu_data *data = cookie; 1780df4fabeSYong Wu 1797c3a2ec0SYong Wu for_each_m4u(data) { 1807c3a2ec0SYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, 1817c3a2ec0SYong Wu data->base + REG_MMU_INV_SEL); 1820df4fabeSYong Wu writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); 1830df4fabeSYong Wu wmb(); /* Make sure the tlb flush all done */ 1840df4fabeSYong Wu } 1857c3a2ec0SYong Wu } 1860df4fabeSYong Wu 1871f4fd624SYong Wu static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, 18867caf7e2SYong Wu size_t granule, void *cookie) 1890df4fabeSYong Wu { 1900df4fabeSYong Wu struct mtk_iommu_data *data = cookie; 1911f4fd624SYong Wu unsigned long flags; 1921f4fd624SYong Wu int ret; 1931f4fd624SYong Wu u32 tmp; 1940df4fabeSYong Wu 1957c3a2ec0SYong Wu for_each_m4u(data) { 1961f4fd624SYong Wu spin_lock_irqsave(&data->tlb_lock, flags); 1977c3a2ec0SYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, 1987c3a2ec0SYong Wu data->base + REG_MMU_INV_SEL); 1990df4fabeSYong Wu 2000df4fabeSYong Wu writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A); 2017c3a2ec0SYong Wu writel_relaxed(iova + size - 1, 2027c3a2ec0SYong Wu data->base + REG_MMU_INVLD_END_A); 2037c3a2ec0SYong Wu writel_relaxed(F_MMU_INV_RANGE, 2047c3a2ec0SYong Wu data->base + REG_MMU_INVALIDATE); 2050df4fabeSYong Wu 2061f4fd624SYong Wu /* tlb sync */ 2077c3a2ec0SYong Wu ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, 208c90ae4a6SYong Wu tmp, tmp != 0, 10, 1000); 2090df4fabeSYong Wu if (ret) { 2100df4fabeSYong Wu dev_warn(data->dev, 2110df4fabeSYong Wu "Partial TLB flush timed out, falling back to full flush\n"); 2120df4fabeSYong Wu mtk_iommu_tlb_flush_all(cookie); 2130df4fabeSYong Wu } 2140df4fabeSYong Wu /* Clear the CPE status */ 2150df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_CPE_DONE); 216da3cc91bSYong Wu spin_unlock_irqrestore(&data->tlb_lock, flags); 2170df4fabeSYong Wu } 2187c3a2ec0SYong Wu } 2190df4fabeSYong Wu 2203951c41aSWill Deacon static void mtk_iommu_tlb_flush_page_nosync(struct iommu_iotlb_gather *gather, 2213951c41aSWill Deacon unsigned long iova, size_t granule, 222abfd6fe0SWill Deacon void *cookie) 223abfd6fe0SWill Deacon { 224da3cc91bSYong Wu struct mtk_iommu_data *data = cookie; 225a7a04ea3SYong Wu struct iommu_domain *domain = &data->m4u_dom->domain; 226da3cc91bSYong Wu 227a7a04ea3SYong Wu iommu_iotlb_gather_add_page(domain, gather, iova, granule); 228abfd6fe0SWill Deacon } 229abfd6fe0SWill Deacon 230298f7889SWill Deacon static const struct iommu_flush_ops mtk_iommu_flush_ops = { 2310df4fabeSYong Wu .tlb_flush_all = mtk_iommu_tlb_flush_all, 2321f4fd624SYong Wu .tlb_flush_walk = mtk_iommu_tlb_flush_range_sync, 2331f4fd624SYong Wu .tlb_flush_leaf = mtk_iommu_tlb_flush_range_sync, 234abfd6fe0SWill Deacon .tlb_add_page = mtk_iommu_tlb_flush_page_nosync, 2350df4fabeSYong Wu }; 2360df4fabeSYong Wu 2370df4fabeSYong Wu static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) 2380df4fabeSYong Wu { 2390df4fabeSYong Wu struct mtk_iommu_data *data = dev_id; 2400df4fabeSYong Wu struct mtk_iommu_domain *dom = data->m4u_dom; 2410df4fabeSYong Wu u32 int_state, regval, fault_iova, fault_pa; 2420df4fabeSYong Wu unsigned int fault_larb, fault_port; 2430df4fabeSYong Wu bool layer, write; 2440df4fabeSYong Wu 2450df4fabeSYong Wu /* Read error info from registers */ 2460df4fabeSYong Wu int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1); 24715a01f4cSYong Wu if (int_state & F_REG_MMU0_FAULT_MASK) { 24815a01f4cSYong Wu regval = readl_relaxed(data->base + REG_MMU0_INT_ID); 24915a01f4cSYong Wu fault_iova = readl_relaxed(data->base + REG_MMU0_FAULT_VA); 25015a01f4cSYong Wu fault_pa = readl_relaxed(data->base + REG_MMU0_INVLD_PA); 25115a01f4cSYong Wu } else { 25215a01f4cSYong Wu regval = readl_relaxed(data->base + REG_MMU1_INT_ID); 25315a01f4cSYong Wu fault_iova = readl_relaxed(data->base + REG_MMU1_FAULT_VA); 25415a01f4cSYong Wu fault_pa = readl_relaxed(data->base + REG_MMU1_INVLD_PA); 25515a01f4cSYong Wu } 2560df4fabeSYong Wu layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; 2570df4fabeSYong Wu write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; 25815a01f4cSYong Wu fault_larb = F_MMU_INT_ID_LARB_ID(regval); 25915a01f4cSYong Wu fault_port = F_MMU_INT_ID_PORT_ID(regval); 2600df4fabeSYong Wu 261b3e5eee7SYong Wu fault_larb = data->plat_data->larbid_remap[fault_larb]; 262b3e5eee7SYong Wu 2630df4fabeSYong Wu if (report_iommu_fault(&dom->domain, data->dev, fault_iova, 2640df4fabeSYong Wu write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { 2650df4fabeSYong Wu dev_err_ratelimited( 2660df4fabeSYong Wu data->dev, 2670df4fabeSYong Wu "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d layer=%d %s\n", 2680df4fabeSYong Wu int_state, fault_iova, fault_pa, fault_larb, fault_port, 2690df4fabeSYong Wu layer, write ? "write" : "read"); 2700df4fabeSYong Wu } 2710df4fabeSYong Wu 2720df4fabeSYong Wu /* Interrupt clear */ 2730df4fabeSYong Wu regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL0); 2740df4fabeSYong Wu regval |= F_INT_CLR_BIT; 2750df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); 2760df4fabeSYong Wu 2770df4fabeSYong Wu mtk_iommu_tlb_flush_all(data); 2780df4fabeSYong Wu 2790df4fabeSYong Wu return IRQ_HANDLED; 2800df4fabeSYong Wu } 2810df4fabeSYong Wu 2820df4fabeSYong Wu static void mtk_iommu_config(struct mtk_iommu_data *data, 2830df4fabeSYong Wu struct device *dev, bool enable) 2840df4fabeSYong Wu { 2850df4fabeSYong Wu struct mtk_smi_larb_iommu *larb_mmu; 2860df4fabeSYong Wu unsigned int larbid, portid; 287a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 28858f0d1d5SRobin Murphy int i; 2890df4fabeSYong Wu 29058f0d1d5SRobin Murphy for (i = 0; i < fwspec->num_ids; ++i) { 29158f0d1d5SRobin Murphy larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); 29258f0d1d5SRobin Murphy portid = MTK_M4U_TO_PORT(fwspec->ids[i]); 2931ee9feb2SYong Wu larb_mmu = &data->larb_imu[larbid]; 2940df4fabeSYong Wu 2950df4fabeSYong Wu dev_dbg(dev, "%s iommu port: %d\n", 2960df4fabeSYong Wu enable ? "enable" : "disable", portid); 2970df4fabeSYong Wu 2980df4fabeSYong Wu if (enable) 2990df4fabeSYong Wu larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); 3000df4fabeSYong Wu else 3010df4fabeSYong Wu larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); 3020df4fabeSYong Wu } 3030df4fabeSYong Wu } 3040df4fabeSYong Wu 3054b00f5acSYong Wu static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom) 3060df4fabeSYong Wu { 3074b00f5acSYong Wu struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); 3080df4fabeSYong Wu 3090df4fabeSYong Wu dom->cfg = (struct io_pgtable_cfg) { 3100df4fabeSYong Wu .quirks = IO_PGTABLE_QUIRK_ARM_NS | 3110df4fabeSYong Wu IO_PGTABLE_QUIRK_NO_PERMS | 312b4dad40eSYong Wu IO_PGTABLE_QUIRK_TLBI_ON_MAP | 313b4dad40eSYong Wu IO_PGTABLE_QUIRK_ARM_MTK_EXT, 3140df4fabeSYong Wu .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, 3150df4fabeSYong Wu .ias = 32, 316b4dad40eSYong Wu .oas = 34, 317298f7889SWill Deacon .tlb = &mtk_iommu_flush_ops, 3180df4fabeSYong Wu .iommu_dev = data->dev, 3190df4fabeSYong Wu }; 3200df4fabeSYong Wu 3210df4fabeSYong Wu dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data); 3220df4fabeSYong Wu if (!dom->iop) { 3230df4fabeSYong Wu dev_err(data->dev, "Failed to alloc io pgtable\n"); 3240df4fabeSYong Wu return -EINVAL; 3250df4fabeSYong Wu } 3260df4fabeSYong Wu 3270df4fabeSYong Wu /* Update our support page sizes bitmap */ 328d16e0faaSRobin Murphy dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; 3290df4fabeSYong Wu return 0; 3300df4fabeSYong Wu } 3310df4fabeSYong Wu 3320df4fabeSYong Wu static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) 3330df4fabeSYong Wu { 3340df4fabeSYong Wu struct mtk_iommu_domain *dom; 3350df4fabeSYong Wu 3360df4fabeSYong Wu if (type != IOMMU_DOMAIN_DMA) 3370df4fabeSYong Wu return NULL; 3380df4fabeSYong Wu 3390df4fabeSYong Wu dom = kzalloc(sizeof(*dom), GFP_KERNEL); 3400df4fabeSYong Wu if (!dom) 3410df4fabeSYong Wu return NULL; 3420df4fabeSYong Wu 3434b00f5acSYong Wu if (iommu_get_dma_cookie(&dom->domain)) 3444b00f5acSYong Wu goto free_dom; 3454b00f5acSYong Wu 3464b00f5acSYong Wu if (mtk_iommu_domain_finalise(dom)) 3474b00f5acSYong Wu goto put_dma_cookie; 3480df4fabeSYong Wu 3490df4fabeSYong Wu dom->domain.geometry.aperture_start = 0; 3500df4fabeSYong Wu dom->domain.geometry.aperture_end = DMA_BIT_MASK(32); 3510df4fabeSYong Wu dom->domain.geometry.force_aperture = true; 3520df4fabeSYong Wu 3530df4fabeSYong Wu return &dom->domain; 3544b00f5acSYong Wu 3554b00f5acSYong Wu put_dma_cookie: 3564b00f5acSYong Wu iommu_put_dma_cookie(&dom->domain); 3574b00f5acSYong Wu free_dom: 3584b00f5acSYong Wu kfree(dom); 3594b00f5acSYong Wu return NULL; 3600df4fabeSYong Wu } 3610df4fabeSYong Wu 3620df4fabeSYong Wu static void mtk_iommu_domain_free(struct iommu_domain *domain) 3630df4fabeSYong Wu { 3644b00f5acSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 3654b00f5acSYong Wu 3664b00f5acSYong Wu free_io_pgtable_ops(dom->iop); 3670df4fabeSYong Wu iommu_put_dma_cookie(domain); 3680df4fabeSYong Wu kfree(to_mtk_domain(domain)); 3690df4fabeSYong Wu } 3700df4fabeSYong Wu 3710df4fabeSYong Wu static int mtk_iommu_attach_device(struct iommu_domain *domain, 3720df4fabeSYong Wu struct device *dev) 3730df4fabeSYong Wu { 3743524b559SJoerg Roedel struct mtk_iommu_data *data = dev_iommu_priv_get(dev); 3750df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 3760df4fabeSYong Wu 3774b00f5acSYong Wu if (!data) 3780df4fabeSYong Wu return -ENODEV; 3790df4fabeSYong Wu 3804b00f5acSYong Wu /* Update the pgtable base address register of the M4U HW */ 3810df4fabeSYong Wu if (!data->m4u_dom) { 3820df4fabeSYong Wu data->m4u_dom = dom; 383d1e5f26fSRobin Murphy writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, 3844b00f5acSYong Wu data->base + REG_MMU_PT_BASE_ADDR); 3850df4fabeSYong Wu } 3860df4fabeSYong Wu 3874b00f5acSYong Wu mtk_iommu_config(data, dev, true); 3880df4fabeSYong Wu return 0; 3890df4fabeSYong Wu } 3900df4fabeSYong Wu 3910df4fabeSYong Wu static void mtk_iommu_detach_device(struct iommu_domain *domain, 3920df4fabeSYong Wu struct device *dev) 3930df4fabeSYong Wu { 3943524b559SJoerg Roedel struct mtk_iommu_data *data = dev_iommu_priv_get(dev); 3950df4fabeSYong Wu 39658f0d1d5SRobin Murphy if (!data) 3970df4fabeSYong Wu return; 3980df4fabeSYong Wu 3990df4fabeSYong Wu mtk_iommu_config(data, dev, false); 4000df4fabeSYong Wu } 4010df4fabeSYong Wu 4020df4fabeSYong Wu static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, 403781ca2deSTom Murphy phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 4040df4fabeSYong Wu { 4050df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 406b4dad40eSYong Wu struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); 4070df4fabeSYong Wu 408b4dad40eSYong Wu /* The "4GB mode" M4U physically can not use the lower remap of Dram. */ 409b4dad40eSYong Wu if (data->enable_4GB) 410b4dad40eSYong Wu paddr |= BIT_ULL(32); 411b4dad40eSYong Wu 41260829b4dSYong Wu /* Synchronize with the tlb_lock */ 41360829b4dSYong Wu return dom->iop->map(dom->iop, iova, paddr, size, prot); 4140df4fabeSYong Wu } 4150df4fabeSYong Wu 4160df4fabeSYong Wu static size_t mtk_iommu_unmap(struct iommu_domain *domain, 41756f8af5eSWill Deacon unsigned long iova, size_t size, 41856f8af5eSWill Deacon struct iommu_iotlb_gather *gather) 4190df4fabeSYong Wu { 4200df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 4210df4fabeSYong Wu 42260829b4dSYong Wu return dom->iop->unmap(dom->iop, iova, size, gather); 4230df4fabeSYong Wu } 4240df4fabeSYong Wu 42556f8af5eSWill Deacon static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain) 42656f8af5eSWill Deacon { 4272009122fSYong Wu mtk_iommu_tlb_flush_all(mtk_iommu_get_m4u_data()); 42856f8af5eSWill Deacon } 42956f8af5eSWill Deacon 43056f8af5eSWill Deacon static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, 43156f8af5eSWill Deacon struct iommu_iotlb_gather *gather) 4324d689b61SRobin Murphy { 433da3cc91bSYong Wu struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); 434a7a04ea3SYong Wu size_t length = gather->end - gather->start; 435da3cc91bSYong Wu 436a7a04ea3SYong Wu if (gather->start == ULONG_MAX) 437a7a04ea3SYong Wu return; 438a7a04ea3SYong Wu 4391f4fd624SYong Wu mtk_iommu_tlb_flush_range_sync(gather->start, length, gather->pgsize, 44067caf7e2SYong Wu data); 4414d689b61SRobin Murphy } 4424d689b61SRobin Murphy 4430df4fabeSYong Wu static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, 4440df4fabeSYong Wu dma_addr_t iova) 4450df4fabeSYong Wu { 4460df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 44730e2fccfSYong Wu struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); 4480df4fabeSYong Wu phys_addr_t pa; 4490df4fabeSYong Wu 4500df4fabeSYong Wu pa = dom->iop->iova_to_phys(dom->iop, iova); 451b4dad40eSYong Wu if (data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) 452b4dad40eSYong Wu pa &= ~BIT_ULL(32); 45330e2fccfSYong Wu 4540df4fabeSYong Wu return pa; 4550df4fabeSYong Wu } 4560df4fabeSYong Wu 45780e4592aSJoerg Roedel static struct iommu_device *mtk_iommu_probe_device(struct device *dev) 4580df4fabeSYong Wu { 459a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 460b16c0170SJoerg Roedel struct mtk_iommu_data *data; 4610df4fabeSYong Wu 462a9bf2eecSJoerg Roedel if (!fwspec || fwspec->ops != &mtk_iommu_ops) 46380e4592aSJoerg Roedel return ERR_PTR(-ENODEV); /* Not a iommu client device */ 4640df4fabeSYong Wu 4653524b559SJoerg Roedel data = dev_iommu_priv_get(dev); 466b16c0170SJoerg Roedel 46780e4592aSJoerg Roedel return &data->iommu; 4680df4fabeSYong Wu } 4690df4fabeSYong Wu 47080e4592aSJoerg Roedel static void mtk_iommu_release_device(struct device *dev) 4710df4fabeSYong Wu { 472a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 473b16c0170SJoerg Roedel 474a9bf2eecSJoerg Roedel if (!fwspec || fwspec->ops != &mtk_iommu_ops) 4750df4fabeSYong Wu return; 4760df4fabeSYong Wu 47758f0d1d5SRobin Murphy iommu_fwspec_free(dev); 4780df4fabeSYong Wu } 4790df4fabeSYong Wu 4800df4fabeSYong Wu static struct iommu_group *mtk_iommu_device_group(struct device *dev) 4810df4fabeSYong Wu { 4827c3a2ec0SYong Wu struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); 4830df4fabeSYong Wu 48458f0d1d5SRobin Murphy if (!data) 4850df4fabeSYong Wu return ERR_PTR(-ENODEV); 4860df4fabeSYong Wu 4870df4fabeSYong Wu /* All the client devices are in the same m4u iommu-group */ 4880df4fabeSYong Wu if (!data->m4u_group) { 4890df4fabeSYong Wu data->m4u_group = iommu_group_alloc(); 4900df4fabeSYong Wu if (IS_ERR(data->m4u_group)) 4910df4fabeSYong Wu dev_err(dev, "Failed to allocate M4U IOMMU group\n"); 4923a8d40b6SRobin Murphy } else { 4933a8d40b6SRobin Murphy iommu_group_ref_get(data->m4u_group); 4940df4fabeSYong Wu } 4950df4fabeSYong Wu return data->m4u_group; 4960df4fabeSYong Wu } 4970df4fabeSYong Wu 4980df4fabeSYong Wu static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) 4990df4fabeSYong Wu { 5000df4fabeSYong Wu struct platform_device *m4updev; 5010df4fabeSYong Wu 5020df4fabeSYong Wu if (args->args_count != 1) { 5030df4fabeSYong Wu dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", 5040df4fabeSYong Wu args->args_count); 5050df4fabeSYong Wu return -EINVAL; 5060df4fabeSYong Wu } 5070df4fabeSYong Wu 5083524b559SJoerg Roedel if (!dev_iommu_priv_get(dev)) { 5090df4fabeSYong Wu /* Get the m4u device */ 5100df4fabeSYong Wu m4updev = of_find_device_by_node(args->np); 5110df4fabeSYong Wu if (WARN_ON(!m4updev)) 5120df4fabeSYong Wu return -EINVAL; 5130df4fabeSYong Wu 5143524b559SJoerg Roedel dev_iommu_priv_set(dev, platform_get_drvdata(m4updev)); 5150df4fabeSYong Wu } 5160df4fabeSYong Wu 51758f0d1d5SRobin Murphy return iommu_fwspec_add_ids(dev, args->args, 1); 5180df4fabeSYong Wu } 5190df4fabeSYong Wu 520b65f5016SArvind Yadav static const struct iommu_ops mtk_iommu_ops = { 5210df4fabeSYong Wu .domain_alloc = mtk_iommu_domain_alloc, 5220df4fabeSYong Wu .domain_free = mtk_iommu_domain_free, 5230df4fabeSYong Wu .attach_dev = mtk_iommu_attach_device, 5240df4fabeSYong Wu .detach_dev = mtk_iommu_detach_device, 5250df4fabeSYong Wu .map = mtk_iommu_map, 5260df4fabeSYong Wu .unmap = mtk_iommu_unmap, 52756f8af5eSWill Deacon .flush_iotlb_all = mtk_iommu_flush_iotlb_all, 5284d689b61SRobin Murphy .iotlb_sync = mtk_iommu_iotlb_sync, 5290df4fabeSYong Wu .iova_to_phys = mtk_iommu_iova_to_phys, 53080e4592aSJoerg Roedel .probe_device = mtk_iommu_probe_device, 53180e4592aSJoerg Roedel .release_device = mtk_iommu_release_device, 5320df4fabeSYong Wu .device_group = mtk_iommu_device_group, 5330df4fabeSYong Wu .of_xlate = mtk_iommu_of_xlate, 5340df4fabeSYong Wu .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, 5350df4fabeSYong Wu }; 5360df4fabeSYong Wu 5370df4fabeSYong Wu static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) 5380df4fabeSYong Wu { 5390df4fabeSYong Wu u32 regval; 5400df4fabeSYong Wu int ret; 5410df4fabeSYong Wu 5420df4fabeSYong Wu ret = clk_prepare_enable(data->bclk); 5430df4fabeSYong Wu if (ret) { 5440df4fabeSYong Wu dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret); 5450df4fabeSYong Wu return ret; 5460df4fabeSYong Wu } 5470df4fabeSYong Wu 548cecdce9dSYong Wu if (data->plat_data->m4u_plat == M4U_MT8173) 549acb3c92aSYong Wu regval = F_MMU_PREFETCH_RT_REPLACE_MOD | 550acb3c92aSYong Wu F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173; 551acb3c92aSYong Wu else 552acb3c92aSYong Wu regval = F_MMU_TF_PROT_TO_PROGRAM_ADDR; 5530df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); 5540df4fabeSYong Wu 5550df4fabeSYong Wu regval = F_L2_MULIT_HIT_EN | 5560df4fabeSYong Wu F_TABLE_WALK_FAULT_INT_EN | 5570df4fabeSYong Wu F_PREETCH_FIFO_OVERFLOW_INT_EN | 5580df4fabeSYong Wu F_MISS_FIFO_OVERFLOW_INT_EN | 5590df4fabeSYong Wu F_PREFETCH_FIFO_ERR_INT_EN | 5600df4fabeSYong Wu F_MISS_FIFO_ERR_INT_EN; 5610df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); 5620df4fabeSYong Wu 5630df4fabeSYong Wu regval = F_INT_TRANSLATION_FAULT | 5640df4fabeSYong Wu F_INT_MAIN_MULTI_HIT_FAULT | 5650df4fabeSYong Wu F_INT_INVALID_PA_FAULT | 5660df4fabeSYong Wu F_INT_ENTRY_REPLACEMENT_FAULT | 5670df4fabeSYong Wu F_INT_TLB_MISS_FAULT | 5680df4fabeSYong Wu F_INT_MISS_TRANSACTION_FIFO_FAULT | 5690df4fabeSYong Wu F_INT_PRETETCH_TRANSATION_FIFO_FAULT; 5700df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); 5710df4fabeSYong Wu 572cecdce9dSYong Wu if (data->plat_data->m4u_plat == M4U_MT8173) 57370ca608bSYong Wu regval = (data->protect_base >> 1) | (data->enable_4GB << 31); 57470ca608bSYong Wu else 57570ca608bSYong Wu regval = lower_32_bits(data->protect_base) | 57670ca608bSYong Wu upper_32_bits(data->protect_base); 57770ca608bSYong Wu writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR); 57870ca608bSYong Wu 5796b717796SChao Hao if (data->enable_4GB && 5806b717796SChao Hao MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_VLD_PA_RNG)) { 58130e2fccfSYong Wu /* 58230e2fccfSYong Wu * If 4GB mode is enabled, the validate PA range is from 58330e2fccfSYong Wu * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30]. 58430e2fccfSYong Wu */ 58530e2fccfSYong Wu regval = F_MMU_VLD_PA_RNG(7, 4); 58630e2fccfSYong Wu writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG); 58730e2fccfSYong Wu } 5880df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_DCM_DIS); 589e6dec923SYong Wu 5906b717796SChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) { 59175eed350SChao Hao /* The register is called STANDARD_AXI_MODE in this case */ 592*4bb2bf4cSChao Hao regval = 0; 593*4bb2bf4cSChao Hao } else { 594*4bb2bf4cSChao Hao regval = readl_relaxed(data->base + REG_MMU_MISC_CTRL); 595*4bb2bf4cSChao Hao regval &= ~F_MMU_STANDARD_AXI_MODE_MASK; 596*4bb2bf4cSChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, OUT_ORDER_WR_EN)) 597*4bb2bf4cSChao Hao regval &= ~F_MMU_IN_ORDER_WR_EN_MASK; 59875eed350SChao Hao } 599*4bb2bf4cSChao Hao writel_relaxed(regval, data->base + REG_MMU_MISC_CTRL); 6000df4fabeSYong Wu 6010df4fabeSYong Wu if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, 6020df4fabeSYong Wu dev_name(data->dev), (void *)data)) { 6030df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); 6040df4fabeSYong Wu clk_disable_unprepare(data->bclk); 6050df4fabeSYong Wu dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); 6060df4fabeSYong Wu return -ENODEV; 6070df4fabeSYong Wu } 6080df4fabeSYong Wu 6090df4fabeSYong Wu return 0; 6100df4fabeSYong Wu } 6110df4fabeSYong Wu 6120df4fabeSYong Wu static const struct component_master_ops mtk_iommu_com_ops = { 6130df4fabeSYong Wu .bind = mtk_iommu_bind, 6140df4fabeSYong Wu .unbind = mtk_iommu_unbind, 6150df4fabeSYong Wu }; 6160df4fabeSYong Wu 6170df4fabeSYong Wu static int mtk_iommu_probe(struct platform_device *pdev) 6180df4fabeSYong Wu { 6190df4fabeSYong Wu struct mtk_iommu_data *data; 6200df4fabeSYong Wu struct device *dev = &pdev->dev; 6210df4fabeSYong Wu struct resource *res; 622b16c0170SJoerg Roedel resource_size_t ioaddr; 6230df4fabeSYong Wu struct component_match *match = NULL; 6240df4fabeSYong Wu void *protect; 6250b6c0ad3SAndrzej Hajda int i, larb_nr, ret; 6260df4fabeSYong Wu 6270df4fabeSYong Wu data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); 6280df4fabeSYong Wu if (!data) 6290df4fabeSYong Wu return -ENOMEM; 6300df4fabeSYong Wu data->dev = dev; 631cecdce9dSYong Wu data->plat_data = of_device_get_match_data(dev); 6320df4fabeSYong Wu 6330df4fabeSYong Wu /* Protect memory. HW will access here while translation fault.*/ 6340df4fabeSYong Wu protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); 6350df4fabeSYong Wu if (!protect) 6360df4fabeSYong Wu return -ENOMEM; 6370df4fabeSYong Wu data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); 6380df4fabeSYong Wu 63901e23c93SYong Wu /* Whether the current dram is over 4GB */ 64041939980SYong Wu data->enable_4GB = !!(max_pfn > (BIT_ULL(32) >> PAGE_SHIFT)); 6416b717796SChao Hao if (!MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) 642b4dad40eSYong Wu data->enable_4GB = false; 64301e23c93SYong Wu 6440df4fabeSYong Wu res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 6450df4fabeSYong Wu data->base = devm_ioremap_resource(dev, res); 6460df4fabeSYong Wu if (IS_ERR(data->base)) 6470df4fabeSYong Wu return PTR_ERR(data->base); 648b16c0170SJoerg Roedel ioaddr = res->start; 6490df4fabeSYong Wu 6500df4fabeSYong Wu data->irq = platform_get_irq(pdev, 0); 6510df4fabeSYong Wu if (data->irq < 0) 6520df4fabeSYong Wu return data->irq; 6530df4fabeSYong Wu 6546b717796SChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) { 6550df4fabeSYong Wu data->bclk = devm_clk_get(dev, "bclk"); 6560df4fabeSYong Wu if (IS_ERR(data->bclk)) 6570df4fabeSYong Wu return PTR_ERR(data->bclk); 6582aa4c259SYong Wu } 6590df4fabeSYong Wu 6600df4fabeSYong Wu larb_nr = of_count_phandle_with_args(dev->of_node, 6610df4fabeSYong Wu "mediatek,larbs", NULL); 6620df4fabeSYong Wu if (larb_nr < 0) 6630df4fabeSYong Wu return larb_nr; 6640df4fabeSYong Wu 6650df4fabeSYong Wu for (i = 0; i < larb_nr; i++) { 6660df4fabeSYong Wu struct device_node *larbnode; 6670df4fabeSYong Wu struct platform_device *plarbdev; 668e6dec923SYong Wu u32 id; 6690df4fabeSYong Wu 6700df4fabeSYong Wu larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); 6710df4fabeSYong Wu if (!larbnode) 6720df4fabeSYong Wu return -EINVAL; 6730df4fabeSYong Wu 6741eb8e4e2SWen Yang if (!of_device_is_available(larbnode)) { 6751eb8e4e2SWen Yang of_node_put(larbnode); 6760df4fabeSYong Wu continue; 6771eb8e4e2SWen Yang } 6780df4fabeSYong Wu 679e6dec923SYong Wu ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id); 680e6dec923SYong Wu if (ret)/* The id is consecutive if there is no this property */ 681e6dec923SYong Wu id = i; 682e6dec923SYong Wu 6830df4fabeSYong Wu plarbdev = of_find_device_by_node(larbnode); 6841eb8e4e2SWen Yang if (!plarbdev) { 6851eb8e4e2SWen Yang of_node_put(larbnode); 6860df4fabeSYong Wu return -EPROBE_DEFER; 6871eb8e4e2SWen Yang } 6881ee9feb2SYong Wu data->larb_imu[id].dev = &plarbdev->dev; 6890df4fabeSYong Wu 69000c7c81fSRussell King component_match_add_release(dev, &match, release_of, 69100c7c81fSRussell King compare_of, larbnode); 6920df4fabeSYong Wu } 6930df4fabeSYong Wu 6940df4fabeSYong Wu platform_set_drvdata(pdev, data); 6950df4fabeSYong Wu 6960df4fabeSYong Wu ret = mtk_iommu_hw_init(data); 6970df4fabeSYong Wu if (ret) 6980df4fabeSYong Wu return ret; 6990df4fabeSYong Wu 700b16c0170SJoerg Roedel ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, 701b16c0170SJoerg Roedel "mtk-iommu.%pa", &ioaddr); 702b16c0170SJoerg Roedel if (ret) 703b16c0170SJoerg Roedel return ret; 704b16c0170SJoerg Roedel 705b16c0170SJoerg Roedel iommu_device_set_ops(&data->iommu, &mtk_iommu_ops); 706b16c0170SJoerg Roedel iommu_device_set_fwnode(&data->iommu, &pdev->dev.of_node->fwnode); 707b16c0170SJoerg Roedel 708b16c0170SJoerg Roedel ret = iommu_device_register(&data->iommu); 709b16c0170SJoerg Roedel if (ret) 710b16c0170SJoerg Roedel return ret; 711b16c0170SJoerg Roedel 712da3cc91bSYong Wu spin_lock_init(&data->tlb_lock); 7137c3a2ec0SYong Wu list_add_tail(&data->list, &m4ulist); 7147c3a2ec0SYong Wu 7150df4fabeSYong Wu if (!iommu_present(&platform_bus_type)) 7160df4fabeSYong Wu bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); 7170df4fabeSYong Wu 7180df4fabeSYong Wu return component_master_add_with_match(dev, &mtk_iommu_com_ops, match); 7190df4fabeSYong Wu } 7200df4fabeSYong Wu 7210df4fabeSYong Wu static int mtk_iommu_remove(struct platform_device *pdev) 7220df4fabeSYong Wu { 7230df4fabeSYong Wu struct mtk_iommu_data *data = platform_get_drvdata(pdev); 7240df4fabeSYong Wu 725b16c0170SJoerg Roedel iommu_device_sysfs_remove(&data->iommu); 726b16c0170SJoerg Roedel iommu_device_unregister(&data->iommu); 727b16c0170SJoerg Roedel 7280df4fabeSYong Wu if (iommu_present(&platform_bus_type)) 7290df4fabeSYong Wu bus_set_iommu(&platform_bus_type, NULL); 7300df4fabeSYong Wu 7310df4fabeSYong Wu clk_disable_unprepare(data->bclk); 7320df4fabeSYong Wu devm_free_irq(&pdev->dev, data->irq, data); 7330df4fabeSYong Wu component_master_del(&pdev->dev, &mtk_iommu_com_ops); 7340df4fabeSYong Wu return 0; 7350df4fabeSYong Wu } 7360df4fabeSYong Wu 737fd99f796SArnd Bergmann static int __maybe_unused mtk_iommu_suspend(struct device *dev) 7380df4fabeSYong Wu { 7390df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 7400df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 7410df4fabeSYong Wu void __iomem *base = data->base; 7420df4fabeSYong Wu 74375eed350SChao Hao reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL); 7440df4fabeSYong Wu reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); 7450df4fabeSYong Wu reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); 7460df4fabeSYong Wu reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0); 7470df4fabeSYong Wu reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); 74870ca608bSYong Wu reg->ivrp_paddr = readl_relaxed(base + REG_MMU_IVRP_PADDR); 749b9475b34SYong Wu reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG); 7506254b64fSYong Wu clk_disable_unprepare(data->bclk); 7510df4fabeSYong Wu return 0; 7520df4fabeSYong Wu } 7530df4fabeSYong Wu 754fd99f796SArnd Bergmann static int __maybe_unused mtk_iommu_resume(struct device *dev) 7550df4fabeSYong Wu { 7560df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 7570df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 758907ba6a1SYong Wu struct mtk_iommu_domain *m4u_dom = data->m4u_dom; 7590df4fabeSYong Wu void __iomem *base = data->base; 7606254b64fSYong Wu int ret; 7610df4fabeSYong Wu 7626254b64fSYong Wu ret = clk_prepare_enable(data->bclk); 7636254b64fSYong Wu if (ret) { 7646254b64fSYong Wu dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret); 7656254b64fSYong Wu return ret; 7666254b64fSYong Wu } 76775eed350SChao Hao writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL); 7680df4fabeSYong Wu writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); 7690df4fabeSYong Wu writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); 7700df4fabeSYong Wu writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); 7710df4fabeSYong Wu writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); 77270ca608bSYong Wu writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR); 773b9475b34SYong Wu writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG); 774907ba6a1SYong Wu if (m4u_dom) 775d1e5f26fSRobin Murphy writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, 776e6dec923SYong Wu base + REG_MMU_PT_BASE_ADDR); 7770df4fabeSYong Wu return 0; 7780df4fabeSYong Wu } 7790df4fabeSYong Wu 780e6dec923SYong Wu static const struct dev_pm_ops mtk_iommu_pm_ops = { 7816254b64fSYong Wu SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume) 7820df4fabeSYong Wu }; 7830df4fabeSYong Wu 784cecdce9dSYong Wu static const struct mtk_iommu_plat_data mt2712_data = { 785cecdce9dSYong Wu .m4u_plat = M4U_MT2712, 7866b717796SChao Hao .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG, 787b3e5eee7SYong Wu .larbid_remap = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, 788cecdce9dSYong Wu }; 789cecdce9dSYong Wu 790cecdce9dSYong Wu static const struct mtk_iommu_plat_data mt8173_data = { 791cecdce9dSYong Wu .m4u_plat = M4U_MT8173, 7926b717796SChao Hao .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI, 793b3e5eee7SYong Wu .larbid_remap = {0, 1, 2, 3, 4, 5}, /* Linear mapping. */ 794cecdce9dSYong Wu }; 795cecdce9dSYong Wu 796907ba6a1SYong Wu static const struct mtk_iommu_plat_data mt8183_data = { 797907ba6a1SYong Wu .m4u_plat = M4U_MT8183, 7986b717796SChao Hao .flags = RESET_AXI, 799907ba6a1SYong Wu .larbid_remap = {0, 4, 5, 6, 7, 2, 3, 1}, 800907ba6a1SYong Wu }; 801907ba6a1SYong Wu 8020df4fabeSYong Wu static const struct of_device_id mtk_iommu_of_ids[] = { 803cecdce9dSYong Wu { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, 804cecdce9dSYong Wu { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, 805907ba6a1SYong Wu { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, 8060df4fabeSYong Wu {} 8070df4fabeSYong Wu }; 8080df4fabeSYong Wu 8090df4fabeSYong Wu static struct platform_driver mtk_iommu_driver = { 8100df4fabeSYong Wu .probe = mtk_iommu_probe, 8110df4fabeSYong Wu .remove = mtk_iommu_remove, 8120df4fabeSYong Wu .driver = { 8130df4fabeSYong Wu .name = "mtk-iommu", 814e6dec923SYong Wu .of_match_table = of_match_ptr(mtk_iommu_of_ids), 8150df4fabeSYong Wu .pm = &mtk_iommu_pm_ops, 8160df4fabeSYong Wu } 8170df4fabeSYong Wu }; 8180df4fabeSYong Wu 819e6dec923SYong Wu static int __init mtk_iommu_init(void) 8200df4fabeSYong Wu { 8210df4fabeSYong Wu int ret; 8220df4fabeSYong Wu 8230df4fabeSYong Wu ret = platform_driver_register(&mtk_iommu_driver); 824e6dec923SYong Wu if (ret != 0) 825e6dec923SYong Wu pr_err("Failed to register MTK IOMMU driver\n"); 826e6dec923SYong Wu 8270df4fabeSYong Wu return ret; 8280df4fabeSYong Wu } 8290df4fabeSYong Wu 830e6dec923SYong Wu subsys_initcall(mtk_iommu_init) 831