11802d0beSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 20df4fabeSYong Wu /* 30df4fabeSYong Wu * Copyright (c) 2015-2016 MediaTek Inc. 40df4fabeSYong Wu * Author: Yong Wu <yong.wu@mediatek.com> 50df4fabeSYong Wu */ 657c8a661SMike Rapoport #include <linux/memblock.h> 70df4fabeSYong Wu #include <linux/bug.h> 80df4fabeSYong Wu #include <linux/clk.h> 90df4fabeSYong Wu #include <linux/component.h> 100df4fabeSYong Wu #include <linux/device.h> 110df4fabeSYong Wu #include <linux/dma-iommu.h> 120df4fabeSYong Wu #include <linux/err.h> 130df4fabeSYong Wu #include <linux/interrupt.h> 140df4fabeSYong Wu #include <linux/io.h> 150df4fabeSYong Wu #include <linux/iommu.h> 160df4fabeSYong Wu #include <linux/iopoll.h> 170df4fabeSYong Wu #include <linux/list.h> 180df4fabeSYong Wu #include <linux/of_address.h> 190df4fabeSYong Wu #include <linux/of_iommu.h> 200df4fabeSYong Wu #include <linux/of_irq.h> 210df4fabeSYong Wu #include <linux/of_platform.h> 220df4fabeSYong Wu #include <linux/platform_device.h> 230df4fabeSYong Wu #include <linux/slab.h> 240df4fabeSYong Wu #include <linux/spinlock.h> 250df4fabeSYong Wu #include <asm/barrier.h> 260df4fabeSYong Wu #include <soc/mediatek/smi.h> 270df4fabeSYong Wu 289ca340c9SHonghui Zhang #include "mtk_iommu.h" 290df4fabeSYong Wu 300df4fabeSYong Wu #define REG_MMU_PT_BASE_ADDR 0x000 31907ba6a1SYong Wu #define MMU_PT_ADDR_MASK GENMASK(31, 7) 320df4fabeSYong Wu 330df4fabeSYong Wu #define REG_MMU_INVALIDATE 0x020 340df4fabeSYong Wu #define F_ALL_INVLD 0x2 350df4fabeSYong Wu #define F_MMU_INV_RANGE 0x1 360df4fabeSYong Wu 370df4fabeSYong Wu #define REG_MMU_INVLD_START_A 0x024 380df4fabeSYong Wu #define REG_MMU_INVLD_END_A 0x028 390df4fabeSYong Wu 400df4fabeSYong Wu #define REG_MMU_INV_SEL 0x038 410df4fabeSYong Wu #define F_INVLD_EN0 BIT(0) 420df4fabeSYong Wu #define F_INVLD_EN1 BIT(1) 430df4fabeSYong Wu 440df4fabeSYong Wu #define REG_MMU_STANDARD_AXI_MODE 0x048 450df4fabeSYong Wu #define REG_MMU_DCM_DIS 0x050 460df4fabeSYong Wu 470df4fabeSYong Wu #define REG_MMU_CTRL_REG 0x110 48acb3c92aSYong Wu #define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4) 490df4fabeSYong Wu #define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4) 50acb3c92aSYong Wu #define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5) 510df4fabeSYong Wu 520df4fabeSYong Wu #define REG_MMU_IVRP_PADDR 0x114 5370ca608bSYong Wu 5430e2fccfSYong Wu #define REG_MMU_VLD_PA_RNG 0x118 5530e2fccfSYong Wu #define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA)) 560df4fabeSYong Wu 570df4fabeSYong Wu #define REG_MMU_INT_CONTROL0 0x120 580df4fabeSYong Wu #define F_L2_MULIT_HIT_EN BIT(0) 590df4fabeSYong Wu #define F_TABLE_WALK_FAULT_INT_EN BIT(1) 600df4fabeSYong Wu #define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2) 610df4fabeSYong Wu #define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3) 620df4fabeSYong Wu #define F_PREFETCH_FIFO_ERR_INT_EN BIT(5) 630df4fabeSYong Wu #define F_MISS_FIFO_ERR_INT_EN BIT(6) 640df4fabeSYong Wu #define F_INT_CLR_BIT BIT(12) 650df4fabeSYong Wu 660df4fabeSYong Wu #define REG_MMU_INT_MAIN_CONTROL 0x124 6715a01f4cSYong Wu /* mmu0 | mmu1 */ 6815a01f4cSYong Wu #define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7)) 6915a01f4cSYong Wu #define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8)) 7015a01f4cSYong Wu #define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9)) 7115a01f4cSYong Wu #define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10)) 7215a01f4cSYong Wu #define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11)) 7315a01f4cSYong Wu #define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12)) 7415a01f4cSYong Wu #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13)) 750df4fabeSYong Wu 760df4fabeSYong Wu #define REG_MMU_CPE_DONE 0x12C 770df4fabeSYong Wu 780df4fabeSYong Wu #define REG_MMU_FAULT_ST1 0x134 7915a01f4cSYong Wu #define F_REG_MMU0_FAULT_MASK GENMASK(6, 0) 8015a01f4cSYong Wu #define F_REG_MMU1_FAULT_MASK GENMASK(13, 7) 810df4fabeSYong Wu 8215a01f4cSYong Wu #define REG_MMU0_FAULT_VA 0x13c 830df4fabeSYong Wu #define F_MMU_FAULT_VA_WRITE_BIT BIT(1) 840df4fabeSYong Wu #define F_MMU_FAULT_VA_LAYER_BIT BIT(0) 850df4fabeSYong Wu 8615a01f4cSYong Wu #define REG_MMU0_INVLD_PA 0x140 8715a01f4cSYong Wu #define REG_MMU1_FAULT_VA 0x144 8815a01f4cSYong Wu #define REG_MMU1_INVLD_PA 0x148 8915a01f4cSYong Wu #define REG_MMU0_INT_ID 0x150 9015a01f4cSYong Wu #define REG_MMU1_INT_ID 0x154 9115a01f4cSYong Wu #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) 9215a01f4cSYong Wu #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) 930df4fabeSYong Wu 940df4fabeSYong Wu #define MTK_PROTECT_PA_ALIGN 128 950df4fabeSYong Wu 96a9467d95SYong Wu /* 97a9467d95SYong Wu * Get the local arbiter ID and the portid within the larb arbiter 98a9467d95SYong Wu * from mtk_m4u_id which is defined by MTK_M4U_ID. 99a9467d95SYong Wu */ 100e6dec923SYong Wu #define MTK_M4U_TO_LARB(id) (((id) >> 5) & 0xf) 101a9467d95SYong Wu #define MTK_M4U_TO_PORT(id) ((id) & 0x1f) 102a9467d95SYong Wu 1030df4fabeSYong Wu struct mtk_iommu_domain { 1040df4fabeSYong Wu spinlock_t pgtlock; /* lock for page table */ 1050df4fabeSYong Wu 1060df4fabeSYong Wu struct io_pgtable_cfg cfg; 1070df4fabeSYong Wu struct io_pgtable_ops *iop; 1080df4fabeSYong Wu 1090df4fabeSYong Wu struct iommu_domain domain; 1100df4fabeSYong Wu }; 1110df4fabeSYong Wu 112b65f5016SArvind Yadav static const struct iommu_ops mtk_iommu_ops; 1130df4fabeSYong Wu 11476ce6546SYong Wu /* 11576ce6546SYong Wu * In M4U 4GB mode, the physical address is remapped as below: 11676ce6546SYong Wu * 11776ce6546SYong Wu * CPU Physical address: 11876ce6546SYong Wu * ==================== 11976ce6546SYong Wu * 12076ce6546SYong Wu * 0 1G 2G 3G 4G 5G 12176ce6546SYong Wu * |---A---|---B---|---C---|---D---|---E---| 12276ce6546SYong Wu * +--I/O--+------------Memory-------------+ 12376ce6546SYong Wu * 12476ce6546SYong Wu * IOMMU output physical address: 12576ce6546SYong Wu * ============================= 12676ce6546SYong Wu * 12776ce6546SYong Wu * 4G 5G 6G 7G 8G 12876ce6546SYong Wu * |---E---|---B---|---C---|---D---| 12976ce6546SYong Wu * +------------Memory-------------+ 13076ce6546SYong Wu * 13176ce6546SYong Wu * The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the 13276ce6546SYong Wu * bit32 of the CPU physical address always is needed to set, and for Region 13376ce6546SYong Wu * 'E', the CPU physical address keep as is. 13476ce6546SYong Wu * Additionally, The iommu consumers always use the CPU phyiscal address. 13576ce6546SYong Wu */ 136b4dad40eSYong Wu #define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL 13776ce6546SYong Wu 1387c3a2ec0SYong Wu static LIST_HEAD(m4ulist); /* List all the M4U HWs */ 1397c3a2ec0SYong Wu 1407c3a2ec0SYong Wu #define for_each_m4u(data) list_for_each_entry(data, &m4ulist, list) 1417c3a2ec0SYong Wu 1427c3a2ec0SYong Wu /* 1437c3a2ec0SYong Wu * There may be 1 or 2 M4U HWs, But we always expect they are in the same domain 1447c3a2ec0SYong Wu * for the performance. 1457c3a2ec0SYong Wu * 1467c3a2ec0SYong Wu * Here always return the mtk_iommu_data of the first probed M4U where the 1477c3a2ec0SYong Wu * iommu domain information is recorded. 1487c3a2ec0SYong Wu */ 1497c3a2ec0SYong Wu static struct mtk_iommu_data *mtk_iommu_get_m4u_data(void) 1507c3a2ec0SYong Wu { 1517c3a2ec0SYong Wu struct mtk_iommu_data *data; 1527c3a2ec0SYong Wu 1537c3a2ec0SYong Wu for_each_m4u(data) 1547c3a2ec0SYong Wu return data; 1557c3a2ec0SYong Wu 1567c3a2ec0SYong Wu return NULL; 1577c3a2ec0SYong Wu } 1587c3a2ec0SYong Wu 1590df4fabeSYong Wu static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) 1600df4fabeSYong Wu { 1610df4fabeSYong Wu return container_of(dom, struct mtk_iommu_domain, domain); 1620df4fabeSYong Wu } 1630df4fabeSYong Wu 1640df4fabeSYong Wu static void mtk_iommu_tlb_flush_all(void *cookie) 1650df4fabeSYong Wu { 1660df4fabeSYong Wu struct mtk_iommu_data *data = cookie; 1670df4fabeSYong Wu 1687c3a2ec0SYong Wu for_each_m4u(data) { 1697c3a2ec0SYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, 1707c3a2ec0SYong Wu data->base + REG_MMU_INV_SEL); 1710df4fabeSYong Wu writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); 1720df4fabeSYong Wu wmb(); /* Make sure the tlb flush all done */ 1730df4fabeSYong Wu } 1747c3a2ec0SYong Wu } 1750df4fabeSYong Wu 1760df4fabeSYong Wu static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size, 1770df4fabeSYong Wu size_t granule, bool leaf, 1780df4fabeSYong Wu void *cookie) 1790df4fabeSYong Wu { 1800df4fabeSYong Wu struct mtk_iommu_data *data = cookie; 1810df4fabeSYong Wu 1827c3a2ec0SYong Wu for_each_m4u(data) { 1837c3a2ec0SYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, 1847c3a2ec0SYong Wu data->base + REG_MMU_INV_SEL); 1850df4fabeSYong Wu 1860df4fabeSYong Wu writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A); 1877c3a2ec0SYong Wu writel_relaxed(iova + size - 1, 1887c3a2ec0SYong Wu data->base + REG_MMU_INVLD_END_A); 1897c3a2ec0SYong Wu writel_relaxed(F_MMU_INV_RANGE, 1907c3a2ec0SYong Wu data->base + REG_MMU_INVALIDATE); 19198a8f63eSRobin Murphy data->tlb_flush_active = true; 1920df4fabeSYong Wu } 1937c3a2ec0SYong Wu } 1940df4fabeSYong Wu 1950df4fabeSYong Wu static void mtk_iommu_tlb_sync(void *cookie) 1960df4fabeSYong Wu { 1970df4fabeSYong Wu struct mtk_iommu_data *data = cookie; 1980df4fabeSYong Wu int ret; 1990df4fabeSYong Wu u32 tmp; 2000df4fabeSYong Wu 2017c3a2ec0SYong Wu for_each_m4u(data) { 20298a8f63eSRobin Murphy /* Avoid timing out if there's nothing to wait for */ 20398a8f63eSRobin Murphy if (!data->tlb_flush_active) 20498a8f63eSRobin Murphy return; 20598a8f63eSRobin Murphy 2067c3a2ec0SYong Wu ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, 2077c3a2ec0SYong Wu tmp, tmp != 0, 10, 100000); 2080df4fabeSYong Wu if (ret) { 2090df4fabeSYong Wu dev_warn(data->dev, 2100df4fabeSYong Wu "Partial TLB flush timed out, falling back to full flush\n"); 2110df4fabeSYong Wu mtk_iommu_tlb_flush_all(cookie); 2120df4fabeSYong Wu } 2130df4fabeSYong Wu /* Clear the CPE status */ 2140df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_CPE_DONE); 21598a8f63eSRobin Murphy data->tlb_flush_active = false; 2160df4fabeSYong Wu } 2177c3a2ec0SYong Wu } 2180df4fabeSYong Wu 2190df4fabeSYong Wu static const struct iommu_gather_ops mtk_iommu_gather_ops = { 2200df4fabeSYong Wu .tlb_flush_all = mtk_iommu_tlb_flush_all, 2210df4fabeSYong Wu .tlb_add_flush = mtk_iommu_tlb_add_flush_nosync, 2220df4fabeSYong Wu .tlb_sync = mtk_iommu_tlb_sync, 2230df4fabeSYong Wu }; 2240df4fabeSYong Wu 2250df4fabeSYong Wu static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) 2260df4fabeSYong Wu { 2270df4fabeSYong Wu struct mtk_iommu_data *data = dev_id; 2280df4fabeSYong Wu struct mtk_iommu_domain *dom = data->m4u_dom; 2290df4fabeSYong Wu u32 int_state, regval, fault_iova, fault_pa; 2300df4fabeSYong Wu unsigned int fault_larb, fault_port; 2310df4fabeSYong Wu bool layer, write; 2320df4fabeSYong Wu 2330df4fabeSYong Wu /* Read error info from registers */ 2340df4fabeSYong Wu int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1); 23515a01f4cSYong Wu if (int_state & F_REG_MMU0_FAULT_MASK) { 23615a01f4cSYong Wu regval = readl_relaxed(data->base + REG_MMU0_INT_ID); 23715a01f4cSYong Wu fault_iova = readl_relaxed(data->base + REG_MMU0_FAULT_VA); 23815a01f4cSYong Wu fault_pa = readl_relaxed(data->base + REG_MMU0_INVLD_PA); 23915a01f4cSYong Wu } else { 24015a01f4cSYong Wu regval = readl_relaxed(data->base + REG_MMU1_INT_ID); 24115a01f4cSYong Wu fault_iova = readl_relaxed(data->base + REG_MMU1_FAULT_VA); 24215a01f4cSYong Wu fault_pa = readl_relaxed(data->base + REG_MMU1_INVLD_PA); 24315a01f4cSYong Wu } 2440df4fabeSYong Wu layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; 2450df4fabeSYong Wu write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; 24615a01f4cSYong Wu fault_larb = F_MMU_INT_ID_LARB_ID(regval); 24715a01f4cSYong Wu fault_port = F_MMU_INT_ID_PORT_ID(regval); 2480df4fabeSYong Wu 249b3e5eee7SYong Wu fault_larb = data->plat_data->larbid_remap[fault_larb]; 250b3e5eee7SYong Wu 2510df4fabeSYong Wu if (report_iommu_fault(&dom->domain, data->dev, fault_iova, 2520df4fabeSYong Wu write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { 2530df4fabeSYong Wu dev_err_ratelimited( 2540df4fabeSYong Wu data->dev, 2550df4fabeSYong Wu "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d layer=%d %s\n", 2560df4fabeSYong Wu int_state, fault_iova, fault_pa, fault_larb, fault_port, 2570df4fabeSYong Wu layer, write ? "write" : "read"); 2580df4fabeSYong Wu } 2590df4fabeSYong Wu 2600df4fabeSYong Wu /* Interrupt clear */ 2610df4fabeSYong Wu regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL0); 2620df4fabeSYong Wu regval |= F_INT_CLR_BIT; 2630df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); 2640df4fabeSYong Wu 2650df4fabeSYong Wu mtk_iommu_tlb_flush_all(data); 2660df4fabeSYong Wu 2670df4fabeSYong Wu return IRQ_HANDLED; 2680df4fabeSYong Wu } 2690df4fabeSYong Wu 2700df4fabeSYong Wu static void mtk_iommu_config(struct mtk_iommu_data *data, 2710df4fabeSYong Wu struct device *dev, bool enable) 2720df4fabeSYong Wu { 2730df4fabeSYong Wu struct mtk_smi_larb_iommu *larb_mmu; 2740df4fabeSYong Wu unsigned int larbid, portid; 275a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 27658f0d1d5SRobin Murphy int i; 2770df4fabeSYong Wu 27858f0d1d5SRobin Murphy for (i = 0; i < fwspec->num_ids; ++i) { 27958f0d1d5SRobin Murphy larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); 28058f0d1d5SRobin Murphy portid = MTK_M4U_TO_PORT(fwspec->ids[i]); 281*1ee9feb2SYong Wu larb_mmu = &data->larb_imu[larbid]; 2820df4fabeSYong Wu 2830df4fabeSYong Wu dev_dbg(dev, "%s iommu port: %d\n", 2840df4fabeSYong Wu enable ? "enable" : "disable", portid); 2850df4fabeSYong Wu 2860df4fabeSYong Wu if (enable) 2870df4fabeSYong Wu larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); 2880df4fabeSYong Wu else 2890df4fabeSYong Wu larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); 2900df4fabeSYong Wu } 2910df4fabeSYong Wu } 2920df4fabeSYong Wu 2934b00f5acSYong Wu static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom) 2940df4fabeSYong Wu { 2954b00f5acSYong Wu struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); 2960df4fabeSYong Wu 2970df4fabeSYong Wu spin_lock_init(&dom->pgtlock); 2980df4fabeSYong Wu 2990df4fabeSYong Wu dom->cfg = (struct io_pgtable_cfg) { 3000df4fabeSYong Wu .quirks = IO_PGTABLE_QUIRK_ARM_NS | 3010df4fabeSYong Wu IO_PGTABLE_QUIRK_NO_PERMS | 302b4dad40eSYong Wu IO_PGTABLE_QUIRK_TLBI_ON_MAP | 303b4dad40eSYong Wu IO_PGTABLE_QUIRK_ARM_MTK_EXT, 3040df4fabeSYong Wu .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, 3050df4fabeSYong Wu .ias = 32, 306b4dad40eSYong Wu .oas = 34, 3070df4fabeSYong Wu .tlb = &mtk_iommu_gather_ops, 3080df4fabeSYong Wu .iommu_dev = data->dev, 3090df4fabeSYong Wu }; 3100df4fabeSYong Wu 3110df4fabeSYong Wu dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data); 3120df4fabeSYong Wu if (!dom->iop) { 3130df4fabeSYong Wu dev_err(data->dev, "Failed to alloc io pgtable\n"); 3140df4fabeSYong Wu return -EINVAL; 3150df4fabeSYong Wu } 3160df4fabeSYong Wu 3170df4fabeSYong Wu /* Update our support page sizes bitmap */ 318d16e0faaSRobin Murphy dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; 3190df4fabeSYong Wu return 0; 3200df4fabeSYong Wu } 3210df4fabeSYong Wu 3220df4fabeSYong Wu static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) 3230df4fabeSYong Wu { 3240df4fabeSYong Wu struct mtk_iommu_domain *dom; 3250df4fabeSYong Wu 3260df4fabeSYong Wu if (type != IOMMU_DOMAIN_DMA) 3270df4fabeSYong Wu return NULL; 3280df4fabeSYong Wu 3290df4fabeSYong Wu dom = kzalloc(sizeof(*dom), GFP_KERNEL); 3300df4fabeSYong Wu if (!dom) 3310df4fabeSYong Wu return NULL; 3320df4fabeSYong Wu 3334b00f5acSYong Wu if (iommu_get_dma_cookie(&dom->domain)) 3344b00f5acSYong Wu goto free_dom; 3354b00f5acSYong Wu 3364b00f5acSYong Wu if (mtk_iommu_domain_finalise(dom)) 3374b00f5acSYong Wu goto put_dma_cookie; 3380df4fabeSYong Wu 3390df4fabeSYong Wu dom->domain.geometry.aperture_start = 0; 3400df4fabeSYong Wu dom->domain.geometry.aperture_end = DMA_BIT_MASK(32); 3410df4fabeSYong Wu dom->domain.geometry.force_aperture = true; 3420df4fabeSYong Wu 3430df4fabeSYong Wu return &dom->domain; 3444b00f5acSYong Wu 3454b00f5acSYong Wu put_dma_cookie: 3464b00f5acSYong Wu iommu_put_dma_cookie(&dom->domain); 3474b00f5acSYong Wu free_dom: 3484b00f5acSYong Wu kfree(dom); 3494b00f5acSYong Wu return NULL; 3500df4fabeSYong Wu } 3510df4fabeSYong Wu 3520df4fabeSYong Wu static void mtk_iommu_domain_free(struct iommu_domain *domain) 3530df4fabeSYong Wu { 3544b00f5acSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 3554b00f5acSYong Wu 3564b00f5acSYong Wu free_io_pgtable_ops(dom->iop); 3570df4fabeSYong Wu iommu_put_dma_cookie(domain); 3580df4fabeSYong Wu kfree(to_mtk_domain(domain)); 3590df4fabeSYong Wu } 3600df4fabeSYong Wu 3610df4fabeSYong Wu static int mtk_iommu_attach_device(struct iommu_domain *domain, 3620df4fabeSYong Wu struct device *dev) 3630df4fabeSYong Wu { 3640df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 365a9bf2eecSJoerg Roedel struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv; 3660df4fabeSYong Wu 3674b00f5acSYong Wu if (!data) 3680df4fabeSYong Wu return -ENODEV; 3690df4fabeSYong Wu 3704b00f5acSYong Wu /* Update the pgtable base address register of the M4U HW */ 3710df4fabeSYong Wu if (!data->m4u_dom) { 3720df4fabeSYong Wu data->m4u_dom = dom; 373907ba6a1SYong Wu writel(dom->cfg.arm_v7s_cfg.ttbr[0] & MMU_PT_ADDR_MASK, 3744b00f5acSYong Wu data->base + REG_MMU_PT_BASE_ADDR); 3750df4fabeSYong Wu } 3760df4fabeSYong Wu 3774b00f5acSYong Wu mtk_iommu_config(data, dev, true); 3780df4fabeSYong Wu return 0; 3790df4fabeSYong Wu } 3800df4fabeSYong Wu 3810df4fabeSYong Wu static void mtk_iommu_detach_device(struct iommu_domain *domain, 3820df4fabeSYong Wu struct device *dev) 3830df4fabeSYong Wu { 384a9bf2eecSJoerg Roedel struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv; 3850df4fabeSYong Wu 38658f0d1d5SRobin Murphy if (!data) 3870df4fabeSYong Wu return; 3880df4fabeSYong Wu 3890df4fabeSYong Wu mtk_iommu_config(data, dev, false); 3900df4fabeSYong Wu } 3910df4fabeSYong Wu 3920df4fabeSYong Wu static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, 3930df4fabeSYong Wu phys_addr_t paddr, size_t size, int prot) 3940df4fabeSYong Wu { 3950df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 396b4dad40eSYong Wu struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); 3970df4fabeSYong Wu unsigned long flags; 3980df4fabeSYong Wu int ret; 3990df4fabeSYong Wu 400b4dad40eSYong Wu /* The "4GB mode" M4U physically can not use the lower remap of Dram. */ 401b4dad40eSYong Wu if (data->enable_4GB) 402b4dad40eSYong Wu paddr |= BIT_ULL(32); 403b4dad40eSYong Wu 4040df4fabeSYong Wu spin_lock_irqsave(&dom->pgtlock, flags); 405b4dad40eSYong Wu ret = dom->iop->map(dom->iop, iova, paddr, size, prot); 4060df4fabeSYong Wu spin_unlock_irqrestore(&dom->pgtlock, flags); 4070df4fabeSYong Wu 4080df4fabeSYong Wu return ret; 4090df4fabeSYong Wu } 4100df4fabeSYong Wu 4110df4fabeSYong Wu static size_t mtk_iommu_unmap(struct iommu_domain *domain, 4120df4fabeSYong Wu unsigned long iova, size_t size) 4130df4fabeSYong Wu { 4140df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 4150df4fabeSYong Wu unsigned long flags; 4160df4fabeSYong Wu size_t unmapsz; 4170df4fabeSYong Wu 4180df4fabeSYong Wu spin_lock_irqsave(&dom->pgtlock, flags); 4190df4fabeSYong Wu unmapsz = dom->iop->unmap(dom->iop, iova, size); 4200df4fabeSYong Wu spin_unlock_irqrestore(&dom->pgtlock, flags); 4210df4fabeSYong Wu 4220df4fabeSYong Wu return unmapsz; 4230df4fabeSYong Wu } 4240df4fabeSYong Wu 4254d689b61SRobin Murphy static void mtk_iommu_iotlb_sync(struct iommu_domain *domain) 4264d689b61SRobin Murphy { 4274d689b61SRobin Murphy mtk_iommu_tlb_sync(mtk_iommu_get_m4u_data()); 4284d689b61SRobin Murphy } 4294d689b61SRobin Murphy 4300df4fabeSYong Wu static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, 4310df4fabeSYong Wu dma_addr_t iova) 4320df4fabeSYong Wu { 4330df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 43430e2fccfSYong Wu struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); 4350df4fabeSYong Wu unsigned long flags; 4360df4fabeSYong Wu phys_addr_t pa; 4370df4fabeSYong Wu 4380df4fabeSYong Wu spin_lock_irqsave(&dom->pgtlock, flags); 4390df4fabeSYong Wu pa = dom->iop->iova_to_phys(dom->iop, iova); 4400df4fabeSYong Wu spin_unlock_irqrestore(&dom->pgtlock, flags); 4410df4fabeSYong Wu 442b4dad40eSYong Wu if (data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) 443b4dad40eSYong Wu pa &= ~BIT_ULL(32); 44430e2fccfSYong Wu 4450df4fabeSYong Wu return pa; 4460df4fabeSYong Wu } 4470df4fabeSYong Wu 4480df4fabeSYong Wu static int mtk_iommu_add_device(struct device *dev) 4490df4fabeSYong Wu { 450a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 451b16c0170SJoerg Roedel struct mtk_iommu_data *data; 4520df4fabeSYong Wu struct iommu_group *group; 4530df4fabeSYong Wu 454a9bf2eecSJoerg Roedel if (!fwspec || fwspec->ops != &mtk_iommu_ops) 45558f0d1d5SRobin Murphy return -ENODEV; /* Not a iommu client device */ 4560df4fabeSYong Wu 457a9bf2eecSJoerg Roedel data = fwspec->iommu_priv; 458b16c0170SJoerg Roedel iommu_device_link(&data->iommu, dev); 459b16c0170SJoerg Roedel 4600df4fabeSYong Wu group = iommu_group_get_for_dev(dev); 4610df4fabeSYong Wu if (IS_ERR(group)) 4620df4fabeSYong Wu return PTR_ERR(group); 4630df4fabeSYong Wu 4640df4fabeSYong Wu iommu_group_put(group); 4650df4fabeSYong Wu return 0; 4660df4fabeSYong Wu } 4670df4fabeSYong Wu 4680df4fabeSYong Wu static void mtk_iommu_remove_device(struct device *dev) 4690df4fabeSYong Wu { 470a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 471b16c0170SJoerg Roedel struct mtk_iommu_data *data; 472b16c0170SJoerg Roedel 473a9bf2eecSJoerg Roedel if (!fwspec || fwspec->ops != &mtk_iommu_ops) 4740df4fabeSYong Wu return; 4750df4fabeSYong Wu 476a9bf2eecSJoerg Roedel data = fwspec->iommu_priv; 477b16c0170SJoerg Roedel iommu_device_unlink(&data->iommu, dev); 478b16c0170SJoerg Roedel 4790df4fabeSYong Wu iommu_group_remove_device(dev); 48058f0d1d5SRobin Murphy iommu_fwspec_free(dev); 4810df4fabeSYong Wu } 4820df4fabeSYong Wu 4830df4fabeSYong Wu static struct iommu_group *mtk_iommu_device_group(struct device *dev) 4840df4fabeSYong Wu { 4857c3a2ec0SYong Wu struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); 4860df4fabeSYong Wu 48758f0d1d5SRobin Murphy if (!data) 4880df4fabeSYong Wu return ERR_PTR(-ENODEV); 4890df4fabeSYong Wu 4900df4fabeSYong Wu /* All the client devices are in the same m4u iommu-group */ 4910df4fabeSYong Wu if (!data->m4u_group) { 4920df4fabeSYong Wu data->m4u_group = iommu_group_alloc(); 4930df4fabeSYong Wu if (IS_ERR(data->m4u_group)) 4940df4fabeSYong Wu dev_err(dev, "Failed to allocate M4U IOMMU group\n"); 4953a8d40b6SRobin Murphy } else { 4963a8d40b6SRobin Murphy iommu_group_ref_get(data->m4u_group); 4970df4fabeSYong Wu } 4980df4fabeSYong Wu return data->m4u_group; 4990df4fabeSYong Wu } 5000df4fabeSYong Wu 5010df4fabeSYong Wu static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) 5020df4fabeSYong Wu { 503a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 5040df4fabeSYong Wu struct platform_device *m4updev; 5050df4fabeSYong Wu 5060df4fabeSYong Wu if (args->args_count != 1) { 5070df4fabeSYong Wu dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", 5080df4fabeSYong Wu args->args_count); 5090df4fabeSYong Wu return -EINVAL; 5100df4fabeSYong Wu } 5110df4fabeSYong Wu 512a9bf2eecSJoerg Roedel if (!fwspec->iommu_priv) { 5130df4fabeSYong Wu /* Get the m4u device */ 5140df4fabeSYong Wu m4updev = of_find_device_by_node(args->np); 5150df4fabeSYong Wu if (WARN_ON(!m4updev)) 5160df4fabeSYong Wu return -EINVAL; 5170df4fabeSYong Wu 518a9bf2eecSJoerg Roedel fwspec->iommu_priv = platform_get_drvdata(m4updev); 5190df4fabeSYong Wu } 5200df4fabeSYong Wu 52158f0d1d5SRobin Murphy return iommu_fwspec_add_ids(dev, args->args, 1); 5220df4fabeSYong Wu } 5230df4fabeSYong Wu 524b65f5016SArvind Yadav static const struct iommu_ops mtk_iommu_ops = { 5250df4fabeSYong Wu .domain_alloc = mtk_iommu_domain_alloc, 5260df4fabeSYong Wu .domain_free = mtk_iommu_domain_free, 5270df4fabeSYong Wu .attach_dev = mtk_iommu_attach_device, 5280df4fabeSYong Wu .detach_dev = mtk_iommu_detach_device, 5290df4fabeSYong Wu .map = mtk_iommu_map, 5300df4fabeSYong Wu .unmap = mtk_iommu_unmap, 5314d689b61SRobin Murphy .flush_iotlb_all = mtk_iommu_iotlb_sync, 5324d689b61SRobin Murphy .iotlb_sync = mtk_iommu_iotlb_sync, 5330df4fabeSYong Wu .iova_to_phys = mtk_iommu_iova_to_phys, 5340df4fabeSYong Wu .add_device = mtk_iommu_add_device, 5350df4fabeSYong Wu .remove_device = mtk_iommu_remove_device, 5360df4fabeSYong Wu .device_group = mtk_iommu_device_group, 5370df4fabeSYong Wu .of_xlate = mtk_iommu_of_xlate, 5380df4fabeSYong Wu .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, 5390df4fabeSYong Wu }; 5400df4fabeSYong Wu 5410df4fabeSYong Wu static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) 5420df4fabeSYong Wu { 5430df4fabeSYong Wu u32 regval; 5440df4fabeSYong Wu int ret; 5450df4fabeSYong Wu 5460df4fabeSYong Wu ret = clk_prepare_enable(data->bclk); 5470df4fabeSYong Wu if (ret) { 5480df4fabeSYong Wu dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret); 5490df4fabeSYong Wu return ret; 5500df4fabeSYong Wu } 5510df4fabeSYong Wu 552cecdce9dSYong Wu if (data->plat_data->m4u_plat == M4U_MT8173) 553acb3c92aSYong Wu regval = F_MMU_PREFETCH_RT_REPLACE_MOD | 554acb3c92aSYong Wu F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173; 555acb3c92aSYong Wu else 556acb3c92aSYong Wu regval = F_MMU_TF_PROT_TO_PROGRAM_ADDR; 5570df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); 5580df4fabeSYong Wu 5590df4fabeSYong Wu regval = F_L2_MULIT_HIT_EN | 5600df4fabeSYong Wu F_TABLE_WALK_FAULT_INT_EN | 5610df4fabeSYong Wu F_PREETCH_FIFO_OVERFLOW_INT_EN | 5620df4fabeSYong Wu F_MISS_FIFO_OVERFLOW_INT_EN | 5630df4fabeSYong Wu F_PREFETCH_FIFO_ERR_INT_EN | 5640df4fabeSYong Wu F_MISS_FIFO_ERR_INT_EN; 5650df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); 5660df4fabeSYong Wu 5670df4fabeSYong Wu regval = F_INT_TRANSLATION_FAULT | 5680df4fabeSYong Wu F_INT_MAIN_MULTI_HIT_FAULT | 5690df4fabeSYong Wu F_INT_INVALID_PA_FAULT | 5700df4fabeSYong Wu F_INT_ENTRY_REPLACEMENT_FAULT | 5710df4fabeSYong Wu F_INT_TLB_MISS_FAULT | 5720df4fabeSYong Wu F_INT_MISS_TRANSACTION_FIFO_FAULT | 5730df4fabeSYong Wu F_INT_PRETETCH_TRANSATION_FIFO_FAULT; 5740df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); 5750df4fabeSYong Wu 576cecdce9dSYong Wu if (data->plat_data->m4u_plat == M4U_MT8173) 57770ca608bSYong Wu regval = (data->protect_base >> 1) | (data->enable_4GB << 31); 57870ca608bSYong Wu else 57970ca608bSYong Wu regval = lower_32_bits(data->protect_base) | 58070ca608bSYong Wu upper_32_bits(data->protect_base); 58170ca608bSYong Wu writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR); 58270ca608bSYong Wu 5832b326d8bSYong Wu if (data->enable_4GB && data->plat_data->has_vld_pa_rng) { 58430e2fccfSYong Wu /* 58530e2fccfSYong Wu * If 4GB mode is enabled, the validate PA range is from 58630e2fccfSYong Wu * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30]. 58730e2fccfSYong Wu */ 58830e2fccfSYong Wu regval = F_MMU_VLD_PA_RNG(7, 4); 58930e2fccfSYong Wu writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG); 59030e2fccfSYong Wu } 5910df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_DCM_DIS); 592e6dec923SYong Wu 59350822b0bSYong Wu if (data->plat_data->reset_axi) 5940df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE); 5950df4fabeSYong Wu 5960df4fabeSYong Wu if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, 5970df4fabeSYong Wu dev_name(data->dev), (void *)data)) { 5980df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); 5990df4fabeSYong Wu clk_disable_unprepare(data->bclk); 6000df4fabeSYong Wu dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); 6010df4fabeSYong Wu return -ENODEV; 6020df4fabeSYong Wu } 6030df4fabeSYong Wu 6040df4fabeSYong Wu return 0; 6050df4fabeSYong Wu } 6060df4fabeSYong Wu 6070df4fabeSYong Wu static const struct component_master_ops mtk_iommu_com_ops = { 6080df4fabeSYong Wu .bind = mtk_iommu_bind, 6090df4fabeSYong Wu .unbind = mtk_iommu_unbind, 6100df4fabeSYong Wu }; 6110df4fabeSYong Wu 6120df4fabeSYong Wu static int mtk_iommu_probe(struct platform_device *pdev) 6130df4fabeSYong Wu { 6140df4fabeSYong Wu struct mtk_iommu_data *data; 6150df4fabeSYong Wu struct device *dev = &pdev->dev; 6160df4fabeSYong Wu struct resource *res; 617b16c0170SJoerg Roedel resource_size_t ioaddr; 6180df4fabeSYong Wu struct component_match *match = NULL; 6190df4fabeSYong Wu void *protect; 6200b6c0ad3SAndrzej Hajda int i, larb_nr, ret; 6210df4fabeSYong Wu 6220df4fabeSYong Wu data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); 6230df4fabeSYong Wu if (!data) 6240df4fabeSYong Wu return -ENOMEM; 6250df4fabeSYong Wu data->dev = dev; 626cecdce9dSYong Wu data->plat_data = of_device_get_match_data(dev); 6270df4fabeSYong Wu 6280df4fabeSYong Wu /* Protect memory. HW will access here while translation fault.*/ 6290df4fabeSYong Wu protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); 6300df4fabeSYong Wu if (!protect) 6310df4fabeSYong Wu return -ENOMEM; 6320df4fabeSYong Wu data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); 6330df4fabeSYong Wu 63401e23c93SYong Wu /* Whether the current dram is over 4GB */ 63541939980SYong Wu data->enable_4GB = !!(max_pfn > (BIT_ULL(32) >> PAGE_SHIFT)); 636b4dad40eSYong Wu if (!data->plat_data->has_4gb_mode) 637b4dad40eSYong Wu data->enable_4GB = false; 63801e23c93SYong Wu 6390df4fabeSYong Wu res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 6400df4fabeSYong Wu data->base = devm_ioremap_resource(dev, res); 6410df4fabeSYong Wu if (IS_ERR(data->base)) 6420df4fabeSYong Wu return PTR_ERR(data->base); 643b16c0170SJoerg Roedel ioaddr = res->start; 6440df4fabeSYong Wu 6450df4fabeSYong Wu data->irq = platform_get_irq(pdev, 0); 6460df4fabeSYong Wu if (data->irq < 0) 6470df4fabeSYong Wu return data->irq; 6480df4fabeSYong Wu 6492aa4c259SYong Wu if (data->plat_data->has_bclk) { 6500df4fabeSYong Wu data->bclk = devm_clk_get(dev, "bclk"); 6510df4fabeSYong Wu if (IS_ERR(data->bclk)) 6520df4fabeSYong Wu return PTR_ERR(data->bclk); 6532aa4c259SYong Wu } 6540df4fabeSYong Wu 6550df4fabeSYong Wu larb_nr = of_count_phandle_with_args(dev->of_node, 6560df4fabeSYong Wu "mediatek,larbs", NULL); 6570df4fabeSYong Wu if (larb_nr < 0) 6580df4fabeSYong Wu return larb_nr; 6590df4fabeSYong Wu 6600df4fabeSYong Wu for (i = 0; i < larb_nr; i++) { 6610df4fabeSYong Wu struct device_node *larbnode; 6620df4fabeSYong Wu struct platform_device *plarbdev; 663e6dec923SYong Wu u32 id; 6640df4fabeSYong Wu 6650df4fabeSYong Wu larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); 6660df4fabeSYong Wu if (!larbnode) 6670df4fabeSYong Wu return -EINVAL; 6680df4fabeSYong Wu 6691eb8e4e2SWen Yang if (!of_device_is_available(larbnode)) { 6701eb8e4e2SWen Yang of_node_put(larbnode); 6710df4fabeSYong Wu continue; 6721eb8e4e2SWen Yang } 6730df4fabeSYong Wu 674e6dec923SYong Wu ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id); 675e6dec923SYong Wu if (ret)/* The id is consecutive if there is no this property */ 676e6dec923SYong Wu id = i; 677e6dec923SYong Wu 6780df4fabeSYong Wu plarbdev = of_find_device_by_node(larbnode); 6791eb8e4e2SWen Yang if (!plarbdev) { 6801eb8e4e2SWen Yang of_node_put(larbnode); 6810df4fabeSYong Wu return -EPROBE_DEFER; 6821eb8e4e2SWen Yang } 683*1ee9feb2SYong Wu data->larb_imu[id].dev = &plarbdev->dev; 6840df4fabeSYong Wu 68500c7c81fSRussell King component_match_add_release(dev, &match, release_of, 68600c7c81fSRussell King compare_of, larbnode); 6870df4fabeSYong Wu } 6880df4fabeSYong Wu 6890df4fabeSYong Wu platform_set_drvdata(pdev, data); 6900df4fabeSYong Wu 6910df4fabeSYong Wu ret = mtk_iommu_hw_init(data); 6920df4fabeSYong Wu if (ret) 6930df4fabeSYong Wu return ret; 6940df4fabeSYong Wu 695b16c0170SJoerg Roedel ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, 696b16c0170SJoerg Roedel "mtk-iommu.%pa", &ioaddr); 697b16c0170SJoerg Roedel if (ret) 698b16c0170SJoerg Roedel return ret; 699b16c0170SJoerg Roedel 700b16c0170SJoerg Roedel iommu_device_set_ops(&data->iommu, &mtk_iommu_ops); 701b16c0170SJoerg Roedel iommu_device_set_fwnode(&data->iommu, &pdev->dev.of_node->fwnode); 702b16c0170SJoerg Roedel 703b16c0170SJoerg Roedel ret = iommu_device_register(&data->iommu); 704b16c0170SJoerg Roedel if (ret) 705b16c0170SJoerg Roedel return ret; 706b16c0170SJoerg Roedel 7077c3a2ec0SYong Wu list_add_tail(&data->list, &m4ulist); 7087c3a2ec0SYong Wu 7090df4fabeSYong Wu if (!iommu_present(&platform_bus_type)) 7100df4fabeSYong Wu bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); 7110df4fabeSYong Wu 7120df4fabeSYong Wu return component_master_add_with_match(dev, &mtk_iommu_com_ops, match); 7130df4fabeSYong Wu } 7140df4fabeSYong Wu 7150df4fabeSYong Wu static int mtk_iommu_remove(struct platform_device *pdev) 7160df4fabeSYong Wu { 7170df4fabeSYong Wu struct mtk_iommu_data *data = platform_get_drvdata(pdev); 7180df4fabeSYong Wu 719b16c0170SJoerg Roedel iommu_device_sysfs_remove(&data->iommu); 720b16c0170SJoerg Roedel iommu_device_unregister(&data->iommu); 721b16c0170SJoerg Roedel 7220df4fabeSYong Wu if (iommu_present(&platform_bus_type)) 7230df4fabeSYong Wu bus_set_iommu(&platform_bus_type, NULL); 7240df4fabeSYong Wu 7250df4fabeSYong Wu clk_disable_unprepare(data->bclk); 7260df4fabeSYong Wu devm_free_irq(&pdev->dev, data->irq, data); 7270df4fabeSYong Wu component_master_del(&pdev->dev, &mtk_iommu_com_ops); 7280df4fabeSYong Wu return 0; 7290df4fabeSYong Wu } 7300df4fabeSYong Wu 731fd99f796SArnd Bergmann static int __maybe_unused mtk_iommu_suspend(struct device *dev) 7320df4fabeSYong Wu { 7330df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 7340df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 7350df4fabeSYong Wu void __iomem *base = data->base; 7360df4fabeSYong Wu 7370df4fabeSYong Wu reg->standard_axi_mode = readl_relaxed(base + 7380df4fabeSYong Wu REG_MMU_STANDARD_AXI_MODE); 7390df4fabeSYong Wu reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); 7400df4fabeSYong Wu reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); 7410df4fabeSYong Wu reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0); 7420df4fabeSYong Wu reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); 74370ca608bSYong Wu reg->ivrp_paddr = readl_relaxed(base + REG_MMU_IVRP_PADDR); 744b9475b34SYong Wu reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG); 7456254b64fSYong Wu clk_disable_unprepare(data->bclk); 7460df4fabeSYong Wu return 0; 7470df4fabeSYong Wu } 7480df4fabeSYong Wu 749fd99f796SArnd Bergmann static int __maybe_unused mtk_iommu_resume(struct device *dev) 7500df4fabeSYong Wu { 7510df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 7520df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 753907ba6a1SYong Wu struct mtk_iommu_domain *m4u_dom = data->m4u_dom; 7540df4fabeSYong Wu void __iomem *base = data->base; 7556254b64fSYong Wu int ret; 7560df4fabeSYong Wu 7576254b64fSYong Wu ret = clk_prepare_enable(data->bclk); 7586254b64fSYong Wu if (ret) { 7596254b64fSYong Wu dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret); 7606254b64fSYong Wu return ret; 7616254b64fSYong Wu } 7620df4fabeSYong Wu writel_relaxed(reg->standard_axi_mode, 7630df4fabeSYong Wu base + REG_MMU_STANDARD_AXI_MODE); 7640df4fabeSYong Wu writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); 7650df4fabeSYong Wu writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); 7660df4fabeSYong Wu writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); 7670df4fabeSYong Wu writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); 76870ca608bSYong Wu writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR); 769b9475b34SYong Wu writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG); 770907ba6a1SYong Wu if (m4u_dom) 771907ba6a1SYong Wu writel(m4u_dom->cfg.arm_v7s_cfg.ttbr[0] & MMU_PT_ADDR_MASK, 772e6dec923SYong Wu base + REG_MMU_PT_BASE_ADDR); 7730df4fabeSYong Wu return 0; 7740df4fabeSYong Wu } 7750df4fabeSYong Wu 776e6dec923SYong Wu static const struct dev_pm_ops mtk_iommu_pm_ops = { 7776254b64fSYong Wu SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume) 7780df4fabeSYong Wu }; 7790df4fabeSYong Wu 780cecdce9dSYong Wu static const struct mtk_iommu_plat_data mt2712_data = { 781cecdce9dSYong Wu .m4u_plat = M4U_MT2712, 782b4dad40eSYong Wu .has_4gb_mode = true, 7832aa4c259SYong Wu .has_bclk = true, 7842b326d8bSYong Wu .has_vld_pa_rng = true, 785b3e5eee7SYong Wu .larbid_remap = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, 786cecdce9dSYong Wu }; 787cecdce9dSYong Wu 788cecdce9dSYong Wu static const struct mtk_iommu_plat_data mt8173_data = { 789cecdce9dSYong Wu .m4u_plat = M4U_MT8173, 790b4dad40eSYong Wu .has_4gb_mode = true, 7912aa4c259SYong Wu .has_bclk = true, 79250822b0bSYong Wu .reset_axi = true, 793b3e5eee7SYong Wu .larbid_remap = {0, 1, 2, 3, 4, 5}, /* Linear mapping. */ 794cecdce9dSYong Wu }; 795cecdce9dSYong Wu 796907ba6a1SYong Wu static const struct mtk_iommu_plat_data mt8183_data = { 797907ba6a1SYong Wu .m4u_plat = M4U_MT8183, 798907ba6a1SYong Wu .reset_axi = true, 799907ba6a1SYong Wu .larbid_remap = {0, 4, 5, 6, 7, 2, 3, 1}, 800907ba6a1SYong Wu }; 801907ba6a1SYong Wu 8020df4fabeSYong Wu static const struct of_device_id mtk_iommu_of_ids[] = { 803cecdce9dSYong Wu { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, 804cecdce9dSYong Wu { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, 805907ba6a1SYong Wu { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, 8060df4fabeSYong Wu {} 8070df4fabeSYong Wu }; 8080df4fabeSYong Wu 8090df4fabeSYong Wu static struct platform_driver mtk_iommu_driver = { 8100df4fabeSYong Wu .probe = mtk_iommu_probe, 8110df4fabeSYong Wu .remove = mtk_iommu_remove, 8120df4fabeSYong Wu .driver = { 8130df4fabeSYong Wu .name = "mtk-iommu", 814e6dec923SYong Wu .of_match_table = of_match_ptr(mtk_iommu_of_ids), 8150df4fabeSYong Wu .pm = &mtk_iommu_pm_ops, 8160df4fabeSYong Wu } 8170df4fabeSYong Wu }; 8180df4fabeSYong Wu 819e6dec923SYong Wu static int __init mtk_iommu_init(void) 8200df4fabeSYong Wu { 8210df4fabeSYong Wu int ret; 8220df4fabeSYong Wu 8230df4fabeSYong Wu ret = platform_driver_register(&mtk_iommu_driver); 824e6dec923SYong Wu if (ret != 0) 825e6dec923SYong Wu pr_err("Failed to register MTK IOMMU driver\n"); 826e6dec923SYong Wu 8270df4fabeSYong Wu return ret; 8280df4fabeSYong Wu } 8290df4fabeSYong Wu 830e6dec923SYong Wu subsys_initcall(mtk_iommu_init) 831