11802d0beSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 20df4fabeSYong Wu /* 30df4fabeSYong Wu * Copyright (c) 2015-2016 MediaTek Inc. 40df4fabeSYong Wu * Author: Yong Wu <yong.wu@mediatek.com> 50df4fabeSYong Wu */ 6ef0f0986SYong Wu #include <linux/bitfield.h> 70df4fabeSYong Wu #include <linux/bug.h> 80df4fabeSYong Wu #include <linux/clk.h> 90df4fabeSYong Wu #include <linux/component.h> 100df4fabeSYong Wu #include <linux/device.h> 11803cf9e5SYong Wu #include <linux/dma-direct.h> 120df4fabeSYong Wu #include <linux/err.h> 130df4fabeSYong Wu #include <linux/interrupt.h> 140df4fabeSYong Wu #include <linux/io.h> 150df4fabeSYong Wu #include <linux/iommu.h> 160df4fabeSYong Wu #include <linux/iopoll.h> 170df4fabeSYong Wu #include <linux/list.h> 18c2c59456SMiles Chen #include <linux/mfd/syscon.h> 1918d8c74eSYong Wu #include <linux/module.h> 200df4fabeSYong Wu #include <linux/of_address.h> 210df4fabeSYong Wu #include <linux/of_irq.h> 220df4fabeSYong Wu #include <linux/of_platform.h> 230df4fabeSYong Wu #include <linux/platform_device.h> 24baf94e6eSYong Wu #include <linux/pm_runtime.h> 25c2c59456SMiles Chen #include <linux/regmap.h> 260df4fabeSYong Wu #include <linux/slab.h> 270df4fabeSYong Wu #include <linux/spinlock.h> 28c2c59456SMiles Chen #include <linux/soc/mediatek/infracfg.h> 290df4fabeSYong Wu #include <asm/barrier.h> 300df4fabeSYong Wu #include <soc/mediatek/smi.h> 310df4fabeSYong Wu 329ca340c9SHonghui Zhang #include "mtk_iommu.h" 330df4fabeSYong Wu 340df4fabeSYong Wu #define REG_MMU_PT_BASE_ADDR 0x000 35907ba6a1SYong Wu #define MMU_PT_ADDR_MASK GENMASK(31, 7) 360df4fabeSYong Wu 370df4fabeSYong Wu #define REG_MMU_INVALIDATE 0x020 380df4fabeSYong Wu #define F_ALL_INVLD 0x2 390df4fabeSYong Wu #define F_MMU_INV_RANGE 0x1 400df4fabeSYong Wu 410df4fabeSYong Wu #define REG_MMU_INVLD_START_A 0x024 420df4fabeSYong Wu #define REG_MMU_INVLD_END_A 0x028 430df4fabeSYong Wu 44068c86e9SChao Hao #define REG_MMU_INV_SEL_GEN2 0x02c 45b053bc71SChao Hao #define REG_MMU_INV_SEL_GEN1 0x038 460df4fabeSYong Wu #define F_INVLD_EN0 BIT(0) 470df4fabeSYong Wu #define F_INVLD_EN1 BIT(1) 480df4fabeSYong Wu 4975eed350SChao Hao #define REG_MMU_MISC_CTRL 0x048 504bb2bf4cSChao Hao #define F_MMU_IN_ORDER_WR_EN_MASK (BIT(1) | BIT(17)) 514bb2bf4cSChao Hao #define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19)) 524bb2bf4cSChao Hao 530df4fabeSYong Wu #define REG_MMU_DCM_DIS 0x050 549a87005eSYong Wu #define F_MMU_DCM BIT(8) 559a87005eSYong Wu 5635c1b48dSChao Hao #define REG_MMU_WR_LEN_CTRL 0x054 5735c1b48dSChao Hao #define F_MMU_WR_THROT_DIS_MASK (BIT(5) | BIT(21)) 580df4fabeSYong Wu 590df4fabeSYong Wu #define REG_MMU_CTRL_REG 0x110 60acb3c92aSYong Wu #define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4) 610df4fabeSYong Wu #define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4) 62acb3c92aSYong Wu #define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5) 630df4fabeSYong Wu 640df4fabeSYong Wu #define REG_MMU_IVRP_PADDR 0x114 6570ca608bSYong Wu 6630e2fccfSYong Wu #define REG_MMU_VLD_PA_RNG 0x118 6730e2fccfSYong Wu #define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA)) 680df4fabeSYong Wu 690df4fabeSYong Wu #define REG_MMU_INT_CONTROL0 0x120 700df4fabeSYong Wu #define F_L2_MULIT_HIT_EN BIT(0) 710df4fabeSYong Wu #define F_TABLE_WALK_FAULT_INT_EN BIT(1) 720df4fabeSYong Wu #define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2) 730df4fabeSYong Wu #define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3) 740df4fabeSYong Wu #define F_PREFETCH_FIFO_ERR_INT_EN BIT(5) 750df4fabeSYong Wu #define F_MISS_FIFO_ERR_INT_EN BIT(6) 760df4fabeSYong Wu #define F_INT_CLR_BIT BIT(12) 770df4fabeSYong Wu 780df4fabeSYong Wu #define REG_MMU_INT_MAIN_CONTROL 0x124 7915a01f4cSYong Wu /* mmu0 | mmu1 */ 8015a01f4cSYong Wu #define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7)) 8115a01f4cSYong Wu #define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8)) 8215a01f4cSYong Wu #define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9)) 8315a01f4cSYong Wu #define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10)) 8415a01f4cSYong Wu #define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11)) 8515a01f4cSYong Wu #define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12)) 8615a01f4cSYong Wu #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13)) 870df4fabeSYong Wu 880df4fabeSYong Wu #define REG_MMU_CPE_DONE 0x12C 890df4fabeSYong Wu 900df4fabeSYong Wu #define REG_MMU_FAULT_ST1 0x134 9115a01f4cSYong Wu #define F_REG_MMU0_FAULT_MASK GENMASK(6, 0) 9215a01f4cSYong Wu #define F_REG_MMU1_FAULT_MASK GENMASK(13, 7) 930df4fabeSYong Wu 9415a01f4cSYong Wu #define REG_MMU0_FAULT_VA 0x13c 95ef0f0986SYong Wu #define F_MMU_INVAL_VA_31_12_MASK GENMASK(31, 12) 96ef0f0986SYong Wu #define F_MMU_INVAL_VA_34_32_MASK GENMASK(11, 9) 97ef0f0986SYong Wu #define F_MMU_INVAL_PA_34_32_MASK GENMASK(8, 6) 980df4fabeSYong Wu #define F_MMU_FAULT_VA_WRITE_BIT BIT(1) 990df4fabeSYong Wu #define F_MMU_FAULT_VA_LAYER_BIT BIT(0) 1000df4fabeSYong Wu 10115a01f4cSYong Wu #define REG_MMU0_INVLD_PA 0x140 10215a01f4cSYong Wu #define REG_MMU1_FAULT_VA 0x144 10315a01f4cSYong Wu #define REG_MMU1_INVLD_PA 0x148 10415a01f4cSYong Wu #define REG_MMU0_INT_ID 0x150 10515a01f4cSYong Wu #define REG_MMU1_INT_ID 0x154 10637276e00SChao Hao #define F_MMU_INT_ID_COMM_ID(a) (((a) >> 9) & 0x7) 10737276e00SChao Hao #define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3) 108*9ec30c09SYong Wu #define F_MMU_INT_ID_COMM_ID_EXT(a) (((a) >> 10) & 0x7) 109*9ec30c09SYong Wu #define F_MMU_INT_ID_SUB_COMM_ID_EXT(a) (((a) >> 7) & 0x7) 11015a01f4cSYong Wu #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) 11115a01f4cSYong Wu #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) 1120df4fabeSYong Wu 113829316b3SChao Hao #define MTK_PROTECT_PA_ALIGN 256 1140df4fabeSYong Wu 1156b717796SChao Hao #define HAS_4GB_MODE BIT(0) 1166b717796SChao Hao /* HW will use the EMI clock if there isn't the "bclk". */ 1176b717796SChao Hao #define HAS_BCLK BIT(1) 1186b717796SChao Hao #define HAS_VLD_PA_RNG BIT(2) 1196b717796SChao Hao #define RESET_AXI BIT(3) 1204bb2bf4cSChao Hao #define OUT_ORDER_WR_EN BIT(4) 121*9ec30c09SYong Wu #define HAS_SUB_COMM_2BITS BIT(5) 122*9ec30c09SYong Wu #define HAS_SUB_COMM_3BITS BIT(6) 123*9ec30c09SYong Wu #define WR_THROT_EN BIT(7) 124*9ec30c09SYong Wu #define HAS_LEGACY_IVRP_PADDR BIT(8) 125*9ec30c09SYong Wu #define IOVA_34_EN BIT(9) 126*9ec30c09SYong Wu #define SHARE_PGTABLE BIT(10) /* 2 HW share pgtable */ 127*9ec30c09SYong Wu #define DCM_DISABLE BIT(11) 128*9ec30c09SYong Wu #define STD_AXI_MODE BIT(12) /* For non MM iommu */ 1296b717796SChao Hao 1306b717796SChao Hao #define MTK_IOMMU_HAS_FLAG(pdata, _x) \ 1316b717796SChao Hao ((((pdata)->flags) & (_x)) == (_x)) 1326b717796SChao Hao 1330df4fabeSYong Wu struct mtk_iommu_domain { 1340df4fabeSYong Wu struct io_pgtable_cfg cfg; 1350df4fabeSYong Wu struct io_pgtable_ops *iop; 1360df4fabeSYong Wu 13708500c43SYong Wu struct mtk_iommu_data *data; 1380df4fabeSYong Wu struct iommu_domain domain; 139ddf67a87SYong Wu 140ddf67a87SYong Wu struct mutex mutex; /* Protect "data" in this structure */ 1410df4fabeSYong Wu }; 1420df4fabeSYong Wu 143b65f5016SArvind Yadav static const struct iommu_ops mtk_iommu_ops; 1440df4fabeSYong Wu 1457f37a91dSYong Wu static int mtk_iommu_hw_init(const struct mtk_iommu_data *data); 1467f37a91dSYong Wu 147bfed8731SYong Wu #define MTK_IOMMU_TLB_ADDR(iova) ({ \ 148bfed8731SYong Wu dma_addr_t _addr = iova; \ 149bfed8731SYong Wu ((lower_32_bits(_addr) & GENMASK(31, 12)) | upper_32_bits(_addr));\ 150bfed8731SYong Wu }) 151bfed8731SYong Wu 15276ce6546SYong Wu /* 15376ce6546SYong Wu * In M4U 4GB mode, the physical address is remapped as below: 15476ce6546SYong Wu * 15576ce6546SYong Wu * CPU Physical address: 15676ce6546SYong Wu * ==================== 15776ce6546SYong Wu * 15876ce6546SYong Wu * 0 1G 2G 3G 4G 5G 15976ce6546SYong Wu * |---A---|---B---|---C---|---D---|---E---| 16076ce6546SYong Wu * +--I/O--+------------Memory-------------+ 16176ce6546SYong Wu * 16276ce6546SYong Wu * IOMMU output physical address: 16376ce6546SYong Wu * ============================= 16476ce6546SYong Wu * 16576ce6546SYong Wu * 4G 5G 6G 7G 8G 16676ce6546SYong Wu * |---E---|---B---|---C---|---D---| 16776ce6546SYong Wu * +------------Memory-------------+ 16876ce6546SYong Wu * 16976ce6546SYong Wu * The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the 17076ce6546SYong Wu * bit32 of the CPU physical address always is needed to set, and for Region 17176ce6546SYong Wu * 'E', the CPU physical address keep as is. 17276ce6546SYong Wu * Additionally, The iommu consumers always use the CPU phyiscal address. 17376ce6546SYong Wu */ 174b4dad40eSYong Wu #define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL 17576ce6546SYong Wu 1767c3a2ec0SYong Wu static LIST_HEAD(m4ulist); /* List all the M4U HWs */ 1777c3a2ec0SYong Wu 1789e3a2a64SYong Wu #define for_each_m4u(data, head) list_for_each_entry(data, head, list) 1797c3a2ec0SYong Wu 180585e58f4SYong Wu struct mtk_iommu_iova_region { 181585e58f4SYong Wu dma_addr_t iova_base; 182585e58f4SYong Wu unsigned long long size; 183585e58f4SYong Wu }; 184585e58f4SYong Wu 185585e58f4SYong Wu static const struct mtk_iommu_iova_region single_domain[] = { 186585e58f4SYong Wu {.iova_base = 0, .size = SZ_4G}, 187585e58f4SYong Wu }; 188585e58f4SYong Wu 1899e3489e0SYong Wu static const struct mtk_iommu_iova_region mt8192_multi_dom[] = { 190129a3b88SYong Wu { .iova_base = 0x0, .size = SZ_4G}, /* 0 ~ 4G */ 1919e3489e0SYong Wu #if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) 192129a3b88SYong Wu { .iova_base = SZ_4G, .size = SZ_4G}, /* 4G ~ 8G */ 193129a3b88SYong Wu { .iova_base = SZ_4G * 2, .size = SZ_4G}, /* 8G ~ 12G */ 194129a3b88SYong Wu { .iova_base = SZ_4G * 3, .size = SZ_4G}, /* 12G ~ 16G */ 195129a3b88SYong Wu 1969e3489e0SYong Wu { .iova_base = 0x240000000ULL, .size = 0x4000000}, /* CCU0 */ 1979e3489e0SYong Wu { .iova_base = 0x244000000ULL, .size = 0x4000000}, /* CCU1 */ 1989e3489e0SYong Wu #endif 1999e3489e0SYong Wu }; 2009e3489e0SYong Wu 2019e3a2a64SYong Wu /* If 2 M4U share a domain(use the same hwlist), Put the corresponding info in first data.*/ 2029e3a2a64SYong Wu static struct mtk_iommu_data *mtk_iommu_get_frst_data(struct list_head *hwlist) 2037c3a2ec0SYong Wu { 2049e3a2a64SYong Wu return list_first_entry(hwlist, struct mtk_iommu_data, list); 2057c3a2ec0SYong Wu } 2067c3a2ec0SYong Wu 2070df4fabeSYong Wu static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) 2080df4fabeSYong Wu { 2090df4fabeSYong Wu return container_of(dom, struct mtk_iommu_domain, domain); 2100df4fabeSYong Wu } 2110df4fabeSYong Wu 2120954d61aSYong Wu static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) 2130df4fabeSYong Wu { 21415672b6dSYong Wu unsigned long flags; 215c0b57581SYong Wu 21615672b6dSYong Wu spin_lock_irqsave(&data->tlb_lock, flags); 2177c3a2ec0SYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, 218b053bc71SChao Hao data->base + data->plat_data->inv_sel_reg); 2190df4fabeSYong Wu writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); 2200df4fabeSYong Wu wmb(); /* Make sure the tlb flush all done */ 22115672b6dSYong Wu spin_unlock_irqrestore(&data->tlb_lock, flags); 2227c3a2ec0SYong Wu } 2230df4fabeSYong Wu 2241f4fd624SYong Wu static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, 2250954d61aSYong Wu struct mtk_iommu_data *data) 2260df4fabeSYong Wu { 2279e3a2a64SYong Wu struct list_head *head = data->hw_list; 2281f4fd624SYong Wu unsigned long flags; 2291f4fd624SYong Wu int ret; 2301f4fd624SYong Wu u32 tmp; 2310df4fabeSYong Wu 2329e3a2a64SYong Wu for_each_m4u(data, head) { 233c0b57581SYong Wu if (pm_runtime_get_if_in_use(data->dev) <= 0) 234c0b57581SYong Wu continue; 235c0b57581SYong Wu 2361f4fd624SYong Wu spin_lock_irqsave(&data->tlb_lock, flags); 2377c3a2ec0SYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, 238b053bc71SChao Hao data->base + data->plat_data->inv_sel_reg); 2390df4fabeSYong Wu 240bfed8731SYong Wu writel_relaxed(MTK_IOMMU_TLB_ADDR(iova), 241bfed8731SYong Wu data->base + REG_MMU_INVLD_START_A); 242bfed8731SYong Wu writel_relaxed(MTK_IOMMU_TLB_ADDR(iova + size - 1), 2437c3a2ec0SYong Wu data->base + REG_MMU_INVLD_END_A); 2447c3a2ec0SYong Wu writel_relaxed(F_MMU_INV_RANGE, 2457c3a2ec0SYong Wu data->base + REG_MMU_INVALIDATE); 2460df4fabeSYong Wu 2471f4fd624SYong Wu /* tlb sync */ 2487c3a2ec0SYong Wu ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, 249c90ae4a6SYong Wu tmp, tmp != 0, 10, 1000); 25015672b6dSYong Wu 25115672b6dSYong Wu /* Clear the CPE status */ 25215672b6dSYong Wu writel_relaxed(0, data->base + REG_MMU_CPE_DONE); 25315672b6dSYong Wu spin_unlock_irqrestore(&data->tlb_lock, flags); 25415672b6dSYong Wu 2550df4fabeSYong Wu if (ret) { 2560df4fabeSYong Wu dev_warn(data->dev, 2570df4fabeSYong Wu "Partial TLB flush timed out, falling back to full flush\n"); 2580954d61aSYong Wu mtk_iommu_tlb_flush_all(data); 2590df4fabeSYong Wu } 260c0b57581SYong Wu 261c0b57581SYong Wu pm_runtime_put(data->dev); 2620df4fabeSYong Wu } 2637c3a2ec0SYong Wu } 2640df4fabeSYong Wu 2650df4fabeSYong Wu static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) 2660df4fabeSYong Wu { 2670df4fabeSYong Wu struct mtk_iommu_data *data = dev_id; 2680df4fabeSYong Wu struct mtk_iommu_domain *dom = data->m4u_dom; 26937276e00SChao Hao unsigned int fault_larb, fault_port, sub_comm = 0; 270ef0f0986SYong Wu u32 int_state, regval, va34_32, pa34_32; 271ef0f0986SYong Wu u64 fault_iova, fault_pa; 2720df4fabeSYong Wu bool layer, write; 2730df4fabeSYong Wu 2740df4fabeSYong Wu /* Read error info from registers */ 2750df4fabeSYong Wu int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1); 27615a01f4cSYong Wu if (int_state & F_REG_MMU0_FAULT_MASK) { 27715a01f4cSYong Wu regval = readl_relaxed(data->base + REG_MMU0_INT_ID); 27815a01f4cSYong Wu fault_iova = readl_relaxed(data->base + REG_MMU0_FAULT_VA); 27915a01f4cSYong Wu fault_pa = readl_relaxed(data->base + REG_MMU0_INVLD_PA); 28015a01f4cSYong Wu } else { 28115a01f4cSYong Wu regval = readl_relaxed(data->base + REG_MMU1_INT_ID); 28215a01f4cSYong Wu fault_iova = readl_relaxed(data->base + REG_MMU1_FAULT_VA); 28315a01f4cSYong Wu fault_pa = readl_relaxed(data->base + REG_MMU1_INVLD_PA); 28415a01f4cSYong Wu } 2850df4fabeSYong Wu layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; 2860df4fabeSYong Wu write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; 287ef0f0986SYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN)) { 288ef0f0986SYong Wu va34_32 = FIELD_GET(F_MMU_INVAL_VA_34_32_MASK, fault_iova); 289ef0f0986SYong Wu fault_iova = fault_iova & F_MMU_INVAL_VA_31_12_MASK; 290ef0f0986SYong Wu fault_iova |= (u64)va34_32 << 32; 291ef0f0986SYong Wu } 29282e51771SYong Wu pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova); 29382e51771SYong Wu fault_pa |= (u64)pa34_32 << 32; 294ef0f0986SYong Wu 29515a01f4cSYong Wu fault_port = F_MMU_INT_ID_PORT_ID(regval); 296*9ec30c09SYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM_2BITS)) { 29737276e00SChao Hao fault_larb = F_MMU_INT_ID_COMM_ID(regval); 29837276e00SChao Hao sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval); 299*9ec30c09SYong Wu } else if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM_3BITS)) { 300*9ec30c09SYong Wu fault_larb = F_MMU_INT_ID_COMM_ID_EXT(regval); 301*9ec30c09SYong Wu sub_comm = F_MMU_INT_ID_SUB_COMM_ID_EXT(regval); 30237276e00SChao Hao } else { 30337276e00SChao Hao fault_larb = F_MMU_INT_ID_LARB_ID(regval); 30437276e00SChao Hao } 30537276e00SChao Hao fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm]; 306b3e5eee7SYong Wu 3070df4fabeSYong Wu if (report_iommu_fault(&dom->domain, data->dev, fault_iova, 3080df4fabeSYong Wu write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { 3090df4fabeSYong Wu dev_err_ratelimited( 3100df4fabeSYong Wu data->dev, 311ef0f0986SYong Wu "fault type=0x%x iova=0x%llx pa=0x%llx larb=%d port=%d layer=%d %s\n", 3120df4fabeSYong Wu int_state, fault_iova, fault_pa, fault_larb, fault_port, 3130df4fabeSYong Wu layer, write ? "write" : "read"); 3140df4fabeSYong Wu } 3150df4fabeSYong Wu 3160df4fabeSYong Wu /* Interrupt clear */ 3170df4fabeSYong Wu regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL0); 3180df4fabeSYong Wu regval |= F_INT_CLR_BIT; 3190df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); 3200df4fabeSYong Wu 3210df4fabeSYong Wu mtk_iommu_tlb_flush_all(data); 3220df4fabeSYong Wu 3230df4fabeSYong Wu return IRQ_HANDLED; 3240df4fabeSYong Wu } 3250df4fabeSYong Wu 326803cf9e5SYong Wu static int mtk_iommu_get_domain_id(struct device *dev, 327803cf9e5SYong Wu const struct mtk_iommu_plat_data *plat_data) 328803cf9e5SYong Wu { 329803cf9e5SYong Wu const struct mtk_iommu_iova_region *rgn = plat_data->iova_region; 330803cf9e5SYong Wu const struct bus_dma_region *dma_rgn = dev->dma_range_map; 331803cf9e5SYong Wu int i, candidate = -1; 332803cf9e5SYong Wu dma_addr_t dma_end; 333803cf9e5SYong Wu 334803cf9e5SYong Wu if (!dma_rgn || plat_data->iova_region_nr == 1) 335803cf9e5SYong Wu return 0; 336803cf9e5SYong Wu 337803cf9e5SYong Wu dma_end = dma_rgn->dma_start + dma_rgn->size - 1; 338803cf9e5SYong Wu for (i = 0; i < plat_data->iova_region_nr; i++, rgn++) { 339803cf9e5SYong Wu /* Best fit. */ 340803cf9e5SYong Wu if (dma_rgn->dma_start == rgn->iova_base && 341803cf9e5SYong Wu dma_end == rgn->iova_base + rgn->size - 1) 342803cf9e5SYong Wu return i; 343803cf9e5SYong Wu /* ok if it is inside this region. */ 344803cf9e5SYong Wu if (dma_rgn->dma_start >= rgn->iova_base && 345803cf9e5SYong Wu dma_end < rgn->iova_base + rgn->size) 346803cf9e5SYong Wu candidate = i; 347803cf9e5SYong Wu } 348803cf9e5SYong Wu 349803cf9e5SYong Wu if (candidate >= 0) 350803cf9e5SYong Wu return candidate; 351803cf9e5SYong Wu dev_err(dev, "Can NOT find the iommu domain id(%pad 0x%llx).\n", 352803cf9e5SYong Wu &dma_rgn->dma_start, dma_rgn->size); 353803cf9e5SYong Wu return -EINVAL; 354803cf9e5SYong Wu } 355803cf9e5SYong Wu 3568d2c749eSYong Wu static void mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, 3578d2c749eSYong Wu bool enable, unsigned int domid) 3580df4fabeSYong Wu { 3590df4fabeSYong Wu struct mtk_smi_larb_iommu *larb_mmu; 3600df4fabeSYong Wu unsigned int larbid, portid; 361a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 3628d2c749eSYong Wu const struct mtk_iommu_iova_region *region; 36358f0d1d5SRobin Murphy int i; 3640df4fabeSYong Wu 36558f0d1d5SRobin Murphy for (i = 0; i < fwspec->num_ids; ++i) { 36658f0d1d5SRobin Murphy larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); 36758f0d1d5SRobin Murphy portid = MTK_M4U_TO_PORT(fwspec->ids[i]); 3688d2c749eSYong Wu 3691ee9feb2SYong Wu larb_mmu = &data->larb_imu[larbid]; 3700df4fabeSYong Wu 3718d2c749eSYong Wu region = data->plat_data->iova_region + domid; 3728d2c749eSYong Wu larb_mmu->bank[portid] = upper_32_bits(region->iova_base); 3738d2c749eSYong Wu 3748d2c749eSYong Wu dev_dbg(dev, "%s iommu for larb(%s) port %d dom %d bank %d.\n", 3758d2c749eSYong Wu enable ? "enable" : "disable", dev_name(larb_mmu->dev), 3768d2c749eSYong Wu portid, domid, larb_mmu->bank[portid]); 3770df4fabeSYong Wu 3780df4fabeSYong Wu if (enable) 3790df4fabeSYong Wu larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); 3800df4fabeSYong Wu else 3810df4fabeSYong Wu larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); 3820df4fabeSYong Wu } 3830df4fabeSYong Wu } 3840df4fabeSYong Wu 3854f956c97SYong Wu static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, 386c3045f39SYong Wu struct mtk_iommu_data *data, 387c3045f39SYong Wu unsigned int domid) 3880df4fabeSYong Wu { 389c3045f39SYong Wu const struct mtk_iommu_iova_region *region; 390c3045f39SYong Wu 391c3045f39SYong Wu /* Use the exist domain as there is only one pgtable here. */ 392c3045f39SYong Wu if (data->m4u_dom) { 393c3045f39SYong Wu dom->iop = data->m4u_dom->iop; 394c3045f39SYong Wu dom->cfg = data->m4u_dom->cfg; 395c3045f39SYong Wu dom->domain.pgsize_bitmap = data->m4u_dom->cfg.pgsize_bitmap; 396c3045f39SYong Wu goto update_iova_region; 397c3045f39SYong Wu } 398c3045f39SYong Wu 3990df4fabeSYong Wu dom->cfg = (struct io_pgtable_cfg) { 4000df4fabeSYong Wu .quirks = IO_PGTABLE_QUIRK_ARM_NS | 4010df4fabeSYong Wu IO_PGTABLE_QUIRK_NO_PERMS | 402b4dad40eSYong Wu IO_PGTABLE_QUIRK_ARM_MTK_EXT, 4030df4fabeSYong Wu .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, 4042f317da4SYong Wu .ias = MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN) ? 34 : 32, 4050df4fabeSYong Wu .iommu_dev = data->dev, 4060df4fabeSYong Wu }; 4070df4fabeSYong Wu 4089bdfe4c1SYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) 4099bdfe4c1SYong Wu dom->cfg.oas = data->enable_4GB ? 33 : 32; 4109bdfe4c1SYong Wu else 4119bdfe4c1SYong Wu dom->cfg.oas = 35; 4129bdfe4c1SYong Wu 4130df4fabeSYong Wu dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data); 4140df4fabeSYong Wu if (!dom->iop) { 4150df4fabeSYong Wu dev_err(data->dev, "Failed to alloc io pgtable\n"); 4160df4fabeSYong Wu return -EINVAL; 4170df4fabeSYong Wu } 4180df4fabeSYong Wu 4190df4fabeSYong Wu /* Update our support page sizes bitmap */ 420d16e0faaSRobin Murphy dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; 421b7875eb9SYong Wu 422c3045f39SYong Wu update_iova_region: 423c3045f39SYong Wu /* Update the iova region for this domain */ 424c3045f39SYong Wu region = data->plat_data->iova_region + domid; 425c3045f39SYong Wu dom->domain.geometry.aperture_start = region->iova_base; 426c3045f39SYong Wu dom->domain.geometry.aperture_end = region->iova_base + region->size - 1; 427b7875eb9SYong Wu dom->domain.geometry.force_aperture = true; 4280df4fabeSYong Wu return 0; 4290df4fabeSYong Wu } 4300df4fabeSYong Wu 4310df4fabeSYong Wu static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) 4320df4fabeSYong Wu { 4330df4fabeSYong Wu struct mtk_iommu_domain *dom; 4340df4fabeSYong Wu 4350df4fabeSYong Wu if (type != IOMMU_DOMAIN_DMA) 4360df4fabeSYong Wu return NULL; 4370df4fabeSYong Wu 4380df4fabeSYong Wu dom = kzalloc(sizeof(*dom), GFP_KERNEL); 4390df4fabeSYong Wu if (!dom) 4400df4fabeSYong Wu return NULL; 441ddf67a87SYong Wu mutex_init(&dom->mutex); 4420df4fabeSYong Wu 4434f956c97SYong Wu return &dom->domain; 4444f956c97SYong Wu } 4454f956c97SYong Wu 4460df4fabeSYong Wu static void mtk_iommu_domain_free(struct iommu_domain *domain) 4470df4fabeSYong Wu { 4480df4fabeSYong Wu kfree(to_mtk_domain(domain)); 4490df4fabeSYong Wu } 4500df4fabeSYong Wu 4510df4fabeSYong Wu static int mtk_iommu_attach_device(struct iommu_domain *domain, 4520df4fabeSYong Wu struct device *dev) 4530df4fabeSYong Wu { 454645b87c1SYong Wu struct mtk_iommu_data *data = dev_iommu_priv_get(dev), *frstdata; 4550df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 4569e3a2a64SYong Wu struct list_head *hw_list = data->hw_list; 457c0b57581SYong Wu struct device *m4udev = data->dev; 458803cf9e5SYong Wu int ret, domid; 4590df4fabeSYong Wu 460803cf9e5SYong Wu domid = mtk_iommu_get_domain_id(dev, data->plat_data); 461803cf9e5SYong Wu if (domid < 0) 462803cf9e5SYong Wu return domid; 463803cf9e5SYong Wu 464ddf67a87SYong Wu mutex_lock(&dom->mutex); 4654f956c97SYong Wu if (!dom->data) { 466645b87c1SYong Wu /* Data is in the frstdata in sharing pgtable case. */ 4679e3a2a64SYong Wu frstdata = mtk_iommu_get_frst_data(hw_list); 468645b87c1SYong Wu 469ddf67a87SYong Wu ret = mtk_iommu_domain_finalise(dom, frstdata, domid); 470ddf67a87SYong Wu if (ret) { 471ddf67a87SYong Wu mutex_unlock(&dom->mutex); 4724f956c97SYong Wu return -ENODEV; 473ddf67a87SYong Wu } 4744f956c97SYong Wu dom->data = data; 4754f956c97SYong Wu } 476ddf67a87SYong Wu mutex_unlock(&dom->mutex); 4774f956c97SYong Wu 4780e5a3f2eSYong Wu mutex_lock(&data->mutex); 4797f37a91dSYong Wu if (!data->m4u_dom) { /* Initialize the M4U HW */ 480c0b57581SYong Wu ret = pm_runtime_resume_and_get(m4udev); 481c0b57581SYong Wu if (ret < 0) 4820e5a3f2eSYong Wu goto err_unlock; 483c0b57581SYong Wu 484c0b57581SYong Wu ret = mtk_iommu_hw_init(data); 485c0b57581SYong Wu if (ret) { 486c0b57581SYong Wu pm_runtime_put(m4udev); 4870e5a3f2eSYong Wu goto err_unlock; 488c0b57581SYong Wu } 4890df4fabeSYong Wu data->m4u_dom = dom; 490d1e5f26fSRobin Murphy writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, 4914b00f5acSYong Wu data->base + REG_MMU_PT_BASE_ADDR); 492c0b57581SYong Wu 493c0b57581SYong Wu pm_runtime_put(m4udev); 4940df4fabeSYong Wu } 4950e5a3f2eSYong Wu mutex_unlock(&data->mutex); 4960df4fabeSYong Wu 4978d2c749eSYong Wu mtk_iommu_config(data, dev, true, domid); 4980df4fabeSYong Wu return 0; 4990e5a3f2eSYong Wu 5000e5a3f2eSYong Wu err_unlock: 5010e5a3f2eSYong Wu mutex_unlock(&data->mutex); 5020e5a3f2eSYong Wu return ret; 5030df4fabeSYong Wu } 5040df4fabeSYong Wu 5050df4fabeSYong Wu static void mtk_iommu_detach_device(struct iommu_domain *domain, 5060df4fabeSYong Wu struct device *dev) 5070df4fabeSYong Wu { 5083524b559SJoerg Roedel struct mtk_iommu_data *data = dev_iommu_priv_get(dev); 5090df4fabeSYong Wu 5108d2c749eSYong Wu mtk_iommu_config(data, dev, false, 0); 5110df4fabeSYong Wu } 5120df4fabeSYong Wu 5130df4fabeSYong Wu static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, 514781ca2deSTom Murphy phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 5150df4fabeSYong Wu { 5160df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 5170df4fabeSYong Wu 518b4dad40eSYong Wu /* The "4GB mode" M4U physically can not use the lower remap of Dram. */ 51908500c43SYong Wu if (dom->data->enable_4GB) 520b4dad40eSYong Wu paddr |= BIT_ULL(32); 521b4dad40eSYong Wu 52260829b4dSYong Wu /* Synchronize with the tlb_lock */ 523f34ce7a7SBaolin Wang return dom->iop->map(dom->iop, iova, paddr, size, prot, gfp); 5240df4fabeSYong Wu } 5250df4fabeSYong Wu 5260df4fabeSYong Wu static size_t mtk_iommu_unmap(struct iommu_domain *domain, 52756f8af5eSWill Deacon unsigned long iova, size_t size, 52856f8af5eSWill Deacon struct iommu_iotlb_gather *gather) 5290df4fabeSYong Wu { 5300df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 5310df4fabeSYong Wu 5323136895cSRobin Murphy iommu_iotlb_gather_add_range(gather, iova, size); 53360829b4dSYong Wu return dom->iop->unmap(dom->iop, iova, size, gather); 5340df4fabeSYong Wu } 5350df4fabeSYong Wu 53656f8af5eSWill Deacon static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain) 53756f8af5eSWill Deacon { 53808500c43SYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 53908500c43SYong Wu 54008500c43SYong Wu mtk_iommu_tlb_flush_all(dom->data); 54156f8af5eSWill Deacon } 54256f8af5eSWill Deacon 54356f8af5eSWill Deacon static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, 54456f8af5eSWill Deacon struct iommu_iotlb_gather *gather) 5454d689b61SRobin Murphy { 54608500c43SYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 547862c3715SYong Wu size_t length = gather->end - gather->start + 1; 548da3cc91bSYong Wu 549e6d25e7dSYong Wu mtk_iommu_tlb_flush_range_sync(gather->start, length, dom->data); 5504d689b61SRobin Murphy } 5514d689b61SRobin Murphy 55220143451SYong Wu static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, 55320143451SYong Wu size_t size) 55420143451SYong Wu { 55508500c43SYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 55620143451SYong Wu 557e6d25e7dSYong Wu mtk_iommu_tlb_flush_range_sync(iova, size, dom->data); 55820143451SYong Wu } 55920143451SYong Wu 5600df4fabeSYong Wu static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, 5610df4fabeSYong Wu dma_addr_t iova) 5620df4fabeSYong Wu { 5630df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 5640df4fabeSYong Wu phys_addr_t pa; 5650df4fabeSYong Wu 5660df4fabeSYong Wu pa = dom->iop->iova_to_phys(dom->iop, iova); 567f13efafcSArnd Bergmann if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT) && 568f13efafcSArnd Bergmann dom->data->enable_4GB && 569f13efafcSArnd Bergmann pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) 570b4dad40eSYong Wu pa &= ~BIT_ULL(32); 57130e2fccfSYong Wu 5720df4fabeSYong Wu return pa; 5730df4fabeSYong Wu } 5740df4fabeSYong Wu 57580e4592aSJoerg Roedel static struct iommu_device *mtk_iommu_probe_device(struct device *dev) 5760df4fabeSYong Wu { 577a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 578b16c0170SJoerg Roedel struct mtk_iommu_data *data; 579635319a4SYong Wu struct device_link *link; 580635319a4SYong Wu struct device *larbdev; 581635319a4SYong Wu unsigned int larbid, larbidx, i; 5820df4fabeSYong Wu 583a9bf2eecSJoerg Roedel if (!fwspec || fwspec->ops != &mtk_iommu_ops) 58480e4592aSJoerg Roedel return ERR_PTR(-ENODEV); /* Not a iommu client device */ 5850df4fabeSYong Wu 5863524b559SJoerg Roedel data = dev_iommu_priv_get(dev); 587b16c0170SJoerg Roedel 588635319a4SYong Wu /* 589635319a4SYong Wu * Link the consumer device with the smi-larb device(supplier). 590635319a4SYong Wu * The device that connects with each a larb is a independent HW. 591635319a4SYong Wu * All the ports in each a device should be in the same larbs. 592635319a4SYong Wu */ 593635319a4SYong Wu larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); 594635319a4SYong Wu for (i = 1; i < fwspec->num_ids; i++) { 595635319a4SYong Wu larbidx = MTK_M4U_TO_LARB(fwspec->ids[i]); 596635319a4SYong Wu if (larbid != larbidx) { 597635319a4SYong Wu dev_err(dev, "Can only use one larb. Fail@larb%d-%d.\n", 598635319a4SYong Wu larbid, larbidx); 599635319a4SYong Wu return ERR_PTR(-EINVAL); 600635319a4SYong Wu } 601635319a4SYong Wu } 602635319a4SYong Wu larbdev = data->larb_imu[larbid].dev; 603635319a4SYong Wu link = device_link_add(dev, larbdev, 604635319a4SYong Wu DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS); 605635319a4SYong Wu if (!link) 606635319a4SYong Wu dev_err(dev, "Unable to link %s\n", dev_name(larbdev)); 60780e4592aSJoerg Roedel return &data->iommu; 6080df4fabeSYong Wu } 6090df4fabeSYong Wu 61080e4592aSJoerg Roedel static void mtk_iommu_release_device(struct device *dev) 6110df4fabeSYong Wu { 612a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 613635319a4SYong Wu struct mtk_iommu_data *data; 614635319a4SYong Wu struct device *larbdev; 615635319a4SYong Wu unsigned int larbid; 616b16c0170SJoerg Roedel 617a9bf2eecSJoerg Roedel if (!fwspec || fwspec->ops != &mtk_iommu_ops) 6180df4fabeSYong Wu return; 6190df4fabeSYong Wu 620635319a4SYong Wu data = dev_iommu_priv_get(dev); 621635319a4SYong Wu larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); 622635319a4SYong Wu larbdev = data->larb_imu[larbid].dev; 623635319a4SYong Wu device_link_remove(dev, larbdev); 624635319a4SYong Wu 62558f0d1d5SRobin Murphy iommu_fwspec_free(dev); 6260df4fabeSYong Wu } 6270df4fabeSYong Wu 6280df4fabeSYong Wu static struct iommu_group *mtk_iommu_device_group(struct device *dev) 6290df4fabeSYong Wu { 6309e3a2a64SYong Wu struct mtk_iommu_data *c_data = dev_iommu_priv_get(dev), *data; 6319e3a2a64SYong Wu struct list_head *hw_list = c_data->hw_list; 632c3045f39SYong Wu struct iommu_group *group; 633803cf9e5SYong Wu int domid; 6340df4fabeSYong Wu 6359e3a2a64SYong Wu data = mtk_iommu_get_frst_data(hw_list); 63658f0d1d5SRobin Murphy if (!data) 6370df4fabeSYong Wu return ERR_PTR(-ENODEV); 6380df4fabeSYong Wu 639803cf9e5SYong Wu domid = mtk_iommu_get_domain_id(dev, data->plat_data); 640803cf9e5SYong Wu if (domid < 0) 641803cf9e5SYong Wu return ERR_PTR(domid); 642803cf9e5SYong Wu 6430e5a3f2eSYong Wu mutex_lock(&data->mutex); 644c3045f39SYong Wu group = data->m4u_group[domid]; 645c3045f39SYong Wu if (!group) { 646c3045f39SYong Wu group = iommu_group_alloc(); 647c3045f39SYong Wu if (!IS_ERR(group)) 648c3045f39SYong Wu data->m4u_group[domid] = group; 6493a8d40b6SRobin Murphy } else { 650c3045f39SYong Wu iommu_group_ref_get(group); 6510df4fabeSYong Wu } 6520e5a3f2eSYong Wu mutex_unlock(&data->mutex); 653c3045f39SYong Wu return group; 6540df4fabeSYong Wu } 6550df4fabeSYong Wu 6560df4fabeSYong Wu static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) 6570df4fabeSYong Wu { 6580df4fabeSYong Wu struct platform_device *m4updev; 6590df4fabeSYong Wu 6600df4fabeSYong Wu if (args->args_count != 1) { 6610df4fabeSYong Wu dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", 6620df4fabeSYong Wu args->args_count); 6630df4fabeSYong Wu return -EINVAL; 6640df4fabeSYong Wu } 6650df4fabeSYong Wu 6663524b559SJoerg Roedel if (!dev_iommu_priv_get(dev)) { 6670df4fabeSYong Wu /* Get the m4u device */ 6680df4fabeSYong Wu m4updev = of_find_device_by_node(args->np); 6690df4fabeSYong Wu if (WARN_ON(!m4updev)) 6700df4fabeSYong Wu return -EINVAL; 6710df4fabeSYong Wu 6723524b559SJoerg Roedel dev_iommu_priv_set(dev, platform_get_drvdata(m4updev)); 6730df4fabeSYong Wu } 6740df4fabeSYong Wu 67558f0d1d5SRobin Murphy return iommu_fwspec_add_ids(dev, args->args, 1); 6760df4fabeSYong Wu } 6770df4fabeSYong Wu 678ab1d5281SYong Wu static void mtk_iommu_get_resv_regions(struct device *dev, 679ab1d5281SYong Wu struct list_head *head) 680ab1d5281SYong Wu { 681ab1d5281SYong Wu struct mtk_iommu_data *data = dev_iommu_priv_get(dev); 682ab1d5281SYong Wu unsigned int domid = mtk_iommu_get_domain_id(dev, data->plat_data), i; 683ab1d5281SYong Wu const struct mtk_iommu_iova_region *resv, *curdom; 684ab1d5281SYong Wu struct iommu_resv_region *region; 685ab1d5281SYong Wu int prot = IOMMU_WRITE | IOMMU_READ; 686ab1d5281SYong Wu 6877a566173SColin Ian King if ((int)domid < 0) 688ab1d5281SYong Wu return; 689ab1d5281SYong Wu curdom = data->plat_data->iova_region + domid; 690ab1d5281SYong Wu for (i = 0; i < data->plat_data->iova_region_nr; i++) { 691ab1d5281SYong Wu resv = data->plat_data->iova_region + i; 692ab1d5281SYong Wu 693ab1d5281SYong Wu /* Only reserve when the region is inside the current domain */ 694ab1d5281SYong Wu if (resv->iova_base <= curdom->iova_base || 695ab1d5281SYong Wu resv->iova_base + resv->size >= curdom->iova_base + curdom->size) 696ab1d5281SYong Wu continue; 697ab1d5281SYong Wu 698ab1d5281SYong Wu region = iommu_alloc_resv_region(resv->iova_base, resv->size, 699ab1d5281SYong Wu prot, IOMMU_RESV_RESERVED); 700ab1d5281SYong Wu if (!region) 701ab1d5281SYong Wu return; 702ab1d5281SYong Wu 703ab1d5281SYong Wu list_add_tail(®ion->list, head); 704ab1d5281SYong Wu } 705ab1d5281SYong Wu } 706ab1d5281SYong Wu 707b65f5016SArvind Yadav static const struct iommu_ops mtk_iommu_ops = { 7080df4fabeSYong Wu .domain_alloc = mtk_iommu_domain_alloc, 70980e4592aSJoerg Roedel .probe_device = mtk_iommu_probe_device, 71080e4592aSJoerg Roedel .release_device = mtk_iommu_release_device, 7110df4fabeSYong Wu .device_group = mtk_iommu_device_group, 7120df4fabeSYong Wu .of_xlate = mtk_iommu_of_xlate, 713ab1d5281SYong Wu .get_resv_regions = mtk_iommu_get_resv_regions, 714ab1d5281SYong Wu .put_resv_regions = generic_iommu_put_resv_regions, 7150df4fabeSYong Wu .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, 71618d8c74eSYong Wu .owner = THIS_MODULE, 7179a630a4bSLu Baolu .default_domain_ops = &(const struct iommu_domain_ops) { 7189a630a4bSLu Baolu .attach_dev = mtk_iommu_attach_device, 7199a630a4bSLu Baolu .detach_dev = mtk_iommu_detach_device, 7209a630a4bSLu Baolu .map = mtk_iommu_map, 7219a630a4bSLu Baolu .unmap = mtk_iommu_unmap, 7229a630a4bSLu Baolu .flush_iotlb_all = mtk_iommu_flush_iotlb_all, 7239a630a4bSLu Baolu .iotlb_sync = mtk_iommu_iotlb_sync, 7249a630a4bSLu Baolu .iotlb_sync_map = mtk_iommu_sync_map, 7259a630a4bSLu Baolu .iova_to_phys = mtk_iommu_iova_to_phys, 7269a630a4bSLu Baolu .free = mtk_iommu_domain_free, 7279a630a4bSLu Baolu } 7280df4fabeSYong Wu }; 7290df4fabeSYong Wu 7300df4fabeSYong Wu static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) 7310df4fabeSYong Wu { 7320df4fabeSYong Wu u32 regval; 7330df4fabeSYong Wu 73486444413SChao Hao if (data->plat_data->m4u_plat == M4U_MT8173) { 735acb3c92aSYong Wu regval = F_MMU_PREFETCH_RT_REPLACE_MOD | 736acb3c92aSYong Wu F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173; 73786444413SChao Hao } else { 73886444413SChao Hao regval = readl_relaxed(data->base + REG_MMU_CTRL_REG); 73986444413SChao Hao regval |= F_MMU_TF_PROT_TO_PROGRAM_ADDR; 74086444413SChao Hao } 7410df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); 7420df4fabeSYong Wu 7430df4fabeSYong Wu regval = F_L2_MULIT_HIT_EN | 7440df4fabeSYong Wu F_TABLE_WALK_FAULT_INT_EN | 7450df4fabeSYong Wu F_PREETCH_FIFO_OVERFLOW_INT_EN | 7460df4fabeSYong Wu F_MISS_FIFO_OVERFLOW_INT_EN | 7470df4fabeSYong Wu F_PREFETCH_FIFO_ERR_INT_EN | 7480df4fabeSYong Wu F_MISS_FIFO_ERR_INT_EN; 7490df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); 7500df4fabeSYong Wu 7510df4fabeSYong Wu regval = F_INT_TRANSLATION_FAULT | 7520df4fabeSYong Wu F_INT_MAIN_MULTI_HIT_FAULT | 7530df4fabeSYong Wu F_INT_INVALID_PA_FAULT | 7540df4fabeSYong Wu F_INT_ENTRY_REPLACEMENT_FAULT | 7550df4fabeSYong Wu F_INT_TLB_MISS_FAULT | 7560df4fabeSYong Wu F_INT_MISS_TRANSACTION_FIFO_FAULT | 7570df4fabeSYong Wu F_INT_PRETETCH_TRANSATION_FIFO_FAULT; 7580df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); 7590df4fabeSYong Wu 760d1b5ef00SFabien Parent if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_LEGACY_IVRP_PADDR)) 76170ca608bSYong Wu regval = (data->protect_base >> 1) | (data->enable_4GB << 31); 76270ca608bSYong Wu else 76370ca608bSYong Wu regval = lower_32_bits(data->protect_base) | 76470ca608bSYong Wu upper_32_bits(data->protect_base); 76570ca608bSYong Wu writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR); 76670ca608bSYong Wu 7676b717796SChao Hao if (data->enable_4GB && 7686b717796SChao Hao MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_VLD_PA_RNG)) { 76930e2fccfSYong Wu /* 77030e2fccfSYong Wu * If 4GB mode is enabled, the validate PA range is from 77130e2fccfSYong Wu * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30]. 77230e2fccfSYong Wu */ 77330e2fccfSYong Wu regval = F_MMU_VLD_PA_RNG(7, 4); 77430e2fccfSYong Wu writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG); 77530e2fccfSYong Wu } 7769a87005eSYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, DCM_DISABLE)) 7779a87005eSYong Wu writel_relaxed(F_MMU_DCM, data->base + REG_MMU_DCM_DIS); 7789a87005eSYong Wu else 7790df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_DCM_DIS); 7809a87005eSYong Wu 78135c1b48dSChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, WR_THROT_EN)) { 78235c1b48dSChao Hao /* write command throttling mode */ 78335c1b48dSChao Hao regval = readl_relaxed(data->base + REG_MMU_WR_LEN_CTRL); 78435c1b48dSChao Hao regval &= ~F_MMU_WR_THROT_DIS_MASK; 78535c1b48dSChao Hao writel_relaxed(regval, data->base + REG_MMU_WR_LEN_CTRL); 78635c1b48dSChao Hao } 787e6dec923SYong Wu 7886b717796SChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) { 78975eed350SChao Hao /* The register is called STANDARD_AXI_MODE in this case */ 7904bb2bf4cSChao Hao regval = 0; 7914bb2bf4cSChao Hao } else { 7924bb2bf4cSChao Hao regval = readl_relaxed(data->base + REG_MMU_MISC_CTRL); 793d265a4adSYong Wu if (!MTK_IOMMU_HAS_FLAG(data->plat_data, STD_AXI_MODE)) 7944bb2bf4cSChao Hao regval &= ~F_MMU_STANDARD_AXI_MODE_MASK; 7954bb2bf4cSChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, OUT_ORDER_WR_EN)) 7964bb2bf4cSChao Hao regval &= ~F_MMU_IN_ORDER_WR_EN_MASK; 79775eed350SChao Hao } 7984bb2bf4cSChao Hao writel_relaxed(regval, data->base + REG_MMU_MISC_CTRL); 7990df4fabeSYong Wu 8000df4fabeSYong Wu if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, 8010df4fabeSYong Wu dev_name(data->dev), (void *)data)) { 8020df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); 8030df4fabeSYong Wu dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); 8040df4fabeSYong Wu return -ENODEV; 8050df4fabeSYong Wu } 8060df4fabeSYong Wu 8070df4fabeSYong Wu return 0; 8080df4fabeSYong Wu } 8090df4fabeSYong Wu 8100df4fabeSYong Wu static const struct component_master_ops mtk_iommu_com_ops = { 8110df4fabeSYong Wu .bind = mtk_iommu_bind, 8120df4fabeSYong Wu .unbind = mtk_iommu_unbind, 8130df4fabeSYong Wu }; 8140df4fabeSYong Wu 8150df4fabeSYong Wu static int mtk_iommu_probe(struct platform_device *pdev) 8160df4fabeSYong Wu { 8170df4fabeSYong Wu struct mtk_iommu_data *data; 8180df4fabeSYong Wu struct device *dev = &pdev->dev; 819baf94e6eSYong Wu struct device_node *larbnode, *smicomm_node; 820baf94e6eSYong Wu struct platform_device *plarbdev; 821baf94e6eSYong Wu struct device_link *link; 8220df4fabeSYong Wu struct resource *res; 823b16c0170SJoerg Roedel resource_size_t ioaddr; 8240df4fabeSYong Wu struct component_match *match = NULL; 825c2c59456SMiles Chen struct regmap *infracfg; 8260df4fabeSYong Wu void *protect; 8270b6c0ad3SAndrzej Hajda int i, larb_nr, ret; 828c2c59456SMiles Chen u32 val; 829c2c59456SMiles Chen char *p; 8300df4fabeSYong Wu 8310df4fabeSYong Wu data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); 8320df4fabeSYong Wu if (!data) 8330df4fabeSYong Wu return -ENOMEM; 8340df4fabeSYong Wu data->dev = dev; 835cecdce9dSYong Wu data->plat_data = of_device_get_match_data(dev); 8360df4fabeSYong Wu 8370df4fabeSYong Wu /* Protect memory. HW will access here while translation fault.*/ 8380df4fabeSYong Wu protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); 8390df4fabeSYong Wu if (!protect) 8400df4fabeSYong Wu return -ENOMEM; 8410df4fabeSYong Wu data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); 8420df4fabeSYong Wu 843c2c59456SMiles Chen if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) { 844c2c59456SMiles Chen switch (data->plat_data->m4u_plat) { 845c2c59456SMiles Chen case M4U_MT2712: 846c2c59456SMiles Chen p = "mediatek,mt2712-infracfg"; 847c2c59456SMiles Chen break; 848c2c59456SMiles Chen case M4U_MT8173: 849c2c59456SMiles Chen p = "mediatek,mt8173-infracfg"; 850c2c59456SMiles Chen break; 851c2c59456SMiles Chen default: 852c2c59456SMiles Chen p = NULL; 853c2c59456SMiles Chen } 854c2c59456SMiles Chen 855c2c59456SMiles Chen infracfg = syscon_regmap_lookup_by_compatible(p); 856c2c59456SMiles Chen 857c2c59456SMiles Chen if (IS_ERR(infracfg)) 858c2c59456SMiles Chen return PTR_ERR(infracfg); 859c2c59456SMiles Chen 860c2c59456SMiles Chen ret = regmap_read(infracfg, REG_INFRA_MISC, &val); 861c2c59456SMiles Chen if (ret) 862c2c59456SMiles Chen return ret; 863c2c59456SMiles Chen data->enable_4GB = !!(val & F_DDR_4GB_SUPPORT_EN); 864c2c59456SMiles Chen } 86501e23c93SYong Wu 8660df4fabeSYong Wu res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 8670df4fabeSYong Wu data->base = devm_ioremap_resource(dev, res); 8680df4fabeSYong Wu if (IS_ERR(data->base)) 8690df4fabeSYong Wu return PTR_ERR(data->base); 870b16c0170SJoerg Roedel ioaddr = res->start; 8710df4fabeSYong Wu 8720df4fabeSYong Wu data->irq = platform_get_irq(pdev, 0); 8730df4fabeSYong Wu if (data->irq < 0) 8740df4fabeSYong Wu return data->irq; 8750df4fabeSYong Wu 8766b717796SChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) { 8770df4fabeSYong Wu data->bclk = devm_clk_get(dev, "bclk"); 8780df4fabeSYong Wu if (IS_ERR(data->bclk)) 8790df4fabeSYong Wu return PTR_ERR(data->bclk); 8802aa4c259SYong Wu } 8810df4fabeSYong Wu 8820df4fabeSYong Wu larb_nr = of_count_phandle_with_args(dev->of_node, 8830df4fabeSYong Wu "mediatek,larbs", NULL); 8840df4fabeSYong Wu if (larb_nr < 0) 8850df4fabeSYong Wu return larb_nr; 8860df4fabeSYong Wu 8870df4fabeSYong Wu for (i = 0; i < larb_nr; i++) { 888e6dec923SYong Wu u32 id; 8890df4fabeSYong Wu 8900df4fabeSYong Wu larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); 8910df4fabeSYong Wu if (!larbnode) 8920df4fabeSYong Wu return -EINVAL; 8930df4fabeSYong Wu 8941eb8e4e2SWen Yang if (!of_device_is_available(larbnode)) { 8951eb8e4e2SWen Yang of_node_put(larbnode); 8960df4fabeSYong Wu continue; 8971eb8e4e2SWen Yang } 8980df4fabeSYong Wu 899e6dec923SYong Wu ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id); 900e6dec923SYong Wu if (ret)/* The id is consecutive if there is no this property */ 901e6dec923SYong Wu id = i; 902e6dec923SYong Wu 9030df4fabeSYong Wu plarbdev = of_find_device_by_node(larbnode); 9041eb8e4e2SWen Yang if (!plarbdev) { 9051eb8e4e2SWen Yang of_node_put(larbnode); 9062fb0feedSYong Wu return -ENODEV; 9071eb8e4e2SWen Yang } 9087d09aaf8SYong Wu if (!plarbdev->dev.driver) { 9097d09aaf8SYong Wu of_node_put(larbnode); 9107d09aaf8SYong Wu return -EPROBE_DEFER; 9117d09aaf8SYong Wu } 9121ee9feb2SYong Wu data->larb_imu[id].dev = &plarbdev->dev; 9130df4fabeSYong Wu 9144811a485SYong Wu component_match_add_release(dev, &match, component_release_of, 9154811a485SYong Wu component_compare_of, larbnode); 9160df4fabeSYong Wu } 9170df4fabeSYong Wu 918baf94e6eSYong Wu /* Get smi-common dev from the last larb. */ 919baf94e6eSYong Wu smicomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0); 920baf94e6eSYong Wu if (!smicomm_node) 921baf94e6eSYong Wu return -EINVAL; 922baf94e6eSYong Wu 923baf94e6eSYong Wu plarbdev = of_find_device_by_node(smicomm_node); 924baf94e6eSYong Wu of_node_put(smicomm_node); 925baf94e6eSYong Wu data->smicomm_dev = &plarbdev->dev; 926baf94e6eSYong Wu 927c0b57581SYong Wu pm_runtime_enable(dev); 928c0b57581SYong Wu 929baf94e6eSYong Wu link = device_link_add(data->smicomm_dev, dev, 930baf94e6eSYong Wu DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); 931baf94e6eSYong Wu if (!link) { 932a92a90acSDan Carpenter dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev)); 933a92a90acSDan Carpenter ret = -EINVAL; 934c0b57581SYong Wu goto out_runtime_disable; 935baf94e6eSYong Wu } 936baf94e6eSYong Wu 9370df4fabeSYong Wu platform_set_drvdata(pdev, data); 9380e5a3f2eSYong Wu mutex_init(&data->mutex); 9390df4fabeSYong Wu 940b16c0170SJoerg Roedel ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, 941b16c0170SJoerg Roedel "mtk-iommu.%pa", &ioaddr); 942b16c0170SJoerg Roedel if (ret) 943baf94e6eSYong Wu goto out_link_remove; 944b16c0170SJoerg Roedel 9452d471b20SRobin Murphy ret = iommu_device_register(&data->iommu, &mtk_iommu_ops, dev); 946b16c0170SJoerg Roedel if (ret) 947986d9ec5SYong Wu goto out_sysfs_remove; 948b16c0170SJoerg Roedel 949da3cc91bSYong Wu spin_lock_init(&data->tlb_lock); 9509e3a2a64SYong Wu 9519e3a2a64SYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, SHARE_PGTABLE)) { 9529e3a2a64SYong Wu list_add_tail(&data->list, data->plat_data->hw_list); 9539e3a2a64SYong Wu data->hw_list = data->plat_data->hw_list; 9549e3a2a64SYong Wu } else { 9559e3a2a64SYong Wu INIT_LIST_HEAD(&data->hw_list_head); 9569e3a2a64SYong Wu list_add_tail(&data->list, &data->hw_list_head); 9579e3a2a64SYong Wu data->hw_list = &data->hw_list_head; 9589e3a2a64SYong Wu } 9597c3a2ec0SYong Wu 960986d9ec5SYong Wu if (!iommu_present(&platform_bus_type)) { 961986d9ec5SYong Wu ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); 962986d9ec5SYong Wu if (ret) 963986d9ec5SYong Wu goto out_list_del; 964986d9ec5SYong Wu } 9650df4fabeSYong Wu 966986d9ec5SYong Wu ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); 967986d9ec5SYong Wu if (ret) 968986d9ec5SYong Wu goto out_bus_set_null; 969986d9ec5SYong Wu return ret; 970986d9ec5SYong Wu 971986d9ec5SYong Wu out_bus_set_null: 972986d9ec5SYong Wu bus_set_iommu(&platform_bus_type, NULL); 973986d9ec5SYong Wu out_list_del: 974986d9ec5SYong Wu list_del(&data->list); 975986d9ec5SYong Wu iommu_device_unregister(&data->iommu); 976986d9ec5SYong Wu out_sysfs_remove: 977986d9ec5SYong Wu iommu_device_sysfs_remove(&data->iommu); 978baf94e6eSYong Wu out_link_remove: 979baf94e6eSYong Wu device_link_remove(data->smicomm_dev, dev); 980c0b57581SYong Wu out_runtime_disable: 981c0b57581SYong Wu pm_runtime_disable(dev); 982986d9ec5SYong Wu return ret; 9830df4fabeSYong Wu } 9840df4fabeSYong Wu 9850df4fabeSYong Wu static int mtk_iommu_remove(struct platform_device *pdev) 9860df4fabeSYong Wu { 9870df4fabeSYong Wu struct mtk_iommu_data *data = platform_get_drvdata(pdev); 9880df4fabeSYong Wu 989b16c0170SJoerg Roedel iommu_device_sysfs_remove(&data->iommu); 990b16c0170SJoerg Roedel iommu_device_unregister(&data->iommu); 991b16c0170SJoerg Roedel 992ee55f75eSYong Wu list_del(&data->list); 9930df4fabeSYong Wu 994baf94e6eSYong Wu device_link_remove(data->smicomm_dev, &pdev->dev); 995c0b57581SYong Wu pm_runtime_disable(&pdev->dev); 9960df4fabeSYong Wu devm_free_irq(&pdev->dev, data->irq, data); 9970df4fabeSYong Wu component_master_del(&pdev->dev, &mtk_iommu_com_ops); 9980df4fabeSYong Wu return 0; 9990df4fabeSYong Wu } 10000df4fabeSYong Wu 100134665c79SYong Wu static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev) 10020df4fabeSYong Wu { 10030df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 10040df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 10050df4fabeSYong Wu void __iomem *base = data->base; 10060df4fabeSYong Wu 100735c1b48dSChao Hao reg->wr_len_ctrl = readl_relaxed(base + REG_MMU_WR_LEN_CTRL); 100875eed350SChao Hao reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL); 10090df4fabeSYong Wu reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); 10100df4fabeSYong Wu reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); 10110df4fabeSYong Wu reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0); 10120df4fabeSYong Wu reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); 101370ca608bSYong Wu reg->ivrp_paddr = readl_relaxed(base + REG_MMU_IVRP_PADDR); 1014b9475b34SYong Wu reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG); 10156254b64fSYong Wu clk_disable_unprepare(data->bclk); 10160df4fabeSYong Wu return 0; 10170df4fabeSYong Wu } 10180df4fabeSYong Wu 101934665c79SYong Wu static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) 10200df4fabeSYong Wu { 10210df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 10220df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 1023907ba6a1SYong Wu struct mtk_iommu_domain *m4u_dom = data->m4u_dom; 10240df4fabeSYong Wu void __iomem *base = data->base; 10256254b64fSYong Wu int ret; 10260df4fabeSYong Wu 10276254b64fSYong Wu ret = clk_prepare_enable(data->bclk); 10286254b64fSYong Wu if (ret) { 10296254b64fSYong Wu dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret); 10306254b64fSYong Wu return ret; 10316254b64fSYong Wu } 1032b34ea31fSDafna Hirschfeld 1033b34ea31fSDafna Hirschfeld /* 1034b34ea31fSDafna Hirschfeld * Uppon first resume, only enable the clk and return, since the values of the 1035b34ea31fSDafna Hirschfeld * registers are not yet set. 1036b34ea31fSDafna Hirschfeld */ 1037b34ea31fSDafna Hirschfeld if (!m4u_dom) 1038b34ea31fSDafna Hirschfeld return 0; 1039b34ea31fSDafna Hirschfeld 104035c1b48dSChao Hao writel_relaxed(reg->wr_len_ctrl, base + REG_MMU_WR_LEN_CTRL); 104175eed350SChao Hao writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL); 10420df4fabeSYong Wu writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); 10430df4fabeSYong Wu writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); 10440df4fabeSYong Wu writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); 10450df4fabeSYong Wu writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); 104670ca608bSYong Wu writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR); 1047b9475b34SYong Wu writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG); 1048c0b57581SYong Wu writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, base + REG_MMU_PT_BASE_ADDR); 10494f23f6d4SYong Wu 10504f23f6d4SYong Wu /* 10514f23f6d4SYong Wu * Users may allocate dma buffer before they call pm_runtime_get, 10524f23f6d4SYong Wu * in which case it will lack the necessary tlb flush. 10534f23f6d4SYong Wu * Thus, make sure to update the tlb after each PM resume. 10544f23f6d4SYong Wu */ 10554f23f6d4SYong Wu mtk_iommu_tlb_flush_all(data); 10560df4fabeSYong Wu return 0; 10570df4fabeSYong Wu } 10580df4fabeSYong Wu 1059e6dec923SYong Wu static const struct dev_pm_ops mtk_iommu_pm_ops = { 106034665c79SYong Wu SET_RUNTIME_PM_OPS(mtk_iommu_runtime_suspend, mtk_iommu_runtime_resume, NULL) 106134665c79SYong Wu SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 106234665c79SYong Wu pm_runtime_force_resume) 10630df4fabeSYong Wu }; 10640df4fabeSYong Wu 1065cecdce9dSYong Wu static const struct mtk_iommu_plat_data mt2712_data = { 1066cecdce9dSYong Wu .m4u_plat = M4U_MT2712, 10679e3a2a64SYong Wu .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG | SHARE_PGTABLE, 10689e3a2a64SYong Wu .hw_list = &m4ulist, 1069b053bc71SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 1070585e58f4SYong Wu .iova_region = single_domain, 1071585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 107237276e00SChao Hao .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, 1073cecdce9dSYong Wu }; 1074cecdce9dSYong Wu 1075068c86e9SChao Hao static const struct mtk_iommu_plat_data mt6779_data = { 1076068c86e9SChao Hao .m4u_plat = M4U_MT6779, 1077*9ec30c09SYong Wu .flags = HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | WR_THROT_EN, 1078068c86e9SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN2, 1079585e58f4SYong Wu .iova_region = single_domain, 1080585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 1081068c86e9SChao Hao .larbid_remap = {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}}, 1082cecdce9dSYong Wu }; 1083cecdce9dSYong Wu 10843c213562SFabien Parent static const struct mtk_iommu_plat_data mt8167_data = { 10853c213562SFabien Parent .m4u_plat = M4U_MT8167, 10863c213562SFabien Parent .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR, 10873c213562SFabien Parent .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 1088585e58f4SYong Wu .iova_region = single_domain, 1089585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 10903c213562SFabien Parent .larbid_remap = {{0}, {1}, {2}}, /* Linear mapping. */ 10913c213562SFabien Parent }; 10923c213562SFabien Parent 1093cecdce9dSYong Wu static const struct mtk_iommu_plat_data mt8173_data = { 1094cecdce9dSYong Wu .m4u_plat = M4U_MT8173, 1095d1b5ef00SFabien Parent .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | 1096d1b5ef00SFabien Parent HAS_LEGACY_IVRP_PADDR, 1097b053bc71SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 1098585e58f4SYong Wu .iova_region = single_domain, 1099585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 110037276e00SChao Hao .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */ 1101cecdce9dSYong Wu }; 1102cecdce9dSYong Wu 1103907ba6a1SYong Wu static const struct mtk_iommu_plat_data mt8183_data = { 1104907ba6a1SYong Wu .m4u_plat = M4U_MT8183, 11056b717796SChao Hao .flags = RESET_AXI, 1106b053bc71SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 1107585e58f4SYong Wu .iova_region = single_domain, 1108585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 110937276e00SChao Hao .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}}, 1110907ba6a1SYong Wu }; 1111907ba6a1SYong Wu 11129e3489e0SYong Wu static const struct mtk_iommu_plat_data mt8192_data = { 11139e3489e0SYong Wu .m4u_plat = M4U_MT8192, 1114*9ec30c09SYong Wu .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | 11159e3489e0SYong Wu WR_THROT_EN | IOVA_34_EN, 11169e3489e0SYong Wu .inv_sel_reg = REG_MMU_INV_SEL_GEN2, 11179e3489e0SYong Wu .iova_region = mt8192_multi_dom, 11189e3489e0SYong Wu .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), 11199e3489e0SYong Wu .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, 11209e3489e0SYong Wu {0, 14, 16}, {0, 13, 18, 17}}, 11219e3489e0SYong Wu }; 11229e3489e0SYong Wu 11230df4fabeSYong Wu static const struct of_device_id mtk_iommu_of_ids[] = { 1124cecdce9dSYong Wu { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, 1125068c86e9SChao Hao { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data}, 11263c213562SFabien Parent { .compatible = "mediatek,mt8167-m4u", .data = &mt8167_data}, 1127cecdce9dSYong Wu { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, 1128907ba6a1SYong Wu { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, 11299e3489e0SYong Wu { .compatible = "mediatek,mt8192-m4u", .data = &mt8192_data}, 11300df4fabeSYong Wu {} 11310df4fabeSYong Wu }; 11320df4fabeSYong Wu 11330df4fabeSYong Wu static struct platform_driver mtk_iommu_driver = { 11340df4fabeSYong Wu .probe = mtk_iommu_probe, 11350df4fabeSYong Wu .remove = mtk_iommu_remove, 11360df4fabeSYong Wu .driver = { 11370df4fabeSYong Wu .name = "mtk-iommu", 1138f53dd978SKrzysztof Kozlowski .of_match_table = mtk_iommu_of_ids, 11390df4fabeSYong Wu .pm = &mtk_iommu_pm_ops, 11400df4fabeSYong Wu } 11410df4fabeSYong Wu }; 114218d8c74eSYong Wu module_platform_driver(mtk_iommu_driver); 11430df4fabeSYong Wu 114418d8c74eSYong Wu MODULE_DESCRIPTION("IOMMU API for MediaTek M4U implementations"); 114518d8c74eSYong Wu MODULE_LICENSE("GPL v2"); 1146