11802d0beSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 20df4fabeSYong Wu /* 30df4fabeSYong Wu * Copyright (c) 2015-2016 MediaTek Inc. 40df4fabeSYong Wu * Author: Yong Wu <yong.wu@mediatek.com> 50df4fabeSYong Wu */ 6ef0f0986SYong Wu #include <linux/bitfield.h> 70df4fabeSYong Wu #include <linux/bug.h> 80df4fabeSYong Wu #include <linux/clk.h> 90df4fabeSYong Wu #include <linux/component.h> 100df4fabeSYong Wu #include <linux/device.h> 11803cf9e5SYong Wu #include <linux/dma-direct.h> 120df4fabeSYong Wu #include <linux/err.h> 130df4fabeSYong Wu #include <linux/interrupt.h> 140df4fabeSYong Wu #include <linux/io.h> 150df4fabeSYong Wu #include <linux/iommu.h> 160df4fabeSYong Wu #include <linux/iopoll.h> 170df4fabeSYong Wu #include <linux/list.h> 18c2c59456SMiles Chen #include <linux/mfd/syscon.h> 1918d8c74eSYong Wu #include <linux/module.h> 200df4fabeSYong Wu #include <linux/of_address.h> 210df4fabeSYong Wu #include <linux/of_irq.h> 220df4fabeSYong Wu #include <linux/of_platform.h> 230df4fabeSYong Wu #include <linux/platform_device.h> 24baf94e6eSYong Wu #include <linux/pm_runtime.h> 25c2c59456SMiles Chen #include <linux/regmap.h> 260df4fabeSYong Wu #include <linux/slab.h> 270df4fabeSYong Wu #include <linux/spinlock.h> 28c2c59456SMiles Chen #include <linux/soc/mediatek/infracfg.h> 290df4fabeSYong Wu #include <asm/barrier.h> 300df4fabeSYong Wu #include <soc/mediatek/smi.h> 310df4fabeSYong Wu 329ca340c9SHonghui Zhang #include "mtk_iommu.h" 330df4fabeSYong Wu 340df4fabeSYong Wu #define REG_MMU_PT_BASE_ADDR 0x000 35907ba6a1SYong Wu #define MMU_PT_ADDR_MASK GENMASK(31, 7) 360df4fabeSYong Wu 370df4fabeSYong Wu #define REG_MMU_INVALIDATE 0x020 380df4fabeSYong Wu #define F_ALL_INVLD 0x2 390df4fabeSYong Wu #define F_MMU_INV_RANGE 0x1 400df4fabeSYong Wu 410df4fabeSYong Wu #define REG_MMU_INVLD_START_A 0x024 420df4fabeSYong Wu #define REG_MMU_INVLD_END_A 0x028 430df4fabeSYong Wu 44068c86e9SChao Hao #define REG_MMU_INV_SEL_GEN2 0x02c 45b053bc71SChao Hao #define REG_MMU_INV_SEL_GEN1 0x038 460df4fabeSYong Wu #define F_INVLD_EN0 BIT(0) 470df4fabeSYong Wu #define F_INVLD_EN1 BIT(1) 480df4fabeSYong Wu 4975eed350SChao Hao #define REG_MMU_MISC_CTRL 0x048 504bb2bf4cSChao Hao #define F_MMU_IN_ORDER_WR_EN_MASK (BIT(1) | BIT(17)) 514bb2bf4cSChao Hao #define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19)) 524bb2bf4cSChao Hao 530df4fabeSYong Wu #define REG_MMU_DCM_DIS 0x050 549a87005eSYong Wu #define F_MMU_DCM BIT(8) 559a87005eSYong Wu 5635c1b48dSChao Hao #define REG_MMU_WR_LEN_CTRL 0x054 5735c1b48dSChao Hao #define F_MMU_WR_THROT_DIS_MASK (BIT(5) | BIT(21)) 580df4fabeSYong Wu 590df4fabeSYong Wu #define REG_MMU_CTRL_REG 0x110 60acb3c92aSYong Wu #define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4) 610df4fabeSYong Wu #define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4) 62acb3c92aSYong Wu #define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5) 630df4fabeSYong Wu 640df4fabeSYong Wu #define REG_MMU_IVRP_PADDR 0x114 6570ca608bSYong Wu 6630e2fccfSYong Wu #define REG_MMU_VLD_PA_RNG 0x118 6730e2fccfSYong Wu #define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA)) 680df4fabeSYong Wu 690df4fabeSYong Wu #define REG_MMU_INT_CONTROL0 0x120 700df4fabeSYong Wu #define F_L2_MULIT_HIT_EN BIT(0) 710df4fabeSYong Wu #define F_TABLE_WALK_FAULT_INT_EN BIT(1) 720df4fabeSYong Wu #define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2) 730df4fabeSYong Wu #define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3) 740df4fabeSYong Wu #define F_PREFETCH_FIFO_ERR_INT_EN BIT(5) 750df4fabeSYong Wu #define F_MISS_FIFO_ERR_INT_EN BIT(6) 760df4fabeSYong Wu #define F_INT_CLR_BIT BIT(12) 770df4fabeSYong Wu 780df4fabeSYong Wu #define REG_MMU_INT_MAIN_CONTROL 0x124 7915a01f4cSYong Wu /* mmu0 | mmu1 */ 8015a01f4cSYong Wu #define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7)) 8115a01f4cSYong Wu #define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8)) 8215a01f4cSYong Wu #define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9)) 8315a01f4cSYong Wu #define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10)) 8415a01f4cSYong Wu #define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11)) 8515a01f4cSYong Wu #define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12)) 8615a01f4cSYong Wu #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13)) 870df4fabeSYong Wu 880df4fabeSYong Wu #define REG_MMU_CPE_DONE 0x12C 890df4fabeSYong Wu 900df4fabeSYong Wu #define REG_MMU_FAULT_ST1 0x134 9115a01f4cSYong Wu #define F_REG_MMU0_FAULT_MASK GENMASK(6, 0) 9215a01f4cSYong Wu #define F_REG_MMU1_FAULT_MASK GENMASK(13, 7) 930df4fabeSYong Wu 9415a01f4cSYong Wu #define REG_MMU0_FAULT_VA 0x13c 95ef0f0986SYong Wu #define F_MMU_INVAL_VA_31_12_MASK GENMASK(31, 12) 96ef0f0986SYong Wu #define F_MMU_INVAL_VA_34_32_MASK GENMASK(11, 9) 97ef0f0986SYong Wu #define F_MMU_INVAL_PA_34_32_MASK GENMASK(8, 6) 980df4fabeSYong Wu #define F_MMU_FAULT_VA_WRITE_BIT BIT(1) 990df4fabeSYong Wu #define F_MMU_FAULT_VA_LAYER_BIT BIT(0) 1000df4fabeSYong Wu 10115a01f4cSYong Wu #define REG_MMU0_INVLD_PA 0x140 10215a01f4cSYong Wu #define REG_MMU1_FAULT_VA 0x144 10315a01f4cSYong Wu #define REG_MMU1_INVLD_PA 0x148 10415a01f4cSYong Wu #define REG_MMU0_INT_ID 0x150 10515a01f4cSYong Wu #define REG_MMU1_INT_ID 0x154 10637276e00SChao Hao #define F_MMU_INT_ID_COMM_ID(a) (((a) >> 9) & 0x7) 10737276e00SChao Hao #define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3) 1089ec30c09SYong Wu #define F_MMU_INT_ID_COMM_ID_EXT(a) (((a) >> 10) & 0x7) 1099ec30c09SYong Wu #define F_MMU_INT_ID_SUB_COMM_ID_EXT(a) (((a) >> 7) & 0x7) 11015a01f4cSYong Wu #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) 11115a01f4cSYong Wu #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) 1120df4fabeSYong Wu 113829316b3SChao Hao #define MTK_PROTECT_PA_ALIGN 256 1140df4fabeSYong Wu 1156b717796SChao Hao #define HAS_4GB_MODE BIT(0) 1166b717796SChao Hao /* HW will use the EMI clock if there isn't the "bclk". */ 1176b717796SChao Hao #define HAS_BCLK BIT(1) 1186b717796SChao Hao #define HAS_VLD_PA_RNG BIT(2) 1196b717796SChao Hao #define RESET_AXI BIT(3) 1204bb2bf4cSChao Hao #define OUT_ORDER_WR_EN BIT(4) 1219ec30c09SYong Wu #define HAS_SUB_COMM_2BITS BIT(5) 1229ec30c09SYong Wu #define HAS_SUB_COMM_3BITS BIT(6) 1239ec30c09SYong Wu #define WR_THROT_EN BIT(7) 1249ec30c09SYong Wu #define HAS_LEGACY_IVRP_PADDR BIT(8) 1259ec30c09SYong Wu #define IOVA_34_EN BIT(9) 1269ec30c09SYong Wu #define SHARE_PGTABLE BIT(10) /* 2 HW share pgtable */ 1279ec30c09SYong Wu #define DCM_DISABLE BIT(11) 1289ec30c09SYong Wu #define STD_AXI_MODE BIT(12) /* For non MM iommu */ 1298cd1e619SYong Wu /* 2 bits: iommu type */ 1308cd1e619SYong Wu #define MTK_IOMMU_TYPE_MM (0x0 << 13) 1318cd1e619SYong Wu #define MTK_IOMMU_TYPE_INFRA (0x1 << 13) 1328cd1e619SYong Wu #define MTK_IOMMU_TYPE_MASK (0x3 << 13) 133*6077c7e5SYong Wu /* PM and clock always on. e.g. infra iommu */ 134*6077c7e5SYong Wu #define PM_CLK_AO BIT(15) 1356b717796SChao Hao 1368cd1e619SYong Wu #define MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, mask) \ 1378cd1e619SYong Wu ((((pdata)->flags) & (mask)) == (_x)) 1388cd1e619SYong Wu 1398cd1e619SYong Wu #define MTK_IOMMU_HAS_FLAG(pdata, _x) MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, _x) 1408cd1e619SYong Wu #define MTK_IOMMU_IS_TYPE(pdata, _x) MTK_IOMMU_HAS_FLAG_MASK(pdata, _x,\ 1418cd1e619SYong Wu MTK_IOMMU_TYPE_MASK) 1426b717796SChao Hao 143d2e9a110SYong Wu #define MTK_INVALID_LARBID MTK_LARB_NR_MAX 144d2e9a110SYong Wu 1450df4fabeSYong Wu struct mtk_iommu_domain { 1460df4fabeSYong Wu struct io_pgtable_cfg cfg; 1470df4fabeSYong Wu struct io_pgtable_ops *iop; 1480df4fabeSYong Wu 14908500c43SYong Wu struct mtk_iommu_data *data; 1500df4fabeSYong Wu struct iommu_domain domain; 151ddf67a87SYong Wu 152ddf67a87SYong Wu struct mutex mutex; /* Protect "data" in this structure */ 1530df4fabeSYong Wu }; 1540df4fabeSYong Wu 155b65f5016SArvind Yadav static const struct iommu_ops mtk_iommu_ops; 1560df4fabeSYong Wu 1577f37a91dSYong Wu static int mtk_iommu_hw_init(const struct mtk_iommu_data *data); 1587f37a91dSYong Wu 159bfed8731SYong Wu #define MTK_IOMMU_TLB_ADDR(iova) ({ \ 160bfed8731SYong Wu dma_addr_t _addr = iova; \ 161bfed8731SYong Wu ((lower_32_bits(_addr) & GENMASK(31, 12)) | upper_32_bits(_addr));\ 162bfed8731SYong Wu }) 163bfed8731SYong Wu 16476ce6546SYong Wu /* 16576ce6546SYong Wu * In M4U 4GB mode, the physical address is remapped as below: 16676ce6546SYong Wu * 16776ce6546SYong Wu * CPU Physical address: 16876ce6546SYong Wu * ==================== 16976ce6546SYong Wu * 17076ce6546SYong Wu * 0 1G 2G 3G 4G 5G 17176ce6546SYong Wu * |---A---|---B---|---C---|---D---|---E---| 17276ce6546SYong Wu * +--I/O--+------------Memory-------------+ 17376ce6546SYong Wu * 17476ce6546SYong Wu * IOMMU output physical address: 17576ce6546SYong Wu * ============================= 17676ce6546SYong Wu * 17776ce6546SYong Wu * 4G 5G 6G 7G 8G 17876ce6546SYong Wu * |---E---|---B---|---C---|---D---| 17976ce6546SYong Wu * +------------Memory-------------+ 18076ce6546SYong Wu * 18176ce6546SYong Wu * The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the 18276ce6546SYong Wu * bit32 of the CPU physical address always is needed to set, and for Region 18376ce6546SYong Wu * 'E', the CPU physical address keep as is. 18476ce6546SYong Wu * Additionally, The iommu consumers always use the CPU phyiscal address. 18576ce6546SYong Wu */ 186b4dad40eSYong Wu #define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL 18776ce6546SYong Wu 1887c3a2ec0SYong Wu static LIST_HEAD(m4ulist); /* List all the M4U HWs */ 1897c3a2ec0SYong Wu 1909e3a2a64SYong Wu #define for_each_m4u(data, head) list_for_each_entry(data, head, list) 1917c3a2ec0SYong Wu 192585e58f4SYong Wu struct mtk_iommu_iova_region { 193585e58f4SYong Wu dma_addr_t iova_base; 194585e58f4SYong Wu unsigned long long size; 195585e58f4SYong Wu }; 196585e58f4SYong Wu 197585e58f4SYong Wu static const struct mtk_iommu_iova_region single_domain[] = { 198585e58f4SYong Wu {.iova_base = 0, .size = SZ_4G}, 199585e58f4SYong Wu }; 200585e58f4SYong Wu 2019e3489e0SYong Wu static const struct mtk_iommu_iova_region mt8192_multi_dom[] = { 202129a3b88SYong Wu { .iova_base = 0x0, .size = SZ_4G}, /* 0 ~ 4G */ 2039e3489e0SYong Wu #if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) 204129a3b88SYong Wu { .iova_base = SZ_4G, .size = SZ_4G}, /* 4G ~ 8G */ 205129a3b88SYong Wu { .iova_base = SZ_4G * 2, .size = SZ_4G}, /* 8G ~ 12G */ 206129a3b88SYong Wu { .iova_base = SZ_4G * 3, .size = SZ_4G}, /* 12G ~ 16G */ 207129a3b88SYong Wu 2089e3489e0SYong Wu { .iova_base = 0x240000000ULL, .size = 0x4000000}, /* CCU0 */ 2099e3489e0SYong Wu { .iova_base = 0x244000000ULL, .size = 0x4000000}, /* CCU1 */ 2109e3489e0SYong Wu #endif 2119e3489e0SYong Wu }; 2129e3489e0SYong Wu 2139e3a2a64SYong Wu /* If 2 M4U share a domain(use the same hwlist), Put the corresponding info in first data.*/ 2149e3a2a64SYong Wu static struct mtk_iommu_data *mtk_iommu_get_frst_data(struct list_head *hwlist) 2157c3a2ec0SYong Wu { 2169e3a2a64SYong Wu return list_first_entry(hwlist, struct mtk_iommu_data, list); 2177c3a2ec0SYong Wu } 2187c3a2ec0SYong Wu 2190df4fabeSYong Wu static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) 2200df4fabeSYong Wu { 2210df4fabeSYong Wu return container_of(dom, struct mtk_iommu_domain, domain); 2220df4fabeSYong Wu } 2230df4fabeSYong Wu 2240954d61aSYong Wu static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) 2250df4fabeSYong Wu { 22615672b6dSYong Wu unsigned long flags; 227c0b57581SYong Wu 22815672b6dSYong Wu spin_lock_irqsave(&data->tlb_lock, flags); 2297c3a2ec0SYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, 230b053bc71SChao Hao data->base + data->plat_data->inv_sel_reg); 2310df4fabeSYong Wu writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); 2320df4fabeSYong Wu wmb(); /* Make sure the tlb flush all done */ 23315672b6dSYong Wu spin_unlock_irqrestore(&data->tlb_lock, flags); 2347c3a2ec0SYong Wu } 2350df4fabeSYong Wu 2361f4fd624SYong Wu static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, 2370954d61aSYong Wu struct mtk_iommu_data *data) 2380df4fabeSYong Wu { 2399e3a2a64SYong Wu struct list_head *head = data->hw_list; 240*6077c7e5SYong Wu bool check_pm_status; 2411f4fd624SYong Wu unsigned long flags; 2421f4fd624SYong Wu int ret; 2431f4fd624SYong Wu u32 tmp; 2440df4fabeSYong Wu 2459e3a2a64SYong Wu for_each_m4u(data, head) { 246*6077c7e5SYong Wu /* 247*6077c7e5SYong Wu * To avoid resume the iommu device frequently when the iommu device 248*6077c7e5SYong Wu * is not active, it doesn't always call pm_runtime_get here, then tlb 249*6077c7e5SYong Wu * flush depends on the tlb flush all in the runtime resume. 250*6077c7e5SYong Wu * 251*6077c7e5SYong Wu * There are 2 special cases: 252*6077c7e5SYong Wu * 253*6077c7e5SYong Wu * Case1: The iommu dev doesn't have power domain but has bclk. This case 254*6077c7e5SYong Wu * should also avoid the tlb flush while the dev is not active to mute 255*6077c7e5SYong Wu * the tlb timeout log. like mt8173. 256*6077c7e5SYong Wu * 257*6077c7e5SYong Wu * Case2: The power/clock of infra iommu is always on, and it doesn't 258*6077c7e5SYong Wu * have the device link with the master devices. This case should avoid 259*6077c7e5SYong Wu * the PM status check. 260*6077c7e5SYong Wu */ 261*6077c7e5SYong Wu check_pm_status = !MTK_IOMMU_HAS_FLAG(data->plat_data, PM_CLK_AO); 262*6077c7e5SYong Wu 263*6077c7e5SYong Wu if (check_pm_status) { 264c0b57581SYong Wu if (pm_runtime_get_if_in_use(data->dev) <= 0) 265c0b57581SYong Wu continue; 266*6077c7e5SYong Wu } 267c0b57581SYong Wu 2681f4fd624SYong Wu spin_lock_irqsave(&data->tlb_lock, flags); 2697c3a2ec0SYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, 270b053bc71SChao Hao data->base + data->plat_data->inv_sel_reg); 2710df4fabeSYong Wu 272bfed8731SYong Wu writel_relaxed(MTK_IOMMU_TLB_ADDR(iova), 273bfed8731SYong Wu data->base + REG_MMU_INVLD_START_A); 274bfed8731SYong Wu writel_relaxed(MTK_IOMMU_TLB_ADDR(iova + size - 1), 2757c3a2ec0SYong Wu data->base + REG_MMU_INVLD_END_A); 2767c3a2ec0SYong Wu writel_relaxed(F_MMU_INV_RANGE, 2777c3a2ec0SYong Wu data->base + REG_MMU_INVALIDATE); 2780df4fabeSYong Wu 2791f4fd624SYong Wu /* tlb sync */ 2807c3a2ec0SYong Wu ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, 281c90ae4a6SYong Wu tmp, tmp != 0, 10, 1000); 28215672b6dSYong Wu 28315672b6dSYong Wu /* Clear the CPE status */ 28415672b6dSYong Wu writel_relaxed(0, data->base + REG_MMU_CPE_DONE); 28515672b6dSYong Wu spin_unlock_irqrestore(&data->tlb_lock, flags); 28615672b6dSYong Wu 2870df4fabeSYong Wu if (ret) { 2880df4fabeSYong Wu dev_warn(data->dev, 2890df4fabeSYong Wu "Partial TLB flush timed out, falling back to full flush\n"); 2900954d61aSYong Wu mtk_iommu_tlb_flush_all(data); 2910df4fabeSYong Wu } 292c0b57581SYong Wu 293*6077c7e5SYong Wu if (check_pm_status) 294c0b57581SYong Wu pm_runtime_put(data->dev); 2950df4fabeSYong Wu } 2967c3a2ec0SYong Wu } 2970df4fabeSYong Wu 2980df4fabeSYong Wu static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) 2990df4fabeSYong Wu { 3000df4fabeSYong Wu struct mtk_iommu_data *data = dev_id; 3010df4fabeSYong Wu struct mtk_iommu_domain *dom = data->m4u_dom; 302d2e9a110SYong Wu unsigned int fault_larb = MTK_INVALID_LARBID, fault_port = 0, sub_comm = 0; 303ef0f0986SYong Wu u32 int_state, regval, va34_32, pa34_32; 304ef0f0986SYong Wu u64 fault_iova, fault_pa; 3050df4fabeSYong Wu bool layer, write; 3060df4fabeSYong Wu 3070df4fabeSYong Wu /* Read error info from registers */ 3080df4fabeSYong Wu int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1); 30915a01f4cSYong Wu if (int_state & F_REG_MMU0_FAULT_MASK) { 31015a01f4cSYong Wu regval = readl_relaxed(data->base + REG_MMU0_INT_ID); 31115a01f4cSYong Wu fault_iova = readl_relaxed(data->base + REG_MMU0_FAULT_VA); 31215a01f4cSYong Wu fault_pa = readl_relaxed(data->base + REG_MMU0_INVLD_PA); 31315a01f4cSYong Wu } else { 31415a01f4cSYong Wu regval = readl_relaxed(data->base + REG_MMU1_INT_ID); 31515a01f4cSYong Wu fault_iova = readl_relaxed(data->base + REG_MMU1_FAULT_VA); 31615a01f4cSYong Wu fault_pa = readl_relaxed(data->base + REG_MMU1_INVLD_PA); 31715a01f4cSYong Wu } 3180df4fabeSYong Wu layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; 3190df4fabeSYong Wu write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; 320ef0f0986SYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN)) { 321ef0f0986SYong Wu va34_32 = FIELD_GET(F_MMU_INVAL_VA_34_32_MASK, fault_iova); 322ef0f0986SYong Wu fault_iova = fault_iova & F_MMU_INVAL_VA_31_12_MASK; 323ef0f0986SYong Wu fault_iova |= (u64)va34_32 << 32; 324ef0f0986SYong Wu } 32582e51771SYong Wu pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova); 32682e51771SYong Wu fault_pa |= (u64)pa34_32 << 32; 327ef0f0986SYong Wu 328d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { 32915a01f4cSYong Wu fault_port = F_MMU_INT_ID_PORT_ID(regval); 3309ec30c09SYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM_2BITS)) { 33137276e00SChao Hao fault_larb = F_MMU_INT_ID_COMM_ID(regval); 33237276e00SChao Hao sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval); 3339ec30c09SYong Wu } else if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM_3BITS)) { 3349ec30c09SYong Wu fault_larb = F_MMU_INT_ID_COMM_ID_EXT(regval); 3359ec30c09SYong Wu sub_comm = F_MMU_INT_ID_SUB_COMM_ID_EXT(regval); 33637276e00SChao Hao } else { 33737276e00SChao Hao fault_larb = F_MMU_INT_ID_LARB_ID(regval); 33837276e00SChao Hao } 33937276e00SChao Hao fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm]; 340d2e9a110SYong Wu } 341b3e5eee7SYong Wu 3420df4fabeSYong Wu if (report_iommu_fault(&dom->domain, data->dev, fault_iova, 3430df4fabeSYong Wu write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { 3440df4fabeSYong Wu dev_err_ratelimited( 3450df4fabeSYong Wu data->dev, 346ef0f0986SYong Wu "fault type=0x%x iova=0x%llx pa=0x%llx larb=%d port=%d layer=%d %s\n", 3470df4fabeSYong Wu int_state, fault_iova, fault_pa, fault_larb, fault_port, 3480df4fabeSYong Wu layer, write ? "write" : "read"); 3490df4fabeSYong Wu } 3500df4fabeSYong Wu 3510df4fabeSYong Wu /* Interrupt clear */ 3520df4fabeSYong Wu regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL0); 3530df4fabeSYong Wu regval |= F_INT_CLR_BIT; 3540df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); 3550df4fabeSYong Wu 3560df4fabeSYong Wu mtk_iommu_tlb_flush_all(data); 3570df4fabeSYong Wu 3580df4fabeSYong Wu return IRQ_HANDLED; 3590df4fabeSYong Wu } 3600df4fabeSYong Wu 361803cf9e5SYong Wu static int mtk_iommu_get_domain_id(struct device *dev, 362803cf9e5SYong Wu const struct mtk_iommu_plat_data *plat_data) 363803cf9e5SYong Wu { 364803cf9e5SYong Wu const struct mtk_iommu_iova_region *rgn = plat_data->iova_region; 365803cf9e5SYong Wu const struct bus_dma_region *dma_rgn = dev->dma_range_map; 366803cf9e5SYong Wu int i, candidate = -1; 367803cf9e5SYong Wu dma_addr_t dma_end; 368803cf9e5SYong Wu 369803cf9e5SYong Wu if (!dma_rgn || plat_data->iova_region_nr == 1) 370803cf9e5SYong Wu return 0; 371803cf9e5SYong Wu 372803cf9e5SYong Wu dma_end = dma_rgn->dma_start + dma_rgn->size - 1; 373803cf9e5SYong Wu for (i = 0; i < plat_data->iova_region_nr; i++, rgn++) { 374803cf9e5SYong Wu /* Best fit. */ 375803cf9e5SYong Wu if (dma_rgn->dma_start == rgn->iova_base && 376803cf9e5SYong Wu dma_end == rgn->iova_base + rgn->size - 1) 377803cf9e5SYong Wu return i; 378803cf9e5SYong Wu /* ok if it is inside this region. */ 379803cf9e5SYong Wu if (dma_rgn->dma_start >= rgn->iova_base && 380803cf9e5SYong Wu dma_end < rgn->iova_base + rgn->size) 381803cf9e5SYong Wu candidate = i; 382803cf9e5SYong Wu } 383803cf9e5SYong Wu 384803cf9e5SYong Wu if (candidate >= 0) 385803cf9e5SYong Wu return candidate; 386803cf9e5SYong Wu dev_err(dev, "Can NOT find the iommu domain id(%pad 0x%llx).\n", 387803cf9e5SYong Wu &dma_rgn->dma_start, dma_rgn->size); 388803cf9e5SYong Wu return -EINVAL; 389803cf9e5SYong Wu } 390803cf9e5SYong Wu 3918d2c749eSYong Wu static void mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, 3928d2c749eSYong Wu bool enable, unsigned int domid) 3930df4fabeSYong Wu { 3940df4fabeSYong Wu struct mtk_smi_larb_iommu *larb_mmu; 3950df4fabeSYong Wu unsigned int larbid, portid; 396a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 3978d2c749eSYong Wu const struct mtk_iommu_iova_region *region; 39858f0d1d5SRobin Murphy int i; 3990df4fabeSYong Wu 40058f0d1d5SRobin Murphy for (i = 0; i < fwspec->num_ids; ++i) { 40158f0d1d5SRobin Murphy larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); 40258f0d1d5SRobin Murphy portid = MTK_M4U_TO_PORT(fwspec->ids[i]); 4038d2c749eSYong Wu 404d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { 4051ee9feb2SYong Wu larb_mmu = &data->larb_imu[larbid]; 4060df4fabeSYong Wu 4078d2c749eSYong Wu region = data->plat_data->iova_region + domid; 4088d2c749eSYong Wu larb_mmu->bank[portid] = upper_32_bits(region->iova_base); 4098d2c749eSYong Wu 4108d2c749eSYong Wu dev_dbg(dev, "%s iommu for larb(%s) port %d dom %d bank %d.\n", 4118d2c749eSYong Wu enable ? "enable" : "disable", dev_name(larb_mmu->dev), 4128d2c749eSYong Wu portid, domid, larb_mmu->bank[portid]); 4130df4fabeSYong Wu 4140df4fabeSYong Wu if (enable) 4150df4fabeSYong Wu larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); 4160df4fabeSYong Wu else 4170df4fabeSYong Wu larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); 4180df4fabeSYong Wu } 4190df4fabeSYong Wu } 420d2e9a110SYong Wu } 4210df4fabeSYong Wu 4224f956c97SYong Wu static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, 423c3045f39SYong Wu struct mtk_iommu_data *data, 424c3045f39SYong Wu unsigned int domid) 4250df4fabeSYong Wu { 426c3045f39SYong Wu const struct mtk_iommu_iova_region *region; 427c3045f39SYong Wu 428c3045f39SYong Wu /* Use the exist domain as there is only one pgtable here. */ 429c3045f39SYong Wu if (data->m4u_dom) { 430c3045f39SYong Wu dom->iop = data->m4u_dom->iop; 431c3045f39SYong Wu dom->cfg = data->m4u_dom->cfg; 432c3045f39SYong Wu dom->domain.pgsize_bitmap = data->m4u_dom->cfg.pgsize_bitmap; 433c3045f39SYong Wu goto update_iova_region; 434c3045f39SYong Wu } 435c3045f39SYong Wu 4360df4fabeSYong Wu dom->cfg = (struct io_pgtable_cfg) { 4370df4fabeSYong Wu .quirks = IO_PGTABLE_QUIRK_ARM_NS | 4380df4fabeSYong Wu IO_PGTABLE_QUIRK_NO_PERMS | 439b4dad40eSYong Wu IO_PGTABLE_QUIRK_ARM_MTK_EXT, 4400df4fabeSYong Wu .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, 4412f317da4SYong Wu .ias = MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN) ? 34 : 32, 4420df4fabeSYong Wu .iommu_dev = data->dev, 4430df4fabeSYong Wu }; 4440df4fabeSYong Wu 4459bdfe4c1SYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) 4469bdfe4c1SYong Wu dom->cfg.oas = data->enable_4GB ? 33 : 32; 4479bdfe4c1SYong Wu else 4489bdfe4c1SYong Wu dom->cfg.oas = 35; 4499bdfe4c1SYong Wu 4500df4fabeSYong Wu dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data); 4510df4fabeSYong Wu if (!dom->iop) { 4520df4fabeSYong Wu dev_err(data->dev, "Failed to alloc io pgtable\n"); 4530df4fabeSYong Wu return -EINVAL; 4540df4fabeSYong Wu } 4550df4fabeSYong Wu 4560df4fabeSYong Wu /* Update our support page sizes bitmap */ 457d16e0faaSRobin Murphy dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; 458b7875eb9SYong Wu 459c3045f39SYong Wu update_iova_region: 460c3045f39SYong Wu /* Update the iova region for this domain */ 461c3045f39SYong Wu region = data->plat_data->iova_region + domid; 462c3045f39SYong Wu dom->domain.geometry.aperture_start = region->iova_base; 463c3045f39SYong Wu dom->domain.geometry.aperture_end = region->iova_base + region->size - 1; 464b7875eb9SYong Wu dom->domain.geometry.force_aperture = true; 4650df4fabeSYong Wu return 0; 4660df4fabeSYong Wu } 4670df4fabeSYong Wu 4680df4fabeSYong Wu static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) 4690df4fabeSYong Wu { 4700df4fabeSYong Wu struct mtk_iommu_domain *dom; 4710df4fabeSYong Wu 47232e1cccfSYong Wu if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED) 4730df4fabeSYong Wu return NULL; 4740df4fabeSYong Wu 4750df4fabeSYong Wu dom = kzalloc(sizeof(*dom), GFP_KERNEL); 4760df4fabeSYong Wu if (!dom) 4770df4fabeSYong Wu return NULL; 478ddf67a87SYong Wu mutex_init(&dom->mutex); 4790df4fabeSYong Wu 4804f956c97SYong Wu return &dom->domain; 4814f956c97SYong Wu } 4824f956c97SYong Wu 4830df4fabeSYong Wu static void mtk_iommu_domain_free(struct iommu_domain *domain) 4840df4fabeSYong Wu { 4850df4fabeSYong Wu kfree(to_mtk_domain(domain)); 4860df4fabeSYong Wu } 4870df4fabeSYong Wu 4880df4fabeSYong Wu static int mtk_iommu_attach_device(struct iommu_domain *domain, 4890df4fabeSYong Wu struct device *dev) 4900df4fabeSYong Wu { 491645b87c1SYong Wu struct mtk_iommu_data *data = dev_iommu_priv_get(dev), *frstdata; 4920df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 4939e3a2a64SYong Wu struct list_head *hw_list = data->hw_list; 494c0b57581SYong Wu struct device *m4udev = data->dev; 495803cf9e5SYong Wu int ret, domid; 4960df4fabeSYong Wu 497803cf9e5SYong Wu domid = mtk_iommu_get_domain_id(dev, data->plat_data); 498803cf9e5SYong Wu if (domid < 0) 499803cf9e5SYong Wu return domid; 500803cf9e5SYong Wu 501ddf67a87SYong Wu mutex_lock(&dom->mutex); 5024f956c97SYong Wu if (!dom->data) { 503645b87c1SYong Wu /* Data is in the frstdata in sharing pgtable case. */ 5049e3a2a64SYong Wu frstdata = mtk_iommu_get_frst_data(hw_list); 505645b87c1SYong Wu 506ddf67a87SYong Wu ret = mtk_iommu_domain_finalise(dom, frstdata, domid); 507ddf67a87SYong Wu if (ret) { 508ddf67a87SYong Wu mutex_unlock(&dom->mutex); 5094f956c97SYong Wu return -ENODEV; 510ddf67a87SYong Wu } 5114f956c97SYong Wu dom->data = data; 5124f956c97SYong Wu } 513ddf67a87SYong Wu mutex_unlock(&dom->mutex); 5144f956c97SYong Wu 5150e5a3f2eSYong Wu mutex_lock(&data->mutex); 5167f37a91dSYong Wu if (!data->m4u_dom) { /* Initialize the M4U HW */ 517c0b57581SYong Wu ret = pm_runtime_resume_and_get(m4udev); 518c0b57581SYong Wu if (ret < 0) 5190e5a3f2eSYong Wu goto err_unlock; 520c0b57581SYong Wu 521c0b57581SYong Wu ret = mtk_iommu_hw_init(data); 522c0b57581SYong Wu if (ret) { 523c0b57581SYong Wu pm_runtime_put(m4udev); 5240e5a3f2eSYong Wu goto err_unlock; 525c0b57581SYong Wu } 5260df4fabeSYong Wu data->m4u_dom = dom; 527d1e5f26fSRobin Murphy writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, 5284b00f5acSYong Wu data->base + REG_MMU_PT_BASE_ADDR); 529c0b57581SYong Wu 530c0b57581SYong Wu pm_runtime_put(m4udev); 5310df4fabeSYong Wu } 5320e5a3f2eSYong Wu mutex_unlock(&data->mutex); 5330df4fabeSYong Wu 5348d2c749eSYong Wu mtk_iommu_config(data, dev, true, domid); 5350df4fabeSYong Wu return 0; 5360e5a3f2eSYong Wu 5370e5a3f2eSYong Wu err_unlock: 5380e5a3f2eSYong Wu mutex_unlock(&data->mutex); 5390e5a3f2eSYong Wu return ret; 5400df4fabeSYong Wu } 5410df4fabeSYong Wu 5420df4fabeSYong Wu static void mtk_iommu_detach_device(struct iommu_domain *domain, 5430df4fabeSYong Wu struct device *dev) 5440df4fabeSYong Wu { 5453524b559SJoerg Roedel struct mtk_iommu_data *data = dev_iommu_priv_get(dev); 5460df4fabeSYong Wu 5478d2c749eSYong Wu mtk_iommu_config(data, dev, false, 0); 5480df4fabeSYong Wu } 5490df4fabeSYong Wu 5500df4fabeSYong Wu static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, 551781ca2deSTom Murphy phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 5520df4fabeSYong Wu { 5530df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 5540df4fabeSYong Wu 555b4dad40eSYong Wu /* The "4GB mode" M4U physically can not use the lower remap of Dram. */ 55608500c43SYong Wu if (dom->data->enable_4GB) 557b4dad40eSYong Wu paddr |= BIT_ULL(32); 558b4dad40eSYong Wu 55960829b4dSYong Wu /* Synchronize with the tlb_lock */ 560f34ce7a7SBaolin Wang return dom->iop->map(dom->iop, iova, paddr, size, prot, gfp); 5610df4fabeSYong Wu } 5620df4fabeSYong Wu 5630df4fabeSYong Wu static size_t mtk_iommu_unmap(struct iommu_domain *domain, 56456f8af5eSWill Deacon unsigned long iova, size_t size, 56556f8af5eSWill Deacon struct iommu_iotlb_gather *gather) 5660df4fabeSYong Wu { 5670df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 5680df4fabeSYong Wu 5693136895cSRobin Murphy iommu_iotlb_gather_add_range(gather, iova, size); 57060829b4dSYong Wu return dom->iop->unmap(dom->iop, iova, size, gather); 5710df4fabeSYong Wu } 5720df4fabeSYong Wu 57356f8af5eSWill Deacon static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain) 57456f8af5eSWill Deacon { 57508500c43SYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 57608500c43SYong Wu 57708500c43SYong Wu mtk_iommu_tlb_flush_all(dom->data); 57856f8af5eSWill Deacon } 57956f8af5eSWill Deacon 58056f8af5eSWill Deacon static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, 58156f8af5eSWill Deacon struct iommu_iotlb_gather *gather) 5824d689b61SRobin Murphy { 58308500c43SYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 584862c3715SYong Wu size_t length = gather->end - gather->start + 1; 585da3cc91bSYong Wu 586e6d25e7dSYong Wu mtk_iommu_tlb_flush_range_sync(gather->start, length, dom->data); 5874d689b61SRobin Murphy } 5884d689b61SRobin Murphy 58920143451SYong Wu static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, 59020143451SYong Wu size_t size) 59120143451SYong Wu { 59208500c43SYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 59320143451SYong Wu 594e6d25e7dSYong Wu mtk_iommu_tlb_flush_range_sync(iova, size, dom->data); 59520143451SYong Wu } 59620143451SYong Wu 5970df4fabeSYong Wu static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, 5980df4fabeSYong Wu dma_addr_t iova) 5990df4fabeSYong Wu { 6000df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 6010df4fabeSYong Wu phys_addr_t pa; 6020df4fabeSYong Wu 6030df4fabeSYong Wu pa = dom->iop->iova_to_phys(dom->iop, iova); 604f13efafcSArnd Bergmann if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT) && 605f13efafcSArnd Bergmann dom->data->enable_4GB && 606f13efafcSArnd Bergmann pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) 607b4dad40eSYong Wu pa &= ~BIT_ULL(32); 60830e2fccfSYong Wu 6090df4fabeSYong Wu return pa; 6100df4fabeSYong Wu } 6110df4fabeSYong Wu 61280e4592aSJoerg Roedel static struct iommu_device *mtk_iommu_probe_device(struct device *dev) 6130df4fabeSYong Wu { 614a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 615b16c0170SJoerg Roedel struct mtk_iommu_data *data; 616635319a4SYong Wu struct device_link *link; 617635319a4SYong Wu struct device *larbdev; 618635319a4SYong Wu unsigned int larbid, larbidx, i; 6190df4fabeSYong Wu 620a9bf2eecSJoerg Roedel if (!fwspec || fwspec->ops != &mtk_iommu_ops) 62180e4592aSJoerg Roedel return ERR_PTR(-ENODEV); /* Not a iommu client device */ 6220df4fabeSYong Wu 6233524b559SJoerg Roedel data = dev_iommu_priv_get(dev); 624b16c0170SJoerg Roedel 625d2e9a110SYong Wu if (!MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) 626d2e9a110SYong Wu return &data->iommu; 627d2e9a110SYong Wu 628635319a4SYong Wu /* 629635319a4SYong Wu * Link the consumer device with the smi-larb device(supplier). 630635319a4SYong Wu * The device that connects with each a larb is a independent HW. 631635319a4SYong Wu * All the ports in each a device should be in the same larbs. 632635319a4SYong Wu */ 633635319a4SYong Wu larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); 634635319a4SYong Wu for (i = 1; i < fwspec->num_ids; i++) { 635635319a4SYong Wu larbidx = MTK_M4U_TO_LARB(fwspec->ids[i]); 636635319a4SYong Wu if (larbid != larbidx) { 637635319a4SYong Wu dev_err(dev, "Can only use one larb. Fail@larb%d-%d.\n", 638635319a4SYong Wu larbid, larbidx); 639635319a4SYong Wu return ERR_PTR(-EINVAL); 640635319a4SYong Wu } 641635319a4SYong Wu } 642635319a4SYong Wu larbdev = data->larb_imu[larbid].dev; 643635319a4SYong Wu link = device_link_add(dev, larbdev, 644635319a4SYong Wu DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS); 645635319a4SYong Wu if (!link) 646635319a4SYong Wu dev_err(dev, "Unable to link %s\n", dev_name(larbdev)); 64780e4592aSJoerg Roedel return &data->iommu; 6480df4fabeSYong Wu } 6490df4fabeSYong Wu 65080e4592aSJoerg Roedel static void mtk_iommu_release_device(struct device *dev) 6510df4fabeSYong Wu { 652a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 653635319a4SYong Wu struct mtk_iommu_data *data; 654635319a4SYong Wu struct device *larbdev; 655635319a4SYong Wu unsigned int larbid; 656b16c0170SJoerg Roedel 657a9bf2eecSJoerg Roedel if (!fwspec || fwspec->ops != &mtk_iommu_ops) 6580df4fabeSYong Wu return; 6590df4fabeSYong Wu 660635319a4SYong Wu data = dev_iommu_priv_get(dev); 661d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { 662635319a4SYong Wu larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); 663635319a4SYong Wu larbdev = data->larb_imu[larbid].dev; 664635319a4SYong Wu device_link_remove(dev, larbdev); 665d2e9a110SYong Wu } 666635319a4SYong Wu 66758f0d1d5SRobin Murphy iommu_fwspec_free(dev); 6680df4fabeSYong Wu } 6690df4fabeSYong Wu 6700df4fabeSYong Wu static struct iommu_group *mtk_iommu_device_group(struct device *dev) 6710df4fabeSYong Wu { 6729e3a2a64SYong Wu struct mtk_iommu_data *c_data = dev_iommu_priv_get(dev), *data; 6739e3a2a64SYong Wu struct list_head *hw_list = c_data->hw_list; 674c3045f39SYong Wu struct iommu_group *group; 675803cf9e5SYong Wu int domid; 6760df4fabeSYong Wu 6779e3a2a64SYong Wu data = mtk_iommu_get_frst_data(hw_list); 67858f0d1d5SRobin Murphy if (!data) 6790df4fabeSYong Wu return ERR_PTR(-ENODEV); 6800df4fabeSYong Wu 681803cf9e5SYong Wu domid = mtk_iommu_get_domain_id(dev, data->plat_data); 682803cf9e5SYong Wu if (domid < 0) 683803cf9e5SYong Wu return ERR_PTR(domid); 684803cf9e5SYong Wu 6850e5a3f2eSYong Wu mutex_lock(&data->mutex); 686c3045f39SYong Wu group = data->m4u_group[domid]; 687c3045f39SYong Wu if (!group) { 688c3045f39SYong Wu group = iommu_group_alloc(); 689c3045f39SYong Wu if (!IS_ERR(group)) 690c3045f39SYong Wu data->m4u_group[domid] = group; 6913a8d40b6SRobin Murphy } else { 692c3045f39SYong Wu iommu_group_ref_get(group); 6930df4fabeSYong Wu } 6940e5a3f2eSYong Wu mutex_unlock(&data->mutex); 695c3045f39SYong Wu return group; 6960df4fabeSYong Wu } 6970df4fabeSYong Wu 6980df4fabeSYong Wu static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) 6990df4fabeSYong Wu { 7000df4fabeSYong Wu struct platform_device *m4updev; 7010df4fabeSYong Wu 7020df4fabeSYong Wu if (args->args_count != 1) { 7030df4fabeSYong Wu dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", 7040df4fabeSYong Wu args->args_count); 7050df4fabeSYong Wu return -EINVAL; 7060df4fabeSYong Wu } 7070df4fabeSYong Wu 7083524b559SJoerg Roedel if (!dev_iommu_priv_get(dev)) { 7090df4fabeSYong Wu /* Get the m4u device */ 7100df4fabeSYong Wu m4updev = of_find_device_by_node(args->np); 7110df4fabeSYong Wu if (WARN_ON(!m4updev)) 7120df4fabeSYong Wu return -EINVAL; 7130df4fabeSYong Wu 7143524b559SJoerg Roedel dev_iommu_priv_set(dev, platform_get_drvdata(m4updev)); 7150df4fabeSYong Wu } 7160df4fabeSYong Wu 71758f0d1d5SRobin Murphy return iommu_fwspec_add_ids(dev, args->args, 1); 7180df4fabeSYong Wu } 7190df4fabeSYong Wu 720ab1d5281SYong Wu static void mtk_iommu_get_resv_regions(struct device *dev, 721ab1d5281SYong Wu struct list_head *head) 722ab1d5281SYong Wu { 723ab1d5281SYong Wu struct mtk_iommu_data *data = dev_iommu_priv_get(dev); 724ab1d5281SYong Wu unsigned int domid = mtk_iommu_get_domain_id(dev, data->plat_data), i; 725ab1d5281SYong Wu const struct mtk_iommu_iova_region *resv, *curdom; 726ab1d5281SYong Wu struct iommu_resv_region *region; 727ab1d5281SYong Wu int prot = IOMMU_WRITE | IOMMU_READ; 728ab1d5281SYong Wu 7297a566173SColin Ian King if ((int)domid < 0) 730ab1d5281SYong Wu return; 731ab1d5281SYong Wu curdom = data->plat_data->iova_region + domid; 732ab1d5281SYong Wu for (i = 0; i < data->plat_data->iova_region_nr; i++) { 733ab1d5281SYong Wu resv = data->plat_data->iova_region + i; 734ab1d5281SYong Wu 735ab1d5281SYong Wu /* Only reserve when the region is inside the current domain */ 736ab1d5281SYong Wu if (resv->iova_base <= curdom->iova_base || 737ab1d5281SYong Wu resv->iova_base + resv->size >= curdom->iova_base + curdom->size) 738ab1d5281SYong Wu continue; 739ab1d5281SYong Wu 740ab1d5281SYong Wu region = iommu_alloc_resv_region(resv->iova_base, resv->size, 741ab1d5281SYong Wu prot, IOMMU_RESV_RESERVED); 742ab1d5281SYong Wu if (!region) 743ab1d5281SYong Wu return; 744ab1d5281SYong Wu 745ab1d5281SYong Wu list_add_tail(®ion->list, head); 746ab1d5281SYong Wu } 747ab1d5281SYong Wu } 748ab1d5281SYong Wu 749b65f5016SArvind Yadav static const struct iommu_ops mtk_iommu_ops = { 7500df4fabeSYong Wu .domain_alloc = mtk_iommu_domain_alloc, 75180e4592aSJoerg Roedel .probe_device = mtk_iommu_probe_device, 75280e4592aSJoerg Roedel .release_device = mtk_iommu_release_device, 7530df4fabeSYong Wu .device_group = mtk_iommu_device_group, 7540df4fabeSYong Wu .of_xlate = mtk_iommu_of_xlate, 755ab1d5281SYong Wu .get_resv_regions = mtk_iommu_get_resv_regions, 756ab1d5281SYong Wu .put_resv_regions = generic_iommu_put_resv_regions, 7570df4fabeSYong Wu .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, 75818d8c74eSYong Wu .owner = THIS_MODULE, 7599a630a4bSLu Baolu .default_domain_ops = &(const struct iommu_domain_ops) { 7609a630a4bSLu Baolu .attach_dev = mtk_iommu_attach_device, 7619a630a4bSLu Baolu .detach_dev = mtk_iommu_detach_device, 7629a630a4bSLu Baolu .map = mtk_iommu_map, 7639a630a4bSLu Baolu .unmap = mtk_iommu_unmap, 7649a630a4bSLu Baolu .flush_iotlb_all = mtk_iommu_flush_iotlb_all, 7659a630a4bSLu Baolu .iotlb_sync = mtk_iommu_iotlb_sync, 7669a630a4bSLu Baolu .iotlb_sync_map = mtk_iommu_sync_map, 7679a630a4bSLu Baolu .iova_to_phys = mtk_iommu_iova_to_phys, 7689a630a4bSLu Baolu .free = mtk_iommu_domain_free, 7699a630a4bSLu Baolu } 7700df4fabeSYong Wu }; 7710df4fabeSYong Wu 7720df4fabeSYong Wu static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) 7730df4fabeSYong Wu { 7740df4fabeSYong Wu u32 regval; 7750df4fabeSYong Wu 77686444413SChao Hao if (data->plat_data->m4u_plat == M4U_MT8173) { 777acb3c92aSYong Wu regval = F_MMU_PREFETCH_RT_REPLACE_MOD | 778acb3c92aSYong Wu F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173; 77986444413SChao Hao } else { 78086444413SChao Hao regval = readl_relaxed(data->base + REG_MMU_CTRL_REG); 78186444413SChao Hao regval |= F_MMU_TF_PROT_TO_PROGRAM_ADDR; 78286444413SChao Hao } 7830df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); 7840df4fabeSYong Wu 7850df4fabeSYong Wu regval = F_L2_MULIT_HIT_EN | 7860df4fabeSYong Wu F_TABLE_WALK_FAULT_INT_EN | 7870df4fabeSYong Wu F_PREETCH_FIFO_OVERFLOW_INT_EN | 7880df4fabeSYong Wu F_MISS_FIFO_OVERFLOW_INT_EN | 7890df4fabeSYong Wu F_PREFETCH_FIFO_ERR_INT_EN | 7900df4fabeSYong Wu F_MISS_FIFO_ERR_INT_EN; 7910df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); 7920df4fabeSYong Wu 7930df4fabeSYong Wu regval = F_INT_TRANSLATION_FAULT | 7940df4fabeSYong Wu F_INT_MAIN_MULTI_HIT_FAULT | 7950df4fabeSYong Wu F_INT_INVALID_PA_FAULT | 7960df4fabeSYong Wu F_INT_ENTRY_REPLACEMENT_FAULT | 7970df4fabeSYong Wu F_INT_TLB_MISS_FAULT | 7980df4fabeSYong Wu F_INT_MISS_TRANSACTION_FIFO_FAULT | 7990df4fabeSYong Wu F_INT_PRETETCH_TRANSATION_FIFO_FAULT; 8000df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); 8010df4fabeSYong Wu 802d1b5ef00SFabien Parent if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_LEGACY_IVRP_PADDR)) 80370ca608bSYong Wu regval = (data->protect_base >> 1) | (data->enable_4GB << 31); 80470ca608bSYong Wu else 80570ca608bSYong Wu regval = lower_32_bits(data->protect_base) | 80670ca608bSYong Wu upper_32_bits(data->protect_base); 80770ca608bSYong Wu writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR); 80870ca608bSYong Wu 8096b717796SChao Hao if (data->enable_4GB && 8106b717796SChao Hao MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_VLD_PA_RNG)) { 81130e2fccfSYong Wu /* 81230e2fccfSYong Wu * If 4GB mode is enabled, the validate PA range is from 81330e2fccfSYong Wu * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30]. 81430e2fccfSYong Wu */ 81530e2fccfSYong Wu regval = F_MMU_VLD_PA_RNG(7, 4); 81630e2fccfSYong Wu writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG); 81730e2fccfSYong Wu } 8189a87005eSYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, DCM_DISABLE)) 8199a87005eSYong Wu writel_relaxed(F_MMU_DCM, data->base + REG_MMU_DCM_DIS); 8209a87005eSYong Wu else 8210df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_DCM_DIS); 8229a87005eSYong Wu 82335c1b48dSChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, WR_THROT_EN)) { 82435c1b48dSChao Hao /* write command throttling mode */ 82535c1b48dSChao Hao regval = readl_relaxed(data->base + REG_MMU_WR_LEN_CTRL); 82635c1b48dSChao Hao regval &= ~F_MMU_WR_THROT_DIS_MASK; 82735c1b48dSChao Hao writel_relaxed(regval, data->base + REG_MMU_WR_LEN_CTRL); 82835c1b48dSChao Hao } 829e6dec923SYong Wu 8306b717796SChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) { 83175eed350SChao Hao /* The register is called STANDARD_AXI_MODE in this case */ 8324bb2bf4cSChao Hao regval = 0; 8334bb2bf4cSChao Hao } else { 8344bb2bf4cSChao Hao regval = readl_relaxed(data->base + REG_MMU_MISC_CTRL); 835d265a4adSYong Wu if (!MTK_IOMMU_HAS_FLAG(data->plat_data, STD_AXI_MODE)) 8364bb2bf4cSChao Hao regval &= ~F_MMU_STANDARD_AXI_MODE_MASK; 8374bb2bf4cSChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, OUT_ORDER_WR_EN)) 8384bb2bf4cSChao Hao regval &= ~F_MMU_IN_ORDER_WR_EN_MASK; 83975eed350SChao Hao } 8404bb2bf4cSChao Hao writel_relaxed(regval, data->base + REG_MMU_MISC_CTRL); 8410df4fabeSYong Wu 8420df4fabeSYong Wu if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, 8430df4fabeSYong Wu dev_name(data->dev), (void *)data)) { 8440df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); 8450df4fabeSYong Wu dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); 8460df4fabeSYong Wu return -ENODEV; 8470df4fabeSYong Wu } 8480df4fabeSYong Wu 8490df4fabeSYong Wu return 0; 8500df4fabeSYong Wu } 8510df4fabeSYong Wu 8520df4fabeSYong Wu static const struct component_master_ops mtk_iommu_com_ops = { 8530df4fabeSYong Wu .bind = mtk_iommu_bind, 8540df4fabeSYong Wu .unbind = mtk_iommu_unbind, 8550df4fabeSYong Wu }; 8560df4fabeSYong Wu 857d2e9a110SYong Wu static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **match, 858d2e9a110SYong Wu struct mtk_iommu_data *data) 859d2e9a110SYong Wu { 860f7b71d0dSYong Wu struct device_node *larbnode, *smicomm_node, *smi_subcomm_node; 861d2e9a110SYong Wu struct platform_device *plarbdev; 862d2e9a110SYong Wu struct device_link *link; 863d2e9a110SYong Wu int i, larb_nr, ret; 864d2e9a110SYong Wu 865d2e9a110SYong Wu larb_nr = of_count_phandle_with_args(dev->of_node, "mediatek,larbs", NULL); 866d2e9a110SYong Wu if (larb_nr < 0) 867d2e9a110SYong Wu return larb_nr; 868d2e9a110SYong Wu 869d2e9a110SYong Wu for (i = 0; i < larb_nr; i++) { 870d2e9a110SYong Wu u32 id; 871d2e9a110SYong Wu 872d2e9a110SYong Wu larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); 873d2e9a110SYong Wu if (!larbnode) 874d2e9a110SYong Wu return -EINVAL; 875d2e9a110SYong Wu 876d2e9a110SYong Wu if (!of_device_is_available(larbnode)) { 877d2e9a110SYong Wu of_node_put(larbnode); 878d2e9a110SYong Wu continue; 879d2e9a110SYong Wu } 880d2e9a110SYong Wu 881d2e9a110SYong Wu ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id); 882d2e9a110SYong Wu if (ret)/* The id is consecutive if there is no this property */ 883d2e9a110SYong Wu id = i; 884d2e9a110SYong Wu 885d2e9a110SYong Wu plarbdev = of_find_device_by_node(larbnode); 886d2e9a110SYong Wu if (!plarbdev) { 887d2e9a110SYong Wu of_node_put(larbnode); 888d2e9a110SYong Wu return -ENODEV; 889d2e9a110SYong Wu } 890d2e9a110SYong Wu if (!plarbdev->dev.driver) { 891d2e9a110SYong Wu of_node_put(larbnode); 892d2e9a110SYong Wu return -EPROBE_DEFER; 893d2e9a110SYong Wu } 894d2e9a110SYong Wu data->larb_imu[id].dev = &plarbdev->dev; 895d2e9a110SYong Wu 896d2e9a110SYong Wu component_match_add_release(dev, match, component_release_of, 897d2e9a110SYong Wu component_compare_of, larbnode); 898d2e9a110SYong Wu } 899d2e9a110SYong Wu 900f7b71d0dSYong Wu /* Get smi-(sub)-common dev from the last larb. */ 901f7b71d0dSYong Wu smi_subcomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0); 902f7b71d0dSYong Wu if (!smi_subcomm_node) 903d2e9a110SYong Wu return -EINVAL; 904d2e9a110SYong Wu 905f7b71d0dSYong Wu /* 906f7b71d0dSYong Wu * It may have two level smi-common. the node is smi-sub-common if it 907f7b71d0dSYong Wu * has a new mediatek,smi property. otherwise it is smi-commmon. 908f7b71d0dSYong Wu */ 909f7b71d0dSYong Wu smicomm_node = of_parse_phandle(smi_subcomm_node, "mediatek,smi", 0); 910f7b71d0dSYong Wu if (smicomm_node) 911f7b71d0dSYong Wu of_node_put(smi_subcomm_node); 912f7b71d0dSYong Wu else 913f7b71d0dSYong Wu smicomm_node = smi_subcomm_node; 914f7b71d0dSYong Wu 915d2e9a110SYong Wu plarbdev = of_find_device_by_node(smicomm_node); 916d2e9a110SYong Wu of_node_put(smicomm_node); 917d2e9a110SYong Wu data->smicomm_dev = &plarbdev->dev; 918d2e9a110SYong Wu 919d2e9a110SYong Wu link = device_link_add(data->smicomm_dev, dev, 920d2e9a110SYong Wu DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); 921d2e9a110SYong Wu if (!link) { 922d2e9a110SYong Wu dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev)); 923d2e9a110SYong Wu return -EINVAL; 924d2e9a110SYong Wu } 925d2e9a110SYong Wu return 0; 926d2e9a110SYong Wu } 927d2e9a110SYong Wu 9280df4fabeSYong Wu static int mtk_iommu_probe(struct platform_device *pdev) 9290df4fabeSYong Wu { 9300df4fabeSYong Wu struct mtk_iommu_data *data; 9310df4fabeSYong Wu struct device *dev = &pdev->dev; 9320df4fabeSYong Wu struct resource *res; 933b16c0170SJoerg Roedel resource_size_t ioaddr; 9340df4fabeSYong Wu struct component_match *match = NULL; 935c2c59456SMiles Chen struct regmap *infracfg; 9360df4fabeSYong Wu void *protect; 937d2e9a110SYong Wu int ret; 938c2c59456SMiles Chen u32 val; 939c2c59456SMiles Chen char *p; 9400df4fabeSYong Wu 9410df4fabeSYong Wu data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); 9420df4fabeSYong Wu if (!data) 9430df4fabeSYong Wu return -ENOMEM; 9440df4fabeSYong Wu data->dev = dev; 945cecdce9dSYong Wu data->plat_data = of_device_get_match_data(dev); 9460df4fabeSYong Wu 9470df4fabeSYong Wu /* Protect memory. HW will access here while translation fault.*/ 9480df4fabeSYong Wu protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); 9490df4fabeSYong Wu if (!protect) 9500df4fabeSYong Wu return -ENOMEM; 9510df4fabeSYong Wu data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); 9520df4fabeSYong Wu 953c2c59456SMiles Chen if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) { 954c2c59456SMiles Chen switch (data->plat_data->m4u_plat) { 955c2c59456SMiles Chen case M4U_MT2712: 956c2c59456SMiles Chen p = "mediatek,mt2712-infracfg"; 957c2c59456SMiles Chen break; 958c2c59456SMiles Chen case M4U_MT8173: 959c2c59456SMiles Chen p = "mediatek,mt8173-infracfg"; 960c2c59456SMiles Chen break; 961c2c59456SMiles Chen default: 962c2c59456SMiles Chen p = NULL; 963c2c59456SMiles Chen } 964c2c59456SMiles Chen 965c2c59456SMiles Chen infracfg = syscon_regmap_lookup_by_compatible(p); 966c2c59456SMiles Chen 967c2c59456SMiles Chen if (IS_ERR(infracfg)) 968c2c59456SMiles Chen return PTR_ERR(infracfg); 969c2c59456SMiles Chen 970c2c59456SMiles Chen ret = regmap_read(infracfg, REG_INFRA_MISC, &val); 971c2c59456SMiles Chen if (ret) 972c2c59456SMiles Chen return ret; 973c2c59456SMiles Chen data->enable_4GB = !!(val & F_DDR_4GB_SUPPORT_EN); 974c2c59456SMiles Chen } 97501e23c93SYong Wu 9760df4fabeSYong Wu res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 9770df4fabeSYong Wu data->base = devm_ioremap_resource(dev, res); 9780df4fabeSYong Wu if (IS_ERR(data->base)) 9790df4fabeSYong Wu return PTR_ERR(data->base); 980b16c0170SJoerg Roedel ioaddr = res->start; 9810df4fabeSYong Wu 9820df4fabeSYong Wu data->irq = platform_get_irq(pdev, 0); 9830df4fabeSYong Wu if (data->irq < 0) 9840df4fabeSYong Wu return data->irq; 9850df4fabeSYong Wu 9866b717796SChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) { 9870df4fabeSYong Wu data->bclk = devm_clk_get(dev, "bclk"); 9880df4fabeSYong Wu if (IS_ERR(data->bclk)) 9890df4fabeSYong Wu return PTR_ERR(data->bclk); 9902aa4c259SYong Wu } 9910df4fabeSYong Wu 992c0b57581SYong Wu pm_runtime_enable(dev); 993c0b57581SYong Wu 994d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { 995d2e9a110SYong Wu ret = mtk_iommu_mm_dts_parse(dev, &match, data); 996d2e9a110SYong Wu if (ret) { 997d2e9a110SYong Wu dev_err(dev, "mm dts parse fail(%d).", ret); 998c0b57581SYong Wu goto out_runtime_disable; 999baf94e6eSYong Wu } 1000d2e9a110SYong Wu } 1001baf94e6eSYong Wu 10020df4fabeSYong Wu platform_set_drvdata(pdev, data); 10030e5a3f2eSYong Wu mutex_init(&data->mutex); 10040df4fabeSYong Wu 1005b16c0170SJoerg Roedel ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, 1006b16c0170SJoerg Roedel "mtk-iommu.%pa", &ioaddr); 1007b16c0170SJoerg Roedel if (ret) 1008baf94e6eSYong Wu goto out_link_remove; 1009b16c0170SJoerg Roedel 10102d471b20SRobin Murphy ret = iommu_device_register(&data->iommu, &mtk_iommu_ops, dev); 1011b16c0170SJoerg Roedel if (ret) 1012986d9ec5SYong Wu goto out_sysfs_remove; 1013b16c0170SJoerg Roedel 1014da3cc91bSYong Wu spin_lock_init(&data->tlb_lock); 10159e3a2a64SYong Wu 10169e3a2a64SYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, SHARE_PGTABLE)) { 10179e3a2a64SYong Wu list_add_tail(&data->list, data->plat_data->hw_list); 10189e3a2a64SYong Wu data->hw_list = data->plat_data->hw_list; 10199e3a2a64SYong Wu } else { 10209e3a2a64SYong Wu INIT_LIST_HEAD(&data->hw_list_head); 10219e3a2a64SYong Wu list_add_tail(&data->list, &data->hw_list_head); 10229e3a2a64SYong Wu data->hw_list = &data->hw_list_head; 10239e3a2a64SYong Wu } 10247c3a2ec0SYong Wu 1025986d9ec5SYong Wu if (!iommu_present(&platform_bus_type)) { 1026986d9ec5SYong Wu ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); 1027986d9ec5SYong Wu if (ret) 1028986d9ec5SYong Wu goto out_list_del; 1029986d9ec5SYong Wu } 10300df4fabeSYong Wu 1031d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { 1032986d9ec5SYong Wu ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); 1033986d9ec5SYong Wu if (ret) 1034986d9ec5SYong Wu goto out_bus_set_null; 1035d2e9a110SYong Wu } 1036986d9ec5SYong Wu return ret; 1037986d9ec5SYong Wu 1038986d9ec5SYong Wu out_bus_set_null: 1039986d9ec5SYong Wu bus_set_iommu(&platform_bus_type, NULL); 1040986d9ec5SYong Wu out_list_del: 1041986d9ec5SYong Wu list_del(&data->list); 1042986d9ec5SYong Wu iommu_device_unregister(&data->iommu); 1043986d9ec5SYong Wu out_sysfs_remove: 1044986d9ec5SYong Wu iommu_device_sysfs_remove(&data->iommu); 1045baf94e6eSYong Wu out_link_remove: 1046d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) 1047baf94e6eSYong Wu device_link_remove(data->smicomm_dev, dev); 1048c0b57581SYong Wu out_runtime_disable: 1049c0b57581SYong Wu pm_runtime_disable(dev); 1050986d9ec5SYong Wu return ret; 10510df4fabeSYong Wu } 10520df4fabeSYong Wu 10530df4fabeSYong Wu static int mtk_iommu_remove(struct platform_device *pdev) 10540df4fabeSYong Wu { 10550df4fabeSYong Wu struct mtk_iommu_data *data = platform_get_drvdata(pdev); 10560df4fabeSYong Wu 1057b16c0170SJoerg Roedel iommu_device_sysfs_remove(&data->iommu); 1058b16c0170SJoerg Roedel iommu_device_unregister(&data->iommu); 1059b16c0170SJoerg Roedel 1060ee55f75eSYong Wu list_del(&data->list); 10610df4fabeSYong Wu 1062d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { 1063baf94e6eSYong Wu device_link_remove(data->smicomm_dev, &pdev->dev); 1064d2e9a110SYong Wu component_master_del(&pdev->dev, &mtk_iommu_com_ops); 1065d2e9a110SYong Wu } 1066c0b57581SYong Wu pm_runtime_disable(&pdev->dev); 10670df4fabeSYong Wu devm_free_irq(&pdev->dev, data->irq, data); 10680df4fabeSYong Wu return 0; 10690df4fabeSYong Wu } 10700df4fabeSYong Wu 107134665c79SYong Wu static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev) 10720df4fabeSYong Wu { 10730df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 10740df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 10750df4fabeSYong Wu void __iomem *base = data->base; 10760df4fabeSYong Wu 107735c1b48dSChao Hao reg->wr_len_ctrl = readl_relaxed(base + REG_MMU_WR_LEN_CTRL); 107875eed350SChao Hao reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL); 10790df4fabeSYong Wu reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); 10800df4fabeSYong Wu reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); 10810df4fabeSYong Wu reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0); 10820df4fabeSYong Wu reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); 108370ca608bSYong Wu reg->ivrp_paddr = readl_relaxed(base + REG_MMU_IVRP_PADDR); 1084b9475b34SYong Wu reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG); 10856254b64fSYong Wu clk_disable_unprepare(data->bclk); 10860df4fabeSYong Wu return 0; 10870df4fabeSYong Wu } 10880df4fabeSYong Wu 108934665c79SYong Wu static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) 10900df4fabeSYong Wu { 10910df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 10920df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 1093907ba6a1SYong Wu struct mtk_iommu_domain *m4u_dom = data->m4u_dom; 10940df4fabeSYong Wu void __iomem *base = data->base; 10956254b64fSYong Wu int ret; 10960df4fabeSYong Wu 10976254b64fSYong Wu ret = clk_prepare_enable(data->bclk); 10986254b64fSYong Wu if (ret) { 10996254b64fSYong Wu dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret); 11006254b64fSYong Wu return ret; 11016254b64fSYong Wu } 1102b34ea31fSDafna Hirschfeld 1103b34ea31fSDafna Hirschfeld /* 1104b34ea31fSDafna Hirschfeld * Uppon first resume, only enable the clk and return, since the values of the 1105b34ea31fSDafna Hirschfeld * registers are not yet set. 1106b34ea31fSDafna Hirschfeld */ 1107b34ea31fSDafna Hirschfeld if (!m4u_dom) 1108b34ea31fSDafna Hirschfeld return 0; 1109b34ea31fSDafna Hirschfeld 111035c1b48dSChao Hao writel_relaxed(reg->wr_len_ctrl, base + REG_MMU_WR_LEN_CTRL); 111175eed350SChao Hao writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL); 11120df4fabeSYong Wu writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); 11130df4fabeSYong Wu writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); 11140df4fabeSYong Wu writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); 11150df4fabeSYong Wu writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); 111670ca608bSYong Wu writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR); 1117b9475b34SYong Wu writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG); 1118c0b57581SYong Wu writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, base + REG_MMU_PT_BASE_ADDR); 11194f23f6d4SYong Wu 11204f23f6d4SYong Wu /* 11214f23f6d4SYong Wu * Users may allocate dma buffer before they call pm_runtime_get, 11224f23f6d4SYong Wu * in which case it will lack the necessary tlb flush. 11234f23f6d4SYong Wu * Thus, make sure to update the tlb after each PM resume. 11244f23f6d4SYong Wu */ 11254f23f6d4SYong Wu mtk_iommu_tlb_flush_all(data); 11260df4fabeSYong Wu return 0; 11270df4fabeSYong Wu } 11280df4fabeSYong Wu 1129e6dec923SYong Wu static const struct dev_pm_ops mtk_iommu_pm_ops = { 113034665c79SYong Wu SET_RUNTIME_PM_OPS(mtk_iommu_runtime_suspend, mtk_iommu_runtime_resume, NULL) 113134665c79SYong Wu SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 113234665c79SYong Wu pm_runtime_force_resume) 11330df4fabeSYong Wu }; 11340df4fabeSYong Wu 1135cecdce9dSYong Wu static const struct mtk_iommu_plat_data mt2712_data = { 1136cecdce9dSYong Wu .m4u_plat = M4U_MT2712, 1137d2e9a110SYong Wu .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG | SHARE_PGTABLE | 1138d2e9a110SYong Wu MTK_IOMMU_TYPE_MM, 11399e3a2a64SYong Wu .hw_list = &m4ulist, 1140b053bc71SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 1141585e58f4SYong Wu .iova_region = single_domain, 1142585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 114337276e00SChao Hao .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, 1144cecdce9dSYong Wu }; 1145cecdce9dSYong Wu 1146068c86e9SChao Hao static const struct mtk_iommu_plat_data mt6779_data = { 1147068c86e9SChao Hao .m4u_plat = M4U_MT6779, 1148d2e9a110SYong Wu .flags = HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | WR_THROT_EN | 1149d2e9a110SYong Wu MTK_IOMMU_TYPE_MM, 1150068c86e9SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN2, 1151585e58f4SYong Wu .iova_region = single_domain, 1152585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 1153068c86e9SChao Hao .larbid_remap = {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}}, 1154cecdce9dSYong Wu }; 1155cecdce9dSYong Wu 11563c213562SFabien Parent static const struct mtk_iommu_plat_data mt8167_data = { 11573c213562SFabien Parent .m4u_plat = M4U_MT8167, 1158d2e9a110SYong Wu .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM, 11593c213562SFabien Parent .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 1160585e58f4SYong Wu .iova_region = single_domain, 1161585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 11623c213562SFabien Parent .larbid_remap = {{0}, {1}, {2}}, /* Linear mapping. */ 11633c213562SFabien Parent }; 11643c213562SFabien Parent 1165cecdce9dSYong Wu static const struct mtk_iommu_plat_data mt8173_data = { 1166cecdce9dSYong Wu .m4u_plat = M4U_MT8173, 1167d1b5ef00SFabien Parent .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | 1168d2e9a110SYong Wu HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM, 1169b053bc71SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 1170585e58f4SYong Wu .iova_region = single_domain, 1171585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 117237276e00SChao Hao .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */ 1173cecdce9dSYong Wu }; 1174cecdce9dSYong Wu 1175907ba6a1SYong Wu static const struct mtk_iommu_plat_data mt8183_data = { 1176907ba6a1SYong Wu .m4u_plat = M4U_MT8183, 1177d2e9a110SYong Wu .flags = RESET_AXI | MTK_IOMMU_TYPE_MM, 1178b053bc71SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 1179585e58f4SYong Wu .iova_region = single_domain, 1180585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 118137276e00SChao Hao .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}}, 1182907ba6a1SYong Wu }; 1183907ba6a1SYong Wu 11849e3489e0SYong Wu static const struct mtk_iommu_plat_data mt8192_data = { 11859e3489e0SYong Wu .m4u_plat = M4U_MT8192, 11869ec30c09SYong Wu .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | 1187d2e9a110SYong Wu WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM, 11889e3489e0SYong Wu .inv_sel_reg = REG_MMU_INV_SEL_GEN2, 11899e3489e0SYong Wu .iova_region = mt8192_multi_dom, 11909e3489e0SYong Wu .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), 11919e3489e0SYong Wu .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, 11929e3489e0SYong Wu {0, 14, 16}, {0, 13, 18, 17}}, 11939e3489e0SYong Wu }; 11949e3489e0SYong Wu 11950df4fabeSYong Wu static const struct of_device_id mtk_iommu_of_ids[] = { 1196cecdce9dSYong Wu { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, 1197068c86e9SChao Hao { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data}, 11983c213562SFabien Parent { .compatible = "mediatek,mt8167-m4u", .data = &mt8167_data}, 1199cecdce9dSYong Wu { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, 1200907ba6a1SYong Wu { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, 12019e3489e0SYong Wu { .compatible = "mediatek,mt8192-m4u", .data = &mt8192_data}, 12020df4fabeSYong Wu {} 12030df4fabeSYong Wu }; 12040df4fabeSYong Wu 12050df4fabeSYong Wu static struct platform_driver mtk_iommu_driver = { 12060df4fabeSYong Wu .probe = mtk_iommu_probe, 12070df4fabeSYong Wu .remove = mtk_iommu_remove, 12080df4fabeSYong Wu .driver = { 12090df4fabeSYong Wu .name = "mtk-iommu", 1210f53dd978SKrzysztof Kozlowski .of_match_table = mtk_iommu_of_ids, 12110df4fabeSYong Wu .pm = &mtk_iommu_pm_ops, 12120df4fabeSYong Wu } 12130df4fabeSYong Wu }; 121418d8c74eSYong Wu module_platform_driver(mtk_iommu_driver); 12150df4fabeSYong Wu 121618d8c74eSYong Wu MODULE_DESCRIPTION("IOMMU API for MediaTek M4U implementations"); 121718d8c74eSYong Wu MODULE_LICENSE("GPL v2"); 1218