11802d0beSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 20df4fabeSYong Wu /* 30df4fabeSYong Wu * Copyright (c) 2015-2016 MediaTek Inc. 40df4fabeSYong Wu * Author: Yong Wu <yong.wu@mediatek.com> 50df4fabeSYong Wu */ 6ef0f0986SYong Wu #include <linux/bitfield.h> 70df4fabeSYong Wu #include <linux/bug.h> 80df4fabeSYong Wu #include <linux/clk.h> 90df4fabeSYong Wu #include <linux/component.h> 100df4fabeSYong Wu #include <linux/device.h> 11*803cf9e5SYong Wu #include <linux/dma-direct.h> 120df4fabeSYong Wu #include <linux/dma-iommu.h> 130df4fabeSYong Wu #include <linux/err.h> 140df4fabeSYong Wu #include <linux/interrupt.h> 150df4fabeSYong Wu #include <linux/io.h> 160df4fabeSYong Wu #include <linux/iommu.h> 170df4fabeSYong Wu #include <linux/iopoll.h> 180df4fabeSYong Wu #include <linux/list.h> 19c2c59456SMiles Chen #include <linux/mfd/syscon.h> 200df4fabeSYong Wu #include <linux/of_address.h> 210df4fabeSYong Wu #include <linux/of_iommu.h> 220df4fabeSYong Wu #include <linux/of_irq.h> 230df4fabeSYong Wu #include <linux/of_platform.h> 240df4fabeSYong Wu #include <linux/platform_device.h> 25baf94e6eSYong Wu #include <linux/pm_runtime.h> 26c2c59456SMiles Chen #include <linux/regmap.h> 270df4fabeSYong Wu #include <linux/slab.h> 280df4fabeSYong Wu #include <linux/spinlock.h> 29c2c59456SMiles Chen #include <linux/soc/mediatek/infracfg.h> 300df4fabeSYong Wu #include <asm/barrier.h> 310df4fabeSYong Wu #include <soc/mediatek/smi.h> 320df4fabeSYong Wu 339ca340c9SHonghui Zhang #include "mtk_iommu.h" 340df4fabeSYong Wu 350df4fabeSYong Wu #define REG_MMU_PT_BASE_ADDR 0x000 36907ba6a1SYong Wu #define MMU_PT_ADDR_MASK GENMASK(31, 7) 370df4fabeSYong Wu 380df4fabeSYong Wu #define REG_MMU_INVALIDATE 0x020 390df4fabeSYong Wu #define F_ALL_INVLD 0x2 400df4fabeSYong Wu #define F_MMU_INV_RANGE 0x1 410df4fabeSYong Wu 420df4fabeSYong Wu #define REG_MMU_INVLD_START_A 0x024 430df4fabeSYong Wu #define REG_MMU_INVLD_END_A 0x028 440df4fabeSYong Wu 45068c86e9SChao Hao #define REG_MMU_INV_SEL_GEN2 0x02c 46b053bc71SChao Hao #define REG_MMU_INV_SEL_GEN1 0x038 470df4fabeSYong Wu #define F_INVLD_EN0 BIT(0) 480df4fabeSYong Wu #define F_INVLD_EN1 BIT(1) 490df4fabeSYong Wu 5075eed350SChao Hao #define REG_MMU_MISC_CTRL 0x048 514bb2bf4cSChao Hao #define F_MMU_IN_ORDER_WR_EN_MASK (BIT(1) | BIT(17)) 524bb2bf4cSChao Hao #define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19)) 534bb2bf4cSChao Hao 540df4fabeSYong Wu #define REG_MMU_DCM_DIS 0x050 5535c1b48dSChao Hao #define REG_MMU_WR_LEN_CTRL 0x054 5635c1b48dSChao Hao #define F_MMU_WR_THROT_DIS_MASK (BIT(5) | BIT(21)) 570df4fabeSYong Wu 580df4fabeSYong Wu #define REG_MMU_CTRL_REG 0x110 59acb3c92aSYong Wu #define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4) 600df4fabeSYong Wu #define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4) 61acb3c92aSYong Wu #define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5) 620df4fabeSYong Wu 630df4fabeSYong Wu #define REG_MMU_IVRP_PADDR 0x114 6470ca608bSYong Wu 6530e2fccfSYong Wu #define REG_MMU_VLD_PA_RNG 0x118 6630e2fccfSYong Wu #define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA)) 670df4fabeSYong Wu 680df4fabeSYong Wu #define REG_MMU_INT_CONTROL0 0x120 690df4fabeSYong Wu #define F_L2_MULIT_HIT_EN BIT(0) 700df4fabeSYong Wu #define F_TABLE_WALK_FAULT_INT_EN BIT(1) 710df4fabeSYong Wu #define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2) 720df4fabeSYong Wu #define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3) 730df4fabeSYong Wu #define F_PREFETCH_FIFO_ERR_INT_EN BIT(5) 740df4fabeSYong Wu #define F_MISS_FIFO_ERR_INT_EN BIT(6) 750df4fabeSYong Wu #define F_INT_CLR_BIT BIT(12) 760df4fabeSYong Wu 770df4fabeSYong Wu #define REG_MMU_INT_MAIN_CONTROL 0x124 7815a01f4cSYong Wu /* mmu0 | mmu1 */ 7915a01f4cSYong Wu #define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7)) 8015a01f4cSYong Wu #define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8)) 8115a01f4cSYong Wu #define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9)) 8215a01f4cSYong Wu #define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10)) 8315a01f4cSYong Wu #define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11)) 8415a01f4cSYong Wu #define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12)) 8515a01f4cSYong Wu #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13)) 860df4fabeSYong Wu 870df4fabeSYong Wu #define REG_MMU_CPE_DONE 0x12C 880df4fabeSYong Wu 890df4fabeSYong Wu #define REG_MMU_FAULT_ST1 0x134 9015a01f4cSYong Wu #define F_REG_MMU0_FAULT_MASK GENMASK(6, 0) 9115a01f4cSYong Wu #define F_REG_MMU1_FAULT_MASK GENMASK(13, 7) 920df4fabeSYong Wu 9315a01f4cSYong Wu #define REG_MMU0_FAULT_VA 0x13c 94ef0f0986SYong Wu #define F_MMU_INVAL_VA_31_12_MASK GENMASK(31, 12) 95ef0f0986SYong Wu #define F_MMU_INVAL_VA_34_32_MASK GENMASK(11, 9) 96ef0f0986SYong Wu #define F_MMU_INVAL_PA_34_32_MASK GENMASK(8, 6) 970df4fabeSYong Wu #define F_MMU_FAULT_VA_WRITE_BIT BIT(1) 980df4fabeSYong Wu #define F_MMU_FAULT_VA_LAYER_BIT BIT(0) 990df4fabeSYong Wu 10015a01f4cSYong Wu #define REG_MMU0_INVLD_PA 0x140 10115a01f4cSYong Wu #define REG_MMU1_FAULT_VA 0x144 10215a01f4cSYong Wu #define REG_MMU1_INVLD_PA 0x148 10315a01f4cSYong Wu #define REG_MMU0_INT_ID 0x150 10415a01f4cSYong Wu #define REG_MMU1_INT_ID 0x154 10537276e00SChao Hao #define F_MMU_INT_ID_COMM_ID(a) (((a) >> 9) & 0x7) 10637276e00SChao Hao #define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3) 10715a01f4cSYong Wu #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) 10815a01f4cSYong Wu #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) 1090df4fabeSYong Wu 110829316b3SChao Hao #define MTK_PROTECT_PA_ALIGN 256 1110df4fabeSYong Wu 1126b717796SChao Hao #define HAS_4GB_MODE BIT(0) 1136b717796SChao Hao /* HW will use the EMI clock if there isn't the "bclk". */ 1146b717796SChao Hao #define HAS_BCLK BIT(1) 1156b717796SChao Hao #define HAS_VLD_PA_RNG BIT(2) 1166b717796SChao Hao #define RESET_AXI BIT(3) 1174bb2bf4cSChao Hao #define OUT_ORDER_WR_EN BIT(4) 11837276e00SChao Hao #define HAS_SUB_COMM BIT(5) 11935c1b48dSChao Hao #define WR_THROT_EN BIT(6) 120d1b5ef00SFabien Parent #define HAS_LEGACY_IVRP_PADDR BIT(7) 1212f317da4SYong Wu #define IOVA_34_EN BIT(8) 1226b717796SChao Hao 1236b717796SChao Hao #define MTK_IOMMU_HAS_FLAG(pdata, _x) \ 1246b717796SChao Hao ((((pdata)->flags) & (_x)) == (_x)) 1256b717796SChao Hao 1260df4fabeSYong Wu struct mtk_iommu_domain { 1270df4fabeSYong Wu struct io_pgtable_cfg cfg; 1280df4fabeSYong Wu struct io_pgtable_ops *iop; 1290df4fabeSYong Wu 13008500c43SYong Wu struct mtk_iommu_data *data; 1310df4fabeSYong Wu struct iommu_domain domain; 1320df4fabeSYong Wu }; 1330df4fabeSYong Wu 134b65f5016SArvind Yadav static const struct iommu_ops mtk_iommu_ops; 1350df4fabeSYong Wu 1367f37a91dSYong Wu static int mtk_iommu_hw_init(const struct mtk_iommu_data *data); 1377f37a91dSYong Wu 138bfed8731SYong Wu #define MTK_IOMMU_TLB_ADDR(iova) ({ \ 139bfed8731SYong Wu dma_addr_t _addr = iova; \ 140bfed8731SYong Wu ((lower_32_bits(_addr) & GENMASK(31, 12)) | upper_32_bits(_addr));\ 141bfed8731SYong Wu }) 142bfed8731SYong Wu 14376ce6546SYong Wu /* 14476ce6546SYong Wu * In M4U 4GB mode, the physical address is remapped as below: 14576ce6546SYong Wu * 14676ce6546SYong Wu * CPU Physical address: 14776ce6546SYong Wu * ==================== 14876ce6546SYong Wu * 14976ce6546SYong Wu * 0 1G 2G 3G 4G 5G 15076ce6546SYong Wu * |---A---|---B---|---C---|---D---|---E---| 15176ce6546SYong Wu * +--I/O--+------------Memory-------------+ 15276ce6546SYong Wu * 15376ce6546SYong Wu * IOMMU output physical address: 15476ce6546SYong Wu * ============================= 15576ce6546SYong Wu * 15676ce6546SYong Wu * 4G 5G 6G 7G 8G 15776ce6546SYong Wu * |---E---|---B---|---C---|---D---| 15876ce6546SYong Wu * +------------Memory-------------+ 15976ce6546SYong Wu * 16076ce6546SYong Wu * The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the 16176ce6546SYong Wu * bit32 of the CPU physical address always is needed to set, and for Region 16276ce6546SYong Wu * 'E', the CPU physical address keep as is. 16376ce6546SYong Wu * Additionally, The iommu consumers always use the CPU phyiscal address. 16476ce6546SYong Wu */ 165b4dad40eSYong Wu #define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL 16676ce6546SYong Wu 1677c3a2ec0SYong Wu static LIST_HEAD(m4ulist); /* List all the M4U HWs */ 1687c3a2ec0SYong Wu 1697c3a2ec0SYong Wu #define for_each_m4u(data) list_for_each_entry(data, &m4ulist, list) 1707c3a2ec0SYong Wu 171585e58f4SYong Wu struct mtk_iommu_iova_region { 172585e58f4SYong Wu dma_addr_t iova_base; 173585e58f4SYong Wu unsigned long long size; 174585e58f4SYong Wu }; 175585e58f4SYong Wu 176585e58f4SYong Wu static const struct mtk_iommu_iova_region single_domain[] = { 177585e58f4SYong Wu {.iova_base = 0, .size = SZ_4G}, 178585e58f4SYong Wu }; 179585e58f4SYong Wu 1807c3a2ec0SYong Wu /* 1817c3a2ec0SYong Wu * There may be 1 or 2 M4U HWs, But we always expect they are in the same domain 1827c3a2ec0SYong Wu * for the performance. 1837c3a2ec0SYong Wu * 1847c3a2ec0SYong Wu * Here always return the mtk_iommu_data of the first probed M4U where the 1857c3a2ec0SYong Wu * iommu domain information is recorded. 1867c3a2ec0SYong Wu */ 1877c3a2ec0SYong Wu static struct mtk_iommu_data *mtk_iommu_get_m4u_data(void) 1887c3a2ec0SYong Wu { 1897c3a2ec0SYong Wu struct mtk_iommu_data *data; 1907c3a2ec0SYong Wu 1917c3a2ec0SYong Wu for_each_m4u(data) 1927c3a2ec0SYong Wu return data; 1937c3a2ec0SYong Wu 1947c3a2ec0SYong Wu return NULL; 1957c3a2ec0SYong Wu } 1967c3a2ec0SYong Wu 1970df4fabeSYong Wu static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) 1980df4fabeSYong Wu { 1990df4fabeSYong Wu return container_of(dom, struct mtk_iommu_domain, domain); 2000df4fabeSYong Wu } 2010df4fabeSYong Wu 2020954d61aSYong Wu static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) 2030df4fabeSYong Wu { 2047c3a2ec0SYong Wu for_each_m4u(data) { 205c0b57581SYong Wu if (pm_runtime_get_if_in_use(data->dev) <= 0) 206c0b57581SYong Wu continue; 207c0b57581SYong Wu 2087c3a2ec0SYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, 209b053bc71SChao Hao data->base + data->plat_data->inv_sel_reg); 2100df4fabeSYong Wu writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); 2110df4fabeSYong Wu wmb(); /* Make sure the tlb flush all done */ 212c0b57581SYong Wu 213c0b57581SYong Wu pm_runtime_put(data->dev); 2140df4fabeSYong Wu } 2157c3a2ec0SYong Wu } 2160df4fabeSYong Wu 2171f4fd624SYong Wu static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, 2180954d61aSYong Wu size_t granule, 2190954d61aSYong Wu struct mtk_iommu_data *data) 2200df4fabeSYong Wu { 221c0b57581SYong Wu bool has_pm = !!data->dev->pm_domain; 2221f4fd624SYong Wu unsigned long flags; 2231f4fd624SYong Wu int ret; 2241f4fd624SYong Wu u32 tmp; 2250df4fabeSYong Wu 2267c3a2ec0SYong Wu for_each_m4u(data) { 227c0b57581SYong Wu if (has_pm) { 228c0b57581SYong Wu if (pm_runtime_get_if_in_use(data->dev) <= 0) 229c0b57581SYong Wu continue; 230c0b57581SYong Wu } 231c0b57581SYong Wu 2321f4fd624SYong Wu spin_lock_irqsave(&data->tlb_lock, flags); 2337c3a2ec0SYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, 234b053bc71SChao Hao data->base + data->plat_data->inv_sel_reg); 2350df4fabeSYong Wu 236bfed8731SYong Wu writel_relaxed(MTK_IOMMU_TLB_ADDR(iova), 237bfed8731SYong Wu data->base + REG_MMU_INVLD_START_A); 238bfed8731SYong Wu writel_relaxed(MTK_IOMMU_TLB_ADDR(iova + size - 1), 2397c3a2ec0SYong Wu data->base + REG_MMU_INVLD_END_A); 2407c3a2ec0SYong Wu writel_relaxed(F_MMU_INV_RANGE, 2417c3a2ec0SYong Wu data->base + REG_MMU_INVALIDATE); 2420df4fabeSYong Wu 2431f4fd624SYong Wu /* tlb sync */ 2447c3a2ec0SYong Wu ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, 245c90ae4a6SYong Wu tmp, tmp != 0, 10, 1000); 2460df4fabeSYong Wu if (ret) { 2470df4fabeSYong Wu dev_warn(data->dev, 2480df4fabeSYong Wu "Partial TLB flush timed out, falling back to full flush\n"); 2490954d61aSYong Wu mtk_iommu_tlb_flush_all(data); 2500df4fabeSYong Wu } 2510df4fabeSYong Wu /* Clear the CPE status */ 2520df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_CPE_DONE); 253da3cc91bSYong Wu spin_unlock_irqrestore(&data->tlb_lock, flags); 254c0b57581SYong Wu 255c0b57581SYong Wu if (has_pm) 256c0b57581SYong Wu pm_runtime_put(data->dev); 2570df4fabeSYong Wu } 2587c3a2ec0SYong Wu } 2590df4fabeSYong Wu 2600df4fabeSYong Wu static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) 2610df4fabeSYong Wu { 2620df4fabeSYong Wu struct mtk_iommu_data *data = dev_id; 2630df4fabeSYong Wu struct mtk_iommu_domain *dom = data->m4u_dom; 26437276e00SChao Hao unsigned int fault_larb, fault_port, sub_comm = 0; 265ef0f0986SYong Wu u32 int_state, regval, va34_32, pa34_32; 266ef0f0986SYong Wu u64 fault_iova, fault_pa; 2670df4fabeSYong Wu bool layer, write; 2680df4fabeSYong Wu 2690df4fabeSYong Wu /* Read error info from registers */ 2700df4fabeSYong Wu int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1); 27115a01f4cSYong Wu if (int_state & F_REG_MMU0_FAULT_MASK) { 27215a01f4cSYong Wu regval = readl_relaxed(data->base + REG_MMU0_INT_ID); 27315a01f4cSYong Wu fault_iova = readl_relaxed(data->base + REG_MMU0_FAULT_VA); 27415a01f4cSYong Wu fault_pa = readl_relaxed(data->base + REG_MMU0_INVLD_PA); 27515a01f4cSYong Wu } else { 27615a01f4cSYong Wu regval = readl_relaxed(data->base + REG_MMU1_INT_ID); 27715a01f4cSYong Wu fault_iova = readl_relaxed(data->base + REG_MMU1_FAULT_VA); 27815a01f4cSYong Wu fault_pa = readl_relaxed(data->base + REG_MMU1_INVLD_PA); 27915a01f4cSYong Wu } 2800df4fabeSYong Wu layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; 2810df4fabeSYong Wu write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; 282ef0f0986SYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN)) { 283ef0f0986SYong Wu va34_32 = FIELD_GET(F_MMU_INVAL_VA_34_32_MASK, fault_iova); 284ef0f0986SYong Wu pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova); 285ef0f0986SYong Wu fault_iova = fault_iova & F_MMU_INVAL_VA_31_12_MASK; 286ef0f0986SYong Wu fault_iova |= (u64)va34_32 << 32; 287ef0f0986SYong Wu fault_pa |= (u64)pa34_32 << 32; 288ef0f0986SYong Wu } 289ef0f0986SYong Wu 29015a01f4cSYong Wu fault_port = F_MMU_INT_ID_PORT_ID(regval); 29137276e00SChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM)) { 29237276e00SChao Hao fault_larb = F_MMU_INT_ID_COMM_ID(regval); 29337276e00SChao Hao sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval); 29437276e00SChao Hao } else { 29537276e00SChao Hao fault_larb = F_MMU_INT_ID_LARB_ID(regval); 29637276e00SChao Hao } 29737276e00SChao Hao fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm]; 298b3e5eee7SYong Wu 2990df4fabeSYong Wu if (report_iommu_fault(&dom->domain, data->dev, fault_iova, 3000df4fabeSYong Wu write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { 3010df4fabeSYong Wu dev_err_ratelimited( 3020df4fabeSYong Wu data->dev, 303ef0f0986SYong Wu "fault type=0x%x iova=0x%llx pa=0x%llx larb=%d port=%d layer=%d %s\n", 3040df4fabeSYong Wu int_state, fault_iova, fault_pa, fault_larb, fault_port, 3050df4fabeSYong Wu layer, write ? "write" : "read"); 3060df4fabeSYong Wu } 3070df4fabeSYong Wu 3080df4fabeSYong Wu /* Interrupt clear */ 3090df4fabeSYong Wu regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL0); 3100df4fabeSYong Wu regval |= F_INT_CLR_BIT; 3110df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); 3120df4fabeSYong Wu 3130df4fabeSYong Wu mtk_iommu_tlb_flush_all(data); 3140df4fabeSYong Wu 3150df4fabeSYong Wu return IRQ_HANDLED; 3160df4fabeSYong Wu } 3170df4fabeSYong Wu 318*803cf9e5SYong Wu static int mtk_iommu_get_domain_id(struct device *dev, 319*803cf9e5SYong Wu const struct mtk_iommu_plat_data *plat_data) 320*803cf9e5SYong Wu { 321*803cf9e5SYong Wu const struct mtk_iommu_iova_region *rgn = plat_data->iova_region; 322*803cf9e5SYong Wu const struct bus_dma_region *dma_rgn = dev->dma_range_map; 323*803cf9e5SYong Wu int i, candidate = -1; 324*803cf9e5SYong Wu dma_addr_t dma_end; 325*803cf9e5SYong Wu 326*803cf9e5SYong Wu if (!dma_rgn || plat_data->iova_region_nr == 1) 327*803cf9e5SYong Wu return 0; 328*803cf9e5SYong Wu 329*803cf9e5SYong Wu dma_end = dma_rgn->dma_start + dma_rgn->size - 1; 330*803cf9e5SYong Wu for (i = 0; i < plat_data->iova_region_nr; i++, rgn++) { 331*803cf9e5SYong Wu /* Best fit. */ 332*803cf9e5SYong Wu if (dma_rgn->dma_start == rgn->iova_base && 333*803cf9e5SYong Wu dma_end == rgn->iova_base + rgn->size - 1) 334*803cf9e5SYong Wu return i; 335*803cf9e5SYong Wu /* ok if it is inside this region. */ 336*803cf9e5SYong Wu if (dma_rgn->dma_start >= rgn->iova_base && 337*803cf9e5SYong Wu dma_end < rgn->iova_base + rgn->size) 338*803cf9e5SYong Wu candidate = i; 339*803cf9e5SYong Wu } 340*803cf9e5SYong Wu 341*803cf9e5SYong Wu if (candidate >= 0) 342*803cf9e5SYong Wu return candidate; 343*803cf9e5SYong Wu dev_err(dev, "Can NOT find the iommu domain id(%pad 0x%llx).\n", 344*803cf9e5SYong Wu &dma_rgn->dma_start, dma_rgn->size); 345*803cf9e5SYong Wu return -EINVAL; 346*803cf9e5SYong Wu } 347*803cf9e5SYong Wu 3480df4fabeSYong Wu static void mtk_iommu_config(struct mtk_iommu_data *data, 3490df4fabeSYong Wu struct device *dev, bool enable) 3500df4fabeSYong Wu { 3510df4fabeSYong Wu struct mtk_smi_larb_iommu *larb_mmu; 3520df4fabeSYong Wu unsigned int larbid, portid; 353a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 35458f0d1d5SRobin Murphy int i; 3550df4fabeSYong Wu 35658f0d1d5SRobin Murphy for (i = 0; i < fwspec->num_ids; ++i) { 35758f0d1d5SRobin Murphy larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); 35858f0d1d5SRobin Murphy portid = MTK_M4U_TO_PORT(fwspec->ids[i]); 3591ee9feb2SYong Wu larb_mmu = &data->larb_imu[larbid]; 3600df4fabeSYong Wu 3610df4fabeSYong Wu dev_dbg(dev, "%s iommu port: %d\n", 3620df4fabeSYong Wu enable ? "enable" : "disable", portid); 3630df4fabeSYong Wu 3640df4fabeSYong Wu if (enable) 3650df4fabeSYong Wu larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); 3660df4fabeSYong Wu else 3670df4fabeSYong Wu larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); 3680df4fabeSYong Wu } 3690df4fabeSYong Wu } 3700df4fabeSYong Wu 3714f956c97SYong Wu static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, 3724f956c97SYong Wu struct mtk_iommu_data *data) 3730df4fabeSYong Wu { 3740df4fabeSYong Wu dom->cfg = (struct io_pgtable_cfg) { 3750df4fabeSYong Wu .quirks = IO_PGTABLE_QUIRK_ARM_NS | 3760df4fabeSYong Wu IO_PGTABLE_QUIRK_NO_PERMS | 377b4dad40eSYong Wu IO_PGTABLE_QUIRK_ARM_MTK_EXT, 3780df4fabeSYong Wu .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, 3792f317da4SYong Wu .ias = MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN) ? 34 : 32, 3800df4fabeSYong Wu .iommu_dev = data->dev, 3810df4fabeSYong Wu }; 3820df4fabeSYong Wu 3839bdfe4c1SYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) 3849bdfe4c1SYong Wu dom->cfg.oas = data->enable_4GB ? 33 : 32; 3859bdfe4c1SYong Wu else 3869bdfe4c1SYong Wu dom->cfg.oas = 35; 3879bdfe4c1SYong Wu 3880df4fabeSYong Wu dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data); 3890df4fabeSYong Wu if (!dom->iop) { 3900df4fabeSYong Wu dev_err(data->dev, "Failed to alloc io pgtable\n"); 3910df4fabeSYong Wu return -EINVAL; 3920df4fabeSYong Wu } 3930df4fabeSYong Wu 3940df4fabeSYong Wu /* Update our support page sizes bitmap */ 395d16e0faaSRobin Murphy dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; 396b7875eb9SYong Wu 397b7875eb9SYong Wu dom->domain.geometry.aperture_start = 0; 398b7875eb9SYong Wu dom->domain.geometry.aperture_end = DMA_BIT_MASK(32); 399b7875eb9SYong Wu dom->domain.geometry.force_aperture = true; 4000df4fabeSYong Wu return 0; 4010df4fabeSYong Wu } 4020df4fabeSYong Wu 4030df4fabeSYong Wu static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) 4040df4fabeSYong Wu { 4050df4fabeSYong Wu struct mtk_iommu_domain *dom; 4060df4fabeSYong Wu 4070df4fabeSYong Wu if (type != IOMMU_DOMAIN_DMA) 4080df4fabeSYong Wu return NULL; 4090df4fabeSYong Wu 4100df4fabeSYong Wu dom = kzalloc(sizeof(*dom), GFP_KERNEL); 4110df4fabeSYong Wu if (!dom) 4120df4fabeSYong Wu return NULL; 4130df4fabeSYong Wu 4144f956c97SYong Wu if (iommu_get_dma_cookie(&dom->domain)) { 4154b00f5acSYong Wu kfree(dom); 4164b00f5acSYong Wu return NULL; 4170df4fabeSYong Wu } 4180df4fabeSYong Wu 4194f956c97SYong Wu return &dom->domain; 4204f956c97SYong Wu } 4214f956c97SYong Wu 4220df4fabeSYong Wu static void mtk_iommu_domain_free(struct iommu_domain *domain) 4230df4fabeSYong Wu { 4240df4fabeSYong Wu iommu_put_dma_cookie(domain); 4250df4fabeSYong Wu kfree(to_mtk_domain(domain)); 4260df4fabeSYong Wu } 4270df4fabeSYong Wu 4280df4fabeSYong Wu static int mtk_iommu_attach_device(struct iommu_domain *domain, 4290df4fabeSYong Wu struct device *dev) 4300df4fabeSYong Wu { 4313524b559SJoerg Roedel struct mtk_iommu_data *data = dev_iommu_priv_get(dev); 4320df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 433c0b57581SYong Wu struct device *m4udev = data->dev; 434*803cf9e5SYong Wu int ret, domid; 4350df4fabeSYong Wu 4364b00f5acSYong Wu if (!data) 4370df4fabeSYong Wu return -ENODEV; 4380df4fabeSYong Wu 439*803cf9e5SYong Wu domid = mtk_iommu_get_domain_id(dev, data->plat_data); 440*803cf9e5SYong Wu if (domid < 0) 441*803cf9e5SYong Wu return domid; 442*803cf9e5SYong Wu 4434f956c97SYong Wu if (!dom->data) { 4444f956c97SYong Wu if (mtk_iommu_domain_finalise(dom, data)) 4454f956c97SYong Wu return -ENODEV; 4464f956c97SYong Wu dom->data = data; 4474f956c97SYong Wu } 4484f956c97SYong Wu 4497f37a91dSYong Wu if (!data->m4u_dom) { /* Initialize the M4U HW */ 450c0b57581SYong Wu ret = pm_runtime_resume_and_get(m4udev); 451c0b57581SYong Wu if (ret < 0) 4527f37a91dSYong Wu return ret; 453c0b57581SYong Wu 454c0b57581SYong Wu ret = mtk_iommu_hw_init(data); 455c0b57581SYong Wu if (ret) { 456c0b57581SYong Wu pm_runtime_put(m4udev); 457c0b57581SYong Wu return ret; 458c0b57581SYong Wu } 4590df4fabeSYong Wu data->m4u_dom = dom; 460d1e5f26fSRobin Murphy writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, 4614b00f5acSYong Wu data->base + REG_MMU_PT_BASE_ADDR); 462c0b57581SYong Wu 463c0b57581SYong Wu pm_runtime_put(m4udev); 4640df4fabeSYong Wu } 4650df4fabeSYong Wu 4664b00f5acSYong Wu mtk_iommu_config(data, dev, true); 4670df4fabeSYong Wu return 0; 4680df4fabeSYong Wu } 4690df4fabeSYong Wu 4700df4fabeSYong Wu static void mtk_iommu_detach_device(struct iommu_domain *domain, 4710df4fabeSYong Wu struct device *dev) 4720df4fabeSYong Wu { 4733524b559SJoerg Roedel struct mtk_iommu_data *data = dev_iommu_priv_get(dev); 4740df4fabeSYong Wu 47558f0d1d5SRobin Murphy if (!data) 4760df4fabeSYong Wu return; 4770df4fabeSYong Wu 4780df4fabeSYong Wu mtk_iommu_config(data, dev, false); 4790df4fabeSYong Wu } 4800df4fabeSYong Wu 4810df4fabeSYong Wu static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, 482781ca2deSTom Murphy phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 4830df4fabeSYong Wu { 4840df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 4850df4fabeSYong Wu 486b4dad40eSYong Wu /* The "4GB mode" M4U physically can not use the lower remap of Dram. */ 48708500c43SYong Wu if (dom->data->enable_4GB) 488b4dad40eSYong Wu paddr |= BIT_ULL(32); 489b4dad40eSYong Wu 49060829b4dSYong Wu /* Synchronize with the tlb_lock */ 491f34ce7a7SBaolin Wang return dom->iop->map(dom->iop, iova, paddr, size, prot, gfp); 4920df4fabeSYong Wu } 4930df4fabeSYong Wu 4940df4fabeSYong Wu static size_t mtk_iommu_unmap(struct iommu_domain *domain, 49556f8af5eSWill Deacon unsigned long iova, size_t size, 49656f8af5eSWill Deacon struct iommu_iotlb_gather *gather) 4970df4fabeSYong Wu { 4980df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 499f21ae3b1SYong Wu unsigned long end = iova + size - 1; 5000df4fabeSYong Wu 501f21ae3b1SYong Wu if (gather->start > iova) 502f21ae3b1SYong Wu gather->start = iova; 503f21ae3b1SYong Wu if (gather->end < end) 504f21ae3b1SYong Wu gather->end = end; 50560829b4dSYong Wu return dom->iop->unmap(dom->iop, iova, size, gather); 5060df4fabeSYong Wu } 5070df4fabeSYong Wu 50856f8af5eSWill Deacon static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain) 50956f8af5eSWill Deacon { 51008500c43SYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 51108500c43SYong Wu 51208500c43SYong Wu mtk_iommu_tlb_flush_all(dom->data); 51356f8af5eSWill Deacon } 51456f8af5eSWill Deacon 51556f8af5eSWill Deacon static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, 51656f8af5eSWill Deacon struct iommu_iotlb_gather *gather) 5174d689b61SRobin Murphy { 51808500c43SYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 519862c3715SYong Wu size_t length = gather->end - gather->start + 1; 520da3cc91bSYong Wu 5211f4fd624SYong Wu mtk_iommu_tlb_flush_range_sync(gather->start, length, gather->pgsize, 52208500c43SYong Wu dom->data); 5234d689b61SRobin Murphy } 5244d689b61SRobin Murphy 52520143451SYong Wu static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, 52620143451SYong Wu size_t size) 52720143451SYong Wu { 52808500c43SYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 52920143451SYong Wu 53008500c43SYong Wu mtk_iommu_tlb_flush_range_sync(iova, size, size, dom->data); 53120143451SYong Wu } 53220143451SYong Wu 5330df4fabeSYong Wu static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, 5340df4fabeSYong Wu dma_addr_t iova) 5350df4fabeSYong Wu { 5360df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 5370df4fabeSYong Wu phys_addr_t pa; 5380df4fabeSYong Wu 5390df4fabeSYong Wu pa = dom->iop->iova_to_phys(dom->iop, iova); 54008500c43SYong Wu if (dom->data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) 541b4dad40eSYong Wu pa &= ~BIT_ULL(32); 54230e2fccfSYong Wu 5430df4fabeSYong Wu return pa; 5440df4fabeSYong Wu } 5450df4fabeSYong Wu 54680e4592aSJoerg Roedel static struct iommu_device *mtk_iommu_probe_device(struct device *dev) 5470df4fabeSYong Wu { 548a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 549b16c0170SJoerg Roedel struct mtk_iommu_data *data; 5500df4fabeSYong Wu 551a9bf2eecSJoerg Roedel if (!fwspec || fwspec->ops != &mtk_iommu_ops) 55280e4592aSJoerg Roedel return ERR_PTR(-ENODEV); /* Not a iommu client device */ 5530df4fabeSYong Wu 5543524b559SJoerg Roedel data = dev_iommu_priv_get(dev); 555b16c0170SJoerg Roedel 55680e4592aSJoerg Roedel return &data->iommu; 5570df4fabeSYong Wu } 5580df4fabeSYong Wu 55980e4592aSJoerg Roedel static void mtk_iommu_release_device(struct device *dev) 5600df4fabeSYong Wu { 561a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 562b16c0170SJoerg Roedel 563a9bf2eecSJoerg Roedel if (!fwspec || fwspec->ops != &mtk_iommu_ops) 5640df4fabeSYong Wu return; 5650df4fabeSYong Wu 56658f0d1d5SRobin Murphy iommu_fwspec_free(dev); 5670df4fabeSYong Wu } 5680df4fabeSYong Wu 5690df4fabeSYong Wu static struct iommu_group *mtk_iommu_device_group(struct device *dev) 5700df4fabeSYong Wu { 5717c3a2ec0SYong Wu struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); 572*803cf9e5SYong Wu int domid; 5730df4fabeSYong Wu 57458f0d1d5SRobin Murphy if (!data) 5750df4fabeSYong Wu return ERR_PTR(-ENODEV); 5760df4fabeSYong Wu 577*803cf9e5SYong Wu domid = mtk_iommu_get_domain_id(dev, data->plat_data); 578*803cf9e5SYong Wu if (domid < 0) 579*803cf9e5SYong Wu return ERR_PTR(domid); 580*803cf9e5SYong Wu 5810df4fabeSYong Wu /* All the client devices are in the same m4u iommu-group */ 5820df4fabeSYong Wu if (!data->m4u_group) { 5830df4fabeSYong Wu data->m4u_group = iommu_group_alloc(); 5840df4fabeSYong Wu if (IS_ERR(data->m4u_group)) 5850df4fabeSYong Wu dev_err(dev, "Failed to allocate M4U IOMMU group\n"); 5863a8d40b6SRobin Murphy } else { 5873a8d40b6SRobin Murphy iommu_group_ref_get(data->m4u_group); 5880df4fabeSYong Wu } 5890df4fabeSYong Wu return data->m4u_group; 5900df4fabeSYong Wu } 5910df4fabeSYong Wu 5920df4fabeSYong Wu static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) 5930df4fabeSYong Wu { 5940df4fabeSYong Wu struct platform_device *m4updev; 5950df4fabeSYong Wu 5960df4fabeSYong Wu if (args->args_count != 1) { 5970df4fabeSYong Wu dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", 5980df4fabeSYong Wu args->args_count); 5990df4fabeSYong Wu return -EINVAL; 6000df4fabeSYong Wu } 6010df4fabeSYong Wu 6023524b559SJoerg Roedel if (!dev_iommu_priv_get(dev)) { 6030df4fabeSYong Wu /* Get the m4u device */ 6040df4fabeSYong Wu m4updev = of_find_device_by_node(args->np); 6050df4fabeSYong Wu if (WARN_ON(!m4updev)) 6060df4fabeSYong Wu return -EINVAL; 6070df4fabeSYong Wu 6083524b559SJoerg Roedel dev_iommu_priv_set(dev, platform_get_drvdata(m4updev)); 6090df4fabeSYong Wu } 6100df4fabeSYong Wu 61158f0d1d5SRobin Murphy return iommu_fwspec_add_ids(dev, args->args, 1); 6120df4fabeSYong Wu } 6130df4fabeSYong Wu 614b65f5016SArvind Yadav static const struct iommu_ops mtk_iommu_ops = { 6150df4fabeSYong Wu .domain_alloc = mtk_iommu_domain_alloc, 6160df4fabeSYong Wu .domain_free = mtk_iommu_domain_free, 6170df4fabeSYong Wu .attach_dev = mtk_iommu_attach_device, 6180df4fabeSYong Wu .detach_dev = mtk_iommu_detach_device, 6190df4fabeSYong Wu .map = mtk_iommu_map, 6200df4fabeSYong Wu .unmap = mtk_iommu_unmap, 62156f8af5eSWill Deacon .flush_iotlb_all = mtk_iommu_flush_iotlb_all, 6224d689b61SRobin Murphy .iotlb_sync = mtk_iommu_iotlb_sync, 62320143451SYong Wu .iotlb_sync_map = mtk_iommu_sync_map, 6240df4fabeSYong Wu .iova_to_phys = mtk_iommu_iova_to_phys, 62580e4592aSJoerg Roedel .probe_device = mtk_iommu_probe_device, 62680e4592aSJoerg Roedel .release_device = mtk_iommu_release_device, 6270df4fabeSYong Wu .device_group = mtk_iommu_device_group, 6280df4fabeSYong Wu .of_xlate = mtk_iommu_of_xlate, 6290df4fabeSYong Wu .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, 6300df4fabeSYong Wu }; 6310df4fabeSYong Wu 6320df4fabeSYong Wu static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) 6330df4fabeSYong Wu { 6340df4fabeSYong Wu u32 regval; 6350df4fabeSYong Wu int ret; 6360df4fabeSYong Wu 6370df4fabeSYong Wu ret = clk_prepare_enable(data->bclk); 6380df4fabeSYong Wu if (ret) { 6390df4fabeSYong Wu dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret); 6400df4fabeSYong Wu return ret; 6410df4fabeSYong Wu } 6420df4fabeSYong Wu 64386444413SChao Hao if (data->plat_data->m4u_plat == M4U_MT8173) { 644acb3c92aSYong Wu regval = F_MMU_PREFETCH_RT_REPLACE_MOD | 645acb3c92aSYong Wu F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173; 64686444413SChao Hao } else { 64786444413SChao Hao regval = readl_relaxed(data->base + REG_MMU_CTRL_REG); 64886444413SChao Hao regval |= F_MMU_TF_PROT_TO_PROGRAM_ADDR; 64986444413SChao Hao } 6500df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); 6510df4fabeSYong Wu 6520df4fabeSYong Wu regval = F_L2_MULIT_HIT_EN | 6530df4fabeSYong Wu F_TABLE_WALK_FAULT_INT_EN | 6540df4fabeSYong Wu F_PREETCH_FIFO_OVERFLOW_INT_EN | 6550df4fabeSYong Wu F_MISS_FIFO_OVERFLOW_INT_EN | 6560df4fabeSYong Wu F_PREFETCH_FIFO_ERR_INT_EN | 6570df4fabeSYong Wu F_MISS_FIFO_ERR_INT_EN; 6580df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); 6590df4fabeSYong Wu 6600df4fabeSYong Wu regval = F_INT_TRANSLATION_FAULT | 6610df4fabeSYong Wu F_INT_MAIN_MULTI_HIT_FAULT | 6620df4fabeSYong Wu F_INT_INVALID_PA_FAULT | 6630df4fabeSYong Wu F_INT_ENTRY_REPLACEMENT_FAULT | 6640df4fabeSYong Wu F_INT_TLB_MISS_FAULT | 6650df4fabeSYong Wu F_INT_MISS_TRANSACTION_FIFO_FAULT | 6660df4fabeSYong Wu F_INT_PRETETCH_TRANSATION_FIFO_FAULT; 6670df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); 6680df4fabeSYong Wu 669d1b5ef00SFabien Parent if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_LEGACY_IVRP_PADDR)) 67070ca608bSYong Wu regval = (data->protect_base >> 1) | (data->enable_4GB << 31); 67170ca608bSYong Wu else 67270ca608bSYong Wu regval = lower_32_bits(data->protect_base) | 67370ca608bSYong Wu upper_32_bits(data->protect_base); 67470ca608bSYong Wu writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR); 67570ca608bSYong Wu 6766b717796SChao Hao if (data->enable_4GB && 6776b717796SChao Hao MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_VLD_PA_RNG)) { 67830e2fccfSYong Wu /* 67930e2fccfSYong Wu * If 4GB mode is enabled, the validate PA range is from 68030e2fccfSYong Wu * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30]. 68130e2fccfSYong Wu */ 68230e2fccfSYong Wu regval = F_MMU_VLD_PA_RNG(7, 4); 68330e2fccfSYong Wu writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG); 68430e2fccfSYong Wu } 6850df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_DCM_DIS); 68635c1b48dSChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, WR_THROT_EN)) { 68735c1b48dSChao Hao /* write command throttling mode */ 68835c1b48dSChao Hao regval = readl_relaxed(data->base + REG_MMU_WR_LEN_CTRL); 68935c1b48dSChao Hao regval &= ~F_MMU_WR_THROT_DIS_MASK; 69035c1b48dSChao Hao writel_relaxed(regval, data->base + REG_MMU_WR_LEN_CTRL); 69135c1b48dSChao Hao } 692e6dec923SYong Wu 6936b717796SChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) { 69475eed350SChao Hao /* The register is called STANDARD_AXI_MODE in this case */ 6954bb2bf4cSChao Hao regval = 0; 6964bb2bf4cSChao Hao } else { 6974bb2bf4cSChao Hao regval = readl_relaxed(data->base + REG_MMU_MISC_CTRL); 6984bb2bf4cSChao Hao regval &= ~F_MMU_STANDARD_AXI_MODE_MASK; 6994bb2bf4cSChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, OUT_ORDER_WR_EN)) 7004bb2bf4cSChao Hao regval &= ~F_MMU_IN_ORDER_WR_EN_MASK; 70175eed350SChao Hao } 7024bb2bf4cSChao Hao writel_relaxed(regval, data->base + REG_MMU_MISC_CTRL); 7030df4fabeSYong Wu 7040df4fabeSYong Wu if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, 7050df4fabeSYong Wu dev_name(data->dev), (void *)data)) { 7060df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); 7070df4fabeSYong Wu clk_disable_unprepare(data->bclk); 7080df4fabeSYong Wu dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); 7090df4fabeSYong Wu return -ENODEV; 7100df4fabeSYong Wu } 7110df4fabeSYong Wu 7120df4fabeSYong Wu return 0; 7130df4fabeSYong Wu } 7140df4fabeSYong Wu 7150df4fabeSYong Wu static const struct component_master_ops mtk_iommu_com_ops = { 7160df4fabeSYong Wu .bind = mtk_iommu_bind, 7170df4fabeSYong Wu .unbind = mtk_iommu_unbind, 7180df4fabeSYong Wu }; 7190df4fabeSYong Wu 7200df4fabeSYong Wu static int mtk_iommu_probe(struct platform_device *pdev) 7210df4fabeSYong Wu { 7220df4fabeSYong Wu struct mtk_iommu_data *data; 7230df4fabeSYong Wu struct device *dev = &pdev->dev; 724baf94e6eSYong Wu struct device_node *larbnode, *smicomm_node; 725baf94e6eSYong Wu struct platform_device *plarbdev; 726baf94e6eSYong Wu struct device_link *link; 7270df4fabeSYong Wu struct resource *res; 728b16c0170SJoerg Roedel resource_size_t ioaddr; 7290df4fabeSYong Wu struct component_match *match = NULL; 730c2c59456SMiles Chen struct regmap *infracfg; 7310df4fabeSYong Wu void *protect; 7320b6c0ad3SAndrzej Hajda int i, larb_nr, ret; 733c2c59456SMiles Chen u32 val; 734c2c59456SMiles Chen char *p; 7350df4fabeSYong Wu 7360df4fabeSYong Wu data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); 7370df4fabeSYong Wu if (!data) 7380df4fabeSYong Wu return -ENOMEM; 7390df4fabeSYong Wu data->dev = dev; 740cecdce9dSYong Wu data->plat_data = of_device_get_match_data(dev); 7410df4fabeSYong Wu 7420df4fabeSYong Wu /* Protect memory. HW will access here while translation fault.*/ 7430df4fabeSYong Wu protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); 7440df4fabeSYong Wu if (!protect) 7450df4fabeSYong Wu return -ENOMEM; 7460df4fabeSYong Wu data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); 7470df4fabeSYong Wu 748c2c59456SMiles Chen if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) { 749c2c59456SMiles Chen switch (data->plat_data->m4u_plat) { 750c2c59456SMiles Chen case M4U_MT2712: 751c2c59456SMiles Chen p = "mediatek,mt2712-infracfg"; 752c2c59456SMiles Chen break; 753c2c59456SMiles Chen case M4U_MT8173: 754c2c59456SMiles Chen p = "mediatek,mt8173-infracfg"; 755c2c59456SMiles Chen break; 756c2c59456SMiles Chen default: 757c2c59456SMiles Chen p = NULL; 758c2c59456SMiles Chen } 759c2c59456SMiles Chen 760c2c59456SMiles Chen infracfg = syscon_regmap_lookup_by_compatible(p); 761c2c59456SMiles Chen 762c2c59456SMiles Chen if (IS_ERR(infracfg)) 763c2c59456SMiles Chen return PTR_ERR(infracfg); 764c2c59456SMiles Chen 765c2c59456SMiles Chen ret = regmap_read(infracfg, REG_INFRA_MISC, &val); 766c2c59456SMiles Chen if (ret) 767c2c59456SMiles Chen return ret; 768c2c59456SMiles Chen data->enable_4GB = !!(val & F_DDR_4GB_SUPPORT_EN); 769c2c59456SMiles Chen } 77001e23c93SYong Wu 7710df4fabeSYong Wu res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 7720df4fabeSYong Wu data->base = devm_ioremap_resource(dev, res); 7730df4fabeSYong Wu if (IS_ERR(data->base)) 7740df4fabeSYong Wu return PTR_ERR(data->base); 775b16c0170SJoerg Roedel ioaddr = res->start; 7760df4fabeSYong Wu 7770df4fabeSYong Wu data->irq = platform_get_irq(pdev, 0); 7780df4fabeSYong Wu if (data->irq < 0) 7790df4fabeSYong Wu return data->irq; 7800df4fabeSYong Wu 7816b717796SChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) { 7820df4fabeSYong Wu data->bclk = devm_clk_get(dev, "bclk"); 7830df4fabeSYong Wu if (IS_ERR(data->bclk)) 7840df4fabeSYong Wu return PTR_ERR(data->bclk); 7852aa4c259SYong Wu } 7860df4fabeSYong Wu 7870df4fabeSYong Wu larb_nr = of_count_phandle_with_args(dev->of_node, 7880df4fabeSYong Wu "mediatek,larbs", NULL); 7890df4fabeSYong Wu if (larb_nr < 0) 7900df4fabeSYong Wu return larb_nr; 7910df4fabeSYong Wu 7920df4fabeSYong Wu for (i = 0; i < larb_nr; i++) { 793e6dec923SYong Wu u32 id; 7940df4fabeSYong Wu 7950df4fabeSYong Wu larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); 7960df4fabeSYong Wu if (!larbnode) 7970df4fabeSYong Wu return -EINVAL; 7980df4fabeSYong Wu 7991eb8e4e2SWen Yang if (!of_device_is_available(larbnode)) { 8001eb8e4e2SWen Yang of_node_put(larbnode); 8010df4fabeSYong Wu continue; 8021eb8e4e2SWen Yang } 8030df4fabeSYong Wu 804e6dec923SYong Wu ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id); 805e6dec923SYong Wu if (ret)/* The id is consecutive if there is no this property */ 806e6dec923SYong Wu id = i; 807e6dec923SYong Wu 8080df4fabeSYong Wu plarbdev = of_find_device_by_node(larbnode); 8091eb8e4e2SWen Yang if (!plarbdev) { 8101eb8e4e2SWen Yang of_node_put(larbnode); 8110df4fabeSYong Wu return -EPROBE_DEFER; 8121eb8e4e2SWen Yang } 8131ee9feb2SYong Wu data->larb_imu[id].dev = &plarbdev->dev; 8140df4fabeSYong Wu 81500c7c81fSRussell King component_match_add_release(dev, &match, release_of, 81600c7c81fSRussell King compare_of, larbnode); 8170df4fabeSYong Wu } 8180df4fabeSYong Wu 819baf94e6eSYong Wu /* Get smi-common dev from the last larb. */ 820baf94e6eSYong Wu smicomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0); 821baf94e6eSYong Wu if (!smicomm_node) 822baf94e6eSYong Wu return -EINVAL; 823baf94e6eSYong Wu 824baf94e6eSYong Wu plarbdev = of_find_device_by_node(smicomm_node); 825baf94e6eSYong Wu of_node_put(smicomm_node); 826baf94e6eSYong Wu data->smicomm_dev = &plarbdev->dev; 827baf94e6eSYong Wu 828c0b57581SYong Wu pm_runtime_enable(dev); 829c0b57581SYong Wu 830baf94e6eSYong Wu link = device_link_add(data->smicomm_dev, dev, 831baf94e6eSYong Wu DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); 832baf94e6eSYong Wu if (!link) { 833baf94e6eSYong Wu dev_err(dev, "Unable link %s.\n", dev_name(data->smicomm_dev)); 834c0b57581SYong Wu goto out_runtime_disable; 835baf94e6eSYong Wu } 836baf94e6eSYong Wu 8370df4fabeSYong Wu platform_set_drvdata(pdev, data); 8380df4fabeSYong Wu 839b16c0170SJoerg Roedel ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, 840b16c0170SJoerg Roedel "mtk-iommu.%pa", &ioaddr); 841b16c0170SJoerg Roedel if (ret) 842baf94e6eSYong Wu goto out_link_remove; 843b16c0170SJoerg Roedel 844b16c0170SJoerg Roedel iommu_device_set_ops(&data->iommu, &mtk_iommu_ops); 845b16c0170SJoerg Roedel iommu_device_set_fwnode(&data->iommu, &pdev->dev.of_node->fwnode); 846b16c0170SJoerg Roedel 847b16c0170SJoerg Roedel ret = iommu_device_register(&data->iommu); 848b16c0170SJoerg Roedel if (ret) 849986d9ec5SYong Wu goto out_sysfs_remove; 850b16c0170SJoerg Roedel 851da3cc91bSYong Wu spin_lock_init(&data->tlb_lock); 8527c3a2ec0SYong Wu list_add_tail(&data->list, &m4ulist); 8537c3a2ec0SYong Wu 854986d9ec5SYong Wu if (!iommu_present(&platform_bus_type)) { 855986d9ec5SYong Wu ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); 856986d9ec5SYong Wu if (ret) 857986d9ec5SYong Wu goto out_list_del; 858986d9ec5SYong Wu } 8590df4fabeSYong Wu 860986d9ec5SYong Wu ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); 861986d9ec5SYong Wu if (ret) 862986d9ec5SYong Wu goto out_bus_set_null; 863986d9ec5SYong Wu return ret; 864986d9ec5SYong Wu 865986d9ec5SYong Wu out_bus_set_null: 866986d9ec5SYong Wu bus_set_iommu(&platform_bus_type, NULL); 867986d9ec5SYong Wu out_list_del: 868986d9ec5SYong Wu list_del(&data->list); 869986d9ec5SYong Wu iommu_device_unregister(&data->iommu); 870986d9ec5SYong Wu out_sysfs_remove: 871986d9ec5SYong Wu iommu_device_sysfs_remove(&data->iommu); 872baf94e6eSYong Wu out_link_remove: 873baf94e6eSYong Wu device_link_remove(data->smicomm_dev, dev); 874c0b57581SYong Wu out_runtime_disable: 875c0b57581SYong Wu pm_runtime_disable(dev); 876986d9ec5SYong Wu return ret; 8770df4fabeSYong Wu } 8780df4fabeSYong Wu 8790df4fabeSYong Wu static int mtk_iommu_remove(struct platform_device *pdev) 8800df4fabeSYong Wu { 8810df4fabeSYong Wu struct mtk_iommu_data *data = platform_get_drvdata(pdev); 8820df4fabeSYong Wu 883b16c0170SJoerg Roedel iommu_device_sysfs_remove(&data->iommu); 884b16c0170SJoerg Roedel iommu_device_unregister(&data->iommu); 885b16c0170SJoerg Roedel 8860df4fabeSYong Wu if (iommu_present(&platform_bus_type)) 8870df4fabeSYong Wu bus_set_iommu(&platform_bus_type, NULL); 8880df4fabeSYong Wu 8890df4fabeSYong Wu clk_disable_unprepare(data->bclk); 890baf94e6eSYong Wu device_link_remove(data->smicomm_dev, &pdev->dev); 891c0b57581SYong Wu pm_runtime_disable(&pdev->dev); 8920df4fabeSYong Wu devm_free_irq(&pdev->dev, data->irq, data); 8930df4fabeSYong Wu component_master_del(&pdev->dev, &mtk_iommu_com_ops); 8940df4fabeSYong Wu return 0; 8950df4fabeSYong Wu } 8960df4fabeSYong Wu 89734665c79SYong Wu static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev) 8980df4fabeSYong Wu { 8990df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 9000df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 9010df4fabeSYong Wu void __iomem *base = data->base; 9020df4fabeSYong Wu 90335c1b48dSChao Hao reg->wr_len_ctrl = readl_relaxed(base + REG_MMU_WR_LEN_CTRL); 90475eed350SChao Hao reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL); 9050df4fabeSYong Wu reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); 9060df4fabeSYong Wu reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); 9070df4fabeSYong Wu reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0); 9080df4fabeSYong Wu reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); 90970ca608bSYong Wu reg->ivrp_paddr = readl_relaxed(base + REG_MMU_IVRP_PADDR); 910b9475b34SYong Wu reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG); 9116254b64fSYong Wu clk_disable_unprepare(data->bclk); 9120df4fabeSYong Wu return 0; 9130df4fabeSYong Wu } 9140df4fabeSYong Wu 91534665c79SYong Wu static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) 9160df4fabeSYong Wu { 9170df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 9180df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 919907ba6a1SYong Wu struct mtk_iommu_domain *m4u_dom = data->m4u_dom; 9200df4fabeSYong Wu void __iomem *base = data->base; 9216254b64fSYong Wu int ret; 9220df4fabeSYong Wu 923c0b57581SYong Wu /* Avoid first resume to affect the default value of registers below. */ 924c0b57581SYong Wu if (!m4u_dom) 925c0b57581SYong Wu return 0; 9266254b64fSYong Wu ret = clk_prepare_enable(data->bclk); 9276254b64fSYong Wu if (ret) { 9286254b64fSYong Wu dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret); 9296254b64fSYong Wu return ret; 9306254b64fSYong Wu } 93135c1b48dSChao Hao writel_relaxed(reg->wr_len_ctrl, base + REG_MMU_WR_LEN_CTRL); 93275eed350SChao Hao writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL); 9330df4fabeSYong Wu writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); 9340df4fabeSYong Wu writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); 9350df4fabeSYong Wu writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); 9360df4fabeSYong Wu writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); 93770ca608bSYong Wu writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR); 938b9475b34SYong Wu writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG); 939c0b57581SYong Wu writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, base + REG_MMU_PT_BASE_ADDR); 9400df4fabeSYong Wu return 0; 9410df4fabeSYong Wu } 9420df4fabeSYong Wu 943e6dec923SYong Wu static const struct dev_pm_ops mtk_iommu_pm_ops = { 94434665c79SYong Wu SET_RUNTIME_PM_OPS(mtk_iommu_runtime_suspend, mtk_iommu_runtime_resume, NULL) 94534665c79SYong Wu SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 94634665c79SYong Wu pm_runtime_force_resume) 9470df4fabeSYong Wu }; 9480df4fabeSYong Wu 949cecdce9dSYong Wu static const struct mtk_iommu_plat_data mt2712_data = { 950cecdce9dSYong Wu .m4u_plat = M4U_MT2712, 9516b717796SChao Hao .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG, 952b053bc71SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 953585e58f4SYong Wu .iova_region = single_domain, 954585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 95537276e00SChao Hao .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, 956cecdce9dSYong Wu }; 957cecdce9dSYong Wu 958068c86e9SChao Hao static const struct mtk_iommu_plat_data mt6779_data = { 959068c86e9SChao Hao .m4u_plat = M4U_MT6779, 960068c86e9SChao Hao .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | WR_THROT_EN, 961068c86e9SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN2, 962585e58f4SYong Wu .iova_region = single_domain, 963585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 964068c86e9SChao Hao .larbid_remap = {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}}, 965cecdce9dSYong Wu }; 966cecdce9dSYong Wu 9673c213562SFabien Parent static const struct mtk_iommu_plat_data mt8167_data = { 9683c213562SFabien Parent .m4u_plat = M4U_MT8167, 9693c213562SFabien Parent .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR, 9703c213562SFabien Parent .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 971585e58f4SYong Wu .iova_region = single_domain, 972585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 9733c213562SFabien Parent .larbid_remap = {{0}, {1}, {2}}, /* Linear mapping. */ 9743c213562SFabien Parent }; 9753c213562SFabien Parent 976cecdce9dSYong Wu static const struct mtk_iommu_plat_data mt8173_data = { 977cecdce9dSYong Wu .m4u_plat = M4U_MT8173, 978d1b5ef00SFabien Parent .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | 979d1b5ef00SFabien Parent HAS_LEGACY_IVRP_PADDR, 980b053bc71SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 981585e58f4SYong Wu .iova_region = single_domain, 982585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 98337276e00SChao Hao .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */ 984cecdce9dSYong Wu }; 985cecdce9dSYong Wu 986907ba6a1SYong Wu static const struct mtk_iommu_plat_data mt8183_data = { 987907ba6a1SYong Wu .m4u_plat = M4U_MT8183, 9886b717796SChao Hao .flags = RESET_AXI, 989b053bc71SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 990585e58f4SYong Wu .iova_region = single_domain, 991585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 99237276e00SChao Hao .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}}, 993907ba6a1SYong Wu }; 994907ba6a1SYong Wu 9950df4fabeSYong Wu static const struct of_device_id mtk_iommu_of_ids[] = { 996cecdce9dSYong Wu { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, 997068c86e9SChao Hao { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data}, 9983c213562SFabien Parent { .compatible = "mediatek,mt8167-m4u", .data = &mt8167_data}, 999cecdce9dSYong Wu { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, 1000907ba6a1SYong Wu { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, 10010df4fabeSYong Wu {} 10020df4fabeSYong Wu }; 10030df4fabeSYong Wu 10040df4fabeSYong Wu static struct platform_driver mtk_iommu_driver = { 10050df4fabeSYong Wu .probe = mtk_iommu_probe, 10060df4fabeSYong Wu .remove = mtk_iommu_remove, 10070df4fabeSYong Wu .driver = { 10080df4fabeSYong Wu .name = "mtk-iommu", 1009f53dd978SKrzysztof Kozlowski .of_match_table = mtk_iommu_of_ids, 10100df4fabeSYong Wu .pm = &mtk_iommu_pm_ops, 10110df4fabeSYong Wu } 10120df4fabeSYong Wu }; 10130df4fabeSYong Wu 1014e6dec923SYong Wu static int __init mtk_iommu_init(void) 10150df4fabeSYong Wu { 10160df4fabeSYong Wu int ret; 10170df4fabeSYong Wu 10180df4fabeSYong Wu ret = platform_driver_register(&mtk_iommu_driver); 1019e6dec923SYong Wu if (ret != 0) 1020e6dec923SYong Wu pr_err("Failed to register MTK IOMMU driver\n"); 1021e6dec923SYong Wu 10220df4fabeSYong Wu return ret; 10230df4fabeSYong Wu } 10240df4fabeSYong Wu 1025e6dec923SYong Wu subsys_initcall(mtk_iommu_init) 1026