10df4fabeSYong Wu /* 20df4fabeSYong Wu * Copyright (c) 2015-2016 MediaTek Inc. 30df4fabeSYong Wu * Author: Yong Wu <yong.wu@mediatek.com> 40df4fabeSYong Wu * 50df4fabeSYong Wu * This program is free software; you can redistribute it and/or modify 60df4fabeSYong Wu * it under the terms of the GNU General Public License version 2 as 70df4fabeSYong Wu * published by the Free Software Foundation. 80df4fabeSYong Wu * 90df4fabeSYong Wu * This program is distributed in the hope that it will be useful, 100df4fabeSYong Wu * but WITHOUT ANY WARRANTY; without even the implied warranty of 110df4fabeSYong Wu * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 120df4fabeSYong Wu * GNU General Public License for more details. 130df4fabeSYong Wu */ 1401e23c93SYong Wu #include <linux/bootmem.h> 150df4fabeSYong Wu #include <linux/bug.h> 160df4fabeSYong Wu #include <linux/clk.h> 170df4fabeSYong Wu #include <linux/component.h> 180df4fabeSYong Wu #include <linux/device.h> 190df4fabeSYong Wu #include <linux/dma-iommu.h> 200df4fabeSYong Wu #include <linux/err.h> 210df4fabeSYong Wu #include <linux/interrupt.h> 220df4fabeSYong Wu #include <linux/io.h> 230df4fabeSYong Wu #include <linux/iommu.h> 240df4fabeSYong Wu #include <linux/iopoll.h> 250df4fabeSYong Wu #include <linux/list.h> 260df4fabeSYong Wu #include <linux/of_address.h> 270df4fabeSYong Wu #include <linux/of_iommu.h> 280df4fabeSYong Wu #include <linux/of_irq.h> 290df4fabeSYong Wu #include <linux/of_platform.h> 300df4fabeSYong Wu #include <linux/platform_device.h> 310df4fabeSYong Wu #include <linux/slab.h> 320df4fabeSYong Wu #include <linux/spinlock.h> 330df4fabeSYong Wu #include <asm/barrier.h> 340df4fabeSYong Wu #include <dt-bindings/memory/mt8173-larb-port.h> 350df4fabeSYong Wu #include <soc/mediatek/smi.h> 360df4fabeSYong Wu 379ca340c9SHonghui Zhang #include "mtk_iommu.h" 380df4fabeSYong Wu 390df4fabeSYong Wu #define REG_MMU_PT_BASE_ADDR 0x000 400df4fabeSYong Wu 410df4fabeSYong Wu #define REG_MMU_INVALIDATE 0x020 420df4fabeSYong Wu #define F_ALL_INVLD 0x2 430df4fabeSYong Wu #define F_MMU_INV_RANGE 0x1 440df4fabeSYong Wu 450df4fabeSYong Wu #define REG_MMU_INVLD_START_A 0x024 460df4fabeSYong Wu #define REG_MMU_INVLD_END_A 0x028 470df4fabeSYong Wu 480df4fabeSYong Wu #define REG_MMU_INV_SEL 0x038 490df4fabeSYong Wu #define F_INVLD_EN0 BIT(0) 500df4fabeSYong Wu #define F_INVLD_EN1 BIT(1) 510df4fabeSYong Wu 520df4fabeSYong Wu #define REG_MMU_STANDARD_AXI_MODE 0x048 530df4fabeSYong Wu #define REG_MMU_DCM_DIS 0x050 540df4fabeSYong Wu 550df4fabeSYong Wu #define REG_MMU_CTRL_REG 0x110 560df4fabeSYong Wu #define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4) 570df4fabeSYong Wu #define F_MMU_TF_PROTECT_SEL(prot) (((prot) & 0x3) << 5) 580df4fabeSYong Wu 590df4fabeSYong Wu #define REG_MMU_IVRP_PADDR 0x114 6001e23c93SYong Wu #define F_MMU_IVRP_PA_SET(pa, ext) (((pa) >> 1) | ((!!(ext)) << 31)) 610df4fabeSYong Wu 620df4fabeSYong Wu #define REG_MMU_INT_CONTROL0 0x120 630df4fabeSYong Wu #define F_L2_MULIT_HIT_EN BIT(0) 640df4fabeSYong Wu #define F_TABLE_WALK_FAULT_INT_EN BIT(1) 650df4fabeSYong Wu #define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2) 660df4fabeSYong Wu #define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3) 670df4fabeSYong Wu #define F_PREFETCH_FIFO_ERR_INT_EN BIT(5) 680df4fabeSYong Wu #define F_MISS_FIFO_ERR_INT_EN BIT(6) 690df4fabeSYong Wu #define F_INT_CLR_BIT BIT(12) 700df4fabeSYong Wu 710df4fabeSYong Wu #define REG_MMU_INT_MAIN_CONTROL 0x124 720df4fabeSYong Wu #define F_INT_TRANSLATION_FAULT BIT(0) 730df4fabeSYong Wu #define F_INT_MAIN_MULTI_HIT_FAULT BIT(1) 740df4fabeSYong Wu #define F_INT_INVALID_PA_FAULT BIT(2) 750df4fabeSYong Wu #define F_INT_ENTRY_REPLACEMENT_FAULT BIT(3) 760df4fabeSYong Wu #define F_INT_TLB_MISS_FAULT BIT(4) 770df4fabeSYong Wu #define F_INT_MISS_TRANSACTION_FIFO_FAULT BIT(5) 780df4fabeSYong Wu #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT BIT(6) 790df4fabeSYong Wu 800df4fabeSYong Wu #define REG_MMU_CPE_DONE 0x12C 810df4fabeSYong Wu 820df4fabeSYong Wu #define REG_MMU_FAULT_ST1 0x134 830df4fabeSYong Wu 840df4fabeSYong Wu #define REG_MMU_FAULT_VA 0x13c 850df4fabeSYong Wu #define F_MMU_FAULT_VA_MSK 0xfffff000 860df4fabeSYong Wu #define F_MMU_FAULT_VA_WRITE_BIT BIT(1) 870df4fabeSYong Wu #define F_MMU_FAULT_VA_LAYER_BIT BIT(0) 880df4fabeSYong Wu 890df4fabeSYong Wu #define REG_MMU_INVLD_PA 0x140 900df4fabeSYong Wu #define REG_MMU_INT_ID 0x150 910df4fabeSYong Wu #define F_MMU0_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) 920df4fabeSYong Wu #define F_MMU0_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) 930df4fabeSYong Wu 940df4fabeSYong Wu #define MTK_PROTECT_PA_ALIGN 128 950df4fabeSYong Wu 960df4fabeSYong Wu struct mtk_iommu_domain { 970df4fabeSYong Wu spinlock_t pgtlock; /* lock for page table */ 980df4fabeSYong Wu 990df4fabeSYong Wu struct io_pgtable_cfg cfg; 1000df4fabeSYong Wu struct io_pgtable_ops *iop; 1010df4fabeSYong Wu 1020df4fabeSYong Wu struct iommu_domain domain; 1030df4fabeSYong Wu }; 1040df4fabeSYong Wu 1050df4fabeSYong Wu static struct iommu_ops mtk_iommu_ops; 1060df4fabeSYong Wu 1070df4fabeSYong Wu static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) 1080df4fabeSYong Wu { 1090df4fabeSYong Wu return container_of(dom, struct mtk_iommu_domain, domain); 1100df4fabeSYong Wu } 1110df4fabeSYong Wu 1120df4fabeSYong Wu static void mtk_iommu_tlb_flush_all(void *cookie) 1130df4fabeSYong Wu { 1140df4fabeSYong Wu struct mtk_iommu_data *data = cookie; 1150df4fabeSYong Wu 1160df4fabeSYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL); 1170df4fabeSYong Wu writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); 1180df4fabeSYong Wu wmb(); /* Make sure the tlb flush all done */ 1190df4fabeSYong Wu } 1200df4fabeSYong Wu 1210df4fabeSYong Wu static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size, 1220df4fabeSYong Wu size_t granule, bool leaf, 1230df4fabeSYong Wu void *cookie) 1240df4fabeSYong Wu { 1250df4fabeSYong Wu struct mtk_iommu_data *data = cookie; 1260df4fabeSYong Wu 1270df4fabeSYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL); 1280df4fabeSYong Wu 1290df4fabeSYong Wu writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A); 1300df4fabeSYong Wu writel_relaxed(iova + size - 1, data->base + REG_MMU_INVLD_END_A); 1310df4fabeSYong Wu writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE); 1320df4fabeSYong Wu } 1330df4fabeSYong Wu 1340df4fabeSYong Wu static void mtk_iommu_tlb_sync(void *cookie) 1350df4fabeSYong Wu { 1360df4fabeSYong Wu struct mtk_iommu_data *data = cookie; 1370df4fabeSYong Wu int ret; 1380df4fabeSYong Wu u32 tmp; 1390df4fabeSYong Wu 1400df4fabeSYong Wu ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, tmp, 1410df4fabeSYong Wu tmp != 0, 10, 100000); 1420df4fabeSYong Wu if (ret) { 1430df4fabeSYong Wu dev_warn(data->dev, 1440df4fabeSYong Wu "Partial TLB flush timed out, falling back to full flush\n"); 1450df4fabeSYong Wu mtk_iommu_tlb_flush_all(cookie); 1460df4fabeSYong Wu } 1470df4fabeSYong Wu /* Clear the CPE status */ 1480df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_CPE_DONE); 1490df4fabeSYong Wu } 1500df4fabeSYong Wu 1510df4fabeSYong Wu static const struct iommu_gather_ops mtk_iommu_gather_ops = { 1520df4fabeSYong Wu .tlb_flush_all = mtk_iommu_tlb_flush_all, 1530df4fabeSYong Wu .tlb_add_flush = mtk_iommu_tlb_add_flush_nosync, 1540df4fabeSYong Wu .tlb_sync = mtk_iommu_tlb_sync, 1550df4fabeSYong Wu }; 1560df4fabeSYong Wu 1570df4fabeSYong Wu static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) 1580df4fabeSYong Wu { 1590df4fabeSYong Wu struct mtk_iommu_data *data = dev_id; 1600df4fabeSYong Wu struct mtk_iommu_domain *dom = data->m4u_dom; 1610df4fabeSYong Wu u32 int_state, regval, fault_iova, fault_pa; 1620df4fabeSYong Wu unsigned int fault_larb, fault_port; 1630df4fabeSYong Wu bool layer, write; 1640df4fabeSYong Wu 1650df4fabeSYong Wu /* Read error info from registers */ 1660df4fabeSYong Wu int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1); 1670df4fabeSYong Wu fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA); 1680df4fabeSYong Wu layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; 1690df4fabeSYong Wu write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; 1700df4fabeSYong Wu fault_iova &= F_MMU_FAULT_VA_MSK; 1710df4fabeSYong Wu fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA); 1720df4fabeSYong Wu regval = readl_relaxed(data->base + REG_MMU_INT_ID); 1730df4fabeSYong Wu fault_larb = F_MMU0_INT_ID_LARB_ID(regval); 1740df4fabeSYong Wu fault_port = F_MMU0_INT_ID_PORT_ID(regval); 1750df4fabeSYong Wu 1760df4fabeSYong Wu if (report_iommu_fault(&dom->domain, data->dev, fault_iova, 1770df4fabeSYong Wu write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { 1780df4fabeSYong Wu dev_err_ratelimited( 1790df4fabeSYong Wu data->dev, 1800df4fabeSYong Wu "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d layer=%d %s\n", 1810df4fabeSYong Wu int_state, fault_iova, fault_pa, fault_larb, fault_port, 1820df4fabeSYong Wu layer, write ? "write" : "read"); 1830df4fabeSYong Wu } 1840df4fabeSYong Wu 1850df4fabeSYong Wu /* Interrupt clear */ 1860df4fabeSYong Wu regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL0); 1870df4fabeSYong Wu regval |= F_INT_CLR_BIT; 1880df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); 1890df4fabeSYong Wu 1900df4fabeSYong Wu mtk_iommu_tlb_flush_all(data); 1910df4fabeSYong Wu 1920df4fabeSYong Wu return IRQ_HANDLED; 1930df4fabeSYong Wu } 1940df4fabeSYong Wu 1950df4fabeSYong Wu static void mtk_iommu_config(struct mtk_iommu_data *data, 1960df4fabeSYong Wu struct device *dev, bool enable) 1970df4fabeSYong Wu { 1980df4fabeSYong Wu struct mtk_smi_larb_iommu *larb_mmu; 1990df4fabeSYong Wu unsigned int larbid, portid; 200*58f0d1d5SRobin Murphy struct iommu_fwspec *fwspec = dev->iommu_fwspec; 201*58f0d1d5SRobin Murphy int i; 2020df4fabeSYong Wu 203*58f0d1d5SRobin Murphy for (i = 0; i < fwspec->num_ids; ++i) { 204*58f0d1d5SRobin Murphy larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); 205*58f0d1d5SRobin Murphy portid = MTK_M4U_TO_PORT(fwspec->ids[i]); 2060df4fabeSYong Wu larb_mmu = &data->smi_imu.larb_imu[larbid]; 2070df4fabeSYong Wu 2080df4fabeSYong Wu dev_dbg(dev, "%s iommu port: %d\n", 2090df4fabeSYong Wu enable ? "enable" : "disable", portid); 2100df4fabeSYong Wu 2110df4fabeSYong Wu if (enable) 2120df4fabeSYong Wu larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); 2130df4fabeSYong Wu else 2140df4fabeSYong Wu larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); 2150df4fabeSYong Wu } 2160df4fabeSYong Wu } 2170df4fabeSYong Wu 2180df4fabeSYong Wu static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data) 2190df4fabeSYong Wu { 2200df4fabeSYong Wu struct mtk_iommu_domain *dom = data->m4u_dom; 2210df4fabeSYong Wu 2220df4fabeSYong Wu spin_lock_init(&dom->pgtlock); 2230df4fabeSYong Wu 2240df4fabeSYong Wu dom->cfg = (struct io_pgtable_cfg) { 2250df4fabeSYong Wu .quirks = IO_PGTABLE_QUIRK_ARM_NS | 2260df4fabeSYong Wu IO_PGTABLE_QUIRK_NO_PERMS | 2270df4fabeSYong Wu IO_PGTABLE_QUIRK_TLBI_ON_MAP, 2280df4fabeSYong Wu .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, 2290df4fabeSYong Wu .ias = 32, 2300df4fabeSYong Wu .oas = 32, 2310df4fabeSYong Wu .tlb = &mtk_iommu_gather_ops, 2320df4fabeSYong Wu .iommu_dev = data->dev, 2330df4fabeSYong Wu }; 2340df4fabeSYong Wu 23501e23c93SYong Wu if (data->enable_4GB) 23601e23c93SYong Wu dom->cfg.quirks |= IO_PGTABLE_QUIRK_ARM_MTK_4GB; 23701e23c93SYong Wu 2380df4fabeSYong Wu dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data); 2390df4fabeSYong Wu if (!dom->iop) { 2400df4fabeSYong Wu dev_err(data->dev, "Failed to alloc io pgtable\n"); 2410df4fabeSYong Wu return -EINVAL; 2420df4fabeSYong Wu } 2430df4fabeSYong Wu 2440df4fabeSYong Wu /* Update our support page sizes bitmap */ 245d16e0faaSRobin Murphy dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; 2460df4fabeSYong Wu 2470df4fabeSYong Wu writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0], 2480df4fabeSYong Wu data->base + REG_MMU_PT_BASE_ADDR); 2490df4fabeSYong Wu return 0; 2500df4fabeSYong Wu } 2510df4fabeSYong Wu 2520df4fabeSYong Wu static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) 2530df4fabeSYong Wu { 2540df4fabeSYong Wu struct mtk_iommu_domain *dom; 2550df4fabeSYong Wu 2560df4fabeSYong Wu if (type != IOMMU_DOMAIN_DMA) 2570df4fabeSYong Wu return NULL; 2580df4fabeSYong Wu 2590df4fabeSYong Wu dom = kzalloc(sizeof(*dom), GFP_KERNEL); 2600df4fabeSYong Wu if (!dom) 2610df4fabeSYong Wu return NULL; 2620df4fabeSYong Wu 2630df4fabeSYong Wu if (iommu_get_dma_cookie(&dom->domain)) { 2640df4fabeSYong Wu kfree(dom); 2650df4fabeSYong Wu return NULL; 2660df4fabeSYong Wu } 2670df4fabeSYong Wu 2680df4fabeSYong Wu dom->domain.geometry.aperture_start = 0; 2690df4fabeSYong Wu dom->domain.geometry.aperture_end = DMA_BIT_MASK(32); 2700df4fabeSYong Wu dom->domain.geometry.force_aperture = true; 2710df4fabeSYong Wu 2720df4fabeSYong Wu return &dom->domain; 2730df4fabeSYong Wu } 2740df4fabeSYong Wu 2750df4fabeSYong Wu static void mtk_iommu_domain_free(struct iommu_domain *domain) 2760df4fabeSYong Wu { 2770df4fabeSYong Wu iommu_put_dma_cookie(domain); 2780df4fabeSYong Wu kfree(to_mtk_domain(domain)); 2790df4fabeSYong Wu } 2800df4fabeSYong Wu 2810df4fabeSYong Wu static int mtk_iommu_attach_device(struct iommu_domain *domain, 2820df4fabeSYong Wu struct device *dev) 2830df4fabeSYong Wu { 2840df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 285*58f0d1d5SRobin Murphy struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv; 2860df4fabeSYong Wu int ret; 2870df4fabeSYong Wu 288*58f0d1d5SRobin Murphy if (!data) 2890df4fabeSYong Wu return -ENODEV; 2900df4fabeSYong Wu 2910df4fabeSYong Wu if (!data->m4u_dom) { 2920df4fabeSYong Wu data->m4u_dom = dom; 2930df4fabeSYong Wu ret = mtk_iommu_domain_finalise(data); 2940df4fabeSYong Wu if (ret) { 2950df4fabeSYong Wu data->m4u_dom = NULL; 2960df4fabeSYong Wu return ret; 2970df4fabeSYong Wu } 2980df4fabeSYong Wu } else if (data->m4u_dom != dom) { 2990df4fabeSYong Wu /* All the client devices should be in the same m4u domain */ 3000df4fabeSYong Wu dev_err(dev, "try to attach into the error iommu domain\n"); 3010df4fabeSYong Wu return -EPERM; 3020df4fabeSYong Wu } 3030df4fabeSYong Wu 3040df4fabeSYong Wu mtk_iommu_config(data, dev, true); 3050df4fabeSYong Wu return 0; 3060df4fabeSYong Wu } 3070df4fabeSYong Wu 3080df4fabeSYong Wu static void mtk_iommu_detach_device(struct iommu_domain *domain, 3090df4fabeSYong Wu struct device *dev) 3100df4fabeSYong Wu { 311*58f0d1d5SRobin Murphy struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv; 3120df4fabeSYong Wu 313*58f0d1d5SRobin Murphy if (!data) 3140df4fabeSYong Wu return; 3150df4fabeSYong Wu 3160df4fabeSYong Wu mtk_iommu_config(data, dev, false); 3170df4fabeSYong Wu } 3180df4fabeSYong Wu 3190df4fabeSYong Wu static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, 3200df4fabeSYong Wu phys_addr_t paddr, size_t size, int prot) 3210df4fabeSYong Wu { 3220df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 3230df4fabeSYong Wu unsigned long flags; 3240df4fabeSYong Wu int ret; 3250df4fabeSYong Wu 3260df4fabeSYong Wu spin_lock_irqsave(&dom->pgtlock, flags); 3270df4fabeSYong Wu ret = dom->iop->map(dom->iop, iova, paddr, size, prot); 3280df4fabeSYong Wu spin_unlock_irqrestore(&dom->pgtlock, flags); 3290df4fabeSYong Wu 3300df4fabeSYong Wu return ret; 3310df4fabeSYong Wu } 3320df4fabeSYong Wu 3330df4fabeSYong Wu static size_t mtk_iommu_unmap(struct iommu_domain *domain, 3340df4fabeSYong Wu unsigned long iova, size_t size) 3350df4fabeSYong Wu { 3360df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 3370df4fabeSYong Wu unsigned long flags; 3380df4fabeSYong Wu size_t unmapsz; 3390df4fabeSYong Wu 3400df4fabeSYong Wu spin_lock_irqsave(&dom->pgtlock, flags); 3410df4fabeSYong Wu unmapsz = dom->iop->unmap(dom->iop, iova, size); 3420df4fabeSYong Wu spin_unlock_irqrestore(&dom->pgtlock, flags); 3430df4fabeSYong Wu 3440df4fabeSYong Wu return unmapsz; 3450df4fabeSYong Wu } 3460df4fabeSYong Wu 3470df4fabeSYong Wu static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, 3480df4fabeSYong Wu dma_addr_t iova) 3490df4fabeSYong Wu { 3500df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 3510df4fabeSYong Wu unsigned long flags; 3520df4fabeSYong Wu phys_addr_t pa; 3530df4fabeSYong Wu 3540df4fabeSYong Wu spin_lock_irqsave(&dom->pgtlock, flags); 3550df4fabeSYong Wu pa = dom->iop->iova_to_phys(dom->iop, iova); 3560df4fabeSYong Wu spin_unlock_irqrestore(&dom->pgtlock, flags); 3570df4fabeSYong Wu 3580df4fabeSYong Wu return pa; 3590df4fabeSYong Wu } 3600df4fabeSYong Wu 3610df4fabeSYong Wu static int mtk_iommu_add_device(struct device *dev) 3620df4fabeSYong Wu { 3630df4fabeSYong Wu struct iommu_group *group; 3640df4fabeSYong Wu 365*58f0d1d5SRobin Murphy if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops) 366*58f0d1d5SRobin Murphy return -ENODEV; /* Not a iommu client device */ 3670df4fabeSYong Wu 3680df4fabeSYong Wu group = iommu_group_get_for_dev(dev); 3690df4fabeSYong Wu if (IS_ERR(group)) 3700df4fabeSYong Wu return PTR_ERR(group); 3710df4fabeSYong Wu 3720df4fabeSYong Wu iommu_group_put(group); 3730df4fabeSYong Wu return 0; 3740df4fabeSYong Wu } 3750df4fabeSYong Wu 3760df4fabeSYong Wu static void mtk_iommu_remove_device(struct device *dev) 3770df4fabeSYong Wu { 378*58f0d1d5SRobin Murphy if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops) 3790df4fabeSYong Wu return; 3800df4fabeSYong Wu 3810df4fabeSYong Wu iommu_group_remove_device(dev); 382*58f0d1d5SRobin Murphy iommu_fwspec_free(dev); 3830df4fabeSYong Wu } 3840df4fabeSYong Wu 3850df4fabeSYong Wu static struct iommu_group *mtk_iommu_device_group(struct device *dev) 3860df4fabeSYong Wu { 387*58f0d1d5SRobin Murphy struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv; 3880df4fabeSYong Wu 389*58f0d1d5SRobin Murphy if (!data) 3900df4fabeSYong Wu return ERR_PTR(-ENODEV); 3910df4fabeSYong Wu 3920df4fabeSYong Wu /* All the client devices are in the same m4u iommu-group */ 3930df4fabeSYong Wu if (!data->m4u_group) { 3940df4fabeSYong Wu data->m4u_group = iommu_group_alloc(); 3950df4fabeSYong Wu if (IS_ERR(data->m4u_group)) 3960df4fabeSYong Wu dev_err(dev, "Failed to allocate M4U IOMMU group\n"); 3970df4fabeSYong Wu } 3980df4fabeSYong Wu return data->m4u_group; 3990df4fabeSYong Wu } 4000df4fabeSYong Wu 4010df4fabeSYong Wu static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) 4020df4fabeSYong Wu { 4030df4fabeSYong Wu struct platform_device *m4updev; 4040df4fabeSYong Wu 4050df4fabeSYong Wu if (args->args_count != 1) { 4060df4fabeSYong Wu dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", 4070df4fabeSYong Wu args->args_count); 4080df4fabeSYong Wu return -EINVAL; 4090df4fabeSYong Wu } 4100df4fabeSYong Wu 411*58f0d1d5SRobin Murphy if (!dev->iommu_fwspec->iommu_priv) { 4120df4fabeSYong Wu /* Get the m4u device */ 4130df4fabeSYong Wu m4updev = of_find_device_by_node(args->np); 4140df4fabeSYong Wu if (WARN_ON(!m4updev)) 4150df4fabeSYong Wu return -EINVAL; 4160df4fabeSYong Wu 417*58f0d1d5SRobin Murphy dev->iommu_fwspec->iommu_priv = platform_get_drvdata(m4updev); 4180df4fabeSYong Wu } 4190df4fabeSYong Wu 420*58f0d1d5SRobin Murphy return iommu_fwspec_add_ids(dev, args->args, 1); 4210df4fabeSYong Wu } 4220df4fabeSYong Wu 4230df4fabeSYong Wu static struct iommu_ops mtk_iommu_ops = { 4240df4fabeSYong Wu .domain_alloc = mtk_iommu_domain_alloc, 4250df4fabeSYong Wu .domain_free = mtk_iommu_domain_free, 4260df4fabeSYong Wu .attach_dev = mtk_iommu_attach_device, 4270df4fabeSYong Wu .detach_dev = mtk_iommu_detach_device, 4280df4fabeSYong Wu .map = mtk_iommu_map, 4290df4fabeSYong Wu .unmap = mtk_iommu_unmap, 4300df4fabeSYong Wu .map_sg = default_iommu_map_sg, 4310df4fabeSYong Wu .iova_to_phys = mtk_iommu_iova_to_phys, 4320df4fabeSYong Wu .add_device = mtk_iommu_add_device, 4330df4fabeSYong Wu .remove_device = mtk_iommu_remove_device, 4340df4fabeSYong Wu .device_group = mtk_iommu_device_group, 4350df4fabeSYong Wu .of_xlate = mtk_iommu_of_xlate, 4360df4fabeSYong Wu .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, 4370df4fabeSYong Wu }; 4380df4fabeSYong Wu 4390df4fabeSYong Wu static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) 4400df4fabeSYong Wu { 4410df4fabeSYong Wu u32 regval; 4420df4fabeSYong Wu int ret; 4430df4fabeSYong Wu 4440df4fabeSYong Wu ret = clk_prepare_enable(data->bclk); 4450df4fabeSYong Wu if (ret) { 4460df4fabeSYong Wu dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret); 4470df4fabeSYong Wu return ret; 4480df4fabeSYong Wu } 4490df4fabeSYong Wu 4500df4fabeSYong Wu regval = F_MMU_PREFETCH_RT_REPLACE_MOD | 4510df4fabeSYong Wu F_MMU_TF_PROTECT_SEL(2); 4520df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); 4530df4fabeSYong Wu 4540df4fabeSYong Wu regval = F_L2_MULIT_HIT_EN | 4550df4fabeSYong Wu F_TABLE_WALK_FAULT_INT_EN | 4560df4fabeSYong Wu F_PREETCH_FIFO_OVERFLOW_INT_EN | 4570df4fabeSYong Wu F_MISS_FIFO_OVERFLOW_INT_EN | 4580df4fabeSYong Wu F_PREFETCH_FIFO_ERR_INT_EN | 4590df4fabeSYong Wu F_MISS_FIFO_ERR_INT_EN; 4600df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); 4610df4fabeSYong Wu 4620df4fabeSYong Wu regval = F_INT_TRANSLATION_FAULT | 4630df4fabeSYong Wu F_INT_MAIN_MULTI_HIT_FAULT | 4640df4fabeSYong Wu F_INT_INVALID_PA_FAULT | 4650df4fabeSYong Wu F_INT_ENTRY_REPLACEMENT_FAULT | 4660df4fabeSYong Wu F_INT_TLB_MISS_FAULT | 4670df4fabeSYong Wu F_INT_MISS_TRANSACTION_FIFO_FAULT | 4680df4fabeSYong Wu F_INT_PRETETCH_TRANSATION_FIFO_FAULT; 4690df4fabeSYong Wu writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); 4700df4fabeSYong Wu 47101e23c93SYong Wu writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base, data->enable_4GB), 4720df4fabeSYong Wu data->base + REG_MMU_IVRP_PADDR); 4730df4fabeSYong Wu 4740df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_DCM_DIS); 4750df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE); 4760df4fabeSYong Wu 4770df4fabeSYong Wu if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, 4780df4fabeSYong Wu dev_name(data->dev), (void *)data)) { 4790df4fabeSYong Wu writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); 4800df4fabeSYong Wu clk_disable_unprepare(data->bclk); 4810df4fabeSYong Wu dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); 4820df4fabeSYong Wu return -ENODEV; 4830df4fabeSYong Wu } 4840df4fabeSYong Wu 4850df4fabeSYong Wu return 0; 4860df4fabeSYong Wu } 4870df4fabeSYong Wu 4880df4fabeSYong Wu static const struct component_master_ops mtk_iommu_com_ops = { 4890df4fabeSYong Wu .bind = mtk_iommu_bind, 4900df4fabeSYong Wu .unbind = mtk_iommu_unbind, 4910df4fabeSYong Wu }; 4920df4fabeSYong Wu 4930df4fabeSYong Wu static int mtk_iommu_probe(struct platform_device *pdev) 4940df4fabeSYong Wu { 4950df4fabeSYong Wu struct mtk_iommu_data *data; 4960df4fabeSYong Wu struct device *dev = &pdev->dev; 4970df4fabeSYong Wu struct resource *res; 4980df4fabeSYong Wu struct component_match *match = NULL; 4990df4fabeSYong Wu void *protect; 5000b6c0ad3SAndrzej Hajda int i, larb_nr, ret; 5010df4fabeSYong Wu 5020df4fabeSYong Wu data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); 5030df4fabeSYong Wu if (!data) 5040df4fabeSYong Wu return -ENOMEM; 5050df4fabeSYong Wu data->dev = dev; 5060df4fabeSYong Wu 5070df4fabeSYong Wu /* Protect memory. HW will access here while translation fault.*/ 5080df4fabeSYong Wu protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); 5090df4fabeSYong Wu if (!protect) 5100df4fabeSYong Wu return -ENOMEM; 5110df4fabeSYong Wu data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); 5120df4fabeSYong Wu 51301e23c93SYong Wu /* Whether the current dram is over 4GB */ 51401e23c93SYong Wu data->enable_4GB = !!(max_pfn > (0xffffffffUL >> PAGE_SHIFT)); 51501e23c93SYong Wu 5160df4fabeSYong Wu res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 5170df4fabeSYong Wu data->base = devm_ioremap_resource(dev, res); 5180df4fabeSYong Wu if (IS_ERR(data->base)) 5190df4fabeSYong Wu return PTR_ERR(data->base); 5200df4fabeSYong Wu 5210df4fabeSYong Wu data->irq = platform_get_irq(pdev, 0); 5220df4fabeSYong Wu if (data->irq < 0) 5230df4fabeSYong Wu return data->irq; 5240df4fabeSYong Wu 5250df4fabeSYong Wu data->bclk = devm_clk_get(dev, "bclk"); 5260df4fabeSYong Wu if (IS_ERR(data->bclk)) 5270df4fabeSYong Wu return PTR_ERR(data->bclk); 5280df4fabeSYong Wu 5290df4fabeSYong Wu larb_nr = of_count_phandle_with_args(dev->of_node, 5300df4fabeSYong Wu "mediatek,larbs", NULL); 5310df4fabeSYong Wu if (larb_nr < 0) 5320df4fabeSYong Wu return larb_nr; 5330df4fabeSYong Wu data->smi_imu.larb_nr = larb_nr; 5340df4fabeSYong Wu 5350df4fabeSYong Wu for (i = 0; i < larb_nr; i++) { 5360df4fabeSYong Wu struct device_node *larbnode; 5370df4fabeSYong Wu struct platform_device *plarbdev; 5380df4fabeSYong Wu 5390df4fabeSYong Wu larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); 5400df4fabeSYong Wu if (!larbnode) 5410df4fabeSYong Wu return -EINVAL; 5420df4fabeSYong Wu 5430df4fabeSYong Wu if (!of_device_is_available(larbnode)) 5440df4fabeSYong Wu continue; 5450df4fabeSYong Wu 5460df4fabeSYong Wu plarbdev = of_find_device_by_node(larbnode); 5470df4fabeSYong Wu of_node_put(larbnode); 5480df4fabeSYong Wu if (!plarbdev) { 5490df4fabeSYong Wu plarbdev = of_platform_device_create( 5500df4fabeSYong Wu larbnode, NULL, 5510df4fabeSYong Wu platform_bus_type.dev_root); 5523189e490SDan Carpenter if (!plarbdev) 5530df4fabeSYong Wu return -EPROBE_DEFER; 5540df4fabeSYong Wu } 5550df4fabeSYong Wu data->smi_imu.larb_imu[i].dev = &plarbdev->dev; 5560df4fabeSYong Wu 5570df4fabeSYong Wu component_match_add(dev, &match, compare_of, larbnode); 5580df4fabeSYong Wu } 5590df4fabeSYong Wu 5600df4fabeSYong Wu platform_set_drvdata(pdev, data); 5610df4fabeSYong Wu 5620df4fabeSYong Wu ret = mtk_iommu_hw_init(data); 5630df4fabeSYong Wu if (ret) 5640df4fabeSYong Wu return ret; 5650df4fabeSYong Wu 5660df4fabeSYong Wu if (!iommu_present(&platform_bus_type)) 5670df4fabeSYong Wu bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); 5680df4fabeSYong Wu 5690df4fabeSYong Wu return component_master_add_with_match(dev, &mtk_iommu_com_ops, match); 5700df4fabeSYong Wu } 5710df4fabeSYong Wu 5720df4fabeSYong Wu static int mtk_iommu_remove(struct platform_device *pdev) 5730df4fabeSYong Wu { 5740df4fabeSYong Wu struct mtk_iommu_data *data = platform_get_drvdata(pdev); 5750df4fabeSYong Wu 5760df4fabeSYong Wu if (iommu_present(&platform_bus_type)) 5770df4fabeSYong Wu bus_set_iommu(&platform_bus_type, NULL); 5780df4fabeSYong Wu 5790df4fabeSYong Wu free_io_pgtable_ops(data->m4u_dom->iop); 5800df4fabeSYong Wu clk_disable_unprepare(data->bclk); 5810df4fabeSYong Wu devm_free_irq(&pdev->dev, data->irq, data); 5820df4fabeSYong Wu component_master_del(&pdev->dev, &mtk_iommu_com_ops); 5830df4fabeSYong Wu return 0; 5840df4fabeSYong Wu } 5850df4fabeSYong Wu 586fd99f796SArnd Bergmann static int __maybe_unused mtk_iommu_suspend(struct device *dev) 5870df4fabeSYong Wu { 5880df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 5890df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 5900df4fabeSYong Wu void __iomem *base = data->base; 5910df4fabeSYong Wu 5920df4fabeSYong Wu reg->standard_axi_mode = readl_relaxed(base + 5930df4fabeSYong Wu REG_MMU_STANDARD_AXI_MODE); 5940df4fabeSYong Wu reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); 5950df4fabeSYong Wu reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); 5960df4fabeSYong Wu reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0); 5970df4fabeSYong Wu reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); 5980df4fabeSYong Wu return 0; 5990df4fabeSYong Wu } 6000df4fabeSYong Wu 601fd99f796SArnd Bergmann static int __maybe_unused mtk_iommu_resume(struct device *dev) 6020df4fabeSYong Wu { 6030df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 6040df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 6050df4fabeSYong Wu void __iomem *base = data->base; 6060df4fabeSYong Wu 6070df4fabeSYong Wu writel_relaxed(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0], 6080df4fabeSYong Wu base + REG_MMU_PT_BASE_ADDR); 6090df4fabeSYong Wu writel_relaxed(reg->standard_axi_mode, 6100df4fabeSYong Wu base + REG_MMU_STANDARD_AXI_MODE); 6110df4fabeSYong Wu writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); 6120df4fabeSYong Wu writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); 6130df4fabeSYong Wu writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); 6140df4fabeSYong Wu writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); 61501e23c93SYong Wu writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base, data->enable_4GB), 6160df4fabeSYong Wu base + REG_MMU_IVRP_PADDR); 6170df4fabeSYong Wu return 0; 6180df4fabeSYong Wu } 6190df4fabeSYong Wu 6200df4fabeSYong Wu const struct dev_pm_ops mtk_iommu_pm_ops = { 6210df4fabeSYong Wu SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume) 6220df4fabeSYong Wu }; 6230df4fabeSYong Wu 6240df4fabeSYong Wu static const struct of_device_id mtk_iommu_of_ids[] = { 6250df4fabeSYong Wu { .compatible = "mediatek,mt8173-m4u", }, 6260df4fabeSYong Wu {} 6270df4fabeSYong Wu }; 6280df4fabeSYong Wu 6290df4fabeSYong Wu static struct platform_driver mtk_iommu_driver = { 6300df4fabeSYong Wu .probe = mtk_iommu_probe, 6310df4fabeSYong Wu .remove = mtk_iommu_remove, 6320df4fabeSYong Wu .driver = { 6330df4fabeSYong Wu .name = "mtk-iommu", 6340df4fabeSYong Wu .of_match_table = mtk_iommu_of_ids, 6350df4fabeSYong Wu .pm = &mtk_iommu_pm_ops, 6360df4fabeSYong Wu } 6370df4fabeSYong Wu }; 6380df4fabeSYong Wu 6390df4fabeSYong Wu static int mtk_iommu_init_fn(struct device_node *np) 6400df4fabeSYong Wu { 6410df4fabeSYong Wu int ret; 6420df4fabeSYong Wu struct platform_device *pdev; 6430df4fabeSYong Wu 6440df4fabeSYong Wu pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root); 6453189e490SDan Carpenter if (!pdev) 6463189e490SDan Carpenter return -ENOMEM; 6470df4fabeSYong Wu 6480df4fabeSYong Wu ret = platform_driver_register(&mtk_iommu_driver); 6490df4fabeSYong Wu if (ret) { 6500df4fabeSYong Wu pr_err("%s: Failed to register driver\n", __func__); 6510df4fabeSYong Wu return ret; 6520df4fabeSYong Wu } 6530df4fabeSYong Wu 6540df4fabeSYong Wu of_iommu_set_ops(np, &mtk_iommu_ops); 6550df4fabeSYong Wu return 0; 6560df4fabeSYong Wu } 6570df4fabeSYong Wu 6580df4fabeSYong Wu IOMMU_OF_DECLARE(mtkm4u, "mediatek,mt8173-m4u", mtk_iommu_init_fn); 659