11802d0beSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 20df4fabeSYong Wu /* 30df4fabeSYong Wu * Copyright (c) 2015-2016 MediaTek Inc. 40df4fabeSYong Wu * Author: Yong Wu <yong.wu@mediatek.com> 50df4fabeSYong Wu */ 6ef0f0986SYong Wu #include <linux/bitfield.h> 70df4fabeSYong Wu #include <linux/bug.h> 80df4fabeSYong Wu #include <linux/clk.h> 90df4fabeSYong Wu #include <linux/component.h> 100df4fabeSYong Wu #include <linux/device.h> 11803cf9e5SYong Wu #include <linux/dma-direct.h> 120df4fabeSYong Wu #include <linux/err.h> 130df4fabeSYong Wu #include <linux/interrupt.h> 140df4fabeSYong Wu #include <linux/io.h> 150df4fabeSYong Wu #include <linux/iommu.h> 160df4fabeSYong Wu #include <linux/iopoll.h> 176a513de3SYong Wu #include <linux/io-pgtable.h> 180df4fabeSYong Wu #include <linux/list.h> 19c2c59456SMiles Chen #include <linux/mfd/syscon.h> 2018d8c74eSYong Wu #include <linux/module.h> 210df4fabeSYong Wu #include <linux/of_address.h> 220df4fabeSYong Wu #include <linux/of_irq.h> 230df4fabeSYong Wu #include <linux/of_platform.h> 24e7629070SYong Wu #include <linux/pci.h> 250df4fabeSYong Wu #include <linux/platform_device.h> 26baf94e6eSYong Wu #include <linux/pm_runtime.h> 27c2c59456SMiles Chen #include <linux/regmap.h> 280df4fabeSYong Wu #include <linux/slab.h> 290df4fabeSYong Wu #include <linux/spinlock.h> 30c2c59456SMiles Chen #include <linux/soc/mediatek/infracfg.h> 310df4fabeSYong Wu #include <asm/barrier.h> 320df4fabeSYong Wu #include <soc/mediatek/smi.h> 330df4fabeSYong Wu 346a513de3SYong Wu #include <dt-bindings/memory/mtk-memory-port.h> 350df4fabeSYong Wu 360df4fabeSYong Wu #define REG_MMU_PT_BASE_ADDR 0x000 370df4fabeSYong Wu 380df4fabeSYong Wu #define REG_MMU_INVALIDATE 0x020 390df4fabeSYong Wu #define F_ALL_INVLD 0x2 400df4fabeSYong Wu #define F_MMU_INV_RANGE 0x1 410df4fabeSYong Wu 420df4fabeSYong Wu #define REG_MMU_INVLD_START_A 0x024 430df4fabeSYong Wu #define REG_MMU_INVLD_END_A 0x028 440df4fabeSYong Wu 45068c86e9SChao Hao #define REG_MMU_INV_SEL_GEN2 0x02c 46b053bc71SChao Hao #define REG_MMU_INV_SEL_GEN1 0x038 470df4fabeSYong Wu #define F_INVLD_EN0 BIT(0) 480df4fabeSYong Wu #define F_INVLD_EN1 BIT(1) 490df4fabeSYong Wu 5075eed350SChao Hao #define REG_MMU_MISC_CTRL 0x048 514bb2bf4cSChao Hao #define F_MMU_IN_ORDER_WR_EN_MASK (BIT(1) | BIT(17)) 524bb2bf4cSChao Hao #define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19)) 534bb2bf4cSChao Hao 540df4fabeSYong Wu #define REG_MMU_DCM_DIS 0x050 559a87005eSYong Wu #define F_MMU_DCM BIT(8) 569a87005eSYong Wu 5735c1b48dSChao Hao #define REG_MMU_WR_LEN_CTRL 0x054 5835c1b48dSChao Hao #define F_MMU_WR_THROT_DIS_MASK (BIT(5) | BIT(21)) 590df4fabeSYong Wu 600df4fabeSYong Wu #define REG_MMU_CTRL_REG 0x110 61acb3c92aSYong Wu #define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4) 620df4fabeSYong Wu #define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4) 63acb3c92aSYong Wu #define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5) 640df4fabeSYong Wu 650df4fabeSYong Wu #define REG_MMU_IVRP_PADDR 0x114 6670ca608bSYong Wu 6730e2fccfSYong Wu #define REG_MMU_VLD_PA_RNG 0x118 6830e2fccfSYong Wu #define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA)) 690df4fabeSYong Wu 700df4fabeSYong Wu #define REG_MMU_INT_CONTROL0 0x120 710df4fabeSYong Wu #define F_L2_MULIT_HIT_EN BIT(0) 720df4fabeSYong Wu #define F_TABLE_WALK_FAULT_INT_EN BIT(1) 730df4fabeSYong Wu #define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2) 740df4fabeSYong Wu #define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3) 750df4fabeSYong Wu #define F_PREFETCH_FIFO_ERR_INT_EN BIT(5) 760df4fabeSYong Wu #define F_MISS_FIFO_ERR_INT_EN BIT(6) 770df4fabeSYong Wu #define F_INT_CLR_BIT BIT(12) 780df4fabeSYong Wu 790df4fabeSYong Wu #define REG_MMU_INT_MAIN_CONTROL 0x124 8015a01f4cSYong Wu /* mmu0 | mmu1 */ 8115a01f4cSYong Wu #define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7)) 8215a01f4cSYong Wu #define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8)) 8315a01f4cSYong Wu #define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9)) 8415a01f4cSYong Wu #define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10)) 8515a01f4cSYong Wu #define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11)) 8615a01f4cSYong Wu #define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12)) 8715a01f4cSYong Wu #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13)) 880df4fabeSYong Wu 890df4fabeSYong Wu #define REG_MMU_CPE_DONE 0x12C 900df4fabeSYong Wu 910df4fabeSYong Wu #define REG_MMU_FAULT_ST1 0x134 9215a01f4cSYong Wu #define F_REG_MMU0_FAULT_MASK GENMASK(6, 0) 9315a01f4cSYong Wu #define F_REG_MMU1_FAULT_MASK GENMASK(13, 7) 940df4fabeSYong Wu 9515a01f4cSYong Wu #define REG_MMU0_FAULT_VA 0x13c 96ef0f0986SYong Wu #define F_MMU_INVAL_VA_31_12_MASK GENMASK(31, 12) 97ef0f0986SYong Wu #define F_MMU_INVAL_VA_34_32_MASK GENMASK(11, 9) 98ef0f0986SYong Wu #define F_MMU_INVAL_PA_34_32_MASK GENMASK(8, 6) 990df4fabeSYong Wu #define F_MMU_FAULT_VA_WRITE_BIT BIT(1) 1000df4fabeSYong Wu #define F_MMU_FAULT_VA_LAYER_BIT BIT(0) 1010df4fabeSYong Wu 10215a01f4cSYong Wu #define REG_MMU0_INVLD_PA 0x140 10315a01f4cSYong Wu #define REG_MMU1_FAULT_VA 0x144 10415a01f4cSYong Wu #define REG_MMU1_INVLD_PA 0x148 10515a01f4cSYong Wu #define REG_MMU0_INT_ID 0x150 10615a01f4cSYong Wu #define REG_MMU1_INT_ID 0x154 10737276e00SChao Hao #define F_MMU_INT_ID_COMM_ID(a) (((a) >> 9) & 0x7) 10837276e00SChao Hao #define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3) 1099ec30c09SYong Wu #define F_MMU_INT_ID_COMM_ID_EXT(a) (((a) >> 10) & 0x7) 1109ec30c09SYong Wu #define F_MMU_INT_ID_SUB_COMM_ID_EXT(a) (((a) >> 7) & 0x7) 11165df7d82SFabien Parent /* Macro for 5 bits length port ID field (default) */ 11215a01f4cSYong Wu #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) 11315a01f4cSYong Wu #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) 11465df7d82SFabien Parent /* Macro for 6 bits length port ID field */ 11565df7d82SFabien Parent #define F_MMU_INT_ID_LARB_ID_WID_6(a) (((a) >> 8) & 0x7) 11665df7d82SFabien Parent #define F_MMU_INT_ID_PORT_ID_WID_6(a) (((a) >> 2) & 0x3f) 1170df4fabeSYong Wu 118829316b3SChao Hao #define MTK_PROTECT_PA_ALIGN 256 11942d57fc5SYong Wu #define MTK_IOMMU_BANK_SZ 0x1000 1200df4fabeSYong Wu 121f9b8c9b2SYong Wu #define PERICFG_IOMMU_1 0x714 122f9b8c9b2SYong Wu 1236b717796SChao Hao #define HAS_4GB_MODE BIT(0) 1246b717796SChao Hao /* HW will use the EMI clock if there isn't the "bclk". */ 1256b717796SChao Hao #define HAS_BCLK BIT(1) 1266b717796SChao Hao #define HAS_VLD_PA_RNG BIT(2) 1276b717796SChao Hao #define RESET_AXI BIT(3) 1284bb2bf4cSChao Hao #define OUT_ORDER_WR_EN BIT(4) 1299ec30c09SYong Wu #define HAS_SUB_COMM_2BITS BIT(5) 1309ec30c09SYong Wu #define HAS_SUB_COMM_3BITS BIT(6) 1319ec30c09SYong Wu #define WR_THROT_EN BIT(7) 1329ec30c09SYong Wu #define HAS_LEGACY_IVRP_PADDR BIT(8) 1339ec30c09SYong Wu #define IOVA_34_EN BIT(9) 1349ec30c09SYong Wu #define SHARE_PGTABLE BIT(10) /* 2 HW share pgtable */ 1359ec30c09SYong Wu #define DCM_DISABLE BIT(11) 1369ec30c09SYong Wu #define STD_AXI_MODE BIT(12) /* For non MM iommu */ 1378cd1e619SYong Wu /* 2 bits: iommu type */ 1388cd1e619SYong Wu #define MTK_IOMMU_TYPE_MM (0x0 << 13) 1398cd1e619SYong Wu #define MTK_IOMMU_TYPE_INFRA (0x1 << 13) 1408cd1e619SYong Wu #define MTK_IOMMU_TYPE_MASK (0x3 << 13) 1416077c7e5SYong Wu /* PM and clock always on. e.g. infra iommu */ 1426077c7e5SYong Wu #define PM_CLK_AO BIT(15) 143e7629070SYong Wu #define IFA_IOMMU_PCIE_SUPPORT BIT(16) 144301c3ca1SYunfei Wang #define PGTABLE_PA_35_EN BIT(17) 14586580ec9SAngeloGioacchino Del Regno #define TF_PORT_TO_ADDR_MT8173 BIT(18) 14665df7d82SFabien Parent #define INT_ID_PORT_WIDTH_6 BIT(19) 1476b717796SChao Hao 1488cd1e619SYong Wu #define MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, mask) \ 1498cd1e619SYong Wu ((((pdata)->flags) & (mask)) == (_x)) 1508cd1e619SYong Wu 1518cd1e619SYong Wu #define MTK_IOMMU_HAS_FLAG(pdata, _x) MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, _x) 1528cd1e619SYong Wu #define MTK_IOMMU_IS_TYPE(pdata, _x) MTK_IOMMU_HAS_FLAG_MASK(pdata, _x,\ 1538cd1e619SYong Wu MTK_IOMMU_TYPE_MASK) 1546b717796SChao Hao 155d2e9a110SYong Wu #define MTK_INVALID_LARBID MTK_LARB_NR_MAX 156d2e9a110SYong Wu 1579485a04aSYong Wu #define MTK_LARB_COM_MAX 8 1589485a04aSYong Wu #define MTK_LARB_SUBCOM_MAX 8 1599485a04aSYong Wu 1609485a04aSYong Wu #define MTK_IOMMU_GROUP_MAX 8 16199ca0228SYong Wu #define MTK_IOMMU_BANK_MAX 5 1629485a04aSYong Wu 1639485a04aSYong Wu enum mtk_iommu_plat { 1649485a04aSYong Wu M4U_MT2712, 1659485a04aSYong Wu M4U_MT6779, 166717ec15eSAngeloGioacchino Del Regno M4U_MT6795, 1679485a04aSYong Wu M4U_MT8167, 1689485a04aSYong Wu M4U_MT8173, 1699485a04aSYong Wu M4U_MT8183, 170e8d7ccaaSYong Wu M4U_MT8186, 1719485a04aSYong Wu M4U_MT8192, 1729485a04aSYong Wu M4U_MT8195, 1733cd0e4a3SFabien Parent M4U_MT8365, 1749485a04aSYong Wu }; 1759485a04aSYong Wu 1769485a04aSYong Wu struct mtk_iommu_iova_region { 1779485a04aSYong Wu dma_addr_t iova_base; 1789485a04aSYong Wu unsigned long long size; 1799485a04aSYong Wu }; 1809485a04aSYong Wu 1816a513de3SYong Wu struct mtk_iommu_suspend_reg { 1826a513de3SYong Wu u32 misc_ctrl; 1836a513de3SYong Wu u32 dcm_dis; 1846a513de3SYong Wu u32 ctrl_reg; 1856a513de3SYong Wu u32 vld_pa_rng; 1866a513de3SYong Wu u32 wr_len_ctrl; 187d7127de1SYong Wu 188d7127de1SYong Wu u32 int_control[MTK_IOMMU_BANK_MAX]; 189d7127de1SYong Wu u32 int_main_control[MTK_IOMMU_BANK_MAX]; 190d7127de1SYong Wu u32 ivrp_paddr[MTK_IOMMU_BANK_MAX]; 1916a513de3SYong Wu }; 1926a513de3SYong Wu 1939485a04aSYong Wu struct mtk_iommu_plat_data { 1949485a04aSYong Wu enum mtk_iommu_plat m4u_plat; 1959485a04aSYong Wu u32 flags; 1969485a04aSYong Wu u32 inv_sel_reg; 1979485a04aSYong Wu 1989485a04aSYong Wu char *pericfg_comp_str; 1999485a04aSYong Wu struct list_head *hw_list; 2009485a04aSYong Wu unsigned int iova_region_nr; 2019485a04aSYong Wu const struct mtk_iommu_iova_region *iova_region; 20299ca0228SYong Wu 20399ca0228SYong Wu u8 banks_num; 20499ca0228SYong Wu bool banks_enable[MTK_IOMMU_BANK_MAX]; 20557fb481fSYong Wu unsigned int banks_portmsk[MTK_IOMMU_BANK_MAX]; 2069485a04aSYong Wu unsigned char larbid_remap[MTK_LARB_COM_MAX][MTK_LARB_SUBCOM_MAX]; 2079485a04aSYong Wu }; 2089485a04aSYong Wu 20999ca0228SYong Wu struct mtk_iommu_bank_data { 2109485a04aSYong Wu void __iomem *base; 2119485a04aSYong Wu int irq; 21299ca0228SYong Wu u8 id; 21399ca0228SYong Wu struct device *parent_dev; 21499ca0228SYong Wu struct mtk_iommu_data *parent_data; 21599ca0228SYong Wu spinlock_t tlb_lock; /* lock for tlb range flush */ 21699ca0228SYong Wu struct mtk_iommu_domain *m4u_dom; /* Each bank has a domain */ 21799ca0228SYong Wu }; 21899ca0228SYong Wu 21999ca0228SYong Wu struct mtk_iommu_data { 2209485a04aSYong Wu struct device *dev; 2219485a04aSYong Wu struct clk *bclk; 2229485a04aSYong Wu phys_addr_t protect_base; /* protect memory base */ 2239485a04aSYong Wu struct mtk_iommu_suspend_reg reg; 2249485a04aSYong Wu struct iommu_group *m4u_group[MTK_IOMMU_GROUP_MAX]; 2259485a04aSYong Wu bool enable_4GB; 2269485a04aSYong Wu 2279485a04aSYong Wu struct iommu_device iommu; 2289485a04aSYong Wu const struct mtk_iommu_plat_data *plat_data; 2299485a04aSYong Wu struct device *smicomm_dev; 2309485a04aSYong Wu 23199ca0228SYong Wu struct mtk_iommu_bank_data *bank; 2329485a04aSYong Wu struct regmap *pericfg; 2339485a04aSYong Wu struct mutex mutex; /* Protect m4u_group/m4u_dom above */ 2349485a04aSYong Wu 2359485a04aSYong Wu /* 2369485a04aSYong Wu * In the sharing pgtable case, list data->list to the global list like m4ulist. 2379485a04aSYong Wu * In the non-sharing pgtable case, list data->list to the itself hw_list_head. 2389485a04aSYong Wu */ 2399485a04aSYong Wu struct list_head *hw_list; 2409485a04aSYong Wu struct list_head hw_list_head; 2419485a04aSYong Wu struct list_head list; 2429485a04aSYong Wu struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX]; 2439485a04aSYong Wu }; 2449485a04aSYong Wu 2450df4fabeSYong Wu struct mtk_iommu_domain { 2460df4fabeSYong Wu struct io_pgtable_cfg cfg; 2470df4fabeSYong Wu struct io_pgtable_ops *iop; 2480df4fabeSYong Wu 24999ca0228SYong Wu struct mtk_iommu_bank_data *bank; 2500df4fabeSYong Wu struct iommu_domain domain; 251ddf67a87SYong Wu 252ddf67a87SYong Wu struct mutex mutex; /* Protect "data" in this structure */ 2530df4fabeSYong Wu }; 2540df4fabeSYong Wu 2559485a04aSYong Wu static int mtk_iommu_bind(struct device *dev) 2569485a04aSYong Wu { 2579485a04aSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 2589485a04aSYong Wu 2599485a04aSYong Wu return component_bind_all(dev, &data->larb_imu); 2609485a04aSYong Wu } 2619485a04aSYong Wu 2629485a04aSYong Wu static void mtk_iommu_unbind(struct device *dev) 2639485a04aSYong Wu { 2649485a04aSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 2659485a04aSYong Wu 2669485a04aSYong Wu component_unbind_all(dev, &data->larb_imu); 2679485a04aSYong Wu } 2689485a04aSYong Wu 269b65f5016SArvind Yadav static const struct iommu_ops mtk_iommu_ops; 2700df4fabeSYong Wu 271e24453e1SYong Wu static int mtk_iommu_hw_init(const struct mtk_iommu_data *data, unsigned int bankid); 2727f37a91dSYong Wu 273bfed8731SYong Wu #define MTK_IOMMU_TLB_ADDR(iova) ({ \ 274bfed8731SYong Wu dma_addr_t _addr = iova; \ 275bfed8731SYong Wu ((lower_32_bits(_addr) & GENMASK(31, 12)) | upper_32_bits(_addr));\ 276bfed8731SYong Wu }) 277bfed8731SYong Wu 27876ce6546SYong Wu /* 27976ce6546SYong Wu * In M4U 4GB mode, the physical address is remapped as below: 28076ce6546SYong Wu * 28176ce6546SYong Wu * CPU Physical address: 28276ce6546SYong Wu * ==================== 28376ce6546SYong Wu * 28476ce6546SYong Wu * 0 1G 2G 3G 4G 5G 28576ce6546SYong Wu * |---A---|---B---|---C---|---D---|---E---| 28676ce6546SYong Wu * +--I/O--+------------Memory-------------+ 28776ce6546SYong Wu * 28876ce6546SYong Wu * IOMMU output physical address: 28976ce6546SYong Wu * ============================= 29076ce6546SYong Wu * 29176ce6546SYong Wu * 4G 5G 6G 7G 8G 29276ce6546SYong Wu * |---E---|---B---|---C---|---D---| 29376ce6546SYong Wu * +------------Memory-------------+ 29476ce6546SYong Wu * 29576ce6546SYong Wu * The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the 29676ce6546SYong Wu * bit32 of the CPU physical address always is needed to set, and for Region 29776ce6546SYong Wu * 'E', the CPU physical address keep as is. 29876ce6546SYong Wu * Additionally, The iommu consumers always use the CPU phyiscal address. 29976ce6546SYong Wu */ 300b4dad40eSYong Wu #define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL 30176ce6546SYong Wu 3027c3a2ec0SYong Wu static LIST_HEAD(m4ulist); /* List all the M4U HWs */ 3037c3a2ec0SYong Wu 3049e3a2a64SYong Wu #define for_each_m4u(data, head) list_for_each_entry(data, head, list) 3057c3a2ec0SYong Wu 306585e58f4SYong Wu static const struct mtk_iommu_iova_region single_domain[] = { 307585e58f4SYong Wu {.iova_base = 0, .size = SZ_4G}, 308585e58f4SYong Wu }; 309585e58f4SYong Wu 3109e3489e0SYong Wu static const struct mtk_iommu_iova_region mt8192_multi_dom[] = { 311129a3b88SYong Wu { .iova_base = 0x0, .size = SZ_4G}, /* 0 ~ 4G */ 3129e3489e0SYong Wu #if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) 313129a3b88SYong Wu { .iova_base = SZ_4G, .size = SZ_4G}, /* 4G ~ 8G */ 314129a3b88SYong Wu { .iova_base = SZ_4G * 2, .size = SZ_4G}, /* 8G ~ 12G */ 315129a3b88SYong Wu { .iova_base = SZ_4G * 3, .size = SZ_4G}, /* 12G ~ 16G */ 316129a3b88SYong Wu 3179e3489e0SYong Wu { .iova_base = 0x240000000ULL, .size = 0x4000000}, /* CCU0 */ 3189e3489e0SYong Wu { .iova_base = 0x244000000ULL, .size = 0x4000000}, /* CCU1 */ 3199e3489e0SYong Wu #endif 3209e3489e0SYong Wu }; 3219e3489e0SYong Wu 3229e3a2a64SYong Wu /* If 2 M4U share a domain(use the same hwlist), Put the corresponding info in first data.*/ 3239e3a2a64SYong Wu static struct mtk_iommu_data *mtk_iommu_get_frst_data(struct list_head *hwlist) 3247c3a2ec0SYong Wu { 3259e3a2a64SYong Wu return list_first_entry(hwlist, struct mtk_iommu_data, list); 3267c3a2ec0SYong Wu } 3277c3a2ec0SYong Wu 3280df4fabeSYong Wu static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) 3290df4fabeSYong Wu { 3300df4fabeSYong Wu return container_of(dom, struct mtk_iommu_domain, domain); 3310df4fabeSYong Wu } 3320df4fabeSYong Wu 3330954d61aSYong Wu static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) 3340df4fabeSYong Wu { 33599ca0228SYong Wu /* Tlb flush all always is in bank0. */ 33699ca0228SYong Wu struct mtk_iommu_bank_data *bank = &data->bank[0]; 33799ca0228SYong Wu void __iomem *base = bank->base; 33815672b6dSYong Wu unsigned long flags; 339c0b57581SYong Wu 34099ca0228SYong Wu spin_lock_irqsave(&bank->tlb_lock, flags); 341887cf6a7SYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, base + data->plat_data->inv_sel_reg); 342887cf6a7SYong Wu writel_relaxed(F_ALL_INVLD, base + REG_MMU_INVALIDATE); 3430df4fabeSYong Wu wmb(); /* Make sure the tlb flush all done */ 34499ca0228SYong Wu spin_unlock_irqrestore(&bank->tlb_lock, flags); 3457c3a2ec0SYong Wu } 3460df4fabeSYong Wu 3471f4fd624SYong Wu static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, 34899ca0228SYong Wu struct mtk_iommu_bank_data *bank) 3490df4fabeSYong Wu { 35099ca0228SYong Wu struct list_head *head = bank->parent_data->hw_list; 35199ca0228SYong Wu struct mtk_iommu_bank_data *curbank; 35299ca0228SYong Wu struct mtk_iommu_data *data; 3536077c7e5SYong Wu bool check_pm_status; 3541f4fd624SYong Wu unsigned long flags; 355887cf6a7SYong Wu void __iomem *base; 3561f4fd624SYong Wu int ret; 3571f4fd624SYong Wu u32 tmp; 3580df4fabeSYong Wu 3599e3a2a64SYong Wu for_each_m4u(data, head) { 3606077c7e5SYong Wu /* 3616077c7e5SYong Wu * To avoid resume the iommu device frequently when the iommu device 3626077c7e5SYong Wu * is not active, it doesn't always call pm_runtime_get here, then tlb 3636077c7e5SYong Wu * flush depends on the tlb flush all in the runtime resume. 3646077c7e5SYong Wu * 3656077c7e5SYong Wu * There are 2 special cases: 3666077c7e5SYong Wu * 3676077c7e5SYong Wu * Case1: The iommu dev doesn't have power domain but has bclk. This case 3686077c7e5SYong Wu * should also avoid the tlb flush while the dev is not active to mute 3696077c7e5SYong Wu * the tlb timeout log. like mt8173. 3706077c7e5SYong Wu * 3716077c7e5SYong Wu * Case2: The power/clock of infra iommu is always on, and it doesn't 3726077c7e5SYong Wu * have the device link with the master devices. This case should avoid 3736077c7e5SYong Wu * the PM status check. 3746077c7e5SYong Wu */ 3756077c7e5SYong Wu check_pm_status = !MTK_IOMMU_HAS_FLAG(data->plat_data, PM_CLK_AO); 3766077c7e5SYong Wu 3776077c7e5SYong Wu if (check_pm_status) { 378c0b57581SYong Wu if (pm_runtime_get_if_in_use(data->dev) <= 0) 379c0b57581SYong Wu continue; 3806077c7e5SYong Wu } 381c0b57581SYong Wu 38299ca0228SYong Wu curbank = &data->bank[bank->id]; 38399ca0228SYong Wu base = curbank->base; 384887cf6a7SYong Wu 38599ca0228SYong Wu spin_lock_irqsave(&curbank->tlb_lock, flags); 3867c3a2ec0SYong Wu writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, 387887cf6a7SYong Wu base + data->plat_data->inv_sel_reg); 3880df4fabeSYong Wu 389887cf6a7SYong Wu writel_relaxed(MTK_IOMMU_TLB_ADDR(iova), base + REG_MMU_INVLD_START_A); 390bfed8731SYong Wu writel_relaxed(MTK_IOMMU_TLB_ADDR(iova + size - 1), 391887cf6a7SYong Wu base + REG_MMU_INVLD_END_A); 392887cf6a7SYong Wu writel_relaxed(F_MMU_INV_RANGE, base + REG_MMU_INVALIDATE); 3930df4fabeSYong Wu 3941f4fd624SYong Wu /* tlb sync */ 395887cf6a7SYong Wu ret = readl_poll_timeout_atomic(base + REG_MMU_CPE_DONE, 396c90ae4a6SYong Wu tmp, tmp != 0, 10, 1000); 39715672b6dSYong Wu 39815672b6dSYong Wu /* Clear the CPE status */ 399887cf6a7SYong Wu writel_relaxed(0, base + REG_MMU_CPE_DONE); 40099ca0228SYong Wu spin_unlock_irqrestore(&curbank->tlb_lock, flags); 40115672b6dSYong Wu 4020df4fabeSYong Wu if (ret) { 4030df4fabeSYong Wu dev_warn(data->dev, 4040df4fabeSYong Wu "Partial TLB flush timed out, falling back to full flush\n"); 4050954d61aSYong Wu mtk_iommu_tlb_flush_all(data); 4060df4fabeSYong Wu } 407c0b57581SYong Wu 4086077c7e5SYong Wu if (check_pm_status) 409c0b57581SYong Wu pm_runtime_put(data->dev); 4100df4fabeSYong Wu } 4117c3a2ec0SYong Wu } 4120df4fabeSYong Wu 4130df4fabeSYong Wu static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) 4140df4fabeSYong Wu { 41599ca0228SYong Wu struct mtk_iommu_bank_data *bank = dev_id; 41699ca0228SYong Wu struct mtk_iommu_data *data = bank->parent_data; 41799ca0228SYong Wu struct mtk_iommu_domain *dom = bank->m4u_dom; 418d2e9a110SYong Wu unsigned int fault_larb = MTK_INVALID_LARBID, fault_port = 0, sub_comm = 0; 419ef0f0986SYong Wu u32 int_state, regval, va34_32, pa34_32; 420887cf6a7SYong Wu const struct mtk_iommu_plat_data *plat_data = data->plat_data; 42199ca0228SYong Wu void __iomem *base = bank->base; 422ef0f0986SYong Wu u64 fault_iova, fault_pa; 4230df4fabeSYong Wu bool layer, write; 4240df4fabeSYong Wu 4250df4fabeSYong Wu /* Read error info from registers */ 426887cf6a7SYong Wu int_state = readl_relaxed(base + REG_MMU_FAULT_ST1); 42715a01f4cSYong Wu if (int_state & F_REG_MMU0_FAULT_MASK) { 428887cf6a7SYong Wu regval = readl_relaxed(base + REG_MMU0_INT_ID); 429887cf6a7SYong Wu fault_iova = readl_relaxed(base + REG_MMU0_FAULT_VA); 430887cf6a7SYong Wu fault_pa = readl_relaxed(base + REG_MMU0_INVLD_PA); 43115a01f4cSYong Wu } else { 432887cf6a7SYong Wu regval = readl_relaxed(base + REG_MMU1_INT_ID); 433887cf6a7SYong Wu fault_iova = readl_relaxed(base + REG_MMU1_FAULT_VA); 434887cf6a7SYong Wu fault_pa = readl_relaxed(base + REG_MMU1_INVLD_PA); 43515a01f4cSYong Wu } 4360df4fabeSYong Wu layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; 4370df4fabeSYong Wu write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; 438887cf6a7SYong Wu if (MTK_IOMMU_HAS_FLAG(plat_data, IOVA_34_EN)) { 439ef0f0986SYong Wu va34_32 = FIELD_GET(F_MMU_INVAL_VA_34_32_MASK, fault_iova); 440ef0f0986SYong Wu fault_iova = fault_iova & F_MMU_INVAL_VA_31_12_MASK; 441ef0f0986SYong Wu fault_iova |= (u64)va34_32 << 32; 442ef0f0986SYong Wu } 44382e51771SYong Wu pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova); 44482e51771SYong Wu fault_pa |= (u64)pa34_32 << 32; 445ef0f0986SYong Wu 446887cf6a7SYong Wu if (MTK_IOMMU_IS_TYPE(plat_data, MTK_IOMMU_TYPE_MM)) { 447887cf6a7SYong Wu if (MTK_IOMMU_HAS_FLAG(plat_data, HAS_SUB_COMM_2BITS)) { 44837276e00SChao Hao fault_larb = F_MMU_INT_ID_COMM_ID(regval); 44937276e00SChao Hao sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval); 45065df7d82SFabien Parent fault_port = F_MMU_INT_ID_PORT_ID(regval); 451887cf6a7SYong Wu } else if (MTK_IOMMU_HAS_FLAG(plat_data, HAS_SUB_COMM_3BITS)) { 4529ec30c09SYong Wu fault_larb = F_MMU_INT_ID_COMM_ID_EXT(regval); 4539ec30c09SYong Wu sub_comm = F_MMU_INT_ID_SUB_COMM_ID_EXT(regval); 45465df7d82SFabien Parent fault_port = F_MMU_INT_ID_PORT_ID(regval); 45565df7d82SFabien Parent } else if (MTK_IOMMU_HAS_FLAG(plat_data, INT_ID_PORT_WIDTH_6)) { 45665df7d82SFabien Parent fault_port = F_MMU_INT_ID_PORT_ID_WID_6(regval); 45765df7d82SFabien Parent fault_larb = F_MMU_INT_ID_LARB_ID_WID_6(regval); 45837276e00SChao Hao } else { 45965df7d82SFabien Parent fault_port = F_MMU_INT_ID_PORT_ID(regval); 46037276e00SChao Hao fault_larb = F_MMU_INT_ID_LARB_ID(regval); 46137276e00SChao Hao } 46237276e00SChao Hao fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm]; 463d2e9a110SYong Wu } 464b3e5eee7SYong Wu 465*00ef8885SRicardo Ribalda if (!dom || report_iommu_fault(&dom->domain, bank->parent_dev, fault_iova, 4660df4fabeSYong Wu write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { 4670df4fabeSYong Wu dev_err_ratelimited( 46899ca0228SYong Wu bank->parent_dev, 469f9b8c9b2SYong Wu "fault type=0x%x iova=0x%llx pa=0x%llx master=0x%x(larb=%d port=%d) layer=%d %s\n", 470f9b8c9b2SYong Wu int_state, fault_iova, fault_pa, regval, fault_larb, fault_port, 4710df4fabeSYong Wu layer, write ? "write" : "read"); 4720df4fabeSYong Wu } 4730df4fabeSYong Wu 4740df4fabeSYong Wu /* Interrupt clear */ 475887cf6a7SYong Wu regval = readl_relaxed(base + REG_MMU_INT_CONTROL0); 4760df4fabeSYong Wu regval |= F_INT_CLR_BIT; 477887cf6a7SYong Wu writel_relaxed(regval, base + REG_MMU_INT_CONTROL0); 4780df4fabeSYong Wu 4790df4fabeSYong Wu mtk_iommu_tlb_flush_all(data); 4800df4fabeSYong Wu 4810df4fabeSYong Wu return IRQ_HANDLED; 4820df4fabeSYong Wu } 4830df4fabeSYong Wu 48457fb481fSYong Wu static unsigned int mtk_iommu_get_bank_id(struct device *dev, 48557fb481fSYong Wu const struct mtk_iommu_plat_data *plat_data) 48657fb481fSYong Wu { 48757fb481fSYong Wu struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 48857fb481fSYong Wu unsigned int i, portmsk = 0, bankid = 0; 48957fb481fSYong Wu 49057fb481fSYong Wu if (plat_data->banks_num == 1) 49157fb481fSYong Wu return bankid; 49257fb481fSYong Wu 49357fb481fSYong Wu for (i = 0; i < fwspec->num_ids; i++) 49457fb481fSYong Wu portmsk |= BIT(MTK_M4U_TO_PORT(fwspec->ids[i])); 49557fb481fSYong Wu 49657fb481fSYong Wu for (i = 0; i < plat_data->banks_num && i < MTK_IOMMU_BANK_MAX; i++) { 49757fb481fSYong Wu if (!plat_data->banks_enable[i]) 49857fb481fSYong Wu continue; 49957fb481fSYong Wu 50057fb481fSYong Wu if (portmsk & plat_data->banks_portmsk[i]) { 50157fb481fSYong Wu bankid = i; 50257fb481fSYong Wu break; 50357fb481fSYong Wu } 50457fb481fSYong Wu } 50557fb481fSYong Wu return bankid; /* default is 0 */ 50657fb481fSYong Wu } 50757fb481fSYong Wu 508d72e0ff5SYong Wu static int mtk_iommu_get_iova_region_id(struct device *dev, 509803cf9e5SYong Wu const struct mtk_iommu_plat_data *plat_data) 510803cf9e5SYong Wu { 511803cf9e5SYong Wu const struct mtk_iommu_iova_region *rgn = plat_data->iova_region; 512803cf9e5SYong Wu const struct bus_dma_region *dma_rgn = dev->dma_range_map; 513803cf9e5SYong Wu int i, candidate = -1; 514803cf9e5SYong Wu dma_addr_t dma_end; 515803cf9e5SYong Wu 516803cf9e5SYong Wu if (!dma_rgn || plat_data->iova_region_nr == 1) 517803cf9e5SYong Wu return 0; 518803cf9e5SYong Wu 519803cf9e5SYong Wu dma_end = dma_rgn->dma_start + dma_rgn->size - 1; 520803cf9e5SYong Wu for (i = 0; i < plat_data->iova_region_nr; i++, rgn++) { 521803cf9e5SYong Wu /* Best fit. */ 522803cf9e5SYong Wu if (dma_rgn->dma_start == rgn->iova_base && 523803cf9e5SYong Wu dma_end == rgn->iova_base + rgn->size - 1) 524803cf9e5SYong Wu return i; 525803cf9e5SYong Wu /* ok if it is inside this region. */ 526803cf9e5SYong Wu if (dma_rgn->dma_start >= rgn->iova_base && 527803cf9e5SYong Wu dma_end < rgn->iova_base + rgn->size) 528803cf9e5SYong Wu candidate = i; 529803cf9e5SYong Wu } 530803cf9e5SYong Wu 531803cf9e5SYong Wu if (candidate >= 0) 532803cf9e5SYong Wu return candidate; 533803cf9e5SYong Wu dev_err(dev, "Can NOT find the iommu domain id(%pad 0x%llx).\n", 534803cf9e5SYong Wu &dma_rgn->dma_start, dma_rgn->size); 535803cf9e5SYong Wu return -EINVAL; 536803cf9e5SYong Wu } 537803cf9e5SYong Wu 538f9b8c9b2SYong Wu static int mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, 539d72e0ff5SYong Wu bool enable, unsigned int regionid) 5400df4fabeSYong Wu { 5410df4fabeSYong Wu struct mtk_smi_larb_iommu *larb_mmu; 5420df4fabeSYong Wu unsigned int larbid, portid; 543a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 5448d2c749eSYong Wu const struct mtk_iommu_iova_region *region; 545f9b8c9b2SYong Wu u32 peri_mmuen, peri_mmuen_msk; 546f9b8c9b2SYong Wu int i, ret = 0; 5470df4fabeSYong Wu 54858f0d1d5SRobin Murphy for (i = 0; i < fwspec->num_ids; ++i) { 54958f0d1d5SRobin Murphy larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); 55058f0d1d5SRobin Murphy portid = MTK_M4U_TO_PORT(fwspec->ids[i]); 5518d2c749eSYong Wu 552d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { 5531ee9feb2SYong Wu larb_mmu = &data->larb_imu[larbid]; 5540df4fabeSYong Wu 555d72e0ff5SYong Wu region = data->plat_data->iova_region + regionid; 5568d2c749eSYong Wu larb_mmu->bank[portid] = upper_32_bits(region->iova_base); 5578d2c749eSYong Wu 558d72e0ff5SYong Wu dev_dbg(dev, "%s iommu for larb(%s) port %d region %d rgn-bank %d.\n", 5598d2c749eSYong Wu enable ? "enable" : "disable", dev_name(larb_mmu->dev), 560d72e0ff5SYong Wu portid, regionid, larb_mmu->bank[portid]); 5610df4fabeSYong Wu 5620df4fabeSYong Wu if (enable) 5630df4fabeSYong Wu larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); 5640df4fabeSYong Wu else 5650df4fabeSYong Wu larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); 566f9b8c9b2SYong Wu } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA)) { 567f9b8c9b2SYong Wu peri_mmuen_msk = BIT(portid); 568e7629070SYong Wu /* PCI dev has only one output id, enable the next writing bit for PCIe */ 569e7629070SYong Wu if (dev_is_pci(dev)) 570e7629070SYong Wu peri_mmuen_msk |= BIT(portid + 1); 571f9b8c9b2SYong Wu 572e7629070SYong Wu peri_mmuen = enable ? peri_mmuen_msk : 0; 573f9b8c9b2SYong Wu ret = regmap_update_bits(data->pericfg, PERICFG_IOMMU_1, 574f9b8c9b2SYong Wu peri_mmuen_msk, peri_mmuen); 575f9b8c9b2SYong Wu if (ret) 576f9b8c9b2SYong Wu dev_err(dev, "%s iommu(%s) inframaster 0x%x fail(%d).\n", 577f9b8c9b2SYong Wu enable ? "enable" : "disable", 578f9b8c9b2SYong Wu dev_name(data->dev), peri_mmuen_msk, ret); 5790df4fabeSYong Wu } 5800df4fabeSYong Wu } 581f9b8c9b2SYong Wu return ret; 582d2e9a110SYong Wu } 5830df4fabeSYong Wu 5844f956c97SYong Wu static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, 585c3045f39SYong Wu struct mtk_iommu_data *data, 586d72e0ff5SYong Wu unsigned int region_id) 5870df4fabeSYong Wu { 588c3045f39SYong Wu const struct mtk_iommu_iova_region *region; 58999ca0228SYong Wu struct mtk_iommu_domain *m4u_dom; 590c3045f39SYong Wu 59199ca0228SYong Wu /* Always use bank0 in sharing pgtable case */ 59299ca0228SYong Wu m4u_dom = data->bank[0].m4u_dom; 59399ca0228SYong Wu if (m4u_dom) { 59499ca0228SYong Wu dom->iop = m4u_dom->iop; 59599ca0228SYong Wu dom->cfg = m4u_dom->cfg; 59699ca0228SYong Wu dom->domain.pgsize_bitmap = m4u_dom->cfg.pgsize_bitmap; 597c3045f39SYong Wu goto update_iova_region; 598c3045f39SYong Wu } 599c3045f39SYong Wu 6000df4fabeSYong Wu dom->cfg = (struct io_pgtable_cfg) { 6010df4fabeSYong Wu .quirks = IO_PGTABLE_QUIRK_ARM_NS | 6020df4fabeSYong Wu IO_PGTABLE_QUIRK_NO_PERMS | 603b4dad40eSYong Wu IO_PGTABLE_QUIRK_ARM_MTK_EXT, 6040df4fabeSYong Wu .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, 6052f317da4SYong Wu .ias = MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN) ? 34 : 32, 6060df4fabeSYong Wu .iommu_dev = data->dev, 6070df4fabeSYong Wu }; 6080df4fabeSYong Wu 609301c3ca1SYunfei Wang if (MTK_IOMMU_HAS_FLAG(data->plat_data, PGTABLE_PA_35_EN)) 610301c3ca1SYunfei Wang dom->cfg.quirks |= IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT; 611301c3ca1SYunfei Wang 6129bdfe4c1SYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) 6139bdfe4c1SYong Wu dom->cfg.oas = data->enable_4GB ? 33 : 32; 6149bdfe4c1SYong Wu else 6159bdfe4c1SYong Wu dom->cfg.oas = 35; 6169bdfe4c1SYong Wu 6170df4fabeSYong Wu dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data); 6180df4fabeSYong Wu if (!dom->iop) { 6190df4fabeSYong Wu dev_err(data->dev, "Failed to alloc io pgtable\n"); 6200df4fabeSYong Wu return -EINVAL; 6210df4fabeSYong Wu } 6220df4fabeSYong Wu 6230df4fabeSYong Wu /* Update our support page sizes bitmap */ 624d16e0faaSRobin Murphy dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; 625b7875eb9SYong Wu 626c3045f39SYong Wu update_iova_region: 627c3045f39SYong Wu /* Update the iova region for this domain */ 628d72e0ff5SYong Wu region = data->plat_data->iova_region + region_id; 629c3045f39SYong Wu dom->domain.geometry.aperture_start = region->iova_base; 630c3045f39SYong Wu dom->domain.geometry.aperture_end = region->iova_base + region->size - 1; 631b7875eb9SYong Wu dom->domain.geometry.force_aperture = true; 6320df4fabeSYong Wu return 0; 6330df4fabeSYong Wu } 6340df4fabeSYong Wu 6350df4fabeSYong Wu static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) 6360df4fabeSYong Wu { 6370df4fabeSYong Wu struct mtk_iommu_domain *dom; 6380df4fabeSYong Wu 63932e1cccfSYong Wu if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED) 6400df4fabeSYong Wu return NULL; 6410df4fabeSYong Wu 6420df4fabeSYong Wu dom = kzalloc(sizeof(*dom), GFP_KERNEL); 6430df4fabeSYong Wu if (!dom) 6440df4fabeSYong Wu return NULL; 645ddf67a87SYong Wu mutex_init(&dom->mutex); 6460df4fabeSYong Wu 6474f956c97SYong Wu return &dom->domain; 6484f956c97SYong Wu } 6494f956c97SYong Wu 6500df4fabeSYong Wu static void mtk_iommu_domain_free(struct iommu_domain *domain) 6510df4fabeSYong Wu { 6520df4fabeSYong Wu kfree(to_mtk_domain(domain)); 6530df4fabeSYong Wu } 6540df4fabeSYong Wu 6550df4fabeSYong Wu static int mtk_iommu_attach_device(struct iommu_domain *domain, 6560df4fabeSYong Wu struct device *dev) 6570df4fabeSYong Wu { 658645b87c1SYong Wu struct mtk_iommu_data *data = dev_iommu_priv_get(dev), *frstdata; 6590df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 6609e3a2a64SYong Wu struct list_head *hw_list = data->hw_list; 661c0b57581SYong Wu struct device *m4udev = data->dev; 66299ca0228SYong Wu struct mtk_iommu_bank_data *bank; 66357fb481fSYong Wu unsigned int bankid; 664d72e0ff5SYong Wu int ret, region_id; 6650df4fabeSYong Wu 666d72e0ff5SYong Wu region_id = mtk_iommu_get_iova_region_id(dev, data->plat_data); 667d72e0ff5SYong Wu if (region_id < 0) 668d72e0ff5SYong Wu return region_id; 669803cf9e5SYong Wu 67057fb481fSYong Wu bankid = mtk_iommu_get_bank_id(dev, data->plat_data); 671ddf67a87SYong Wu mutex_lock(&dom->mutex); 67299ca0228SYong Wu if (!dom->bank) { 673645b87c1SYong Wu /* Data is in the frstdata in sharing pgtable case. */ 6749e3a2a64SYong Wu frstdata = mtk_iommu_get_frst_data(hw_list); 675645b87c1SYong Wu 676d72e0ff5SYong Wu ret = mtk_iommu_domain_finalise(dom, frstdata, region_id); 677ddf67a87SYong Wu if (ret) { 678ddf67a87SYong Wu mutex_unlock(&dom->mutex); 6794f956c97SYong Wu return -ENODEV; 680ddf67a87SYong Wu } 68199ca0228SYong Wu dom->bank = &data->bank[bankid]; 6824f956c97SYong Wu } 683ddf67a87SYong Wu mutex_unlock(&dom->mutex); 6844f956c97SYong Wu 6850e5a3f2eSYong Wu mutex_lock(&data->mutex); 68699ca0228SYong Wu bank = &data->bank[bankid]; 687e24453e1SYong Wu if (!bank->m4u_dom) { /* Initialize the M4U HW for each a BANK */ 688c0b57581SYong Wu ret = pm_runtime_resume_and_get(m4udev); 689e24453e1SYong Wu if (ret < 0) { 690e24453e1SYong Wu dev_err(m4udev, "pm get fail(%d) in attach.\n", ret); 6910e5a3f2eSYong Wu goto err_unlock; 692e24453e1SYong Wu } 693c0b57581SYong Wu 694e24453e1SYong Wu ret = mtk_iommu_hw_init(data, bankid); 695c0b57581SYong Wu if (ret) { 696c0b57581SYong Wu pm_runtime_put(m4udev); 6970e5a3f2eSYong Wu goto err_unlock; 698c0b57581SYong Wu } 69999ca0228SYong Wu bank->m4u_dom = dom; 700301c3ca1SYunfei Wang writel(dom->cfg.arm_v7s_cfg.ttbr, bank->base + REG_MMU_PT_BASE_ADDR); 701c0b57581SYong Wu 702c0b57581SYong Wu pm_runtime_put(m4udev); 7030df4fabeSYong Wu } 7040e5a3f2eSYong Wu mutex_unlock(&data->mutex); 7050df4fabeSYong Wu 706d72e0ff5SYong Wu return mtk_iommu_config(data, dev, true, region_id); 7070e5a3f2eSYong Wu 7080e5a3f2eSYong Wu err_unlock: 7090e5a3f2eSYong Wu mutex_unlock(&data->mutex); 7100e5a3f2eSYong Wu return ret; 7110df4fabeSYong Wu } 7120df4fabeSYong Wu 7130df4fabeSYong Wu static void mtk_iommu_detach_device(struct iommu_domain *domain, 7140df4fabeSYong Wu struct device *dev) 7150df4fabeSYong Wu { 7163524b559SJoerg Roedel struct mtk_iommu_data *data = dev_iommu_priv_get(dev); 7170df4fabeSYong Wu 7188d2c749eSYong Wu mtk_iommu_config(data, dev, false, 0); 7190df4fabeSYong Wu } 7200df4fabeSYong Wu 7210df4fabeSYong Wu static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, 722781ca2deSTom Murphy phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 7230df4fabeSYong Wu { 7240df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 7250df4fabeSYong Wu 726b4dad40eSYong Wu /* The "4GB mode" M4U physically can not use the lower remap of Dram. */ 72799ca0228SYong Wu if (dom->bank->parent_data->enable_4GB) 728b4dad40eSYong Wu paddr |= BIT_ULL(32); 729b4dad40eSYong Wu 73060829b4dSYong Wu /* Synchronize with the tlb_lock */ 731f34ce7a7SBaolin Wang return dom->iop->map(dom->iop, iova, paddr, size, prot, gfp); 7320df4fabeSYong Wu } 7330df4fabeSYong Wu 7340df4fabeSYong Wu static size_t mtk_iommu_unmap(struct iommu_domain *domain, 73556f8af5eSWill Deacon unsigned long iova, size_t size, 73656f8af5eSWill Deacon struct iommu_iotlb_gather *gather) 7370df4fabeSYong Wu { 7380df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 7390df4fabeSYong Wu 7403136895cSRobin Murphy iommu_iotlb_gather_add_range(gather, iova, size); 74160829b4dSYong Wu return dom->iop->unmap(dom->iop, iova, size, gather); 7420df4fabeSYong Wu } 7430df4fabeSYong Wu 74456f8af5eSWill Deacon static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain) 74556f8af5eSWill Deacon { 74608500c43SYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 74708500c43SYong Wu 74899ca0228SYong Wu mtk_iommu_tlb_flush_all(dom->bank->parent_data); 74956f8af5eSWill Deacon } 75056f8af5eSWill Deacon 75156f8af5eSWill Deacon static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, 75256f8af5eSWill Deacon struct iommu_iotlb_gather *gather) 7534d689b61SRobin Murphy { 75408500c43SYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 755862c3715SYong Wu size_t length = gather->end - gather->start + 1; 756da3cc91bSYong Wu 75799ca0228SYong Wu mtk_iommu_tlb_flush_range_sync(gather->start, length, dom->bank); 7584d689b61SRobin Murphy } 7594d689b61SRobin Murphy 76020143451SYong Wu static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, 76120143451SYong Wu size_t size) 76220143451SYong Wu { 76308500c43SYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 76420143451SYong Wu 76599ca0228SYong Wu mtk_iommu_tlb_flush_range_sync(iova, size, dom->bank); 76620143451SYong Wu } 76720143451SYong Wu 7680df4fabeSYong Wu static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, 7690df4fabeSYong Wu dma_addr_t iova) 7700df4fabeSYong Wu { 7710df4fabeSYong Wu struct mtk_iommu_domain *dom = to_mtk_domain(domain); 7720df4fabeSYong Wu phys_addr_t pa; 7730df4fabeSYong Wu 7740df4fabeSYong Wu pa = dom->iop->iova_to_phys(dom->iop, iova); 775f13efafcSArnd Bergmann if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT) && 77699ca0228SYong Wu dom->bank->parent_data->enable_4GB && 777f13efafcSArnd Bergmann pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) 778b4dad40eSYong Wu pa &= ~BIT_ULL(32); 77930e2fccfSYong Wu 7800df4fabeSYong Wu return pa; 7810df4fabeSYong Wu } 7820df4fabeSYong Wu 78380e4592aSJoerg Roedel static struct iommu_device *mtk_iommu_probe_device(struct device *dev) 7840df4fabeSYong Wu { 785a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 786b16c0170SJoerg Roedel struct mtk_iommu_data *data; 787635319a4SYong Wu struct device_link *link; 788635319a4SYong Wu struct device *larbdev; 789635319a4SYong Wu unsigned int larbid, larbidx, i; 7900df4fabeSYong Wu 791a9bf2eecSJoerg Roedel if (!fwspec || fwspec->ops != &mtk_iommu_ops) 79280e4592aSJoerg Roedel return ERR_PTR(-ENODEV); /* Not a iommu client device */ 7930df4fabeSYong Wu 7943524b559SJoerg Roedel data = dev_iommu_priv_get(dev); 795b16c0170SJoerg Roedel 796d2e9a110SYong Wu if (!MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) 797d2e9a110SYong Wu return &data->iommu; 798d2e9a110SYong Wu 799635319a4SYong Wu /* 800635319a4SYong Wu * Link the consumer device with the smi-larb device(supplier). 801635319a4SYong Wu * The device that connects with each a larb is a independent HW. 802635319a4SYong Wu * All the ports in each a device should be in the same larbs. 803635319a4SYong Wu */ 804635319a4SYong Wu larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); 805de78657eSMiles Chen if (larbid >= MTK_LARB_NR_MAX) 806de78657eSMiles Chen return ERR_PTR(-EINVAL); 807de78657eSMiles Chen 808635319a4SYong Wu for (i = 1; i < fwspec->num_ids; i++) { 809635319a4SYong Wu larbidx = MTK_M4U_TO_LARB(fwspec->ids[i]); 810635319a4SYong Wu if (larbid != larbidx) { 811635319a4SYong Wu dev_err(dev, "Can only use one larb. Fail@larb%d-%d.\n", 812635319a4SYong Wu larbid, larbidx); 813635319a4SYong Wu return ERR_PTR(-EINVAL); 814635319a4SYong Wu } 815635319a4SYong Wu } 816635319a4SYong Wu larbdev = data->larb_imu[larbid].dev; 817de78657eSMiles Chen if (!larbdev) 818de78657eSMiles Chen return ERR_PTR(-EINVAL); 819de78657eSMiles Chen 820635319a4SYong Wu link = device_link_add(dev, larbdev, 821635319a4SYong Wu DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS); 822635319a4SYong Wu if (!link) 823635319a4SYong Wu dev_err(dev, "Unable to link %s\n", dev_name(larbdev)); 82480e4592aSJoerg Roedel return &data->iommu; 8250df4fabeSYong Wu } 8260df4fabeSYong Wu 82780e4592aSJoerg Roedel static void mtk_iommu_release_device(struct device *dev) 8280df4fabeSYong Wu { 829a9bf2eecSJoerg Roedel struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 830635319a4SYong Wu struct mtk_iommu_data *data; 831635319a4SYong Wu struct device *larbdev; 832635319a4SYong Wu unsigned int larbid; 833b16c0170SJoerg Roedel 834635319a4SYong Wu data = dev_iommu_priv_get(dev); 835d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { 836635319a4SYong Wu larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); 837635319a4SYong Wu larbdev = data->larb_imu[larbid].dev; 838635319a4SYong Wu device_link_remove(dev, larbdev); 839d2e9a110SYong Wu } 8400df4fabeSYong Wu } 8410df4fabeSYong Wu 84257fb481fSYong Wu static int mtk_iommu_get_group_id(struct device *dev, const struct mtk_iommu_plat_data *plat_data) 84357fb481fSYong Wu { 84457fb481fSYong Wu unsigned int bankid; 84557fb481fSYong Wu 84657fb481fSYong Wu /* 84757fb481fSYong Wu * If the bank function is enabled, each bank is a iommu group/domain. 84857fb481fSYong Wu * Otherwise, each iova region is a iommu group/domain. 84957fb481fSYong Wu */ 85057fb481fSYong Wu bankid = mtk_iommu_get_bank_id(dev, plat_data); 85157fb481fSYong Wu if (bankid) 85257fb481fSYong Wu return bankid; 85357fb481fSYong Wu 85457fb481fSYong Wu return mtk_iommu_get_iova_region_id(dev, plat_data); 85557fb481fSYong Wu } 85657fb481fSYong Wu 8570df4fabeSYong Wu static struct iommu_group *mtk_iommu_device_group(struct device *dev) 8580df4fabeSYong Wu { 8599e3a2a64SYong Wu struct mtk_iommu_data *c_data = dev_iommu_priv_get(dev), *data; 8609e3a2a64SYong Wu struct list_head *hw_list = c_data->hw_list; 861c3045f39SYong Wu struct iommu_group *group; 86257fb481fSYong Wu int groupid; 8630df4fabeSYong Wu 8649e3a2a64SYong Wu data = mtk_iommu_get_frst_data(hw_list); 86558f0d1d5SRobin Murphy if (!data) 8660df4fabeSYong Wu return ERR_PTR(-ENODEV); 8670df4fabeSYong Wu 86857fb481fSYong Wu groupid = mtk_iommu_get_group_id(dev, data->plat_data); 86957fb481fSYong Wu if (groupid < 0) 87057fb481fSYong Wu return ERR_PTR(groupid); 871803cf9e5SYong Wu 8720e5a3f2eSYong Wu mutex_lock(&data->mutex); 87357fb481fSYong Wu group = data->m4u_group[groupid]; 874c3045f39SYong Wu if (!group) { 875c3045f39SYong Wu group = iommu_group_alloc(); 876c3045f39SYong Wu if (!IS_ERR(group)) 87757fb481fSYong Wu data->m4u_group[groupid] = group; 8783a8d40b6SRobin Murphy } else { 879c3045f39SYong Wu iommu_group_ref_get(group); 8800df4fabeSYong Wu } 8810e5a3f2eSYong Wu mutex_unlock(&data->mutex); 882c3045f39SYong Wu return group; 8830df4fabeSYong Wu } 8840df4fabeSYong Wu 8850df4fabeSYong Wu static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) 8860df4fabeSYong Wu { 8870df4fabeSYong Wu struct platform_device *m4updev; 8880df4fabeSYong Wu 8890df4fabeSYong Wu if (args->args_count != 1) { 8900df4fabeSYong Wu dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", 8910df4fabeSYong Wu args->args_count); 8920df4fabeSYong Wu return -EINVAL; 8930df4fabeSYong Wu } 8940df4fabeSYong Wu 8953524b559SJoerg Roedel if (!dev_iommu_priv_get(dev)) { 8960df4fabeSYong Wu /* Get the m4u device */ 8970df4fabeSYong Wu m4updev = of_find_device_by_node(args->np); 8980df4fabeSYong Wu if (WARN_ON(!m4updev)) 8990df4fabeSYong Wu return -EINVAL; 9000df4fabeSYong Wu 9013524b559SJoerg Roedel dev_iommu_priv_set(dev, platform_get_drvdata(m4updev)); 9020df4fabeSYong Wu } 9030df4fabeSYong Wu 90458f0d1d5SRobin Murphy return iommu_fwspec_add_ids(dev, args->args, 1); 9050df4fabeSYong Wu } 9060df4fabeSYong Wu 907ab1d5281SYong Wu static void mtk_iommu_get_resv_regions(struct device *dev, 908ab1d5281SYong Wu struct list_head *head) 909ab1d5281SYong Wu { 910ab1d5281SYong Wu struct mtk_iommu_data *data = dev_iommu_priv_get(dev); 911d72e0ff5SYong Wu unsigned int regionid = mtk_iommu_get_iova_region_id(dev, data->plat_data), i; 912ab1d5281SYong Wu const struct mtk_iommu_iova_region *resv, *curdom; 913ab1d5281SYong Wu struct iommu_resv_region *region; 914ab1d5281SYong Wu int prot = IOMMU_WRITE | IOMMU_READ; 915ab1d5281SYong Wu 916d72e0ff5SYong Wu if ((int)regionid < 0) 917ab1d5281SYong Wu return; 918d72e0ff5SYong Wu curdom = data->plat_data->iova_region + regionid; 919ab1d5281SYong Wu for (i = 0; i < data->plat_data->iova_region_nr; i++) { 920ab1d5281SYong Wu resv = data->plat_data->iova_region + i; 921ab1d5281SYong Wu 922ab1d5281SYong Wu /* Only reserve when the region is inside the current domain */ 923ab1d5281SYong Wu if (resv->iova_base <= curdom->iova_base || 924ab1d5281SYong Wu resv->iova_base + resv->size >= curdom->iova_base + curdom->size) 925ab1d5281SYong Wu continue; 926ab1d5281SYong Wu 927ab1d5281SYong Wu region = iommu_alloc_resv_region(resv->iova_base, resv->size, 9280251d010SLu Baolu prot, IOMMU_RESV_RESERVED, 9290251d010SLu Baolu GFP_KERNEL); 930ab1d5281SYong Wu if (!region) 931ab1d5281SYong Wu return; 932ab1d5281SYong Wu 933ab1d5281SYong Wu list_add_tail(®ion->list, head); 934ab1d5281SYong Wu } 935ab1d5281SYong Wu } 936ab1d5281SYong Wu 937b65f5016SArvind Yadav static const struct iommu_ops mtk_iommu_ops = { 9380df4fabeSYong Wu .domain_alloc = mtk_iommu_domain_alloc, 93980e4592aSJoerg Roedel .probe_device = mtk_iommu_probe_device, 94080e4592aSJoerg Roedel .release_device = mtk_iommu_release_device, 9410df4fabeSYong Wu .device_group = mtk_iommu_device_group, 9420df4fabeSYong Wu .of_xlate = mtk_iommu_of_xlate, 943ab1d5281SYong Wu .get_resv_regions = mtk_iommu_get_resv_regions, 9440df4fabeSYong Wu .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, 94518d8c74eSYong Wu .owner = THIS_MODULE, 9469a630a4bSLu Baolu .default_domain_ops = &(const struct iommu_domain_ops) { 9479a630a4bSLu Baolu .attach_dev = mtk_iommu_attach_device, 9489a630a4bSLu Baolu .detach_dev = mtk_iommu_detach_device, 9499a630a4bSLu Baolu .map = mtk_iommu_map, 9509a630a4bSLu Baolu .unmap = mtk_iommu_unmap, 9519a630a4bSLu Baolu .flush_iotlb_all = mtk_iommu_flush_iotlb_all, 9529a630a4bSLu Baolu .iotlb_sync = mtk_iommu_iotlb_sync, 9539a630a4bSLu Baolu .iotlb_sync_map = mtk_iommu_sync_map, 9549a630a4bSLu Baolu .iova_to_phys = mtk_iommu_iova_to_phys, 9559a630a4bSLu Baolu .free = mtk_iommu_domain_free, 9569a630a4bSLu Baolu } 9570df4fabeSYong Wu }; 9580df4fabeSYong Wu 959e24453e1SYong Wu static int mtk_iommu_hw_init(const struct mtk_iommu_data *data, unsigned int bankid) 9600df4fabeSYong Wu { 961e24453e1SYong Wu const struct mtk_iommu_bank_data *bankx = &data->bank[bankid]; 96299ca0228SYong Wu const struct mtk_iommu_bank_data *bank0 = &data->bank[0]; 9630df4fabeSYong Wu u32 regval; 9640df4fabeSYong Wu 965e24453e1SYong Wu /* 966e24453e1SYong Wu * Global control settings are in bank0. May re-init these global registers 967e24453e1SYong Wu * since no sure if there is bank0 consumers. 968e24453e1SYong Wu */ 96986580ec9SAngeloGioacchino Del Regno if (MTK_IOMMU_HAS_FLAG(data->plat_data, TF_PORT_TO_ADDR_MT8173)) { 970acb3c92aSYong Wu regval = F_MMU_PREFETCH_RT_REPLACE_MOD | 971acb3c92aSYong Wu F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173; 97286444413SChao Hao } else { 97399ca0228SYong Wu regval = readl_relaxed(bank0->base + REG_MMU_CTRL_REG); 97486444413SChao Hao regval |= F_MMU_TF_PROT_TO_PROGRAM_ADDR; 97586444413SChao Hao } 97699ca0228SYong Wu writel_relaxed(regval, bank0->base + REG_MMU_CTRL_REG); 9770df4fabeSYong Wu 9786b717796SChao Hao if (data->enable_4GB && 9796b717796SChao Hao MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_VLD_PA_RNG)) { 98030e2fccfSYong Wu /* 98130e2fccfSYong Wu * If 4GB mode is enabled, the validate PA range is from 98230e2fccfSYong Wu * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30]. 98330e2fccfSYong Wu */ 98430e2fccfSYong Wu regval = F_MMU_VLD_PA_RNG(7, 4); 98599ca0228SYong Wu writel_relaxed(regval, bank0->base + REG_MMU_VLD_PA_RNG); 98630e2fccfSYong Wu } 9879a87005eSYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, DCM_DISABLE)) 98899ca0228SYong Wu writel_relaxed(F_MMU_DCM, bank0->base + REG_MMU_DCM_DIS); 9899a87005eSYong Wu else 99099ca0228SYong Wu writel_relaxed(0, bank0->base + REG_MMU_DCM_DIS); 9919a87005eSYong Wu 99235c1b48dSChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, WR_THROT_EN)) { 99335c1b48dSChao Hao /* write command throttling mode */ 99499ca0228SYong Wu regval = readl_relaxed(bank0->base + REG_MMU_WR_LEN_CTRL); 99535c1b48dSChao Hao regval &= ~F_MMU_WR_THROT_DIS_MASK; 99699ca0228SYong Wu writel_relaxed(regval, bank0->base + REG_MMU_WR_LEN_CTRL); 99735c1b48dSChao Hao } 998e6dec923SYong Wu 9996b717796SChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) { 100075eed350SChao Hao /* The register is called STANDARD_AXI_MODE in this case */ 10014bb2bf4cSChao Hao regval = 0; 10024bb2bf4cSChao Hao } else { 100399ca0228SYong Wu regval = readl_relaxed(bank0->base + REG_MMU_MISC_CTRL); 1004d265a4adSYong Wu if (!MTK_IOMMU_HAS_FLAG(data->plat_data, STD_AXI_MODE)) 10054bb2bf4cSChao Hao regval &= ~F_MMU_STANDARD_AXI_MODE_MASK; 10064bb2bf4cSChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, OUT_ORDER_WR_EN)) 10074bb2bf4cSChao Hao regval &= ~F_MMU_IN_ORDER_WR_EN_MASK; 100875eed350SChao Hao } 100999ca0228SYong Wu writel_relaxed(regval, bank0->base + REG_MMU_MISC_CTRL); 10100df4fabeSYong Wu 1011e24453e1SYong Wu /* Independent settings for each bank */ 1012634f57dfSYong Wu regval = F_L2_MULIT_HIT_EN | 1013634f57dfSYong Wu F_TABLE_WALK_FAULT_INT_EN | 1014634f57dfSYong Wu F_PREETCH_FIFO_OVERFLOW_INT_EN | 1015634f57dfSYong Wu F_MISS_FIFO_OVERFLOW_INT_EN | 1016634f57dfSYong Wu F_PREFETCH_FIFO_ERR_INT_EN | 1017634f57dfSYong Wu F_MISS_FIFO_ERR_INT_EN; 1018e24453e1SYong Wu writel_relaxed(regval, bankx->base + REG_MMU_INT_CONTROL0); 1019634f57dfSYong Wu 1020634f57dfSYong Wu regval = F_INT_TRANSLATION_FAULT | 1021634f57dfSYong Wu F_INT_MAIN_MULTI_HIT_FAULT | 1022634f57dfSYong Wu F_INT_INVALID_PA_FAULT | 1023634f57dfSYong Wu F_INT_ENTRY_REPLACEMENT_FAULT | 1024634f57dfSYong Wu F_INT_TLB_MISS_FAULT | 1025634f57dfSYong Wu F_INT_MISS_TRANSACTION_FIFO_FAULT | 1026634f57dfSYong Wu F_INT_PRETETCH_TRANSATION_FIFO_FAULT; 1027e24453e1SYong Wu writel_relaxed(regval, bankx->base + REG_MMU_INT_MAIN_CONTROL); 1028634f57dfSYong Wu 1029634f57dfSYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_LEGACY_IVRP_PADDR)) 1030634f57dfSYong Wu regval = (data->protect_base >> 1) | (data->enable_4GB << 31); 1031634f57dfSYong Wu else 1032634f57dfSYong Wu regval = lower_32_bits(data->protect_base) | 1033634f57dfSYong Wu upper_32_bits(data->protect_base); 1034e24453e1SYong Wu writel_relaxed(regval, bankx->base + REG_MMU_IVRP_PADDR); 1035634f57dfSYong Wu 1036e24453e1SYong Wu if (devm_request_irq(bankx->parent_dev, bankx->irq, mtk_iommu_isr, 0, 1037e24453e1SYong Wu dev_name(bankx->parent_dev), (void *)bankx)) { 1038e24453e1SYong Wu writel_relaxed(0, bankx->base + REG_MMU_PT_BASE_ADDR); 1039e24453e1SYong Wu dev_err(bankx->parent_dev, "Failed @ IRQ-%d Request\n", bankx->irq); 10400df4fabeSYong Wu return -ENODEV; 10410df4fabeSYong Wu } 10420df4fabeSYong Wu 10430df4fabeSYong Wu return 0; 10440df4fabeSYong Wu } 10450df4fabeSYong Wu 10460df4fabeSYong Wu static const struct component_master_ops mtk_iommu_com_ops = { 10470df4fabeSYong Wu .bind = mtk_iommu_bind, 10480df4fabeSYong Wu .unbind = mtk_iommu_unbind, 10490df4fabeSYong Wu }; 10500df4fabeSYong Wu 1051d2e9a110SYong Wu static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **match, 1052d2e9a110SYong Wu struct mtk_iommu_data *data) 1053d2e9a110SYong Wu { 10546cde583dSYong Wu struct device_node *larbnode, *frst_avail_smicomm_node = NULL; 1055dcb40e9fSYong Wu struct platform_device *plarbdev, *pcommdev; 1056d2e9a110SYong Wu struct device_link *link; 1057d2e9a110SYong Wu int i, larb_nr, ret; 1058d2e9a110SYong Wu 1059d2e9a110SYong Wu larb_nr = of_count_phandle_with_args(dev->of_node, "mediatek,larbs", NULL); 1060d2e9a110SYong Wu if (larb_nr < 0) 1061d2e9a110SYong Wu return larb_nr; 1062ef693a84SGuenter Roeck if (larb_nr == 0 || larb_nr > MTK_LARB_NR_MAX) 1063ef693a84SGuenter Roeck return -EINVAL; 1064d2e9a110SYong Wu 1065d2e9a110SYong Wu for (i = 0; i < larb_nr; i++) { 10666cde583dSYong Wu struct device_node *smicomm_node, *smi_subcomm_node; 1067d2e9a110SYong Wu u32 id; 1068d2e9a110SYong Wu 1069d2e9a110SYong Wu larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); 107026593928SYong Wu if (!larbnode) { 107126593928SYong Wu ret = -EINVAL; 107226593928SYong Wu goto err_larbdev_put; 107326593928SYong Wu } 1074d2e9a110SYong Wu 1075d2e9a110SYong Wu if (!of_device_is_available(larbnode)) { 1076d2e9a110SYong Wu of_node_put(larbnode); 1077d2e9a110SYong Wu continue; 1078d2e9a110SYong Wu } 1079d2e9a110SYong Wu 1080d2e9a110SYong Wu ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id); 1081d2e9a110SYong Wu if (ret)/* The id is consecutive if there is no this property */ 1082d2e9a110SYong Wu id = i; 1083ef693a84SGuenter Roeck if (id >= MTK_LARB_NR_MAX) { 1084ef693a84SGuenter Roeck of_node_put(larbnode); 1085ef693a84SGuenter Roeck ret = -EINVAL; 1086ef693a84SGuenter Roeck goto err_larbdev_put; 1087ef693a84SGuenter Roeck } 1088d2e9a110SYong Wu 1089d2e9a110SYong Wu plarbdev = of_find_device_by_node(larbnode); 1090d2e9a110SYong Wu of_node_put(larbnode); 109126593928SYong Wu if (!plarbdev) { 109226593928SYong Wu ret = -ENODEV; 109326593928SYong Wu goto err_larbdev_put; 1094d2e9a110SYong Wu } 1095ef693a84SGuenter Roeck if (data->larb_imu[id].dev) { 1096ef693a84SGuenter Roeck platform_device_put(plarbdev); 1097ef693a84SGuenter Roeck ret = -EEXIST; 1098ef693a84SGuenter Roeck goto err_larbdev_put; 1099ef693a84SGuenter Roeck } 1100d2e9a110SYong Wu data->larb_imu[id].dev = &plarbdev->dev; 1101d2e9a110SYong Wu 110226593928SYong Wu if (!plarbdev->dev.driver) { 110326593928SYong Wu ret = -EPROBE_DEFER; 110426593928SYong Wu goto err_larbdev_put; 110526593928SYong Wu } 110626593928SYong Wu 1107f7b71d0dSYong Wu /* Get smi-(sub)-common dev from the last larb. */ 1108f7b71d0dSYong Wu smi_subcomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0); 11096cde583dSYong Wu if (!smi_subcomm_node) { 11106cde583dSYong Wu ret = -EINVAL; 11116cde583dSYong Wu goto err_larbdev_put; 11126cde583dSYong Wu } 1113d2e9a110SYong Wu 1114f7b71d0dSYong Wu /* 1115f7b71d0dSYong Wu * It may have two level smi-common. the node is smi-sub-common if it 1116f7b71d0dSYong Wu * has a new mediatek,smi property. otherwise it is smi-commmon. 1117f7b71d0dSYong Wu */ 1118f7b71d0dSYong Wu smicomm_node = of_parse_phandle(smi_subcomm_node, "mediatek,smi", 0); 1119f7b71d0dSYong Wu if (smicomm_node) 1120f7b71d0dSYong Wu of_node_put(smi_subcomm_node); 1121f7b71d0dSYong Wu else 1122f7b71d0dSYong Wu smicomm_node = smi_subcomm_node; 1123f7b71d0dSYong Wu 11246cde583dSYong Wu /* 11256cde583dSYong Wu * All the larbs that connect to one IOMMU must connect with the same 11266cde583dSYong Wu * smi-common. 11276cde583dSYong Wu */ 11286cde583dSYong Wu if (!frst_avail_smicomm_node) { 11296cde583dSYong Wu frst_avail_smicomm_node = smicomm_node; 11306cde583dSYong Wu } else if (frst_avail_smicomm_node != smicomm_node) { 11316cde583dSYong Wu dev_err(dev, "mediatek,smi property is not right @larb%d.", id); 1132d2e9a110SYong Wu of_node_put(smicomm_node); 11336cde583dSYong Wu ret = -EINVAL; 11346cde583dSYong Wu goto err_larbdev_put; 11356cde583dSYong Wu } else { 11366cde583dSYong Wu of_node_put(smicomm_node); 11376cde583dSYong Wu } 11386cde583dSYong Wu 11396cde583dSYong Wu component_match_add(dev, match, component_compare_dev, &plarbdev->dev); 11406cde583dSYong Wu platform_device_put(plarbdev); 11416cde583dSYong Wu } 11426cde583dSYong Wu 11436cde583dSYong Wu if (!frst_avail_smicomm_node) 11446cde583dSYong Wu return -EINVAL; 11456cde583dSYong Wu 11466cde583dSYong Wu pcommdev = of_find_device_by_node(frst_avail_smicomm_node); 11476cde583dSYong Wu of_node_put(frst_avail_smicomm_node); 1148dcb40e9fSYong Wu if (!pcommdev) 1149dcb40e9fSYong Wu return -ENODEV; 1150dcb40e9fSYong Wu data->smicomm_dev = &pcommdev->dev; 1151d2e9a110SYong Wu 1152d2e9a110SYong Wu link = device_link_add(data->smicomm_dev, dev, 1153d2e9a110SYong Wu DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); 1154dcb40e9fSYong Wu platform_device_put(pcommdev); 1155d2e9a110SYong Wu if (!link) { 1156d2e9a110SYong Wu dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev)); 1157d2e9a110SYong Wu return -EINVAL; 1158d2e9a110SYong Wu } 1159d2e9a110SYong Wu return 0; 116026593928SYong Wu 116126593928SYong Wu err_larbdev_put: 116226593928SYong Wu /* id may be not linear mapping, loop whole the array */ 116326593928SYong Wu for (i = MTK_LARB_NR_MAX - 1; i >= 0; i++) { 116426593928SYong Wu if (!data->larb_imu[i].dev) 116526593928SYong Wu continue; 116626593928SYong Wu put_device(data->larb_imu[i].dev); 116726593928SYong Wu } 116826593928SYong Wu return ret; 1169d2e9a110SYong Wu } 1170d2e9a110SYong Wu 11710df4fabeSYong Wu static int mtk_iommu_probe(struct platform_device *pdev) 11720df4fabeSYong Wu { 11730df4fabeSYong Wu struct mtk_iommu_data *data; 11740df4fabeSYong Wu struct device *dev = &pdev->dev; 11750df4fabeSYong Wu struct resource *res; 1176b16c0170SJoerg Roedel resource_size_t ioaddr; 11770df4fabeSYong Wu struct component_match *match = NULL; 1178c2c59456SMiles Chen struct regmap *infracfg; 11790df4fabeSYong Wu void *protect; 118042d57fc5SYong Wu int ret, banks_num, i = 0; 1181c2c59456SMiles Chen u32 val; 1182c2c59456SMiles Chen char *p; 118399ca0228SYong Wu struct mtk_iommu_bank_data *bank; 118499ca0228SYong Wu void __iomem *base; 11850df4fabeSYong Wu 11860df4fabeSYong Wu data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); 11870df4fabeSYong Wu if (!data) 11880df4fabeSYong Wu return -ENOMEM; 11890df4fabeSYong Wu data->dev = dev; 1190cecdce9dSYong Wu data->plat_data = of_device_get_match_data(dev); 11910df4fabeSYong Wu 11920df4fabeSYong Wu /* Protect memory. HW will access here while translation fault.*/ 11930df4fabeSYong Wu protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); 11940df4fabeSYong Wu if (!protect) 11950df4fabeSYong Wu return -ENOMEM; 11960df4fabeSYong Wu data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); 11970df4fabeSYong Wu 1198c2c59456SMiles Chen if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) { 11997d748ffdSAngeloGioacchino Del Regno infracfg = syscon_regmap_lookup_by_phandle(dev->of_node, "mediatek,infracfg"); 12007d748ffdSAngeloGioacchino Del Regno if (IS_ERR(infracfg)) { 12017d748ffdSAngeloGioacchino Del Regno /* 12027d748ffdSAngeloGioacchino Del Regno * Legacy devicetrees will not specify a phandle to 12037d748ffdSAngeloGioacchino Del Regno * mediatek,infracfg: in that case, we use the older 12047d748ffdSAngeloGioacchino Del Regno * way to retrieve a syscon to infra. 12057d748ffdSAngeloGioacchino Del Regno * 12067d748ffdSAngeloGioacchino Del Regno * This is for retrocompatibility purposes only, hence 12077d748ffdSAngeloGioacchino Del Regno * no more compatibles shall be added to this. 12087d748ffdSAngeloGioacchino Del Regno */ 1209c2c59456SMiles Chen switch (data->plat_data->m4u_plat) { 1210c2c59456SMiles Chen case M4U_MT2712: 1211c2c59456SMiles Chen p = "mediatek,mt2712-infracfg"; 1212c2c59456SMiles Chen break; 1213c2c59456SMiles Chen case M4U_MT8173: 1214c2c59456SMiles Chen p = "mediatek,mt8173-infracfg"; 1215c2c59456SMiles Chen break; 1216c2c59456SMiles Chen default: 1217c2c59456SMiles Chen p = NULL; 1218c2c59456SMiles Chen } 1219c2c59456SMiles Chen 1220c2c59456SMiles Chen infracfg = syscon_regmap_lookup_by_compatible(p); 1221c2c59456SMiles Chen if (IS_ERR(infracfg)) 1222c2c59456SMiles Chen return PTR_ERR(infracfg); 12237d748ffdSAngeloGioacchino Del Regno } 1224c2c59456SMiles Chen 1225c2c59456SMiles Chen ret = regmap_read(infracfg, REG_INFRA_MISC, &val); 1226c2c59456SMiles Chen if (ret) 1227c2c59456SMiles Chen return ret; 1228c2c59456SMiles Chen data->enable_4GB = !!(val & F_DDR_4GB_SUPPORT_EN); 1229c2c59456SMiles Chen } 123001e23c93SYong Wu 123142d57fc5SYong Wu banks_num = data->plat_data->banks_num; 12320df4fabeSYong Wu res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 123373b6924cSYang Yingliang if (!res) 123473b6924cSYang Yingliang return -EINVAL; 123542d57fc5SYong Wu if (resource_size(res) < banks_num * MTK_IOMMU_BANK_SZ) { 123642d57fc5SYong Wu dev_err(dev, "banknr %d. res %pR is not enough.\n", banks_num, res); 123742d57fc5SYong Wu return -EINVAL; 123842d57fc5SYong Wu } 123999ca0228SYong Wu base = devm_ioremap_resource(dev, res); 124099ca0228SYong Wu if (IS_ERR(base)) 124199ca0228SYong Wu return PTR_ERR(base); 1242b16c0170SJoerg Roedel ioaddr = res->start; 12430df4fabeSYong Wu 124499ca0228SYong Wu data->bank = devm_kmalloc(dev, banks_num * sizeof(*data->bank), GFP_KERNEL); 124599ca0228SYong Wu if (!data->bank) 124699ca0228SYong Wu return -ENOMEM; 124799ca0228SYong Wu 124842d57fc5SYong Wu do { 124942d57fc5SYong Wu if (!data->plat_data->banks_enable[i]) 125042d57fc5SYong Wu continue; 125142d57fc5SYong Wu bank = &data->bank[i]; 125242d57fc5SYong Wu bank->id = i; 125342d57fc5SYong Wu bank->base = base + i * MTK_IOMMU_BANK_SZ; 125499ca0228SYong Wu bank->m4u_dom = NULL; 125542d57fc5SYong Wu 125642d57fc5SYong Wu bank->irq = platform_get_irq(pdev, i); 125799ca0228SYong Wu if (bank->irq < 0) 125899ca0228SYong Wu return bank->irq; 125999ca0228SYong Wu bank->parent_dev = dev; 126099ca0228SYong Wu bank->parent_data = data; 126199ca0228SYong Wu spin_lock_init(&bank->tlb_lock); 126242d57fc5SYong Wu } while (++i < banks_num); 12630df4fabeSYong Wu 12646b717796SChao Hao if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) { 12650df4fabeSYong Wu data->bclk = devm_clk_get(dev, "bclk"); 12660df4fabeSYong Wu if (IS_ERR(data->bclk)) 12670df4fabeSYong Wu return PTR_ERR(data->bclk); 12682aa4c259SYong Wu } 12690df4fabeSYong Wu 1270c0b57581SYong Wu pm_runtime_enable(dev); 1271c0b57581SYong Wu 1272d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { 1273d2e9a110SYong Wu ret = mtk_iommu_mm_dts_parse(dev, &match, data); 1274d2e9a110SYong Wu if (ret) { 12753168010dSNícolas F. R. A. Prado dev_err_probe(dev, ret, "mm dts parse fail\n"); 1276c0b57581SYong Wu goto out_runtime_disable; 1277baf94e6eSYong Wu } 127821fd9be4SAngeloGioacchino Del Regno } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA)) { 127921fd9be4SAngeloGioacchino Del Regno p = data->plat_data->pericfg_comp_str; 128021fd9be4SAngeloGioacchino Del Regno data->pericfg = syscon_regmap_lookup_by_compatible(p); 128121fd9be4SAngeloGioacchino Del Regno if (IS_ERR(data->pericfg)) { 128221fd9be4SAngeloGioacchino Del Regno ret = PTR_ERR(data->pericfg); 1283f9b8c9b2SYong Wu goto out_runtime_disable; 1284f9b8c9b2SYong Wu } 1285d2e9a110SYong Wu } 1286baf94e6eSYong Wu 12870df4fabeSYong Wu platform_set_drvdata(pdev, data); 12880e5a3f2eSYong Wu mutex_init(&data->mutex); 12890df4fabeSYong Wu 1290b16c0170SJoerg Roedel ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, 1291b16c0170SJoerg Roedel "mtk-iommu.%pa", &ioaddr); 1292b16c0170SJoerg Roedel if (ret) 1293baf94e6eSYong Wu goto out_link_remove; 1294b16c0170SJoerg Roedel 12952d471b20SRobin Murphy ret = iommu_device_register(&data->iommu, &mtk_iommu_ops, dev); 1296b16c0170SJoerg Roedel if (ret) 1297986d9ec5SYong Wu goto out_sysfs_remove; 1298b16c0170SJoerg Roedel 12999e3a2a64SYong Wu if (MTK_IOMMU_HAS_FLAG(data->plat_data, SHARE_PGTABLE)) { 13009e3a2a64SYong Wu list_add_tail(&data->list, data->plat_data->hw_list); 13019e3a2a64SYong Wu data->hw_list = data->plat_data->hw_list; 13029e3a2a64SYong Wu } else { 13039e3a2a64SYong Wu INIT_LIST_HEAD(&data->hw_list_head); 13049e3a2a64SYong Wu list_add_tail(&data->list, &data->hw_list_head); 13059e3a2a64SYong Wu data->hw_list = &data->hw_list_head; 13069e3a2a64SYong Wu } 13077c3a2ec0SYong Wu 1308d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { 1309986d9ec5SYong Wu ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); 1310986d9ec5SYong Wu if (ret) 1311e7629070SYong Wu goto out_list_del; 1312e7629070SYong Wu } 1313986d9ec5SYong Wu return ret; 1314986d9ec5SYong Wu 1315986d9ec5SYong Wu out_list_del: 1316986d9ec5SYong Wu list_del(&data->list); 1317986d9ec5SYong Wu iommu_device_unregister(&data->iommu); 1318986d9ec5SYong Wu out_sysfs_remove: 1319986d9ec5SYong Wu iommu_device_sysfs_remove(&data->iommu); 1320baf94e6eSYong Wu out_link_remove: 1321d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) 1322baf94e6eSYong Wu device_link_remove(data->smicomm_dev, dev); 1323c0b57581SYong Wu out_runtime_disable: 1324c0b57581SYong Wu pm_runtime_disable(dev); 1325986d9ec5SYong Wu return ret; 13260df4fabeSYong Wu } 13270df4fabeSYong Wu 13280df4fabeSYong Wu static int mtk_iommu_remove(struct platform_device *pdev) 13290df4fabeSYong Wu { 13300df4fabeSYong Wu struct mtk_iommu_data *data = platform_get_drvdata(pdev); 133142d57fc5SYong Wu struct mtk_iommu_bank_data *bank; 133242d57fc5SYong Wu int i; 13330df4fabeSYong Wu 1334b16c0170SJoerg Roedel iommu_device_sysfs_remove(&data->iommu); 1335b16c0170SJoerg Roedel iommu_device_unregister(&data->iommu); 1336b16c0170SJoerg Roedel 1337ee55f75eSYong Wu list_del(&data->list); 13380df4fabeSYong Wu 1339d2e9a110SYong Wu if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { 1340baf94e6eSYong Wu device_link_remove(data->smicomm_dev, &pdev->dev); 1341d2e9a110SYong Wu component_master_del(&pdev->dev, &mtk_iommu_com_ops); 1342d2e9a110SYong Wu } 1343c0b57581SYong Wu pm_runtime_disable(&pdev->dev); 134442d57fc5SYong Wu for (i = 0; i < data->plat_data->banks_num; i++) { 134542d57fc5SYong Wu bank = &data->bank[i]; 134642d57fc5SYong Wu if (!bank->m4u_dom) 134742d57fc5SYong Wu continue; 134899ca0228SYong Wu devm_free_irq(&pdev->dev, bank->irq, bank); 134942d57fc5SYong Wu } 13500df4fabeSYong Wu return 0; 13510df4fabeSYong Wu } 13520df4fabeSYong Wu 135334665c79SYong Wu static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev) 13540df4fabeSYong Wu { 13550df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 13560df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 1357d7127de1SYong Wu void __iomem *base; 1358d7127de1SYong Wu int i = 0; 13590df4fabeSYong Wu 1360d7127de1SYong Wu base = data->bank[i].base; 136135c1b48dSChao Hao reg->wr_len_ctrl = readl_relaxed(base + REG_MMU_WR_LEN_CTRL); 136275eed350SChao Hao reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL); 13630df4fabeSYong Wu reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); 13640df4fabeSYong Wu reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); 1365b9475b34SYong Wu reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG); 1366d7127de1SYong Wu do { 1367d7127de1SYong Wu if (!data->plat_data->banks_enable[i]) 1368d7127de1SYong Wu continue; 1369d7127de1SYong Wu base = data->bank[i].base; 1370d7127de1SYong Wu reg->int_control[i] = readl_relaxed(base + REG_MMU_INT_CONTROL0); 1371d7127de1SYong Wu reg->int_main_control[i] = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); 1372d7127de1SYong Wu reg->ivrp_paddr[i] = readl_relaxed(base + REG_MMU_IVRP_PADDR); 1373d7127de1SYong Wu } while (++i < data->plat_data->banks_num); 13746254b64fSYong Wu clk_disable_unprepare(data->bclk); 13750df4fabeSYong Wu return 0; 13760df4fabeSYong Wu } 13770df4fabeSYong Wu 137834665c79SYong Wu static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) 13790df4fabeSYong Wu { 13800df4fabeSYong Wu struct mtk_iommu_data *data = dev_get_drvdata(dev); 13810df4fabeSYong Wu struct mtk_iommu_suspend_reg *reg = &data->reg; 1382d7127de1SYong Wu struct mtk_iommu_domain *m4u_dom; 1383d7127de1SYong Wu void __iomem *base; 1384d7127de1SYong Wu int ret, i = 0; 13850df4fabeSYong Wu 13866254b64fSYong Wu ret = clk_prepare_enable(data->bclk); 13876254b64fSYong Wu if (ret) { 13886254b64fSYong Wu dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret); 13896254b64fSYong Wu return ret; 13906254b64fSYong Wu } 1391b34ea31fSDafna Hirschfeld 1392b34ea31fSDafna Hirschfeld /* 1393b34ea31fSDafna Hirschfeld * Uppon first resume, only enable the clk and return, since the values of the 1394b34ea31fSDafna Hirschfeld * registers are not yet set. 1395b34ea31fSDafna Hirschfeld */ 1396d7127de1SYong Wu if (!reg->wr_len_ctrl) 1397b34ea31fSDafna Hirschfeld return 0; 1398b34ea31fSDafna Hirschfeld 1399d7127de1SYong Wu base = data->bank[i].base; 140035c1b48dSChao Hao writel_relaxed(reg->wr_len_ctrl, base + REG_MMU_WR_LEN_CTRL); 140175eed350SChao Hao writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL); 14020df4fabeSYong Wu writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); 14030df4fabeSYong Wu writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); 1404b9475b34SYong Wu writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG); 1405d7127de1SYong Wu do { 1406d7127de1SYong Wu m4u_dom = data->bank[i].m4u_dom; 1407d7127de1SYong Wu if (!data->plat_data->banks_enable[i] || !m4u_dom) 1408d7127de1SYong Wu continue; 1409d7127de1SYong Wu base = data->bank[i].base; 1410d7127de1SYong Wu writel_relaxed(reg->int_control[i], base + REG_MMU_INT_CONTROL0); 1411d7127de1SYong Wu writel_relaxed(reg->int_main_control[i], base + REG_MMU_INT_MAIN_CONTROL); 1412d7127de1SYong Wu writel_relaxed(reg->ivrp_paddr[i], base + REG_MMU_IVRP_PADDR); 1413301c3ca1SYunfei Wang writel(m4u_dom->cfg.arm_v7s_cfg.ttbr, base + REG_MMU_PT_BASE_ADDR); 1414d7127de1SYong Wu } while (++i < data->plat_data->banks_num); 14154f23f6d4SYong Wu 14164f23f6d4SYong Wu /* 14174f23f6d4SYong Wu * Users may allocate dma buffer before they call pm_runtime_get, 14184f23f6d4SYong Wu * in which case it will lack the necessary tlb flush. 14194f23f6d4SYong Wu * Thus, make sure to update the tlb after each PM resume. 14204f23f6d4SYong Wu */ 14214f23f6d4SYong Wu mtk_iommu_tlb_flush_all(data); 14220df4fabeSYong Wu return 0; 14230df4fabeSYong Wu } 14240df4fabeSYong Wu 1425e6dec923SYong Wu static const struct dev_pm_ops mtk_iommu_pm_ops = { 142634665c79SYong Wu SET_RUNTIME_PM_OPS(mtk_iommu_runtime_suspend, mtk_iommu_runtime_resume, NULL) 142734665c79SYong Wu SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 142834665c79SYong Wu pm_runtime_force_resume) 14290df4fabeSYong Wu }; 14300df4fabeSYong Wu 1431cecdce9dSYong Wu static const struct mtk_iommu_plat_data mt2712_data = { 1432cecdce9dSYong Wu .m4u_plat = M4U_MT2712, 1433d2e9a110SYong Wu .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG | SHARE_PGTABLE | 1434d2e9a110SYong Wu MTK_IOMMU_TYPE_MM, 14359e3a2a64SYong Wu .hw_list = &m4ulist, 1436b053bc71SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 1437585e58f4SYong Wu .iova_region = single_domain, 143899ca0228SYong Wu .banks_num = 1, 143999ca0228SYong Wu .banks_enable = {true}, 1440585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 144137276e00SChao Hao .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, 1442cecdce9dSYong Wu }; 1443cecdce9dSYong Wu 1444068c86e9SChao Hao static const struct mtk_iommu_plat_data mt6779_data = { 1445068c86e9SChao Hao .m4u_plat = M4U_MT6779, 1446d2e9a110SYong Wu .flags = HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | WR_THROT_EN | 1447301c3ca1SYunfei Wang MTK_IOMMU_TYPE_MM | PGTABLE_PA_35_EN, 1448068c86e9SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN2, 144999ca0228SYong Wu .banks_num = 1, 145099ca0228SYong Wu .banks_enable = {true}, 1451585e58f4SYong Wu .iova_region = single_domain, 1452585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 1453068c86e9SChao Hao .larbid_remap = {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}}, 1454cecdce9dSYong Wu }; 1455cecdce9dSYong Wu 1456717ec15eSAngeloGioacchino Del Regno static const struct mtk_iommu_plat_data mt6795_data = { 1457717ec15eSAngeloGioacchino Del Regno .m4u_plat = M4U_MT6795, 1458717ec15eSAngeloGioacchino Del Regno .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | 1459717ec15eSAngeloGioacchino Del Regno HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM | 1460717ec15eSAngeloGioacchino Del Regno TF_PORT_TO_ADDR_MT8173, 1461717ec15eSAngeloGioacchino Del Regno .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 1462717ec15eSAngeloGioacchino Del Regno .banks_num = 1, 1463717ec15eSAngeloGioacchino Del Regno .banks_enable = {true}, 1464717ec15eSAngeloGioacchino Del Regno .iova_region = single_domain, 1465717ec15eSAngeloGioacchino Del Regno .iova_region_nr = ARRAY_SIZE(single_domain), 1466717ec15eSAngeloGioacchino Del Regno .larbid_remap = {{0}, {1}, {2}, {3}, {4}}, /* Linear mapping. */ 1467717ec15eSAngeloGioacchino Del Regno }; 1468717ec15eSAngeloGioacchino Del Regno 14693c213562SFabien Parent static const struct mtk_iommu_plat_data mt8167_data = { 14703c213562SFabien Parent .m4u_plat = M4U_MT8167, 1471d2e9a110SYong Wu .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM, 14723c213562SFabien Parent .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 147399ca0228SYong Wu .banks_num = 1, 147499ca0228SYong Wu .banks_enable = {true}, 1475585e58f4SYong Wu .iova_region = single_domain, 1476585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 14773c213562SFabien Parent .larbid_remap = {{0}, {1}, {2}}, /* Linear mapping. */ 14783c213562SFabien Parent }; 14793c213562SFabien Parent 1480cecdce9dSYong Wu static const struct mtk_iommu_plat_data mt8173_data = { 1481cecdce9dSYong Wu .m4u_plat = M4U_MT8173, 1482d1b5ef00SFabien Parent .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | 148386580ec9SAngeloGioacchino Del Regno HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM | 148486580ec9SAngeloGioacchino Del Regno TF_PORT_TO_ADDR_MT8173, 1485b053bc71SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 148699ca0228SYong Wu .banks_num = 1, 148799ca0228SYong Wu .banks_enable = {true}, 1488585e58f4SYong Wu .iova_region = single_domain, 1489585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 149037276e00SChao Hao .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */ 1491cecdce9dSYong Wu }; 1492cecdce9dSYong Wu 1493907ba6a1SYong Wu static const struct mtk_iommu_plat_data mt8183_data = { 1494907ba6a1SYong Wu .m4u_plat = M4U_MT8183, 1495d2e9a110SYong Wu .flags = RESET_AXI | MTK_IOMMU_TYPE_MM, 1496b053bc71SChao Hao .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 149799ca0228SYong Wu .banks_num = 1, 149899ca0228SYong Wu .banks_enable = {true}, 1499585e58f4SYong Wu .iova_region = single_domain, 1500585e58f4SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 150137276e00SChao Hao .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}}, 1502907ba6a1SYong Wu }; 1503907ba6a1SYong Wu 1504e8d7ccaaSYong Wu static const struct mtk_iommu_plat_data mt8186_data_mm = { 1505e8d7ccaaSYong Wu .m4u_plat = M4U_MT8186, 1506e8d7ccaaSYong Wu .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | 1507e8d7ccaaSYong Wu WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM, 1508e8d7ccaaSYong Wu .larbid_remap = {{0}, {1, MTK_INVALID_LARBID, 8}, {4}, {7}, {2}, {9, 11, 19, 20}, 1509e8d7ccaaSYong Wu {MTK_INVALID_LARBID, 14, 16}, 1510e8d7ccaaSYong Wu {MTK_INVALID_LARBID, 13, MTK_INVALID_LARBID, 17}}, 1511e8d7ccaaSYong Wu .inv_sel_reg = REG_MMU_INV_SEL_GEN2, 1512e8d7ccaaSYong Wu .banks_num = 1, 1513e8d7ccaaSYong Wu .banks_enable = {true}, 1514e8d7ccaaSYong Wu .iova_region = mt8192_multi_dom, 1515e8d7ccaaSYong Wu .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), 1516e8d7ccaaSYong Wu }; 1517e8d7ccaaSYong Wu 15189e3489e0SYong Wu static const struct mtk_iommu_plat_data mt8192_data = { 15199e3489e0SYong Wu .m4u_plat = M4U_MT8192, 15209ec30c09SYong Wu .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | 1521d2e9a110SYong Wu WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM, 15229e3489e0SYong Wu .inv_sel_reg = REG_MMU_INV_SEL_GEN2, 152399ca0228SYong Wu .banks_num = 1, 152499ca0228SYong Wu .banks_enable = {true}, 15259e3489e0SYong Wu .iova_region = mt8192_multi_dom, 15269e3489e0SYong Wu .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), 15279e3489e0SYong Wu .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, 15289e3489e0SYong Wu {0, 14, 16}, {0, 13, 18, 17}}, 15299e3489e0SYong Wu }; 15309e3489e0SYong Wu 1531ef68a193SYong Wu static const struct mtk_iommu_plat_data mt8195_data_infra = { 1532ef68a193SYong Wu .m4u_plat = M4U_MT8195, 1533ef68a193SYong Wu .flags = WR_THROT_EN | DCM_DISABLE | STD_AXI_MODE | PM_CLK_AO | 1534ef68a193SYong Wu MTK_IOMMU_TYPE_INFRA | IFA_IOMMU_PCIE_SUPPORT, 1535ef68a193SYong Wu .pericfg_comp_str = "mediatek,mt8195-pericfg_ao", 1536ef68a193SYong Wu .inv_sel_reg = REG_MMU_INV_SEL_GEN2, 15377597e3c5SYong Wu .banks_num = 5, 15387597e3c5SYong Wu .banks_enable = {true, false, false, false, true}, 15397597e3c5SYong Wu .banks_portmsk = {[0] = GENMASK(19, 16), /* PCIe */ 15407597e3c5SYong Wu [4] = GENMASK(31, 20), /* USB */ 15417597e3c5SYong Wu }, 1542ef68a193SYong Wu .iova_region = single_domain, 1543ef68a193SYong Wu .iova_region_nr = ARRAY_SIZE(single_domain), 1544ef68a193SYong Wu }; 1545ef68a193SYong Wu 1546ef68a193SYong Wu static const struct mtk_iommu_plat_data mt8195_data_vdo = { 1547ef68a193SYong Wu .m4u_plat = M4U_MT8195, 1548ef68a193SYong Wu .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | 1549ef68a193SYong Wu WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | MTK_IOMMU_TYPE_MM, 1550ef68a193SYong Wu .hw_list = &m4ulist, 1551ef68a193SYong Wu .inv_sel_reg = REG_MMU_INV_SEL_GEN2, 155299ca0228SYong Wu .banks_num = 1, 155399ca0228SYong Wu .banks_enable = {true}, 1554ef68a193SYong Wu .iova_region = mt8192_multi_dom, 1555ef68a193SYong Wu .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), 1556ef68a193SYong Wu .larbid_remap = {{2, 0}, {21}, {24}, {7}, {19}, {9, 10, 11}, 1557ef68a193SYong Wu {13, 17, 15/* 17b */, 25}, {5}}, 1558ef68a193SYong Wu }; 1559ef68a193SYong Wu 1560ef68a193SYong Wu static const struct mtk_iommu_plat_data mt8195_data_vpp = { 1561ef68a193SYong Wu .m4u_plat = M4U_MT8195, 1562ef68a193SYong Wu .flags = HAS_BCLK | HAS_SUB_COMM_3BITS | OUT_ORDER_WR_EN | 1563ef68a193SYong Wu WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | MTK_IOMMU_TYPE_MM, 1564ef68a193SYong Wu .hw_list = &m4ulist, 1565ef68a193SYong Wu .inv_sel_reg = REG_MMU_INV_SEL_GEN2, 156699ca0228SYong Wu .banks_num = 1, 156799ca0228SYong Wu .banks_enable = {true}, 1568ef68a193SYong Wu .iova_region = mt8192_multi_dom, 1569ef68a193SYong Wu .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), 1570ef68a193SYong Wu .larbid_remap = {{1}, {3}, 1571ef68a193SYong Wu {22, MTK_INVALID_LARBID, MTK_INVALID_LARBID, MTK_INVALID_LARBID, 23}, 1572ef68a193SYong Wu {8}, {20}, {12}, 1573ef68a193SYong Wu /* 16: 16a; 29: 16b; 30: CCUtop0; 31: CCUtop1 */ 1574ef68a193SYong Wu {14, 16, 29, 26, 30, 31, 18}, 1575ef68a193SYong Wu {4, MTK_INVALID_LARBID, MTK_INVALID_LARBID, MTK_INVALID_LARBID, 6}}, 1576ef68a193SYong Wu }; 1577ef68a193SYong Wu 15783cd0e4a3SFabien Parent static const struct mtk_iommu_plat_data mt8365_data = { 15793cd0e4a3SFabien Parent .m4u_plat = M4U_MT8365, 15803cd0e4a3SFabien Parent .flags = RESET_AXI | INT_ID_PORT_WIDTH_6, 15813cd0e4a3SFabien Parent .inv_sel_reg = REG_MMU_INV_SEL_GEN1, 15823cd0e4a3SFabien Parent .banks_num = 1, 15833cd0e4a3SFabien Parent .banks_enable = {true}, 15843cd0e4a3SFabien Parent .iova_region = single_domain, 15853cd0e4a3SFabien Parent .iova_region_nr = ARRAY_SIZE(single_domain), 15863cd0e4a3SFabien Parent .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */ 15873cd0e4a3SFabien Parent }; 15883cd0e4a3SFabien Parent 15890df4fabeSYong Wu static const struct of_device_id mtk_iommu_of_ids[] = { 1590cecdce9dSYong Wu { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, 1591068c86e9SChao Hao { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data}, 1592717ec15eSAngeloGioacchino Del Regno { .compatible = "mediatek,mt6795-m4u", .data = &mt6795_data}, 15933c213562SFabien Parent { .compatible = "mediatek,mt8167-m4u", .data = &mt8167_data}, 1594cecdce9dSYong Wu { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, 1595907ba6a1SYong Wu { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, 1596e8d7ccaaSYong Wu { .compatible = "mediatek,mt8186-iommu-mm", .data = &mt8186_data_mm}, /* mm: m4u */ 15979e3489e0SYong Wu { .compatible = "mediatek,mt8192-m4u", .data = &mt8192_data}, 1598ef68a193SYong Wu { .compatible = "mediatek,mt8195-iommu-infra", .data = &mt8195_data_infra}, 1599ef68a193SYong Wu { .compatible = "mediatek,mt8195-iommu-vdo", .data = &mt8195_data_vdo}, 1600ef68a193SYong Wu { .compatible = "mediatek,mt8195-iommu-vpp", .data = &mt8195_data_vpp}, 16013cd0e4a3SFabien Parent { .compatible = "mediatek,mt8365-m4u", .data = &mt8365_data}, 16020df4fabeSYong Wu {} 16030df4fabeSYong Wu }; 16040df4fabeSYong Wu 16050df4fabeSYong Wu static struct platform_driver mtk_iommu_driver = { 16060df4fabeSYong Wu .probe = mtk_iommu_probe, 16070df4fabeSYong Wu .remove = mtk_iommu_remove, 16080df4fabeSYong Wu .driver = { 16090df4fabeSYong Wu .name = "mtk-iommu", 1610f53dd978SKrzysztof Kozlowski .of_match_table = mtk_iommu_of_ids, 16110df4fabeSYong Wu .pm = &mtk_iommu_pm_ops, 16120df4fabeSYong Wu } 16130df4fabeSYong Wu }; 161418d8c74eSYong Wu module_platform_driver(mtk_iommu_driver); 16150df4fabeSYong Wu 161618d8c74eSYong Wu MODULE_DESCRIPTION("IOMMU API for MediaTek M4U implementations"); 161718d8c74eSYong Wu MODULE_LICENSE("GPL v2"); 1618