1e86d1aa8SWill Deacon /* SPDX-License-Identifier: GPL-2.0-only */ 2e86d1aa8SWill Deacon /* 3e86d1aa8SWill Deacon * IOMMU API for ARM architected SMMU implementations. 4e86d1aa8SWill Deacon * 5e86d1aa8SWill Deacon * Copyright (C) 2013 ARM Limited 6e86d1aa8SWill Deacon * 7e86d1aa8SWill Deacon * Author: Will Deacon <will.deacon@arm.com> 8e86d1aa8SWill Deacon */ 9e86d1aa8SWill Deacon 10e86d1aa8SWill Deacon #ifndef _ARM_SMMU_H 11e86d1aa8SWill Deacon #define _ARM_SMMU_H 12e86d1aa8SWill Deacon 13e86d1aa8SWill Deacon #include <linux/atomic.h> 14e86d1aa8SWill Deacon #include <linux/bitfield.h> 15e86d1aa8SWill Deacon #include <linux/bits.h> 16e86d1aa8SWill Deacon #include <linux/clk.h> 17e86d1aa8SWill Deacon #include <linux/device.h> 18e86d1aa8SWill Deacon #include <linux/io-64-nonatomic-hi-lo.h> 19e86d1aa8SWill Deacon #include <linux/io-pgtable.h> 20e86d1aa8SWill Deacon #include <linux/iommu.h> 21e86d1aa8SWill Deacon #include <linux/irqreturn.h> 22e86d1aa8SWill Deacon #include <linux/mutex.h> 23e86d1aa8SWill Deacon #include <linux/spinlock.h> 24e86d1aa8SWill Deacon #include <linux/types.h> 25e86d1aa8SWill Deacon 26e86d1aa8SWill Deacon /* Configuration registers */ 27e86d1aa8SWill Deacon #define ARM_SMMU_GR0_sCR0 0x0 28e86d1aa8SWill Deacon #define ARM_SMMU_sCR0_VMID16EN BIT(31) 29e86d1aa8SWill Deacon #define ARM_SMMU_sCR0_BSU GENMASK(15, 14) 30e86d1aa8SWill Deacon #define ARM_SMMU_sCR0_FB BIT(13) 31e86d1aa8SWill Deacon #define ARM_SMMU_sCR0_PTM BIT(12) 32e86d1aa8SWill Deacon #define ARM_SMMU_sCR0_VMIDPNE BIT(11) 33e86d1aa8SWill Deacon #define ARM_SMMU_sCR0_USFCFG BIT(10) 34e86d1aa8SWill Deacon #define ARM_SMMU_sCR0_GCFGFIE BIT(5) 35e86d1aa8SWill Deacon #define ARM_SMMU_sCR0_GCFGFRE BIT(4) 36e86d1aa8SWill Deacon #define ARM_SMMU_sCR0_EXIDENABLE BIT(3) 37e86d1aa8SWill Deacon #define ARM_SMMU_sCR0_GFIE BIT(2) 38e86d1aa8SWill Deacon #define ARM_SMMU_sCR0_GFRE BIT(1) 39e86d1aa8SWill Deacon #define ARM_SMMU_sCR0_CLIENTPD BIT(0) 40e86d1aa8SWill Deacon 41e86d1aa8SWill Deacon /* Auxiliary Configuration register */ 42e86d1aa8SWill Deacon #define ARM_SMMU_GR0_sACR 0x10 43e86d1aa8SWill Deacon 44e86d1aa8SWill Deacon /* Identification registers */ 45e86d1aa8SWill Deacon #define ARM_SMMU_GR0_ID0 0x20 46e86d1aa8SWill Deacon #define ARM_SMMU_ID0_S1TS BIT(30) 47e86d1aa8SWill Deacon #define ARM_SMMU_ID0_S2TS BIT(29) 48e86d1aa8SWill Deacon #define ARM_SMMU_ID0_NTS BIT(28) 49e86d1aa8SWill Deacon #define ARM_SMMU_ID0_SMS BIT(27) 50e86d1aa8SWill Deacon #define ARM_SMMU_ID0_ATOSNS BIT(26) 51e86d1aa8SWill Deacon #define ARM_SMMU_ID0_PTFS_NO_AARCH32 BIT(25) 52e86d1aa8SWill Deacon #define ARM_SMMU_ID0_PTFS_NO_AARCH32S BIT(24) 53e86d1aa8SWill Deacon #define ARM_SMMU_ID0_NUMIRPT GENMASK(23, 16) 54e86d1aa8SWill Deacon #define ARM_SMMU_ID0_CTTW BIT(14) 55e86d1aa8SWill Deacon #define ARM_SMMU_ID0_NUMSIDB GENMASK(12, 9) 56e86d1aa8SWill Deacon #define ARM_SMMU_ID0_EXIDS BIT(8) 57e86d1aa8SWill Deacon #define ARM_SMMU_ID0_NUMSMRG GENMASK(7, 0) 58e86d1aa8SWill Deacon 59e86d1aa8SWill Deacon #define ARM_SMMU_GR0_ID1 0x24 60e86d1aa8SWill Deacon #define ARM_SMMU_ID1_PAGESIZE BIT(31) 61e86d1aa8SWill Deacon #define ARM_SMMU_ID1_NUMPAGENDXB GENMASK(30, 28) 62e86d1aa8SWill Deacon #define ARM_SMMU_ID1_NUMS2CB GENMASK(23, 16) 63e86d1aa8SWill Deacon #define ARM_SMMU_ID1_NUMCB GENMASK(7, 0) 64e86d1aa8SWill Deacon 65e86d1aa8SWill Deacon #define ARM_SMMU_GR0_ID2 0x28 66e86d1aa8SWill Deacon #define ARM_SMMU_ID2_VMID16 BIT(15) 67e86d1aa8SWill Deacon #define ARM_SMMU_ID2_PTFS_64K BIT(14) 68e86d1aa8SWill Deacon #define ARM_SMMU_ID2_PTFS_16K BIT(13) 69e86d1aa8SWill Deacon #define ARM_SMMU_ID2_PTFS_4K BIT(12) 70e86d1aa8SWill Deacon #define ARM_SMMU_ID2_UBS GENMASK(11, 8) 71e86d1aa8SWill Deacon #define ARM_SMMU_ID2_OAS GENMASK(7, 4) 72e86d1aa8SWill Deacon #define ARM_SMMU_ID2_IAS GENMASK(3, 0) 73e86d1aa8SWill Deacon 74e86d1aa8SWill Deacon #define ARM_SMMU_GR0_ID3 0x2c 75e86d1aa8SWill Deacon #define ARM_SMMU_GR0_ID4 0x30 76e86d1aa8SWill Deacon #define ARM_SMMU_GR0_ID5 0x34 77e86d1aa8SWill Deacon #define ARM_SMMU_GR0_ID6 0x38 78e86d1aa8SWill Deacon 79e86d1aa8SWill Deacon #define ARM_SMMU_GR0_ID7 0x3c 80e86d1aa8SWill Deacon #define ARM_SMMU_ID7_MAJOR GENMASK(7, 4) 81e86d1aa8SWill Deacon #define ARM_SMMU_ID7_MINOR GENMASK(3, 0) 82e86d1aa8SWill Deacon 83e86d1aa8SWill Deacon #define ARM_SMMU_GR0_sGFSR 0x48 84e86d1aa8SWill Deacon #define ARM_SMMU_sGFSR_USF BIT(1) 85e86d1aa8SWill Deacon 86e86d1aa8SWill Deacon #define ARM_SMMU_GR0_sGFSYNR0 0x50 87e86d1aa8SWill Deacon #define ARM_SMMU_GR0_sGFSYNR1 0x54 88e86d1aa8SWill Deacon #define ARM_SMMU_GR0_sGFSYNR2 0x58 89e86d1aa8SWill Deacon 90e86d1aa8SWill Deacon /* Global TLB invalidation */ 91e86d1aa8SWill Deacon #define ARM_SMMU_GR0_TLBIVMID 0x64 92e86d1aa8SWill Deacon #define ARM_SMMU_GR0_TLBIALLNSNH 0x68 93e86d1aa8SWill Deacon #define ARM_SMMU_GR0_TLBIALLH 0x6c 94e86d1aa8SWill Deacon #define ARM_SMMU_GR0_sTLBGSYNC 0x70 95e86d1aa8SWill Deacon 96e86d1aa8SWill Deacon #define ARM_SMMU_GR0_sTLBGSTATUS 0x74 97e86d1aa8SWill Deacon #define ARM_SMMU_sTLBGSTATUS_GSACTIVE BIT(0) 98e86d1aa8SWill Deacon 99e86d1aa8SWill Deacon /* Stream mapping registers */ 100e86d1aa8SWill Deacon #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2)) 101e86d1aa8SWill Deacon #define ARM_SMMU_SMR_VALID BIT(31) 102e86d1aa8SWill Deacon #define ARM_SMMU_SMR_MASK GENMASK(31, 16) 103e86d1aa8SWill Deacon #define ARM_SMMU_SMR_ID GENMASK(15, 0) 104e86d1aa8SWill Deacon 105e86d1aa8SWill Deacon #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2)) 106e86d1aa8SWill Deacon #define ARM_SMMU_S2CR_PRIVCFG GENMASK(25, 24) 107e86d1aa8SWill Deacon enum arm_smmu_s2cr_privcfg { 108e86d1aa8SWill Deacon S2CR_PRIVCFG_DEFAULT, 109e86d1aa8SWill Deacon S2CR_PRIVCFG_DIPAN, 110e86d1aa8SWill Deacon S2CR_PRIVCFG_UNPRIV, 111e86d1aa8SWill Deacon S2CR_PRIVCFG_PRIV, 112e86d1aa8SWill Deacon }; 113e86d1aa8SWill Deacon #define ARM_SMMU_S2CR_TYPE GENMASK(17, 16) 114e86d1aa8SWill Deacon enum arm_smmu_s2cr_type { 115e86d1aa8SWill Deacon S2CR_TYPE_TRANS, 116e86d1aa8SWill Deacon S2CR_TYPE_BYPASS, 117e86d1aa8SWill Deacon S2CR_TYPE_FAULT, 118e86d1aa8SWill Deacon }; 119e86d1aa8SWill Deacon #define ARM_SMMU_S2CR_EXIDVALID BIT(10) 120e86d1aa8SWill Deacon #define ARM_SMMU_S2CR_CBNDX GENMASK(7, 0) 121e86d1aa8SWill Deacon 122e86d1aa8SWill Deacon /* Context bank attribute registers */ 123e86d1aa8SWill Deacon #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2)) 124e86d1aa8SWill Deacon #define ARM_SMMU_CBAR_IRPTNDX GENMASK(31, 24) 125e86d1aa8SWill Deacon #define ARM_SMMU_CBAR_TYPE GENMASK(17, 16) 126e86d1aa8SWill Deacon enum arm_smmu_cbar_type { 127e86d1aa8SWill Deacon CBAR_TYPE_S2_TRANS, 128e86d1aa8SWill Deacon CBAR_TYPE_S1_TRANS_S2_BYPASS, 129e86d1aa8SWill Deacon CBAR_TYPE_S1_TRANS_S2_FAULT, 130e86d1aa8SWill Deacon CBAR_TYPE_S1_TRANS_S2_TRANS, 131e86d1aa8SWill Deacon }; 132e86d1aa8SWill Deacon #define ARM_SMMU_CBAR_S1_MEMATTR GENMASK(15, 12) 133e86d1aa8SWill Deacon #define ARM_SMMU_CBAR_S1_MEMATTR_WB 0xf 134e86d1aa8SWill Deacon #define ARM_SMMU_CBAR_S1_BPSHCFG GENMASK(9, 8) 135e86d1aa8SWill Deacon #define ARM_SMMU_CBAR_S1_BPSHCFG_NSH 3 136e86d1aa8SWill Deacon #define ARM_SMMU_CBAR_VMID GENMASK(7, 0) 137e86d1aa8SWill Deacon 138e86d1aa8SWill Deacon #define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2)) 139414ecb03SGeorgi Djakov #define ARM_SMMU_CBFRSYNRA_SID GENMASK(15, 0) 140e86d1aa8SWill Deacon 141e86d1aa8SWill Deacon #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) 142e86d1aa8SWill Deacon #define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16) 143e86d1aa8SWill Deacon #define ARM_SMMU_CBA2R_VA64 BIT(0) 144e86d1aa8SWill Deacon 145e86d1aa8SWill Deacon #define ARM_SMMU_CB_SCTLR 0x0 146e86d1aa8SWill Deacon #define ARM_SMMU_SCTLR_S1_ASIDPNE BIT(12) 147e86d1aa8SWill Deacon #define ARM_SMMU_SCTLR_CFCFG BIT(7) 148bffb2eafSRob Clark #define ARM_SMMU_SCTLR_HUPCF BIT(8) 149e86d1aa8SWill Deacon #define ARM_SMMU_SCTLR_CFIE BIT(6) 150e86d1aa8SWill Deacon #define ARM_SMMU_SCTLR_CFRE BIT(5) 151e86d1aa8SWill Deacon #define ARM_SMMU_SCTLR_E BIT(4) 152e86d1aa8SWill Deacon #define ARM_SMMU_SCTLR_AFE BIT(2) 153e86d1aa8SWill Deacon #define ARM_SMMU_SCTLR_TRE BIT(1) 154e86d1aa8SWill Deacon #define ARM_SMMU_SCTLR_M BIT(0) 155e86d1aa8SWill Deacon 156e86d1aa8SWill Deacon #define ARM_SMMU_CB_ACTLR 0x4 157e86d1aa8SWill Deacon 158e86d1aa8SWill Deacon #define ARM_SMMU_CB_RESUME 0x8 159e86d1aa8SWill Deacon #define ARM_SMMU_RESUME_TERMINATE BIT(0) 160e86d1aa8SWill Deacon 161e86d1aa8SWill Deacon #define ARM_SMMU_CB_TCR2 0x10 162e86d1aa8SWill Deacon #define ARM_SMMU_TCR2_SEP GENMASK(17, 15) 163e86d1aa8SWill Deacon #define ARM_SMMU_TCR2_SEP_UPSTREAM 0x7 164e86d1aa8SWill Deacon #define ARM_SMMU_TCR2_AS BIT(4) 165e86d1aa8SWill Deacon #define ARM_SMMU_TCR2_PASIZE GENMASK(3, 0) 166e86d1aa8SWill Deacon 167e86d1aa8SWill Deacon #define ARM_SMMU_CB_TTBR0 0x20 168e86d1aa8SWill Deacon #define ARM_SMMU_CB_TTBR1 0x28 169e86d1aa8SWill Deacon #define ARM_SMMU_TTBRn_ASID GENMASK_ULL(63, 48) 170e86d1aa8SWill Deacon 171e86d1aa8SWill Deacon #define ARM_SMMU_CB_TCR 0x30 172e86d1aa8SWill Deacon #define ARM_SMMU_TCR_EAE BIT(31) 173e86d1aa8SWill Deacon #define ARM_SMMU_TCR_EPD1 BIT(23) 17467f1a7a3SJordan Crouse #define ARM_SMMU_TCR_A1 BIT(22) 175e86d1aa8SWill Deacon #define ARM_SMMU_TCR_TG0 GENMASK(15, 14) 176e86d1aa8SWill Deacon #define ARM_SMMU_TCR_SH0 GENMASK(13, 12) 177e86d1aa8SWill Deacon #define ARM_SMMU_TCR_ORGN0 GENMASK(11, 10) 178e86d1aa8SWill Deacon #define ARM_SMMU_TCR_IRGN0 GENMASK(9, 8) 17967f1a7a3SJordan Crouse #define ARM_SMMU_TCR_EPD0 BIT(7) 180e86d1aa8SWill Deacon #define ARM_SMMU_TCR_T0SZ GENMASK(5, 0) 181e86d1aa8SWill Deacon 182e86d1aa8SWill Deacon #define ARM_SMMU_VTCR_RES1 BIT(31) 183e86d1aa8SWill Deacon #define ARM_SMMU_VTCR_PS GENMASK(18, 16) 184e86d1aa8SWill Deacon #define ARM_SMMU_VTCR_TG0 ARM_SMMU_TCR_TG0 185e86d1aa8SWill Deacon #define ARM_SMMU_VTCR_SH0 ARM_SMMU_TCR_SH0 186e86d1aa8SWill Deacon #define ARM_SMMU_VTCR_ORGN0 ARM_SMMU_TCR_ORGN0 187e86d1aa8SWill Deacon #define ARM_SMMU_VTCR_IRGN0 ARM_SMMU_TCR_IRGN0 188e86d1aa8SWill Deacon #define ARM_SMMU_VTCR_SL0 GENMASK(7, 6) 189e86d1aa8SWill Deacon #define ARM_SMMU_VTCR_T0SZ ARM_SMMU_TCR_T0SZ 190e86d1aa8SWill Deacon 191e86d1aa8SWill Deacon #define ARM_SMMU_CB_CONTEXTIDR 0x34 192e86d1aa8SWill Deacon #define ARM_SMMU_CB_S1_MAIR0 0x38 193e86d1aa8SWill Deacon #define ARM_SMMU_CB_S1_MAIR1 0x3c 194e86d1aa8SWill Deacon 195e86d1aa8SWill Deacon #define ARM_SMMU_CB_PAR 0x50 196e86d1aa8SWill Deacon #define ARM_SMMU_CB_PAR_F BIT(0) 197e86d1aa8SWill Deacon 198e86d1aa8SWill Deacon #define ARM_SMMU_CB_FSR 0x58 199e86d1aa8SWill Deacon #define ARM_SMMU_FSR_MULTI BIT(31) 200e86d1aa8SWill Deacon #define ARM_SMMU_FSR_SS BIT(30) 201e86d1aa8SWill Deacon #define ARM_SMMU_FSR_UUT BIT(8) 202e86d1aa8SWill Deacon #define ARM_SMMU_FSR_ASF BIT(7) 203e86d1aa8SWill Deacon #define ARM_SMMU_FSR_TLBLKF BIT(6) 204e86d1aa8SWill Deacon #define ARM_SMMU_FSR_TLBMCF BIT(5) 205e86d1aa8SWill Deacon #define ARM_SMMU_FSR_EF BIT(4) 206e86d1aa8SWill Deacon #define ARM_SMMU_FSR_PF BIT(3) 207e86d1aa8SWill Deacon #define ARM_SMMU_FSR_AFF BIT(2) 208e86d1aa8SWill Deacon #define ARM_SMMU_FSR_TF BIT(1) 209e86d1aa8SWill Deacon 210e86d1aa8SWill Deacon #define ARM_SMMU_FSR_IGN (ARM_SMMU_FSR_AFF | \ 211e86d1aa8SWill Deacon ARM_SMMU_FSR_ASF | \ 212e86d1aa8SWill Deacon ARM_SMMU_FSR_TLBMCF | \ 213e86d1aa8SWill Deacon ARM_SMMU_FSR_TLBLKF) 214e86d1aa8SWill Deacon 215e86d1aa8SWill Deacon #define ARM_SMMU_FSR_FAULT (ARM_SMMU_FSR_MULTI | \ 216e86d1aa8SWill Deacon ARM_SMMU_FSR_SS | \ 217e86d1aa8SWill Deacon ARM_SMMU_FSR_UUT | \ 218e86d1aa8SWill Deacon ARM_SMMU_FSR_EF | \ 219e86d1aa8SWill Deacon ARM_SMMU_FSR_PF | \ 220e86d1aa8SWill Deacon ARM_SMMU_FSR_TF | \ 221e86d1aa8SWill Deacon ARM_SMMU_FSR_IGN) 222e86d1aa8SWill Deacon 223e86d1aa8SWill Deacon #define ARM_SMMU_CB_FAR 0x60 224e86d1aa8SWill Deacon 225e86d1aa8SWill Deacon #define ARM_SMMU_CB_FSYNR0 0x68 226e86d1aa8SWill Deacon #define ARM_SMMU_FSYNR0_WNR BIT(4) 227e86d1aa8SWill Deacon 228ab5df7b9SJordan Crouse #define ARM_SMMU_CB_FSYNR1 0x6c 229ab5df7b9SJordan Crouse 230e86d1aa8SWill Deacon #define ARM_SMMU_CB_S1_TLBIVA 0x600 231e86d1aa8SWill Deacon #define ARM_SMMU_CB_S1_TLBIASID 0x610 232e86d1aa8SWill Deacon #define ARM_SMMU_CB_S1_TLBIVAL 0x620 233e86d1aa8SWill Deacon #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630 234e86d1aa8SWill Deacon #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638 235e86d1aa8SWill Deacon #define ARM_SMMU_CB_TLBSYNC 0x7f0 236e86d1aa8SWill Deacon #define ARM_SMMU_CB_TLBSTATUS 0x7f4 237e86d1aa8SWill Deacon #define ARM_SMMU_CB_ATS1PR 0x800 238e86d1aa8SWill Deacon 239e86d1aa8SWill Deacon #define ARM_SMMU_CB_ATSR 0x8f0 240e86d1aa8SWill Deacon #define ARM_SMMU_ATSR_ACTIVE BIT(0) 241e86d1aa8SWill Deacon 242414ecb03SGeorgi Djakov #define ARM_SMMU_RESUME_TERMINATE BIT(0) 243e86d1aa8SWill Deacon 244e86d1aa8SWill Deacon /* Maximum number of context banks per SMMU */ 245e86d1aa8SWill Deacon #define ARM_SMMU_MAX_CBS 128 246e86d1aa8SWill Deacon 247e86d1aa8SWill Deacon #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */ 248e86d1aa8SWill Deacon #define TLB_SPIN_COUNT 10 249e86d1aa8SWill Deacon 250e86d1aa8SWill Deacon /* Shared driver definitions */ 251e86d1aa8SWill Deacon enum arm_smmu_arch_version { 252e86d1aa8SWill Deacon ARM_SMMU_V1, 253e86d1aa8SWill Deacon ARM_SMMU_V1_64K, 254e86d1aa8SWill Deacon ARM_SMMU_V2, 255e86d1aa8SWill Deacon }; 256e86d1aa8SWill Deacon 257e86d1aa8SWill Deacon enum arm_smmu_implementation { 258e86d1aa8SWill Deacon GENERIC_SMMU, 259e86d1aa8SWill Deacon ARM_MMU500, 260e86d1aa8SWill Deacon CAVIUM_SMMUV2, 261e86d1aa8SWill Deacon QCOM_SMMUV2, 262e86d1aa8SWill Deacon }; 263e86d1aa8SWill Deacon 264556db53aSJordan Crouse struct arm_smmu_s2cr { 265556db53aSJordan Crouse struct iommu_group *group; 266556db53aSJordan Crouse int count; 267556db53aSJordan Crouse enum arm_smmu_s2cr_type type; 268556db53aSJordan Crouse enum arm_smmu_s2cr_privcfg privcfg; 269556db53aSJordan Crouse u8 cbndx; 270556db53aSJordan Crouse }; 271556db53aSJordan Crouse 272556db53aSJordan Crouse struct arm_smmu_smr { 273556db53aSJordan Crouse u16 mask; 274556db53aSJordan Crouse u16 id; 275556db53aSJordan Crouse bool valid; 276556db53aSJordan Crouse bool pinned; 277556db53aSJordan Crouse }; 278556db53aSJordan Crouse 279e86d1aa8SWill Deacon struct arm_smmu_device { 280e86d1aa8SWill Deacon struct device *dev; 281e86d1aa8SWill Deacon 282e86d1aa8SWill Deacon void __iomem *base; 283b9b721d1SSai Prakash Ranjan phys_addr_t ioaddr; 284e86d1aa8SWill Deacon unsigned int numpage; 285e86d1aa8SWill Deacon unsigned int pgshift; 286e86d1aa8SWill Deacon 287e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0) 288e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1) 289e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2) 290e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3) 291e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4) 292e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5) 293e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_VMID16 (1 << 6) 294e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7) 295e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8) 296e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9) 297e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10) 298e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11) 299e86d1aa8SWill Deacon #define ARM_SMMU_FEAT_EXIDS (1 << 12) 300e86d1aa8SWill Deacon u32 features; 301e86d1aa8SWill Deacon 302e86d1aa8SWill Deacon enum arm_smmu_arch_version version; 303e86d1aa8SWill Deacon enum arm_smmu_implementation model; 304e86d1aa8SWill Deacon const struct arm_smmu_impl *impl; 305e86d1aa8SWill Deacon 306e86d1aa8SWill Deacon u32 num_context_banks; 307e86d1aa8SWill Deacon u32 num_s2_context_banks; 308e86d1aa8SWill Deacon DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS); 309e86d1aa8SWill Deacon struct arm_smmu_cb *cbs; 310e86d1aa8SWill Deacon atomic_t irptndx; 311e86d1aa8SWill Deacon 312e86d1aa8SWill Deacon u32 num_mapping_groups; 313e86d1aa8SWill Deacon u16 streamid_mask; 314e86d1aa8SWill Deacon u16 smr_mask_mask; 315e86d1aa8SWill Deacon struct arm_smmu_smr *smrs; 316e86d1aa8SWill Deacon struct arm_smmu_s2cr *s2crs; 317e86d1aa8SWill Deacon struct mutex stream_map_mutex; 318e86d1aa8SWill Deacon 319e86d1aa8SWill Deacon unsigned long va_size; 320e86d1aa8SWill Deacon unsigned long ipa_size; 321e86d1aa8SWill Deacon unsigned long pa_size; 322e86d1aa8SWill Deacon unsigned long pgsize_bitmap; 323e86d1aa8SWill Deacon 32497dfad19SRobin Murphy int num_context_irqs; 32597dfad19SRobin Murphy int num_clks; 326e86d1aa8SWill Deacon unsigned int *irqs; 327e86d1aa8SWill Deacon struct clk_bulk_data *clks; 328e86d1aa8SWill Deacon 329e86d1aa8SWill Deacon spinlock_t global_sync_lock; 330e86d1aa8SWill Deacon 331e86d1aa8SWill Deacon /* IOMMU core code handle */ 332e86d1aa8SWill Deacon struct iommu_device iommu; 333e86d1aa8SWill Deacon }; 334e86d1aa8SWill Deacon 335e86d1aa8SWill Deacon enum arm_smmu_context_fmt { 336e86d1aa8SWill Deacon ARM_SMMU_CTX_FMT_NONE, 337e86d1aa8SWill Deacon ARM_SMMU_CTX_FMT_AARCH64, 338e86d1aa8SWill Deacon ARM_SMMU_CTX_FMT_AARCH32_L, 339e86d1aa8SWill Deacon ARM_SMMU_CTX_FMT_AARCH32_S, 340e86d1aa8SWill Deacon }; 341e86d1aa8SWill Deacon 342e86d1aa8SWill Deacon struct arm_smmu_cfg { 343e86d1aa8SWill Deacon u8 cbndx; 344e86d1aa8SWill Deacon u8 irptndx; 345e86d1aa8SWill Deacon union { 346e86d1aa8SWill Deacon u16 asid; 347e86d1aa8SWill Deacon u16 vmid; 348e86d1aa8SWill Deacon }; 349e86d1aa8SWill Deacon enum arm_smmu_cbar_type cbar; 350e86d1aa8SWill Deacon enum arm_smmu_context_fmt fmt; 351ef75702dSSai Prakash Ranjan bool flush_walk_prefer_tlbiasid; 352e86d1aa8SWill Deacon }; 353e86d1aa8SWill Deacon #define ARM_SMMU_INVALID_IRPTNDX 0xff 354e86d1aa8SWill Deacon 355556db53aSJordan Crouse struct arm_smmu_cb { 356556db53aSJordan Crouse u64 ttbr[2]; 357556db53aSJordan Crouse u32 tcr[2]; 358556db53aSJordan Crouse u32 mair[2]; 359556db53aSJordan Crouse struct arm_smmu_cfg *cfg; 360556db53aSJordan Crouse }; 361556db53aSJordan Crouse 362e86d1aa8SWill Deacon enum arm_smmu_domain_stage { 363e86d1aa8SWill Deacon ARM_SMMU_DOMAIN_S1 = 0, 364e86d1aa8SWill Deacon ARM_SMMU_DOMAIN_S2, 365e86d1aa8SWill Deacon ARM_SMMU_DOMAIN_NESTED, 366e86d1aa8SWill Deacon }; 367e86d1aa8SWill Deacon 368e86d1aa8SWill Deacon struct arm_smmu_domain { 369e86d1aa8SWill Deacon struct arm_smmu_device *smmu; 370e86d1aa8SWill Deacon struct io_pgtable_ops *pgtbl_ops; 3714fc52b81SChristoph Hellwig unsigned long pgtbl_quirks; 372e86d1aa8SWill Deacon const struct iommu_flush_ops *flush_ops; 373e86d1aa8SWill Deacon struct arm_smmu_cfg cfg; 374e86d1aa8SWill Deacon enum arm_smmu_domain_stage stage; 375e86d1aa8SWill Deacon struct mutex init_mutex; /* Protects smmu pointer */ 376e86d1aa8SWill Deacon spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */ 377e86d1aa8SWill Deacon struct iommu_domain domain; 378e86d1aa8SWill Deacon }; 379e86d1aa8SWill Deacon 380556db53aSJordan Crouse struct arm_smmu_master_cfg { 381556db53aSJordan Crouse struct arm_smmu_device *smmu; 382556db53aSJordan Crouse s16 smendx[]; 383556db53aSJordan Crouse }; 384556db53aSJordan Crouse 3851226fa0eSRob Clark static inline u32 arm_smmu_lpae_tcr(const struct io_pgtable_cfg *cfg) 386e86d1aa8SWill Deacon { 38767f1a7a3SJordan Crouse u32 tcr = FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) | 388e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) | 389e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) | 390e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) | 391e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz); 39267f1a7a3SJordan Crouse 39367f1a7a3SJordan Crouse /* 39467f1a7a3SJordan Crouse * When TTBR1 is selected shift the TCR fields by 16 bits and disable 39567f1a7a3SJordan Crouse * translation in TTBR0 39667f1a7a3SJordan Crouse */ 39767f1a7a3SJordan Crouse if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) { 39867f1a7a3SJordan Crouse tcr = (tcr << 16) & ~ARM_SMMU_TCR_A1; 39967f1a7a3SJordan Crouse tcr |= ARM_SMMU_TCR_EPD0; 40067f1a7a3SJordan Crouse } else 40167f1a7a3SJordan Crouse tcr |= ARM_SMMU_TCR_EPD1; 40267f1a7a3SJordan Crouse 40367f1a7a3SJordan Crouse return tcr; 404e86d1aa8SWill Deacon } 405e86d1aa8SWill Deacon 4061226fa0eSRob Clark static inline u32 arm_smmu_lpae_tcr2(const struct io_pgtable_cfg *cfg) 407e86d1aa8SWill Deacon { 408e86d1aa8SWill Deacon return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) | 409e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM); 410e86d1aa8SWill Deacon } 411e86d1aa8SWill Deacon 4121226fa0eSRob Clark static inline u32 arm_smmu_lpae_vtcr(const struct io_pgtable_cfg *cfg) 413e86d1aa8SWill Deacon { 414e86d1aa8SWill Deacon return ARM_SMMU_VTCR_RES1 | 415e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) | 416e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_VTCR_TG0, cfg->arm_lpae_s2_cfg.vtcr.tg) | 417e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_VTCR_SH0, cfg->arm_lpae_s2_cfg.vtcr.sh) | 418e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_VTCR_ORGN0, cfg->arm_lpae_s2_cfg.vtcr.orgn) | 419e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_VTCR_IRGN0, cfg->arm_lpae_s2_cfg.vtcr.irgn) | 420e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_VTCR_SL0, cfg->arm_lpae_s2_cfg.vtcr.sl) | 421e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz); 422e86d1aa8SWill Deacon } 423e86d1aa8SWill Deacon 424e86d1aa8SWill Deacon /* Implementation details, yay! */ 425e86d1aa8SWill Deacon struct arm_smmu_impl { 426e86d1aa8SWill Deacon u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset); 427e86d1aa8SWill Deacon void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset, 428e86d1aa8SWill Deacon u32 val); 429e86d1aa8SWill Deacon u64 (*read_reg64)(struct arm_smmu_device *smmu, int page, int offset); 430e86d1aa8SWill Deacon void (*write_reg64)(struct arm_smmu_device *smmu, int page, int offset, 431e86d1aa8SWill Deacon u64 val); 432e86d1aa8SWill Deacon int (*cfg_probe)(struct arm_smmu_device *smmu); 433e86d1aa8SWill Deacon int (*reset)(struct arm_smmu_device *smmu); 434dd147a89SJordan Crouse int (*init_context)(struct arm_smmu_domain *smmu_domain, 435556db53aSJordan Crouse struct io_pgtable_cfg *cfg, struct device *dev); 436e86d1aa8SWill Deacon void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync, 437e86d1aa8SWill Deacon int status); 438e86d1aa8SWill Deacon int (*def_domain_type)(struct device *dev); 439e86d1aa8SWill Deacon irqreturn_t (*global_fault)(int irq, void *dev); 440e86d1aa8SWill Deacon irqreturn_t (*context_fault)(int irq, void *dev); 441*960be6e1SGeorgi Djakov bool context_fault_needs_threaded_irq; 442556db53aSJordan Crouse int (*alloc_context_bank)(struct arm_smmu_domain *smmu_domain, 443556db53aSJordan Crouse struct arm_smmu_device *smmu, 444556db53aSJordan Crouse struct device *dev, int start); 44556b75b51SBjorn Andersson void (*write_s2cr)(struct arm_smmu_device *smmu, int idx); 446bffb2eafSRob Clark void (*write_sctlr)(struct arm_smmu_device *smmu, int idx, u32 reg); 4470d97174aSThierry Reding void (*probe_finalize)(struct arm_smmu_device *smmu, struct device *dev); 448e86d1aa8SWill Deacon }; 449e86d1aa8SWill Deacon 450556db53aSJordan Crouse #define INVALID_SMENDX -1 451556db53aSJordan Crouse #define cfg_smendx(cfg, fw, i) \ 452556db53aSJordan Crouse (i >= fw->num_ids ? INVALID_SMENDX : cfg->smendx[i]) 453556db53aSJordan Crouse #define for_each_cfg_sme(cfg, fw, i, idx) \ 454556db53aSJordan Crouse for (i = 0; idx = cfg_smendx(cfg, fw, i), i < fw->num_ids; ++i) 455556db53aSJordan Crouse 456556db53aSJordan Crouse static inline int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end) 457556db53aSJordan Crouse { 458556db53aSJordan Crouse int idx; 459556db53aSJordan Crouse 460556db53aSJordan Crouse do { 461556db53aSJordan Crouse idx = find_next_zero_bit(map, end, start); 462556db53aSJordan Crouse if (idx == end) 463556db53aSJordan Crouse return -ENOSPC; 464556db53aSJordan Crouse } while (test_and_set_bit(idx, map)); 465556db53aSJordan Crouse 466556db53aSJordan Crouse return idx; 467556db53aSJordan Crouse } 468556db53aSJordan Crouse 469e86d1aa8SWill Deacon static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n) 470e86d1aa8SWill Deacon { 471e86d1aa8SWill Deacon return smmu->base + (n << smmu->pgshift); 472e86d1aa8SWill Deacon } 473e86d1aa8SWill Deacon 474e86d1aa8SWill Deacon static inline u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset) 475e86d1aa8SWill Deacon { 476e86d1aa8SWill Deacon if (smmu->impl && unlikely(smmu->impl->read_reg)) 477e86d1aa8SWill Deacon return smmu->impl->read_reg(smmu, page, offset); 478e86d1aa8SWill Deacon return readl_relaxed(arm_smmu_page(smmu, page) + offset); 479e86d1aa8SWill Deacon } 480e86d1aa8SWill Deacon 481e86d1aa8SWill Deacon static inline void arm_smmu_writel(struct arm_smmu_device *smmu, int page, 482e86d1aa8SWill Deacon int offset, u32 val) 483e86d1aa8SWill Deacon { 484e86d1aa8SWill Deacon if (smmu->impl && unlikely(smmu->impl->write_reg)) 485e86d1aa8SWill Deacon smmu->impl->write_reg(smmu, page, offset, val); 486e86d1aa8SWill Deacon else 487e86d1aa8SWill Deacon writel_relaxed(val, arm_smmu_page(smmu, page) + offset); 488e86d1aa8SWill Deacon } 489e86d1aa8SWill Deacon 490e86d1aa8SWill Deacon static inline u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset) 491e86d1aa8SWill Deacon { 492e86d1aa8SWill Deacon if (smmu->impl && unlikely(smmu->impl->read_reg64)) 493e86d1aa8SWill Deacon return smmu->impl->read_reg64(smmu, page, offset); 494e86d1aa8SWill Deacon return readq_relaxed(arm_smmu_page(smmu, page) + offset); 495e86d1aa8SWill Deacon } 496e86d1aa8SWill Deacon 497e86d1aa8SWill Deacon static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, 498e86d1aa8SWill Deacon int offset, u64 val) 499e86d1aa8SWill Deacon { 500e86d1aa8SWill Deacon if (smmu->impl && unlikely(smmu->impl->write_reg64)) 501e86d1aa8SWill Deacon smmu->impl->write_reg64(smmu, page, offset, val); 502e86d1aa8SWill Deacon else 503e86d1aa8SWill Deacon writeq_relaxed(val, arm_smmu_page(smmu, page) + offset); 504e86d1aa8SWill Deacon } 505e86d1aa8SWill Deacon 506e86d1aa8SWill Deacon #define ARM_SMMU_GR0 0 507e86d1aa8SWill Deacon #define ARM_SMMU_GR1 1 508e86d1aa8SWill Deacon #define ARM_SMMU_CB(s, n) ((s)->numpage + (n)) 509e86d1aa8SWill Deacon 510e86d1aa8SWill Deacon #define arm_smmu_gr0_read(s, o) \ 511e86d1aa8SWill Deacon arm_smmu_readl((s), ARM_SMMU_GR0, (o)) 512e86d1aa8SWill Deacon #define arm_smmu_gr0_write(s, o, v) \ 513e86d1aa8SWill Deacon arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v)) 514e86d1aa8SWill Deacon 515e86d1aa8SWill Deacon #define arm_smmu_gr1_read(s, o) \ 516e86d1aa8SWill Deacon arm_smmu_readl((s), ARM_SMMU_GR1, (o)) 517e86d1aa8SWill Deacon #define arm_smmu_gr1_write(s, o, v) \ 518e86d1aa8SWill Deacon arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v)) 519e86d1aa8SWill Deacon 520e86d1aa8SWill Deacon #define arm_smmu_cb_read(s, n, o) \ 521e86d1aa8SWill Deacon arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o)) 522e86d1aa8SWill Deacon #define arm_smmu_cb_write(s, n, o, v) \ 523e86d1aa8SWill Deacon arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v)) 524e86d1aa8SWill Deacon #define arm_smmu_cb_readq(s, n, o) \ 525e86d1aa8SWill Deacon arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o)) 526e86d1aa8SWill Deacon #define arm_smmu_cb_writeq(s, n, o, v) \ 527e86d1aa8SWill Deacon arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v)) 528e86d1aa8SWill Deacon 529e86d1aa8SWill Deacon struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); 530e86d1aa8SWill Deacon struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu); 531e86d1aa8SWill Deacon struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu); 532e86d1aa8SWill Deacon 533556db53aSJordan Crouse void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx); 534e86d1aa8SWill Deacon int arm_mmu500_reset(struct arm_smmu_device *smmu); 535e86d1aa8SWill Deacon 536e86d1aa8SWill Deacon #endif /* _ARM_SMMU_H */ 537