1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * IOMMU API for ARM architected SMMUv3 implementations. 4 * 5 * Copyright (C) 2015 ARM Limited 6 */ 7 8 #ifndef _ARM_SMMU_V3_H 9 #define _ARM_SMMU_V3_H 10 11 #include <linux/bitfield.h> 12 #include <linux/iommu.h> 13 #include <linux/kernel.h> 14 #include <linux/mmzone.h> 15 #include <linux/sizes.h> 16 17 /* MMIO registers */ 18 #define ARM_SMMU_IDR0 0x0 19 #define IDR0_ST_LVL GENMASK(28, 27) 20 #define IDR0_ST_LVL_2LVL 1 21 #define IDR0_STALL_MODEL GENMASK(25, 24) 22 #define IDR0_STALL_MODEL_STALL 0 23 #define IDR0_STALL_MODEL_FORCE 2 24 #define IDR0_TTENDIAN GENMASK(22, 21) 25 #define IDR0_TTENDIAN_MIXED 0 26 #define IDR0_TTENDIAN_LE 2 27 #define IDR0_TTENDIAN_BE 3 28 #define IDR0_CD2L (1 << 19) 29 #define IDR0_VMID16 (1 << 18) 30 #define IDR0_PRI (1 << 16) 31 #define IDR0_SEV (1 << 14) 32 #define IDR0_MSI (1 << 13) 33 #define IDR0_ASID16 (1 << 12) 34 #define IDR0_ATS (1 << 10) 35 #define IDR0_HYP (1 << 9) 36 #define IDR0_COHACC (1 << 4) 37 #define IDR0_TTF GENMASK(3, 2) 38 #define IDR0_TTF_AARCH64 2 39 #define IDR0_TTF_AARCH32_64 3 40 #define IDR0_S1P (1 << 1) 41 #define IDR0_S2P (1 << 0) 42 43 #define ARM_SMMU_IDR1 0x4 44 #define IDR1_TABLES_PRESET (1 << 30) 45 #define IDR1_QUEUES_PRESET (1 << 29) 46 #define IDR1_REL (1 << 28) 47 #define IDR1_ATTR_TYPES_OVR (1 << 27) 48 #define IDR1_CMDQS GENMASK(25, 21) 49 #define IDR1_EVTQS GENMASK(20, 16) 50 #define IDR1_PRIQS GENMASK(15, 11) 51 #define IDR1_SSIDSIZE GENMASK(10, 6) 52 #define IDR1_SIDSIZE GENMASK(5, 0) 53 54 #define ARM_SMMU_IDR3 0xc 55 #define IDR3_RIL (1 << 10) 56 57 #define ARM_SMMU_IDR5 0x14 58 #define IDR5_STALL_MAX GENMASK(31, 16) 59 #define IDR5_GRAN64K (1 << 6) 60 #define IDR5_GRAN16K (1 << 5) 61 #define IDR5_GRAN4K (1 << 4) 62 #define IDR5_OAS GENMASK(2, 0) 63 #define IDR5_OAS_32_BIT 0 64 #define IDR5_OAS_36_BIT 1 65 #define IDR5_OAS_40_BIT 2 66 #define IDR5_OAS_42_BIT 3 67 #define IDR5_OAS_44_BIT 4 68 #define IDR5_OAS_48_BIT 5 69 #define IDR5_OAS_52_BIT 6 70 #define IDR5_VAX GENMASK(11, 10) 71 #define IDR5_VAX_52_BIT 1 72 73 #define ARM_SMMU_IIDR 0x18 74 #define IIDR_PRODUCTID GENMASK(31, 20) 75 #define IIDR_VARIANT GENMASK(19, 16) 76 #define IIDR_REVISION GENMASK(15, 12) 77 #define IIDR_IMPLEMENTER GENMASK(11, 0) 78 79 #define ARM_SMMU_CR0 0x20 80 #define CR0_ATSCHK (1 << 4) 81 #define CR0_CMDQEN (1 << 3) 82 #define CR0_EVTQEN (1 << 2) 83 #define CR0_PRIQEN (1 << 1) 84 #define CR0_SMMUEN (1 << 0) 85 86 #define ARM_SMMU_CR0ACK 0x24 87 88 #define ARM_SMMU_CR1 0x28 89 #define CR1_TABLE_SH GENMASK(11, 10) 90 #define CR1_TABLE_OC GENMASK(9, 8) 91 #define CR1_TABLE_IC GENMASK(7, 6) 92 #define CR1_QUEUE_SH GENMASK(5, 4) 93 #define CR1_QUEUE_OC GENMASK(3, 2) 94 #define CR1_QUEUE_IC GENMASK(1, 0) 95 /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */ 96 #define CR1_CACHE_NC 0 97 #define CR1_CACHE_WB 1 98 #define CR1_CACHE_WT 2 99 100 #define ARM_SMMU_CR2 0x2c 101 #define CR2_PTM (1 << 2) 102 #define CR2_RECINVSID (1 << 1) 103 #define CR2_E2H (1 << 0) 104 105 #define ARM_SMMU_GBPA 0x44 106 #define GBPA_UPDATE (1 << 31) 107 #define GBPA_ABORT (1 << 20) 108 109 #define ARM_SMMU_IRQ_CTRL 0x50 110 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2) 111 #define IRQ_CTRL_PRIQ_IRQEN (1 << 1) 112 #define IRQ_CTRL_GERROR_IRQEN (1 << 0) 113 114 #define ARM_SMMU_IRQ_CTRLACK 0x54 115 116 #define ARM_SMMU_GERROR 0x60 117 #define GERROR_SFM_ERR (1 << 8) 118 #define GERROR_MSI_GERROR_ABT_ERR (1 << 7) 119 #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6) 120 #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5) 121 #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4) 122 #define GERROR_PRIQ_ABT_ERR (1 << 3) 123 #define GERROR_EVTQ_ABT_ERR (1 << 2) 124 #define GERROR_CMDQ_ERR (1 << 0) 125 #define GERROR_ERR_MASK 0x1fd 126 127 #define ARM_SMMU_GERRORN 0x64 128 129 #define ARM_SMMU_GERROR_IRQ_CFG0 0x68 130 #define ARM_SMMU_GERROR_IRQ_CFG1 0x70 131 #define ARM_SMMU_GERROR_IRQ_CFG2 0x74 132 133 #define ARM_SMMU_STRTAB_BASE 0x80 134 #define STRTAB_BASE_RA (1UL << 62) 135 #define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6) 136 137 #define ARM_SMMU_STRTAB_BASE_CFG 0x88 138 #define STRTAB_BASE_CFG_FMT GENMASK(17, 16) 139 #define STRTAB_BASE_CFG_FMT_LINEAR 0 140 #define STRTAB_BASE_CFG_FMT_2LVL 1 141 #define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6) 142 #define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0) 143 144 #define ARM_SMMU_CMDQ_BASE 0x90 145 #define ARM_SMMU_CMDQ_PROD 0x98 146 #define ARM_SMMU_CMDQ_CONS 0x9c 147 148 #define ARM_SMMU_EVTQ_BASE 0xa0 149 #define ARM_SMMU_EVTQ_PROD 0xa8 150 #define ARM_SMMU_EVTQ_CONS 0xac 151 #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0 152 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8 153 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc 154 155 #define ARM_SMMU_PRIQ_BASE 0xc0 156 #define ARM_SMMU_PRIQ_PROD 0xc8 157 #define ARM_SMMU_PRIQ_CONS 0xcc 158 #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0 159 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8 160 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc 161 162 #define ARM_SMMU_REG_SZ 0xe00 163 164 /* Common MSI config fields */ 165 #define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2) 166 #define MSI_CFG2_SH GENMASK(5, 4) 167 #define MSI_CFG2_MEMATTR GENMASK(3, 0) 168 169 /* Common memory attribute values */ 170 #define ARM_SMMU_SH_NSH 0 171 #define ARM_SMMU_SH_OSH 2 172 #define ARM_SMMU_SH_ISH 3 173 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1 174 #define ARM_SMMU_MEMATTR_OIWB 0xf 175 176 #define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1)) 177 #define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift)) 178 #define Q_OVERFLOW_FLAG (1U << 31) 179 #define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG) 180 #define Q_ENT(q, p) ((q)->base + \ 181 Q_IDX(&((q)->llq), p) * \ 182 (q)->ent_dwords) 183 184 #define Q_BASE_RWA (1UL << 62) 185 #define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5) 186 #define Q_BASE_LOG2SIZE GENMASK(4, 0) 187 188 /* Ensure DMA allocations are naturally aligned */ 189 #ifdef CONFIG_CMA_ALIGNMENT 190 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT) 191 #else 192 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_PAGE_ORDER) 193 #endif 194 195 /* 196 * Stream table. 197 * 198 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries 199 * 2lvl: 128k L1 entries, 200 * 256 lazy entries per table (each table covers a PCI bus) 201 */ 202 #define STRTAB_L1_SZ_SHIFT 20 203 #define STRTAB_SPLIT 8 204 205 #define STRTAB_L1_DESC_DWORDS 1 206 #define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0) 207 #define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6) 208 209 #define STRTAB_STE_DWORDS 8 210 211 struct arm_smmu_ste { 212 __le64 data[STRTAB_STE_DWORDS]; 213 }; 214 215 #define STRTAB_STE_0_V (1UL << 0) 216 #define STRTAB_STE_0_CFG GENMASK_ULL(3, 1) 217 #define STRTAB_STE_0_CFG_ABORT 0 218 #define STRTAB_STE_0_CFG_BYPASS 4 219 #define STRTAB_STE_0_CFG_S1_TRANS 5 220 #define STRTAB_STE_0_CFG_S2_TRANS 6 221 222 #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4) 223 #define STRTAB_STE_0_S1FMT_LINEAR 0 224 #define STRTAB_STE_0_S1FMT_64K_L2 2 225 #define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6) 226 #define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59) 227 228 #define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0) 229 #define STRTAB_STE_1_S1DSS_TERMINATE 0x0 230 #define STRTAB_STE_1_S1DSS_BYPASS 0x1 231 #define STRTAB_STE_1_S1DSS_SSID0 0x2 232 233 #define STRTAB_STE_1_S1C_CACHE_NC 0UL 234 #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL 235 #define STRTAB_STE_1_S1C_CACHE_WT 2UL 236 #define STRTAB_STE_1_S1C_CACHE_WB 3UL 237 #define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2) 238 #define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4) 239 #define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6) 240 241 #define STRTAB_STE_1_S1STALLD (1UL << 27) 242 243 #define STRTAB_STE_1_EATS GENMASK_ULL(29, 28) 244 #define STRTAB_STE_1_EATS_ABT 0UL 245 #define STRTAB_STE_1_EATS_TRANS 1UL 246 #define STRTAB_STE_1_EATS_S1CHK 2UL 247 248 #define STRTAB_STE_1_STRW GENMASK_ULL(31, 30) 249 #define STRTAB_STE_1_STRW_NSEL1 0UL 250 #define STRTAB_STE_1_STRW_EL2 2UL 251 252 #define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44) 253 #define STRTAB_STE_1_SHCFG_INCOMING 1UL 254 255 #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0) 256 #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32) 257 #define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0) 258 #define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6) 259 #define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8) 260 #define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10) 261 #define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12) 262 #define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14) 263 #define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16) 264 #define STRTAB_STE_2_S2AA64 (1UL << 51) 265 #define STRTAB_STE_2_S2ENDI (1UL << 52) 266 #define STRTAB_STE_2_S2PTW (1UL << 54) 267 #define STRTAB_STE_2_S2R (1UL << 58) 268 269 #define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4) 270 271 /* 272 * Context descriptors. 273 * 274 * Linear: when less than 1024 SSIDs are supported 275 * 2lvl: at most 1024 L1 entries, 276 * 1024 lazy entries per table. 277 */ 278 #define CTXDESC_L2_ENTRIES 1024 279 280 #define CTXDESC_L1_DESC_DWORDS 1 281 #define CTXDESC_L1_DESC_V (1UL << 0) 282 #define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12) 283 284 #define CTXDESC_CD_DWORDS 8 285 286 struct arm_smmu_cd { 287 __le64 data[CTXDESC_CD_DWORDS]; 288 }; 289 290 #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) 291 #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6) 292 #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) 293 #define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10) 294 #define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12) 295 #define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14) 296 #define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30) 297 298 #define CTXDESC_CD_0_ENDI (1UL << 15) 299 #define CTXDESC_CD_0_V (1UL << 31) 300 301 #define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32) 302 #define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38) 303 304 #define CTXDESC_CD_0_AA64 (1UL << 41) 305 #define CTXDESC_CD_0_S (1UL << 44) 306 #define CTXDESC_CD_0_R (1UL << 45) 307 #define CTXDESC_CD_0_A (1UL << 46) 308 #define CTXDESC_CD_0_ASET (1UL << 47) 309 #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48) 310 311 #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4) 312 313 /* 314 * When the SMMU only supports linear context descriptor tables, pick a 315 * reasonable size limit (64kB). 316 */ 317 #define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3)) 318 319 /* Command queue */ 320 #define CMDQ_ENT_SZ_SHIFT 4 321 #define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3) 322 #define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT) 323 324 #define CMDQ_CONS_ERR GENMASK(30, 24) 325 #define CMDQ_ERR_CERROR_NONE_IDX 0 326 #define CMDQ_ERR_CERROR_ILL_IDX 1 327 #define CMDQ_ERR_CERROR_ABT_IDX 2 328 #define CMDQ_ERR_CERROR_ATC_INV_IDX 3 329 330 #define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG 331 332 /* 333 * This is used to size the command queue and therefore must be at least 334 * BITS_PER_LONG so that the valid_map works correctly (it relies on the 335 * total number of queue entries being a multiple of BITS_PER_LONG). 336 */ 337 #define CMDQ_BATCH_ENTRIES BITS_PER_LONG 338 339 #define CMDQ_0_OP GENMASK_ULL(7, 0) 340 #define CMDQ_0_SSV (1UL << 11) 341 342 #define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32) 343 #define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0) 344 #define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12) 345 346 #define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12) 347 #define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32) 348 #define CMDQ_CFGI_1_LEAF (1UL << 0) 349 #define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0) 350 351 #define CMDQ_TLBI_0_NUM GENMASK_ULL(16, 12) 352 #define CMDQ_TLBI_RANGE_NUM_MAX 31 353 #define CMDQ_TLBI_0_SCALE GENMASK_ULL(24, 20) 354 #define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32) 355 #define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48) 356 #define CMDQ_TLBI_1_LEAF (1UL << 0) 357 #define CMDQ_TLBI_1_TTL GENMASK_ULL(9, 8) 358 #define CMDQ_TLBI_1_TG GENMASK_ULL(11, 10) 359 #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12) 360 #define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12) 361 362 #define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12) 363 #define CMDQ_ATC_0_SID GENMASK_ULL(63, 32) 364 #define CMDQ_ATC_0_GLOBAL (1UL << 9) 365 #define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0) 366 #define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12) 367 368 #define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12) 369 #define CMDQ_PRI_0_SID GENMASK_ULL(63, 32) 370 #define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0) 371 #define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12) 372 373 #define CMDQ_RESUME_0_RESP_TERM 0UL 374 #define CMDQ_RESUME_0_RESP_RETRY 1UL 375 #define CMDQ_RESUME_0_RESP_ABORT 2UL 376 #define CMDQ_RESUME_0_RESP GENMASK_ULL(13, 12) 377 #define CMDQ_RESUME_0_SID GENMASK_ULL(63, 32) 378 #define CMDQ_RESUME_1_STAG GENMASK_ULL(15, 0) 379 380 #define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12) 381 #define CMDQ_SYNC_0_CS_NONE 0 382 #define CMDQ_SYNC_0_CS_IRQ 1 383 #define CMDQ_SYNC_0_CS_SEV 2 384 #define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22) 385 #define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24) 386 #define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32) 387 #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2) 388 389 /* Event queue */ 390 #define EVTQ_ENT_SZ_SHIFT 5 391 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3) 392 #define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT) 393 394 #define EVTQ_0_ID GENMASK_ULL(7, 0) 395 396 #define EVT_ID_TRANSLATION_FAULT 0x10 397 #define EVT_ID_ADDR_SIZE_FAULT 0x11 398 #define EVT_ID_ACCESS_FAULT 0x12 399 #define EVT_ID_PERMISSION_FAULT 0x13 400 401 #define EVTQ_0_SSV (1UL << 11) 402 #define EVTQ_0_SSID GENMASK_ULL(31, 12) 403 #define EVTQ_0_SID GENMASK_ULL(63, 32) 404 #define EVTQ_1_STAG GENMASK_ULL(15, 0) 405 #define EVTQ_1_STALL (1UL << 31) 406 #define EVTQ_1_PnU (1UL << 33) 407 #define EVTQ_1_InD (1UL << 34) 408 #define EVTQ_1_RnW (1UL << 35) 409 #define EVTQ_1_S2 (1UL << 39) 410 #define EVTQ_1_CLASS GENMASK_ULL(41, 40) 411 #define EVTQ_1_TT_READ (1UL << 44) 412 #define EVTQ_2_ADDR GENMASK_ULL(63, 0) 413 #define EVTQ_3_IPA GENMASK_ULL(51, 12) 414 415 /* PRI queue */ 416 #define PRIQ_ENT_SZ_SHIFT 4 417 #define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3) 418 #define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT) 419 420 #define PRIQ_0_SID GENMASK_ULL(31, 0) 421 #define PRIQ_0_SSID GENMASK_ULL(51, 32) 422 #define PRIQ_0_PERM_PRIV (1UL << 58) 423 #define PRIQ_0_PERM_EXEC (1UL << 59) 424 #define PRIQ_0_PERM_READ (1UL << 60) 425 #define PRIQ_0_PERM_WRITE (1UL << 61) 426 #define PRIQ_0_PRG_LAST (1UL << 62) 427 #define PRIQ_0_SSID_V (1UL << 63) 428 429 #define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0) 430 #define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12) 431 432 /* High-level queue structures */ 433 #define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */ 434 #define ARM_SMMU_POLL_SPIN_COUNT 10 435 436 #define MSI_IOVA_BASE 0x8000000 437 #define MSI_IOVA_LENGTH 0x100000 438 439 enum pri_resp { 440 PRI_RESP_DENY = 0, 441 PRI_RESP_FAIL = 1, 442 PRI_RESP_SUCC = 2, 443 }; 444 445 struct arm_smmu_cmdq_ent { 446 /* Common fields */ 447 u8 opcode; 448 bool substream_valid; 449 450 /* Command-specific fields */ 451 union { 452 #define CMDQ_OP_PREFETCH_CFG 0x1 453 struct { 454 u32 sid; 455 } prefetch; 456 457 #define CMDQ_OP_CFGI_STE 0x3 458 #define CMDQ_OP_CFGI_ALL 0x4 459 #define CMDQ_OP_CFGI_CD 0x5 460 #define CMDQ_OP_CFGI_CD_ALL 0x6 461 struct { 462 u32 sid; 463 u32 ssid; 464 union { 465 bool leaf; 466 u8 span; 467 }; 468 } cfgi; 469 470 #define CMDQ_OP_TLBI_NH_ASID 0x11 471 #define CMDQ_OP_TLBI_NH_VA 0x12 472 #define CMDQ_OP_TLBI_EL2_ALL 0x20 473 #define CMDQ_OP_TLBI_EL2_ASID 0x21 474 #define CMDQ_OP_TLBI_EL2_VA 0x22 475 #define CMDQ_OP_TLBI_S12_VMALL 0x28 476 #define CMDQ_OP_TLBI_S2_IPA 0x2a 477 #define CMDQ_OP_TLBI_NSNH_ALL 0x30 478 struct { 479 u8 num; 480 u8 scale; 481 u16 asid; 482 u16 vmid; 483 bool leaf; 484 u8 ttl; 485 u8 tg; 486 u64 addr; 487 } tlbi; 488 489 #define CMDQ_OP_ATC_INV 0x40 490 #define ATC_INV_SIZE_ALL 52 491 struct { 492 u32 sid; 493 u32 ssid; 494 u64 addr; 495 u8 size; 496 bool global; 497 } atc; 498 499 #define CMDQ_OP_PRI_RESP 0x41 500 struct { 501 u32 sid; 502 u32 ssid; 503 u16 grpid; 504 enum pri_resp resp; 505 } pri; 506 507 #define CMDQ_OP_RESUME 0x44 508 struct { 509 u32 sid; 510 u16 stag; 511 u8 resp; 512 } resume; 513 514 #define CMDQ_OP_CMD_SYNC 0x46 515 struct { 516 u64 msiaddr; 517 } sync; 518 }; 519 }; 520 521 struct arm_smmu_ll_queue { 522 union { 523 u64 val; 524 struct { 525 u32 prod; 526 u32 cons; 527 }; 528 struct { 529 atomic_t prod; 530 atomic_t cons; 531 } atomic; 532 u8 __pad[SMP_CACHE_BYTES]; 533 } ____cacheline_aligned_in_smp; 534 u32 max_n_shift; 535 }; 536 537 struct arm_smmu_queue { 538 struct arm_smmu_ll_queue llq; 539 int irq; /* Wired interrupt */ 540 541 __le64 *base; 542 dma_addr_t base_dma; 543 u64 q_base; 544 545 size_t ent_dwords; 546 547 u32 __iomem *prod_reg; 548 u32 __iomem *cons_reg; 549 }; 550 551 struct arm_smmu_queue_poll { 552 ktime_t timeout; 553 unsigned int delay; 554 unsigned int spin_cnt; 555 bool wfe; 556 }; 557 558 struct arm_smmu_cmdq { 559 struct arm_smmu_queue q; 560 atomic_long_t *valid_map; 561 atomic_t owner_prod; 562 atomic_t lock; 563 }; 564 565 struct arm_smmu_cmdq_batch { 566 u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS]; 567 int num; 568 }; 569 570 struct arm_smmu_evtq { 571 struct arm_smmu_queue q; 572 struct iopf_queue *iopf; 573 u32 max_stalls; 574 }; 575 576 struct arm_smmu_priq { 577 struct arm_smmu_queue q; 578 }; 579 580 /* High-level stream table and context descriptor structures */ 581 struct arm_smmu_strtab_l1_desc { 582 u8 span; 583 584 struct arm_smmu_ste *l2ptr; 585 dma_addr_t l2ptr_dma; 586 }; 587 588 struct arm_smmu_ctx_desc { 589 u16 asid; 590 591 refcount_t refs; 592 struct mm_struct *mm; 593 }; 594 595 struct arm_smmu_l1_ctx_desc { 596 struct arm_smmu_cd *l2ptr; 597 dma_addr_t l2ptr_dma; 598 }; 599 600 struct arm_smmu_ctx_desc_cfg { 601 __le64 *cdtab; 602 dma_addr_t cdtab_dma; 603 struct arm_smmu_l1_ctx_desc *l1_desc; 604 unsigned int num_l1_ents; 605 u8 s1fmt; 606 /* log2 of the maximum number of CDs supported by this table */ 607 u8 s1cdmax; 608 }; 609 610 struct arm_smmu_s2_cfg { 611 u16 vmid; 612 }; 613 614 struct arm_smmu_strtab_cfg { 615 __le64 *strtab; 616 dma_addr_t strtab_dma; 617 struct arm_smmu_strtab_l1_desc *l1_desc; 618 unsigned int num_l1_ents; 619 620 u64 strtab_base; 621 u32 strtab_base_cfg; 622 }; 623 624 /* An SMMUv3 instance */ 625 struct arm_smmu_device { 626 struct device *dev; 627 void __iomem *base; 628 void __iomem *page1; 629 630 #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0) 631 #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1) 632 #define ARM_SMMU_FEAT_TT_LE (1 << 2) 633 #define ARM_SMMU_FEAT_TT_BE (1 << 3) 634 #define ARM_SMMU_FEAT_PRI (1 << 4) 635 #define ARM_SMMU_FEAT_ATS (1 << 5) 636 #define ARM_SMMU_FEAT_SEV (1 << 6) 637 #define ARM_SMMU_FEAT_MSI (1 << 7) 638 #define ARM_SMMU_FEAT_COHERENCY (1 << 8) 639 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9) 640 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10) 641 #define ARM_SMMU_FEAT_STALLS (1 << 11) 642 #define ARM_SMMU_FEAT_HYP (1 << 12) 643 #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13) 644 #define ARM_SMMU_FEAT_VAX (1 << 14) 645 #define ARM_SMMU_FEAT_RANGE_INV (1 << 15) 646 #define ARM_SMMU_FEAT_BTM (1 << 16) 647 #define ARM_SMMU_FEAT_SVA (1 << 17) 648 #define ARM_SMMU_FEAT_E2H (1 << 18) 649 #define ARM_SMMU_FEAT_NESTING (1 << 19) 650 #define ARM_SMMU_FEAT_ATTR_TYPES_OVR (1 << 20) 651 u32 features; 652 653 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0) 654 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1) 655 #define ARM_SMMU_OPT_MSIPOLL (1 << 2) 656 #define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3) 657 u32 options; 658 659 struct arm_smmu_cmdq cmdq; 660 struct arm_smmu_evtq evtq; 661 struct arm_smmu_priq priq; 662 663 int gerr_irq; 664 int combined_irq; 665 666 unsigned long ias; /* IPA */ 667 unsigned long oas; /* PA */ 668 unsigned long pgsize_bitmap; 669 670 #define ARM_SMMU_MAX_ASIDS (1 << 16) 671 unsigned int asid_bits; 672 673 #define ARM_SMMU_MAX_VMIDS (1 << 16) 674 unsigned int vmid_bits; 675 struct ida vmid_map; 676 677 unsigned int ssid_bits; 678 unsigned int sid_bits; 679 680 struct arm_smmu_strtab_cfg strtab_cfg; 681 682 /* IOMMU core code handle */ 683 struct iommu_device iommu; 684 685 struct rb_root streams; 686 struct mutex streams_mutex; 687 }; 688 689 struct arm_smmu_stream { 690 u32 id; 691 struct arm_smmu_master *master; 692 struct rb_node node; 693 }; 694 695 /* SMMU private data for each master */ 696 struct arm_smmu_master { 697 struct arm_smmu_device *smmu; 698 struct device *dev; 699 struct list_head domain_head; 700 struct arm_smmu_stream *streams; 701 /* Locked by the iommu core using the group mutex */ 702 struct arm_smmu_ctx_desc_cfg cd_table; 703 unsigned int num_streams; 704 bool ats_enabled; 705 bool stall_enabled; 706 bool sva_enabled; 707 bool iopf_enabled; 708 struct list_head bonds; 709 unsigned int ssid_bits; 710 }; 711 712 /* SMMU private data for an IOMMU domain */ 713 enum arm_smmu_domain_stage { 714 ARM_SMMU_DOMAIN_S1 = 0, 715 ARM_SMMU_DOMAIN_S2, 716 }; 717 718 struct arm_smmu_domain { 719 struct arm_smmu_device *smmu; 720 struct mutex init_mutex; /* Protects smmu pointer */ 721 722 struct io_pgtable_ops *pgtbl_ops; 723 atomic_t nr_ats_masters; 724 725 enum arm_smmu_domain_stage stage; 726 union { 727 struct arm_smmu_ctx_desc cd; 728 struct arm_smmu_s2_cfg s2_cfg; 729 }; 730 731 struct iommu_domain domain; 732 733 struct list_head devices; 734 spinlock_t devices_lock; 735 736 struct list_head mmu_notifiers; 737 }; 738 739 /* The following are exposed for testing purposes. */ 740 struct arm_smmu_entry_writer_ops; 741 struct arm_smmu_entry_writer { 742 const struct arm_smmu_entry_writer_ops *ops; 743 struct arm_smmu_master *master; 744 }; 745 746 struct arm_smmu_entry_writer_ops { 747 void (*get_used)(const __le64 *entry, __le64 *used); 748 void (*sync)(struct arm_smmu_entry_writer *writer); 749 }; 750 751 #if IS_ENABLED(CONFIG_KUNIT) 752 void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits); 753 void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *cur, 754 const __le64 *target); 755 void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits); 756 void arm_smmu_make_abort_ste(struct arm_smmu_ste *target); 757 void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, 758 struct arm_smmu_ste *target); 759 void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, 760 struct arm_smmu_master *master); 761 void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, 762 struct arm_smmu_master *master, 763 struct arm_smmu_domain *smmu_domain); 764 void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, 765 struct arm_smmu_master *master, struct mm_struct *mm, 766 u16 asid); 767 #endif 768 769 static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) 770 { 771 return container_of(dom, struct arm_smmu_domain, domain); 772 } 773 774 extern struct xarray arm_smmu_asid_xa; 775 extern struct mutex arm_smmu_asid_lock; 776 777 void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid); 778 struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, 779 u32 ssid); 780 struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master, 781 u32 ssid); 782 void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, 783 struct arm_smmu_master *master, 784 struct arm_smmu_domain *smmu_domain); 785 void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid, 786 struct arm_smmu_cd *cdptr, 787 const struct arm_smmu_cd *target); 788 789 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid); 790 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, 791 size_t granule, bool leaf, 792 struct arm_smmu_domain *smmu_domain); 793 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd); 794 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid, 795 unsigned long iova, size_t size); 796 797 #ifdef CONFIG_ARM_SMMU_V3_SVA 798 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu); 799 bool arm_smmu_master_sva_supported(struct arm_smmu_master *master); 800 bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master); 801 int arm_smmu_master_enable_sva(struct arm_smmu_master *master); 802 int arm_smmu_master_disable_sva(struct arm_smmu_master *master); 803 bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master); 804 void arm_smmu_sva_notifier_synchronize(void); 805 struct iommu_domain *arm_smmu_sva_domain_alloc(void); 806 void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, 807 struct device *dev, ioasid_t id); 808 #else /* CONFIG_ARM_SMMU_V3_SVA */ 809 static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) 810 { 811 return false; 812 } 813 814 static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master) 815 { 816 return false; 817 } 818 819 static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master) 820 { 821 return false; 822 } 823 824 static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master) 825 { 826 return -ENODEV; 827 } 828 829 static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master) 830 { 831 return -ENODEV; 832 } 833 834 static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master) 835 { 836 return false; 837 } 838 839 static inline void arm_smmu_sva_notifier_synchronize(void) {} 840 841 static inline struct iommu_domain *arm_smmu_sva_domain_alloc(void) 842 { 843 return NULL; 844 } 845 846 static inline void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, 847 struct device *dev, 848 ioasid_t id) 849 { 850 } 851 #endif /* CONFIG_ARM_SMMU_V3_SVA */ 852 #endif /* _ARM_SMMU_V3_H */ 853