1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * IOMMU API for ARM architected SMMUv3 implementations. 4 * 5 * Copyright (C) 2015 ARM Limited 6 */ 7 8 #ifndef _ARM_SMMU_V3_H 9 #define _ARM_SMMU_V3_H 10 11 #include <linux/bitfield.h> 12 #include <linux/iommu.h> 13 #include <linux/kernel.h> 14 #include <linux/mmzone.h> 15 #include <linux/sizes.h> 16 17 /* MMIO registers */ 18 #define ARM_SMMU_IDR0 0x0 19 #define IDR0_ST_LVL GENMASK(28, 27) 20 #define IDR0_ST_LVL_2LVL 1 21 #define IDR0_STALL_MODEL GENMASK(25, 24) 22 #define IDR0_STALL_MODEL_STALL 0 23 #define IDR0_STALL_MODEL_FORCE 2 24 #define IDR0_TTENDIAN GENMASK(22, 21) 25 #define IDR0_TTENDIAN_MIXED 0 26 #define IDR0_TTENDIAN_LE 2 27 #define IDR0_TTENDIAN_BE 3 28 #define IDR0_CD2L (1 << 19) 29 #define IDR0_VMID16 (1 << 18) 30 #define IDR0_PRI (1 << 16) 31 #define IDR0_SEV (1 << 14) 32 #define IDR0_MSI (1 << 13) 33 #define IDR0_ASID16 (1 << 12) 34 #define IDR0_ATS (1 << 10) 35 #define IDR0_HYP (1 << 9) 36 #define IDR0_COHACC (1 << 4) 37 #define IDR0_TTF GENMASK(3, 2) 38 #define IDR0_TTF_AARCH64 2 39 #define IDR0_TTF_AARCH32_64 3 40 #define IDR0_S1P (1 << 1) 41 #define IDR0_S2P (1 << 0) 42 43 #define ARM_SMMU_IDR1 0x4 44 #define IDR1_TABLES_PRESET (1 << 30) 45 #define IDR1_QUEUES_PRESET (1 << 29) 46 #define IDR1_REL (1 << 28) 47 #define IDR1_ATTR_TYPES_OVR (1 << 27) 48 #define IDR1_CMDQS GENMASK(25, 21) 49 #define IDR1_EVTQS GENMASK(20, 16) 50 #define IDR1_PRIQS GENMASK(15, 11) 51 #define IDR1_SSIDSIZE GENMASK(10, 6) 52 #define IDR1_SIDSIZE GENMASK(5, 0) 53 54 #define ARM_SMMU_IDR3 0xc 55 #define IDR3_RIL (1 << 10) 56 57 #define ARM_SMMU_IDR5 0x14 58 #define IDR5_STALL_MAX GENMASK(31, 16) 59 #define IDR5_GRAN64K (1 << 6) 60 #define IDR5_GRAN16K (1 << 5) 61 #define IDR5_GRAN4K (1 << 4) 62 #define IDR5_OAS GENMASK(2, 0) 63 #define IDR5_OAS_32_BIT 0 64 #define IDR5_OAS_36_BIT 1 65 #define IDR5_OAS_40_BIT 2 66 #define IDR5_OAS_42_BIT 3 67 #define IDR5_OAS_44_BIT 4 68 #define IDR5_OAS_48_BIT 5 69 #define IDR5_OAS_52_BIT 6 70 #define IDR5_VAX GENMASK(11, 10) 71 #define IDR5_VAX_52_BIT 1 72 73 #define ARM_SMMU_IIDR 0x18 74 #define IIDR_PRODUCTID GENMASK(31, 20) 75 #define IIDR_VARIANT GENMASK(19, 16) 76 #define IIDR_REVISION GENMASK(15, 12) 77 #define IIDR_IMPLEMENTER GENMASK(11, 0) 78 79 #define ARM_SMMU_CR0 0x20 80 #define CR0_ATSCHK (1 << 4) 81 #define CR0_CMDQEN (1 << 3) 82 #define CR0_EVTQEN (1 << 2) 83 #define CR0_PRIQEN (1 << 1) 84 #define CR0_SMMUEN (1 << 0) 85 86 #define ARM_SMMU_CR0ACK 0x24 87 88 #define ARM_SMMU_CR1 0x28 89 #define CR1_TABLE_SH GENMASK(11, 10) 90 #define CR1_TABLE_OC GENMASK(9, 8) 91 #define CR1_TABLE_IC GENMASK(7, 6) 92 #define CR1_QUEUE_SH GENMASK(5, 4) 93 #define CR1_QUEUE_OC GENMASK(3, 2) 94 #define CR1_QUEUE_IC GENMASK(1, 0) 95 /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */ 96 #define CR1_CACHE_NC 0 97 #define CR1_CACHE_WB 1 98 #define CR1_CACHE_WT 2 99 100 #define ARM_SMMU_CR2 0x2c 101 #define CR2_PTM (1 << 2) 102 #define CR2_RECINVSID (1 << 1) 103 #define CR2_E2H (1 << 0) 104 105 #define ARM_SMMU_GBPA 0x44 106 #define GBPA_UPDATE (1 << 31) 107 #define GBPA_ABORT (1 << 20) 108 109 #define ARM_SMMU_IRQ_CTRL 0x50 110 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2) 111 #define IRQ_CTRL_PRIQ_IRQEN (1 << 1) 112 #define IRQ_CTRL_GERROR_IRQEN (1 << 0) 113 114 #define ARM_SMMU_IRQ_CTRLACK 0x54 115 116 #define ARM_SMMU_GERROR 0x60 117 #define GERROR_SFM_ERR (1 << 8) 118 #define GERROR_MSI_GERROR_ABT_ERR (1 << 7) 119 #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6) 120 #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5) 121 #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4) 122 #define GERROR_PRIQ_ABT_ERR (1 << 3) 123 #define GERROR_EVTQ_ABT_ERR (1 << 2) 124 #define GERROR_CMDQ_ERR (1 << 0) 125 #define GERROR_ERR_MASK 0x1fd 126 127 #define ARM_SMMU_GERRORN 0x64 128 129 #define ARM_SMMU_GERROR_IRQ_CFG0 0x68 130 #define ARM_SMMU_GERROR_IRQ_CFG1 0x70 131 #define ARM_SMMU_GERROR_IRQ_CFG2 0x74 132 133 #define ARM_SMMU_STRTAB_BASE 0x80 134 #define STRTAB_BASE_RA (1UL << 62) 135 #define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6) 136 137 #define ARM_SMMU_STRTAB_BASE_CFG 0x88 138 #define STRTAB_BASE_CFG_FMT GENMASK(17, 16) 139 #define STRTAB_BASE_CFG_FMT_LINEAR 0 140 #define STRTAB_BASE_CFG_FMT_2LVL 1 141 #define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6) 142 #define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0) 143 144 #define ARM_SMMU_CMDQ_BASE 0x90 145 #define ARM_SMMU_CMDQ_PROD 0x98 146 #define ARM_SMMU_CMDQ_CONS 0x9c 147 148 #define ARM_SMMU_EVTQ_BASE 0xa0 149 #define ARM_SMMU_EVTQ_PROD 0xa8 150 #define ARM_SMMU_EVTQ_CONS 0xac 151 #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0 152 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8 153 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc 154 155 #define ARM_SMMU_PRIQ_BASE 0xc0 156 #define ARM_SMMU_PRIQ_PROD 0xc8 157 #define ARM_SMMU_PRIQ_CONS 0xcc 158 #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0 159 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8 160 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc 161 162 #define ARM_SMMU_REG_SZ 0xe00 163 164 /* Common MSI config fields */ 165 #define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2) 166 #define MSI_CFG2_SH GENMASK(5, 4) 167 #define MSI_CFG2_MEMATTR GENMASK(3, 0) 168 169 /* Common memory attribute values */ 170 #define ARM_SMMU_SH_NSH 0 171 #define ARM_SMMU_SH_OSH 2 172 #define ARM_SMMU_SH_ISH 3 173 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1 174 #define ARM_SMMU_MEMATTR_OIWB 0xf 175 176 #define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1)) 177 #define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift)) 178 #define Q_OVERFLOW_FLAG (1U << 31) 179 #define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG) 180 #define Q_ENT(q, p) ((q)->base + \ 181 Q_IDX(&((q)->llq), p) * \ 182 (q)->ent_dwords) 183 184 #define Q_BASE_RWA (1UL << 62) 185 #define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5) 186 #define Q_BASE_LOG2SIZE GENMASK(4, 0) 187 188 /* Ensure DMA allocations are naturally aligned */ 189 #ifdef CONFIG_CMA_ALIGNMENT 190 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT) 191 #else 192 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_PAGE_ORDER) 193 #endif 194 195 /* 196 * Stream table. 197 * 198 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries 199 * 2lvl: 128k L1 entries, 200 * 256 lazy entries per table (each table covers a PCI bus) 201 */ 202 #define STRTAB_L1_SZ_SHIFT 20 203 #define STRTAB_SPLIT 8 204 205 #define STRTAB_L1_DESC_DWORDS 1 206 #define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0) 207 #define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6) 208 209 #define STRTAB_STE_DWORDS 8 210 211 struct arm_smmu_ste { 212 __le64 data[STRTAB_STE_DWORDS]; 213 }; 214 215 #define STRTAB_STE_0_V (1UL << 0) 216 #define STRTAB_STE_0_CFG GENMASK_ULL(3, 1) 217 #define STRTAB_STE_0_CFG_ABORT 0 218 #define STRTAB_STE_0_CFG_BYPASS 4 219 #define STRTAB_STE_0_CFG_S1_TRANS 5 220 #define STRTAB_STE_0_CFG_S2_TRANS 6 221 222 #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4) 223 #define STRTAB_STE_0_S1FMT_LINEAR 0 224 #define STRTAB_STE_0_S1FMT_64K_L2 2 225 #define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6) 226 #define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59) 227 228 #define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0) 229 #define STRTAB_STE_1_S1DSS_TERMINATE 0x0 230 #define STRTAB_STE_1_S1DSS_BYPASS 0x1 231 #define STRTAB_STE_1_S1DSS_SSID0 0x2 232 233 #define STRTAB_STE_1_S1C_CACHE_NC 0UL 234 #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL 235 #define STRTAB_STE_1_S1C_CACHE_WT 2UL 236 #define STRTAB_STE_1_S1C_CACHE_WB 3UL 237 #define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2) 238 #define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4) 239 #define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6) 240 241 #define STRTAB_STE_1_S1STALLD (1UL << 27) 242 243 #define STRTAB_STE_1_EATS GENMASK_ULL(29, 28) 244 #define STRTAB_STE_1_EATS_ABT 0UL 245 #define STRTAB_STE_1_EATS_TRANS 1UL 246 #define STRTAB_STE_1_EATS_S1CHK 2UL 247 248 #define STRTAB_STE_1_STRW GENMASK_ULL(31, 30) 249 #define STRTAB_STE_1_STRW_NSEL1 0UL 250 #define STRTAB_STE_1_STRW_EL2 2UL 251 252 #define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44) 253 #define STRTAB_STE_1_SHCFG_INCOMING 1UL 254 255 #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0) 256 #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32) 257 #define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0) 258 #define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6) 259 #define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8) 260 #define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10) 261 #define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12) 262 #define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14) 263 #define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16) 264 #define STRTAB_STE_2_S2AA64 (1UL << 51) 265 #define STRTAB_STE_2_S2ENDI (1UL << 52) 266 #define STRTAB_STE_2_S2PTW (1UL << 54) 267 #define STRTAB_STE_2_S2R (1UL << 58) 268 269 #define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4) 270 271 /* 272 * Context descriptors. 273 * 274 * Linear: when less than 1024 SSIDs are supported 275 * 2lvl: at most 1024 L1 entries, 276 * 1024 lazy entries per table. 277 */ 278 #define CTXDESC_SPLIT 10 279 #define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT) 280 281 #define CTXDESC_L1_DESC_DWORDS 1 282 #define CTXDESC_L1_DESC_V (1UL << 0) 283 #define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12) 284 285 #define CTXDESC_CD_DWORDS 8 286 #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) 287 #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6) 288 #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) 289 #define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10) 290 #define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12) 291 #define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14) 292 #define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30) 293 294 #define CTXDESC_CD_0_ENDI (1UL << 15) 295 #define CTXDESC_CD_0_V (1UL << 31) 296 297 #define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32) 298 #define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38) 299 300 #define CTXDESC_CD_0_AA64 (1UL << 41) 301 #define CTXDESC_CD_0_S (1UL << 44) 302 #define CTXDESC_CD_0_R (1UL << 45) 303 #define CTXDESC_CD_0_A (1UL << 46) 304 #define CTXDESC_CD_0_ASET (1UL << 47) 305 #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48) 306 307 #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4) 308 309 /* 310 * When the SMMU only supports linear context descriptor tables, pick a 311 * reasonable size limit (64kB). 312 */ 313 #define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3)) 314 315 /* Command queue */ 316 #define CMDQ_ENT_SZ_SHIFT 4 317 #define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3) 318 #define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT) 319 320 #define CMDQ_CONS_ERR GENMASK(30, 24) 321 #define CMDQ_ERR_CERROR_NONE_IDX 0 322 #define CMDQ_ERR_CERROR_ILL_IDX 1 323 #define CMDQ_ERR_CERROR_ABT_IDX 2 324 #define CMDQ_ERR_CERROR_ATC_INV_IDX 3 325 326 #define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG 327 328 /* 329 * This is used to size the command queue and therefore must be at least 330 * BITS_PER_LONG so that the valid_map works correctly (it relies on the 331 * total number of queue entries being a multiple of BITS_PER_LONG). 332 */ 333 #define CMDQ_BATCH_ENTRIES BITS_PER_LONG 334 335 #define CMDQ_0_OP GENMASK_ULL(7, 0) 336 #define CMDQ_0_SSV (1UL << 11) 337 338 #define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32) 339 #define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0) 340 #define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12) 341 342 #define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12) 343 #define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32) 344 #define CMDQ_CFGI_1_LEAF (1UL << 0) 345 #define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0) 346 347 #define CMDQ_TLBI_0_NUM GENMASK_ULL(16, 12) 348 #define CMDQ_TLBI_RANGE_NUM_MAX 31 349 #define CMDQ_TLBI_0_SCALE GENMASK_ULL(24, 20) 350 #define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32) 351 #define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48) 352 #define CMDQ_TLBI_1_LEAF (1UL << 0) 353 #define CMDQ_TLBI_1_TTL GENMASK_ULL(9, 8) 354 #define CMDQ_TLBI_1_TG GENMASK_ULL(11, 10) 355 #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12) 356 #define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12) 357 358 #define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12) 359 #define CMDQ_ATC_0_SID GENMASK_ULL(63, 32) 360 #define CMDQ_ATC_0_GLOBAL (1UL << 9) 361 #define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0) 362 #define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12) 363 364 #define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12) 365 #define CMDQ_PRI_0_SID GENMASK_ULL(63, 32) 366 #define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0) 367 #define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12) 368 369 #define CMDQ_RESUME_0_RESP_TERM 0UL 370 #define CMDQ_RESUME_0_RESP_RETRY 1UL 371 #define CMDQ_RESUME_0_RESP_ABORT 2UL 372 #define CMDQ_RESUME_0_RESP GENMASK_ULL(13, 12) 373 #define CMDQ_RESUME_0_SID GENMASK_ULL(63, 32) 374 #define CMDQ_RESUME_1_STAG GENMASK_ULL(15, 0) 375 376 #define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12) 377 #define CMDQ_SYNC_0_CS_NONE 0 378 #define CMDQ_SYNC_0_CS_IRQ 1 379 #define CMDQ_SYNC_0_CS_SEV 2 380 #define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22) 381 #define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24) 382 #define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32) 383 #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2) 384 385 /* Event queue */ 386 #define EVTQ_ENT_SZ_SHIFT 5 387 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3) 388 #define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT) 389 390 #define EVTQ_0_ID GENMASK_ULL(7, 0) 391 392 #define EVT_ID_TRANSLATION_FAULT 0x10 393 #define EVT_ID_ADDR_SIZE_FAULT 0x11 394 #define EVT_ID_ACCESS_FAULT 0x12 395 #define EVT_ID_PERMISSION_FAULT 0x13 396 397 #define EVTQ_0_SSV (1UL << 11) 398 #define EVTQ_0_SSID GENMASK_ULL(31, 12) 399 #define EVTQ_0_SID GENMASK_ULL(63, 32) 400 #define EVTQ_1_STAG GENMASK_ULL(15, 0) 401 #define EVTQ_1_STALL (1UL << 31) 402 #define EVTQ_1_PnU (1UL << 33) 403 #define EVTQ_1_InD (1UL << 34) 404 #define EVTQ_1_RnW (1UL << 35) 405 #define EVTQ_1_S2 (1UL << 39) 406 #define EVTQ_1_CLASS GENMASK_ULL(41, 40) 407 #define EVTQ_1_TT_READ (1UL << 44) 408 #define EVTQ_2_ADDR GENMASK_ULL(63, 0) 409 #define EVTQ_3_IPA GENMASK_ULL(51, 12) 410 411 /* PRI queue */ 412 #define PRIQ_ENT_SZ_SHIFT 4 413 #define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3) 414 #define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT) 415 416 #define PRIQ_0_SID GENMASK_ULL(31, 0) 417 #define PRIQ_0_SSID GENMASK_ULL(51, 32) 418 #define PRIQ_0_PERM_PRIV (1UL << 58) 419 #define PRIQ_0_PERM_EXEC (1UL << 59) 420 #define PRIQ_0_PERM_READ (1UL << 60) 421 #define PRIQ_0_PERM_WRITE (1UL << 61) 422 #define PRIQ_0_PRG_LAST (1UL << 62) 423 #define PRIQ_0_SSID_V (1UL << 63) 424 425 #define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0) 426 #define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12) 427 428 /* High-level queue structures */ 429 #define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */ 430 #define ARM_SMMU_POLL_SPIN_COUNT 10 431 432 #define MSI_IOVA_BASE 0x8000000 433 #define MSI_IOVA_LENGTH 0x100000 434 435 enum pri_resp { 436 PRI_RESP_DENY = 0, 437 PRI_RESP_FAIL = 1, 438 PRI_RESP_SUCC = 2, 439 }; 440 441 struct arm_smmu_cmdq_ent { 442 /* Common fields */ 443 u8 opcode; 444 bool substream_valid; 445 446 /* Command-specific fields */ 447 union { 448 #define CMDQ_OP_PREFETCH_CFG 0x1 449 struct { 450 u32 sid; 451 } prefetch; 452 453 #define CMDQ_OP_CFGI_STE 0x3 454 #define CMDQ_OP_CFGI_ALL 0x4 455 #define CMDQ_OP_CFGI_CD 0x5 456 #define CMDQ_OP_CFGI_CD_ALL 0x6 457 struct { 458 u32 sid; 459 u32 ssid; 460 union { 461 bool leaf; 462 u8 span; 463 }; 464 } cfgi; 465 466 #define CMDQ_OP_TLBI_NH_ASID 0x11 467 #define CMDQ_OP_TLBI_NH_VA 0x12 468 #define CMDQ_OP_TLBI_EL2_ALL 0x20 469 #define CMDQ_OP_TLBI_EL2_ASID 0x21 470 #define CMDQ_OP_TLBI_EL2_VA 0x22 471 #define CMDQ_OP_TLBI_S12_VMALL 0x28 472 #define CMDQ_OP_TLBI_S2_IPA 0x2a 473 #define CMDQ_OP_TLBI_NSNH_ALL 0x30 474 struct { 475 u8 num; 476 u8 scale; 477 u16 asid; 478 u16 vmid; 479 bool leaf; 480 u8 ttl; 481 u8 tg; 482 u64 addr; 483 } tlbi; 484 485 #define CMDQ_OP_ATC_INV 0x40 486 #define ATC_INV_SIZE_ALL 52 487 struct { 488 u32 sid; 489 u32 ssid; 490 u64 addr; 491 u8 size; 492 bool global; 493 } atc; 494 495 #define CMDQ_OP_PRI_RESP 0x41 496 struct { 497 u32 sid; 498 u32 ssid; 499 u16 grpid; 500 enum pri_resp resp; 501 } pri; 502 503 #define CMDQ_OP_RESUME 0x44 504 struct { 505 u32 sid; 506 u16 stag; 507 u8 resp; 508 } resume; 509 510 #define CMDQ_OP_CMD_SYNC 0x46 511 struct { 512 u64 msiaddr; 513 } sync; 514 }; 515 }; 516 517 struct arm_smmu_ll_queue { 518 union { 519 u64 val; 520 struct { 521 u32 prod; 522 u32 cons; 523 }; 524 struct { 525 atomic_t prod; 526 atomic_t cons; 527 } atomic; 528 u8 __pad[SMP_CACHE_BYTES]; 529 } ____cacheline_aligned_in_smp; 530 u32 max_n_shift; 531 }; 532 533 struct arm_smmu_queue { 534 struct arm_smmu_ll_queue llq; 535 int irq; /* Wired interrupt */ 536 537 __le64 *base; 538 dma_addr_t base_dma; 539 u64 q_base; 540 541 size_t ent_dwords; 542 543 u32 __iomem *prod_reg; 544 u32 __iomem *cons_reg; 545 }; 546 547 struct arm_smmu_queue_poll { 548 ktime_t timeout; 549 unsigned int delay; 550 unsigned int spin_cnt; 551 bool wfe; 552 }; 553 554 struct arm_smmu_cmdq { 555 struct arm_smmu_queue q; 556 atomic_long_t *valid_map; 557 atomic_t owner_prod; 558 atomic_t lock; 559 }; 560 561 struct arm_smmu_cmdq_batch { 562 u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS]; 563 int num; 564 }; 565 566 struct arm_smmu_evtq { 567 struct arm_smmu_queue q; 568 struct iopf_queue *iopf; 569 u32 max_stalls; 570 }; 571 572 struct arm_smmu_priq { 573 struct arm_smmu_queue q; 574 }; 575 576 /* High-level stream table and context descriptor structures */ 577 struct arm_smmu_strtab_l1_desc { 578 u8 span; 579 580 struct arm_smmu_ste *l2ptr; 581 dma_addr_t l2ptr_dma; 582 }; 583 584 struct arm_smmu_ctx_desc { 585 u16 asid; 586 u64 ttbr; 587 u64 tcr; 588 u64 mair; 589 590 refcount_t refs; 591 struct mm_struct *mm; 592 }; 593 594 struct arm_smmu_l1_ctx_desc { 595 __le64 *l2ptr; 596 dma_addr_t l2ptr_dma; 597 }; 598 599 struct arm_smmu_ctx_desc_cfg { 600 __le64 *cdtab; 601 dma_addr_t cdtab_dma; 602 struct arm_smmu_l1_ctx_desc *l1_desc; 603 unsigned int num_l1_ents; 604 u8 s1fmt; 605 /* log2 of the maximum number of CDs supported by this table */ 606 u8 s1cdmax; 607 /* Whether CD entries in this table have the stall bit set. */ 608 u8 stall_enabled:1; 609 }; 610 611 struct arm_smmu_s2_cfg { 612 u16 vmid; 613 }; 614 615 struct arm_smmu_strtab_cfg { 616 __le64 *strtab; 617 dma_addr_t strtab_dma; 618 struct arm_smmu_strtab_l1_desc *l1_desc; 619 unsigned int num_l1_ents; 620 621 u64 strtab_base; 622 u32 strtab_base_cfg; 623 }; 624 625 /* An SMMUv3 instance */ 626 struct arm_smmu_device { 627 struct device *dev; 628 void __iomem *base; 629 void __iomem *page1; 630 631 #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0) 632 #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1) 633 #define ARM_SMMU_FEAT_TT_LE (1 << 2) 634 #define ARM_SMMU_FEAT_TT_BE (1 << 3) 635 #define ARM_SMMU_FEAT_PRI (1 << 4) 636 #define ARM_SMMU_FEAT_ATS (1 << 5) 637 #define ARM_SMMU_FEAT_SEV (1 << 6) 638 #define ARM_SMMU_FEAT_MSI (1 << 7) 639 #define ARM_SMMU_FEAT_COHERENCY (1 << 8) 640 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9) 641 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10) 642 #define ARM_SMMU_FEAT_STALLS (1 << 11) 643 #define ARM_SMMU_FEAT_HYP (1 << 12) 644 #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13) 645 #define ARM_SMMU_FEAT_VAX (1 << 14) 646 #define ARM_SMMU_FEAT_RANGE_INV (1 << 15) 647 #define ARM_SMMU_FEAT_BTM (1 << 16) 648 #define ARM_SMMU_FEAT_SVA (1 << 17) 649 #define ARM_SMMU_FEAT_E2H (1 << 18) 650 #define ARM_SMMU_FEAT_NESTING (1 << 19) 651 #define ARM_SMMU_FEAT_ATTR_TYPES_OVR (1 << 20) 652 u32 features; 653 654 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0) 655 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1) 656 #define ARM_SMMU_OPT_MSIPOLL (1 << 2) 657 #define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3) 658 u32 options; 659 660 struct arm_smmu_cmdq cmdq; 661 struct arm_smmu_evtq evtq; 662 struct arm_smmu_priq priq; 663 664 int gerr_irq; 665 int combined_irq; 666 667 unsigned long ias; /* IPA */ 668 unsigned long oas; /* PA */ 669 unsigned long pgsize_bitmap; 670 671 #define ARM_SMMU_MAX_ASIDS (1 << 16) 672 unsigned int asid_bits; 673 674 #define ARM_SMMU_MAX_VMIDS (1 << 16) 675 unsigned int vmid_bits; 676 struct ida vmid_map; 677 678 unsigned int ssid_bits; 679 unsigned int sid_bits; 680 681 struct arm_smmu_strtab_cfg strtab_cfg; 682 683 /* IOMMU core code handle */ 684 struct iommu_device iommu; 685 686 struct rb_root streams; 687 struct mutex streams_mutex; 688 }; 689 690 struct arm_smmu_stream { 691 u32 id; 692 struct arm_smmu_master *master; 693 struct rb_node node; 694 }; 695 696 /* SMMU private data for each master */ 697 struct arm_smmu_master { 698 struct arm_smmu_device *smmu; 699 struct device *dev; 700 struct list_head domain_head; 701 struct arm_smmu_stream *streams; 702 /* Locked by the iommu core using the group mutex */ 703 struct arm_smmu_ctx_desc_cfg cd_table; 704 unsigned int num_streams; 705 bool ats_enabled; 706 bool stall_enabled; 707 bool sva_enabled; 708 bool iopf_enabled; 709 struct list_head bonds; 710 unsigned int ssid_bits; 711 }; 712 713 /* SMMU private data for an IOMMU domain */ 714 enum arm_smmu_domain_stage { 715 ARM_SMMU_DOMAIN_S1 = 0, 716 ARM_SMMU_DOMAIN_S2, 717 }; 718 719 struct arm_smmu_domain { 720 struct arm_smmu_device *smmu; 721 struct mutex init_mutex; /* Protects smmu pointer */ 722 723 struct io_pgtable_ops *pgtbl_ops; 724 atomic_t nr_ats_masters; 725 726 enum arm_smmu_domain_stage stage; 727 union { 728 struct arm_smmu_ctx_desc cd; 729 struct arm_smmu_s2_cfg s2_cfg; 730 }; 731 732 struct iommu_domain domain; 733 734 struct list_head devices; 735 spinlock_t devices_lock; 736 737 struct list_head mmu_notifiers; 738 }; 739 740 static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) 741 { 742 return container_of(dom, struct arm_smmu_domain, domain); 743 } 744 745 extern struct xarray arm_smmu_asid_xa; 746 extern struct mutex arm_smmu_asid_lock; 747 extern struct arm_smmu_ctx_desc quiet_cd; 748 749 int arm_smmu_write_ctx_desc(struct arm_smmu_master *smmu_master, int ssid, 750 struct arm_smmu_ctx_desc *cd); 751 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid); 752 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, 753 size_t granule, bool leaf, 754 struct arm_smmu_domain *smmu_domain); 755 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd); 756 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid, 757 unsigned long iova, size_t size); 758 759 #ifdef CONFIG_ARM_SMMU_V3_SVA 760 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu); 761 bool arm_smmu_master_sva_supported(struct arm_smmu_master *master); 762 bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master); 763 int arm_smmu_master_enable_sva(struct arm_smmu_master *master); 764 int arm_smmu_master_disable_sva(struct arm_smmu_master *master); 765 bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master); 766 void arm_smmu_sva_notifier_synchronize(void); 767 struct iommu_domain *arm_smmu_sva_domain_alloc(void); 768 void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, 769 struct device *dev, ioasid_t id); 770 #else /* CONFIG_ARM_SMMU_V3_SVA */ 771 static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) 772 { 773 return false; 774 } 775 776 static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master) 777 { 778 return false; 779 } 780 781 static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master) 782 { 783 return false; 784 } 785 786 static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master) 787 { 788 return -ENODEV; 789 } 790 791 static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master) 792 { 793 return -ENODEV; 794 } 795 796 static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master) 797 { 798 return false; 799 } 800 801 static inline void arm_smmu_sva_notifier_synchronize(void) {} 802 803 static inline struct iommu_domain *arm_smmu_sva_domain_alloc(void) 804 { 805 return NULL; 806 } 807 808 static inline void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, 809 struct device *dev, 810 ioasid_t id) 811 { 812 } 813 #endif /* CONFIG_ARM_SMMU_V3_SVA */ 814 #endif /* _ARM_SMMU_V3_H */ 815