xref: /linux/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h (revision 119ff04864a24470b1e531bb53e5c141aa8fefb0)
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * IOMMU API for ARM architected SMMUv3 implementations.
4  *
5  * Copyright (C) 2015 ARM Limited
6  */
7 
8 #ifndef _ARM_SMMU_V3_H
9 #define _ARM_SMMU_V3_H
10 
11 #include <linux/bitfield.h>
12 #include <linux/iommu.h>
13 #include <linux/kernel.h>
14 #include <linux/mmzone.h>
15 #include <linux/sizes.h>
16 
17 /* MMIO registers */
18 #define ARM_SMMU_IDR0			0x0
19 #define IDR0_ST_LVL			GENMASK(28, 27)
20 #define IDR0_ST_LVL_2LVL		1
21 #define IDR0_STALL_MODEL		GENMASK(25, 24)
22 #define IDR0_STALL_MODEL_STALL		0
23 #define IDR0_STALL_MODEL_FORCE		2
24 #define IDR0_TTENDIAN			GENMASK(22, 21)
25 #define IDR0_TTENDIAN_MIXED		0
26 #define IDR0_TTENDIAN_LE		2
27 #define IDR0_TTENDIAN_BE		3
28 #define IDR0_CD2L			(1 << 19)
29 #define IDR0_VMID16			(1 << 18)
30 #define IDR0_PRI			(1 << 16)
31 #define IDR0_SEV			(1 << 14)
32 #define IDR0_MSI			(1 << 13)
33 #define IDR0_ASID16			(1 << 12)
34 #define IDR0_ATS			(1 << 10)
35 #define IDR0_HYP			(1 << 9)
36 #define IDR0_COHACC			(1 << 4)
37 #define IDR0_TTF			GENMASK(3, 2)
38 #define IDR0_TTF_AARCH64		2
39 #define IDR0_TTF_AARCH32_64		3
40 #define IDR0_S1P			(1 << 1)
41 #define IDR0_S2P			(1 << 0)
42 
43 #define ARM_SMMU_IDR1			0x4
44 #define IDR1_TABLES_PRESET		(1 << 30)
45 #define IDR1_QUEUES_PRESET		(1 << 29)
46 #define IDR1_REL			(1 << 28)
47 #define IDR1_CMDQS			GENMASK(25, 21)
48 #define IDR1_EVTQS			GENMASK(20, 16)
49 #define IDR1_PRIQS			GENMASK(15, 11)
50 #define IDR1_SSIDSIZE			GENMASK(10, 6)
51 #define IDR1_SIDSIZE			GENMASK(5, 0)
52 
53 #define ARM_SMMU_IDR3			0xc
54 #define IDR3_RIL			(1 << 10)
55 
56 #define ARM_SMMU_IDR5			0x14
57 #define IDR5_STALL_MAX			GENMASK(31, 16)
58 #define IDR5_GRAN64K			(1 << 6)
59 #define IDR5_GRAN16K			(1 << 5)
60 #define IDR5_GRAN4K			(1 << 4)
61 #define IDR5_OAS			GENMASK(2, 0)
62 #define IDR5_OAS_32_BIT			0
63 #define IDR5_OAS_36_BIT			1
64 #define IDR5_OAS_40_BIT			2
65 #define IDR5_OAS_42_BIT			3
66 #define IDR5_OAS_44_BIT			4
67 #define IDR5_OAS_48_BIT			5
68 #define IDR5_OAS_52_BIT			6
69 #define IDR5_VAX			GENMASK(11, 10)
70 #define IDR5_VAX_52_BIT			1
71 
72 #define ARM_SMMU_IIDR			0x18
73 #define IIDR_PRODUCTID			GENMASK(31, 20)
74 #define IIDR_VARIANT			GENMASK(19, 16)
75 #define IIDR_REVISION			GENMASK(15, 12)
76 #define IIDR_IMPLEMENTER		GENMASK(11, 0)
77 
78 #define ARM_SMMU_CR0			0x20
79 #define CR0_ATSCHK			(1 << 4)
80 #define CR0_CMDQEN			(1 << 3)
81 #define CR0_EVTQEN			(1 << 2)
82 #define CR0_PRIQEN			(1 << 1)
83 #define CR0_SMMUEN			(1 << 0)
84 
85 #define ARM_SMMU_CR0ACK			0x24
86 
87 #define ARM_SMMU_CR1			0x28
88 #define CR1_TABLE_SH			GENMASK(11, 10)
89 #define CR1_TABLE_OC			GENMASK(9, 8)
90 #define CR1_TABLE_IC			GENMASK(7, 6)
91 #define CR1_QUEUE_SH			GENMASK(5, 4)
92 #define CR1_QUEUE_OC			GENMASK(3, 2)
93 #define CR1_QUEUE_IC			GENMASK(1, 0)
94 /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
95 #define CR1_CACHE_NC			0
96 #define CR1_CACHE_WB			1
97 #define CR1_CACHE_WT			2
98 
99 #define ARM_SMMU_CR2			0x2c
100 #define CR2_PTM				(1 << 2)
101 #define CR2_RECINVSID			(1 << 1)
102 #define CR2_E2H				(1 << 0)
103 
104 #define ARM_SMMU_GBPA			0x44
105 #define GBPA_UPDATE			(1 << 31)
106 #define GBPA_ABORT			(1 << 20)
107 
108 #define ARM_SMMU_IRQ_CTRL		0x50
109 #define IRQ_CTRL_EVTQ_IRQEN		(1 << 2)
110 #define IRQ_CTRL_PRIQ_IRQEN		(1 << 1)
111 #define IRQ_CTRL_GERROR_IRQEN		(1 << 0)
112 
113 #define ARM_SMMU_IRQ_CTRLACK		0x54
114 
115 #define ARM_SMMU_GERROR			0x60
116 #define GERROR_SFM_ERR			(1 << 8)
117 #define GERROR_MSI_GERROR_ABT_ERR	(1 << 7)
118 #define GERROR_MSI_PRIQ_ABT_ERR		(1 << 6)
119 #define GERROR_MSI_EVTQ_ABT_ERR		(1 << 5)
120 #define GERROR_MSI_CMDQ_ABT_ERR		(1 << 4)
121 #define GERROR_PRIQ_ABT_ERR		(1 << 3)
122 #define GERROR_EVTQ_ABT_ERR		(1 << 2)
123 #define GERROR_CMDQ_ERR			(1 << 0)
124 #define GERROR_ERR_MASK			0x1fd
125 
126 #define ARM_SMMU_GERRORN		0x64
127 
128 #define ARM_SMMU_GERROR_IRQ_CFG0	0x68
129 #define ARM_SMMU_GERROR_IRQ_CFG1	0x70
130 #define ARM_SMMU_GERROR_IRQ_CFG2	0x74
131 
132 #define ARM_SMMU_STRTAB_BASE		0x80
133 #define STRTAB_BASE_RA			(1UL << 62)
134 #define STRTAB_BASE_ADDR_MASK		GENMASK_ULL(51, 6)
135 
136 #define ARM_SMMU_STRTAB_BASE_CFG	0x88
137 #define STRTAB_BASE_CFG_FMT		GENMASK(17, 16)
138 #define STRTAB_BASE_CFG_FMT_LINEAR	0
139 #define STRTAB_BASE_CFG_FMT_2LVL	1
140 #define STRTAB_BASE_CFG_SPLIT		GENMASK(10, 6)
141 #define STRTAB_BASE_CFG_LOG2SIZE	GENMASK(5, 0)
142 
143 #define ARM_SMMU_CMDQ_BASE		0x90
144 #define ARM_SMMU_CMDQ_PROD		0x98
145 #define ARM_SMMU_CMDQ_CONS		0x9c
146 
147 #define ARM_SMMU_EVTQ_BASE		0xa0
148 #define ARM_SMMU_EVTQ_PROD		0xa8
149 #define ARM_SMMU_EVTQ_CONS		0xac
150 #define ARM_SMMU_EVTQ_IRQ_CFG0		0xb0
151 #define ARM_SMMU_EVTQ_IRQ_CFG1		0xb8
152 #define ARM_SMMU_EVTQ_IRQ_CFG2		0xbc
153 
154 #define ARM_SMMU_PRIQ_BASE		0xc0
155 #define ARM_SMMU_PRIQ_PROD		0xc8
156 #define ARM_SMMU_PRIQ_CONS		0xcc
157 #define ARM_SMMU_PRIQ_IRQ_CFG0		0xd0
158 #define ARM_SMMU_PRIQ_IRQ_CFG1		0xd8
159 #define ARM_SMMU_PRIQ_IRQ_CFG2		0xdc
160 
161 #define ARM_SMMU_REG_SZ			0xe00
162 
163 /* Common MSI config fields */
164 #define MSI_CFG0_ADDR_MASK		GENMASK_ULL(51, 2)
165 #define MSI_CFG2_SH			GENMASK(5, 4)
166 #define MSI_CFG2_MEMATTR		GENMASK(3, 0)
167 
168 /* Common memory attribute values */
169 #define ARM_SMMU_SH_NSH			0
170 #define ARM_SMMU_SH_OSH			2
171 #define ARM_SMMU_SH_ISH			3
172 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE	0x1
173 #define ARM_SMMU_MEMATTR_OIWB		0xf
174 
175 #define Q_IDX(llq, p)			((p) & ((1 << (llq)->max_n_shift) - 1))
176 #define Q_WRP(llq, p)			((p) & (1 << (llq)->max_n_shift))
177 #define Q_OVERFLOW_FLAG			(1U << 31)
178 #define Q_OVF(p)			((p) & Q_OVERFLOW_FLAG)
179 #define Q_ENT(q, p)			((q)->base +			\
180 					 Q_IDX(&((q)->llq), p) *	\
181 					 (q)->ent_dwords)
182 
183 #define Q_BASE_RWA			(1UL << 62)
184 #define Q_BASE_ADDR_MASK		GENMASK_ULL(51, 5)
185 #define Q_BASE_LOG2SIZE			GENMASK(4, 0)
186 
187 /* Ensure DMA allocations are naturally aligned */
188 #ifdef CONFIG_CMA_ALIGNMENT
189 #define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
190 #else
191 #define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + MAX_PAGE_ORDER)
192 #endif
193 
194 /*
195  * Stream table.
196  *
197  * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
198  * 2lvl: 128k L1 entries,
199  *       256 lazy entries per table (each table covers a PCI bus)
200  */
201 #define STRTAB_L1_SZ_SHIFT		20
202 #define STRTAB_SPLIT			8
203 
204 #define STRTAB_L1_DESC_DWORDS		1
205 #define STRTAB_L1_DESC_SPAN		GENMASK_ULL(4, 0)
206 #define STRTAB_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 6)
207 
208 #define STRTAB_STE_DWORDS		8
209 
210 struct arm_smmu_ste {
211 	__le64 data[STRTAB_STE_DWORDS];
212 };
213 
214 #define STRTAB_STE_0_V			(1UL << 0)
215 #define STRTAB_STE_0_CFG		GENMASK_ULL(3, 1)
216 #define STRTAB_STE_0_CFG_ABORT		0
217 #define STRTAB_STE_0_CFG_BYPASS		4
218 #define STRTAB_STE_0_CFG_S1_TRANS	5
219 #define STRTAB_STE_0_CFG_S2_TRANS	6
220 
221 #define STRTAB_STE_0_S1FMT		GENMASK_ULL(5, 4)
222 #define STRTAB_STE_0_S1FMT_LINEAR	0
223 #define STRTAB_STE_0_S1FMT_64K_L2	2
224 #define STRTAB_STE_0_S1CTXPTR_MASK	GENMASK_ULL(51, 6)
225 #define STRTAB_STE_0_S1CDMAX		GENMASK_ULL(63, 59)
226 
227 #define STRTAB_STE_1_S1DSS		GENMASK_ULL(1, 0)
228 #define STRTAB_STE_1_S1DSS_TERMINATE	0x0
229 #define STRTAB_STE_1_S1DSS_BYPASS	0x1
230 #define STRTAB_STE_1_S1DSS_SSID0	0x2
231 
232 #define STRTAB_STE_1_S1C_CACHE_NC	0UL
233 #define STRTAB_STE_1_S1C_CACHE_WBRA	1UL
234 #define STRTAB_STE_1_S1C_CACHE_WT	2UL
235 #define STRTAB_STE_1_S1C_CACHE_WB	3UL
236 #define STRTAB_STE_1_S1CIR		GENMASK_ULL(3, 2)
237 #define STRTAB_STE_1_S1COR		GENMASK_ULL(5, 4)
238 #define STRTAB_STE_1_S1CSH		GENMASK_ULL(7, 6)
239 
240 #define STRTAB_STE_1_S1STALLD		(1UL << 27)
241 
242 #define STRTAB_STE_1_EATS		GENMASK_ULL(29, 28)
243 #define STRTAB_STE_1_EATS_ABT		0UL
244 #define STRTAB_STE_1_EATS_TRANS		1UL
245 #define STRTAB_STE_1_EATS_S1CHK		2UL
246 
247 #define STRTAB_STE_1_STRW		GENMASK_ULL(31, 30)
248 #define STRTAB_STE_1_STRW_NSEL1		0UL
249 #define STRTAB_STE_1_STRW_EL2		2UL
250 
251 #define STRTAB_STE_1_SHCFG		GENMASK_ULL(45, 44)
252 #define STRTAB_STE_1_SHCFG_INCOMING	1UL
253 
254 #define STRTAB_STE_2_S2VMID		GENMASK_ULL(15, 0)
255 #define STRTAB_STE_2_VTCR		GENMASK_ULL(50, 32)
256 #define STRTAB_STE_2_VTCR_S2T0SZ	GENMASK_ULL(5, 0)
257 #define STRTAB_STE_2_VTCR_S2SL0		GENMASK_ULL(7, 6)
258 #define STRTAB_STE_2_VTCR_S2IR0		GENMASK_ULL(9, 8)
259 #define STRTAB_STE_2_VTCR_S2OR0		GENMASK_ULL(11, 10)
260 #define STRTAB_STE_2_VTCR_S2SH0		GENMASK_ULL(13, 12)
261 #define STRTAB_STE_2_VTCR_S2TG		GENMASK_ULL(15, 14)
262 #define STRTAB_STE_2_VTCR_S2PS		GENMASK_ULL(18, 16)
263 #define STRTAB_STE_2_S2AA64		(1UL << 51)
264 #define STRTAB_STE_2_S2ENDI		(1UL << 52)
265 #define STRTAB_STE_2_S2PTW		(1UL << 54)
266 #define STRTAB_STE_2_S2R		(1UL << 58)
267 
268 #define STRTAB_STE_3_S2TTB_MASK		GENMASK_ULL(51, 4)
269 
270 /*
271  * Context descriptors.
272  *
273  * Linear: when less than 1024 SSIDs are supported
274  * 2lvl: at most 1024 L1 entries,
275  *       1024 lazy entries per table.
276  */
277 #define CTXDESC_SPLIT			10
278 #define CTXDESC_L2_ENTRIES		(1 << CTXDESC_SPLIT)
279 
280 #define CTXDESC_L1_DESC_DWORDS		1
281 #define CTXDESC_L1_DESC_V		(1UL << 0)
282 #define CTXDESC_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 12)
283 
284 #define CTXDESC_CD_DWORDS		8
285 #define CTXDESC_CD_0_TCR_T0SZ		GENMASK_ULL(5, 0)
286 #define CTXDESC_CD_0_TCR_TG0		GENMASK_ULL(7, 6)
287 #define CTXDESC_CD_0_TCR_IRGN0		GENMASK_ULL(9, 8)
288 #define CTXDESC_CD_0_TCR_ORGN0		GENMASK_ULL(11, 10)
289 #define CTXDESC_CD_0_TCR_SH0		GENMASK_ULL(13, 12)
290 #define CTXDESC_CD_0_TCR_EPD0		(1ULL << 14)
291 #define CTXDESC_CD_0_TCR_EPD1		(1ULL << 30)
292 
293 #define CTXDESC_CD_0_ENDI		(1UL << 15)
294 #define CTXDESC_CD_0_V			(1UL << 31)
295 
296 #define CTXDESC_CD_0_TCR_IPS		GENMASK_ULL(34, 32)
297 #define CTXDESC_CD_0_TCR_TBI0		(1ULL << 38)
298 
299 #define CTXDESC_CD_0_AA64		(1UL << 41)
300 #define CTXDESC_CD_0_S			(1UL << 44)
301 #define CTXDESC_CD_0_R			(1UL << 45)
302 #define CTXDESC_CD_0_A			(1UL << 46)
303 #define CTXDESC_CD_0_ASET		(1UL << 47)
304 #define CTXDESC_CD_0_ASID		GENMASK_ULL(63, 48)
305 
306 #define CTXDESC_CD_1_TTB0_MASK		GENMASK_ULL(51, 4)
307 
308 /*
309  * When the SMMU only supports linear context descriptor tables, pick a
310  * reasonable size limit (64kB).
311  */
312 #define CTXDESC_LINEAR_CDMAX		ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
313 
314 /* Command queue */
315 #define CMDQ_ENT_SZ_SHIFT		4
316 #define CMDQ_ENT_DWORDS			((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
317 #define CMDQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
318 
319 #define CMDQ_CONS_ERR			GENMASK(30, 24)
320 #define CMDQ_ERR_CERROR_NONE_IDX	0
321 #define CMDQ_ERR_CERROR_ILL_IDX		1
322 #define CMDQ_ERR_CERROR_ABT_IDX		2
323 #define CMDQ_ERR_CERROR_ATC_INV_IDX	3
324 
325 #define CMDQ_PROD_OWNED_FLAG		Q_OVERFLOW_FLAG
326 
327 /*
328  * This is used to size the command queue and therefore must be at least
329  * BITS_PER_LONG so that the valid_map works correctly (it relies on the
330  * total number of queue entries being a multiple of BITS_PER_LONG).
331  */
332 #define CMDQ_BATCH_ENTRIES		BITS_PER_LONG
333 
334 #define CMDQ_0_OP			GENMASK_ULL(7, 0)
335 #define CMDQ_0_SSV			(1UL << 11)
336 
337 #define CMDQ_PREFETCH_0_SID		GENMASK_ULL(63, 32)
338 #define CMDQ_PREFETCH_1_SIZE		GENMASK_ULL(4, 0)
339 #define CMDQ_PREFETCH_1_ADDR_MASK	GENMASK_ULL(63, 12)
340 
341 #define CMDQ_CFGI_0_SSID		GENMASK_ULL(31, 12)
342 #define CMDQ_CFGI_0_SID			GENMASK_ULL(63, 32)
343 #define CMDQ_CFGI_1_LEAF		(1UL << 0)
344 #define CMDQ_CFGI_1_RANGE		GENMASK_ULL(4, 0)
345 
346 #define CMDQ_TLBI_0_NUM			GENMASK_ULL(16, 12)
347 #define CMDQ_TLBI_RANGE_NUM_MAX		31
348 #define CMDQ_TLBI_0_SCALE		GENMASK_ULL(24, 20)
349 #define CMDQ_TLBI_0_VMID		GENMASK_ULL(47, 32)
350 #define CMDQ_TLBI_0_ASID		GENMASK_ULL(63, 48)
351 #define CMDQ_TLBI_1_LEAF		(1UL << 0)
352 #define CMDQ_TLBI_1_TTL			GENMASK_ULL(9, 8)
353 #define CMDQ_TLBI_1_TG			GENMASK_ULL(11, 10)
354 #define CMDQ_TLBI_1_VA_MASK		GENMASK_ULL(63, 12)
355 #define CMDQ_TLBI_1_IPA_MASK		GENMASK_ULL(51, 12)
356 
357 #define CMDQ_ATC_0_SSID			GENMASK_ULL(31, 12)
358 #define CMDQ_ATC_0_SID			GENMASK_ULL(63, 32)
359 #define CMDQ_ATC_0_GLOBAL		(1UL << 9)
360 #define CMDQ_ATC_1_SIZE			GENMASK_ULL(5, 0)
361 #define CMDQ_ATC_1_ADDR_MASK		GENMASK_ULL(63, 12)
362 
363 #define CMDQ_PRI_0_SSID			GENMASK_ULL(31, 12)
364 #define CMDQ_PRI_0_SID			GENMASK_ULL(63, 32)
365 #define CMDQ_PRI_1_GRPID		GENMASK_ULL(8, 0)
366 #define CMDQ_PRI_1_RESP			GENMASK_ULL(13, 12)
367 
368 #define CMDQ_RESUME_0_RESP_TERM		0UL
369 #define CMDQ_RESUME_0_RESP_RETRY	1UL
370 #define CMDQ_RESUME_0_RESP_ABORT	2UL
371 #define CMDQ_RESUME_0_RESP		GENMASK_ULL(13, 12)
372 #define CMDQ_RESUME_0_SID		GENMASK_ULL(63, 32)
373 #define CMDQ_RESUME_1_STAG		GENMASK_ULL(15, 0)
374 
375 #define CMDQ_SYNC_0_CS			GENMASK_ULL(13, 12)
376 #define CMDQ_SYNC_0_CS_NONE		0
377 #define CMDQ_SYNC_0_CS_IRQ		1
378 #define CMDQ_SYNC_0_CS_SEV		2
379 #define CMDQ_SYNC_0_MSH			GENMASK_ULL(23, 22)
380 #define CMDQ_SYNC_0_MSIATTR		GENMASK_ULL(27, 24)
381 #define CMDQ_SYNC_0_MSIDATA		GENMASK_ULL(63, 32)
382 #define CMDQ_SYNC_1_MSIADDR_MASK	GENMASK_ULL(51, 2)
383 
384 /* Event queue */
385 #define EVTQ_ENT_SZ_SHIFT		5
386 #define EVTQ_ENT_DWORDS			((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
387 #define EVTQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
388 
389 #define EVTQ_0_ID			GENMASK_ULL(7, 0)
390 
391 #define EVT_ID_TRANSLATION_FAULT	0x10
392 #define EVT_ID_ADDR_SIZE_FAULT		0x11
393 #define EVT_ID_ACCESS_FAULT		0x12
394 #define EVT_ID_PERMISSION_FAULT		0x13
395 
396 #define EVTQ_0_SSV			(1UL << 11)
397 #define EVTQ_0_SSID			GENMASK_ULL(31, 12)
398 #define EVTQ_0_SID			GENMASK_ULL(63, 32)
399 #define EVTQ_1_STAG			GENMASK_ULL(15, 0)
400 #define EVTQ_1_STALL			(1UL << 31)
401 #define EVTQ_1_PnU			(1UL << 33)
402 #define EVTQ_1_InD			(1UL << 34)
403 #define EVTQ_1_RnW			(1UL << 35)
404 #define EVTQ_1_S2			(1UL << 39)
405 #define EVTQ_1_CLASS			GENMASK_ULL(41, 40)
406 #define EVTQ_1_TT_READ			(1UL << 44)
407 #define EVTQ_2_ADDR			GENMASK_ULL(63, 0)
408 #define EVTQ_3_IPA			GENMASK_ULL(51, 12)
409 
410 /* PRI queue */
411 #define PRIQ_ENT_SZ_SHIFT		4
412 #define PRIQ_ENT_DWORDS			((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
413 #define PRIQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
414 
415 #define PRIQ_0_SID			GENMASK_ULL(31, 0)
416 #define PRIQ_0_SSID			GENMASK_ULL(51, 32)
417 #define PRIQ_0_PERM_PRIV		(1UL << 58)
418 #define PRIQ_0_PERM_EXEC		(1UL << 59)
419 #define PRIQ_0_PERM_READ		(1UL << 60)
420 #define PRIQ_0_PERM_WRITE		(1UL << 61)
421 #define PRIQ_0_PRG_LAST			(1UL << 62)
422 #define PRIQ_0_SSID_V			(1UL << 63)
423 
424 #define PRIQ_1_PRG_IDX			GENMASK_ULL(8, 0)
425 #define PRIQ_1_ADDR_MASK		GENMASK_ULL(63, 12)
426 
427 /* High-level queue structures */
428 #define ARM_SMMU_POLL_TIMEOUT_US	1000000 /* 1s! */
429 #define ARM_SMMU_POLL_SPIN_COUNT	10
430 
431 #define MSI_IOVA_BASE			0x8000000
432 #define MSI_IOVA_LENGTH			0x100000
433 
434 enum pri_resp {
435 	PRI_RESP_DENY = 0,
436 	PRI_RESP_FAIL = 1,
437 	PRI_RESP_SUCC = 2,
438 };
439 
440 struct arm_smmu_cmdq_ent {
441 	/* Common fields */
442 	u8				opcode;
443 	bool				substream_valid;
444 
445 	/* Command-specific fields */
446 	union {
447 		#define CMDQ_OP_PREFETCH_CFG	0x1
448 		struct {
449 			u32			sid;
450 		} prefetch;
451 
452 		#define CMDQ_OP_CFGI_STE	0x3
453 		#define CMDQ_OP_CFGI_ALL	0x4
454 		#define CMDQ_OP_CFGI_CD		0x5
455 		#define CMDQ_OP_CFGI_CD_ALL	0x6
456 		struct {
457 			u32			sid;
458 			u32			ssid;
459 			union {
460 				bool		leaf;
461 				u8		span;
462 			};
463 		} cfgi;
464 
465 		#define CMDQ_OP_TLBI_NH_ASID	0x11
466 		#define CMDQ_OP_TLBI_NH_VA	0x12
467 		#define CMDQ_OP_TLBI_EL2_ALL	0x20
468 		#define CMDQ_OP_TLBI_EL2_ASID	0x21
469 		#define CMDQ_OP_TLBI_EL2_VA	0x22
470 		#define CMDQ_OP_TLBI_S12_VMALL	0x28
471 		#define CMDQ_OP_TLBI_S2_IPA	0x2a
472 		#define CMDQ_OP_TLBI_NSNH_ALL	0x30
473 		struct {
474 			u8			num;
475 			u8			scale;
476 			u16			asid;
477 			u16			vmid;
478 			bool			leaf;
479 			u8			ttl;
480 			u8			tg;
481 			u64			addr;
482 		} tlbi;
483 
484 		#define CMDQ_OP_ATC_INV		0x40
485 		#define ATC_INV_SIZE_ALL	52
486 		struct {
487 			u32			sid;
488 			u32			ssid;
489 			u64			addr;
490 			u8			size;
491 			bool			global;
492 		} atc;
493 
494 		#define CMDQ_OP_PRI_RESP	0x41
495 		struct {
496 			u32			sid;
497 			u32			ssid;
498 			u16			grpid;
499 			enum pri_resp		resp;
500 		} pri;
501 
502 		#define CMDQ_OP_RESUME		0x44
503 		struct {
504 			u32			sid;
505 			u16			stag;
506 			u8			resp;
507 		} resume;
508 
509 		#define CMDQ_OP_CMD_SYNC	0x46
510 		struct {
511 			u64			msiaddr;
512 		} sync;
513 	};
514 };
515 
516 struct arm_smmu_ll_queue {
517 	union {
518 		u64			val;
519 		struct {
520 			u32		prod;
521 			u32		cons;
522 		};
523 		struct {
524 			atomic_t	prod;
525 			atomic_t	cons;
526 		} atomic;
527 		u8			__pad[SMP_CACHE_BYTES];
528 	} ____cacheline_aligned_in_smp;
529 	u32				max_n_shift;
530 };
531 
532 struct arm_smmu_queue {
533 	struct arm_smmu_ll_queue	llq;
534 	int				irq; /* Wired interrupt */
535 
536 	__le64				*base;
537 	dma_addr_t			base_dma;
538 	u64				q_base;
539 
540 	size_t				ent_dwords;
541 
542 	u32 __iomem			*prod_reg;
543 	u32 __iomem			*cons_reg;
544 };
545 
546 struct arm_smmu_queue_poll {
547 	ktime_t				timeout;
548 	unsigned int			delay;
549 	unsigned int			spin_cnt;
550 	bool				wfe;
551 };
552 
553 struct arm_smmu_cmdq {
554 	struct arm_smmu_queue		q;
555 	atomic_long_t			*valid_map;
556 	atomic_t			owner_prod;
557 	atomic_t			lock;
558 };
559 
560 struct arm_smmu_cmdq_batch {
561 	u64				cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
562 	int				num;
563 };
564 
565 struct arm_smmu_evtq {
566 	struct arm_smmu_queue		q;
567 	struct iopf_queue		*iopf;
568 	u32				max_stalls;
569 };
570 
571 struct arm_smmu_priq {
572 	struct arm_smmu_queue		q;
573 };
574 
575 /* High-level stream table and context descriptor structures */
576 struct arm_smmu_strtab_l1_desc {
577 	u8				span;
578 
579 	struct arm_smmu_ste		*l2ptr;
580 	dma_addr_t			l2ptr_dma;
581 };
582 
583 struct arm_smmu_ctx_desc {
584 	u16				asid;
585 	u64				ttbr;
586 	u64				tcr;
587 	u64				mair;
588 
589 	refcount_t			refs;
590 	struct mm_struct		*mm;
591 };
592 
593 struct arm_smmu_l1_ctx_desc {
594 	__le64				*l2ptr;
595 	dma_addr_t			l2ptr_dma;
596 };
597 
598 struct arm_smmu_ctx_desc_cfg {
599 	__le64				*cdtab;
600 	dma_addr_t			cdtab_dma;
601 	struct arm_smmu_l1_ctx_desc	*l1_desc;
602 	unsigned int			num_l1_ents;
603 	u8				s1fmt;
604 	/* log2 of the maximum number of CDs supported by this table */
605 	u8				s1cdmax;
606 	/* Whether CD entries in this table have the stall bit set. */
607 	u8				stall_enabled:1;
608 };
609 
610 struct arm_smmu_s2_cfg {
611 	u16				vmid;
612 	u64				vttbr;
613 	u64				vtcr;
614 };
615 
616 struct arm_smmu_strtab_cfg {
617 	__le64				*strtab;
618 	dma_addr_t			strtab_dma;
619 	struct arm_smmu_strtab_l1_desc	*l1_desc;
620 	unsigned int			num_l1_ents;
621 
622 	u64				strtab_base;
623 	u32				strtab_base_cfg;
624 };
625 
626 /* An SMMUv3 instance */
627 struct arm_smmu_device {
628 	struct device			*dev;
629 	void __iomem			*base;
630 	void __iomem			*page1;
631 
632 #define ARM_SMMU_FEAT_2_LVL_STRTAB	(1 << 0)
633 #define ARM_SMMU_FEAT_2_LVL_CDTAB	(1 << 1)
634 #define ARM_SMMU_FEAT_TT_LE		(1 << 2)
635 #define ARM_SMMU_FEAT_TT_BE		(1 << 3)
636 #define ARM_SMMU_FEAT_PRI		(1 << 4)
637 #define ARM_SMMU_FEAT_ATS		(1 << 5)
638 #define ARM_SMMU_FEAT_SEV		(1 << 6)
639 #define ARM_SMMU_FEAT_MSI		(1 << 7)
640 #define ARM_SMMU_FEAT_COHERENCY		(1 << 8)
641 #define ARM_SMMU_FEAT_TRANS_S1		(1 << 9)
642 #define ARM_SMMU_FEAT_TRANS_S2		(1 << 10)
643 #define ARM_SMMU_FEAT_STALLS		(1 << 11)
644 #define ARM_SMMU_FEAT_HYP		(1 << 12)
645 #define ARM_SMMU_FEAT_STALL_FORCE	(1 << 13)
646 #define ARM_SMMU_FEAT_VAX		(1 << 14)
647 #define ARM_SMMU_FEAT_RANGE_INV		(1 << 15)
648 #define ARM_SMMU_FEAT_BTM		(1 << 16)
649 #define ARM_SMMU_FEAT_SVA		(1 << 17)
650 #define ARM_SMMU_FEAT_E2H		(1 << 18)
651 #define ARM_SMMU_FEAT_NESTING		(1 << 19)
652 	u32				features;
653 
654 #define ARM_SMMU_OPT_SKIP_PREFETCH	(1 << 0)
655 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY	(1 << 1)
656 #define ARM_SMMU_OPT_MSIPOLL		(1 << 2)
657 #define ARM_SMMU_OPT_CMDQ_FORCE_SYNC	(1 << 3)
658 	u32				options;
659 
660 	struct arm_smmu_cmdq		cmdq;
661 	struct arm_smmu_evtq		evtq;
662 	struct arm_smmu_priq		priq;
663 
664 	int				gerr_irq;
665 	int				combined_irq;
666 
667 	unsigned long			ias; /* IPA */
668 	unsigned long			oas; /* PA */
669 	unsigned long			pgsize_bitmap;
670 
671 #define ARM_SMMU_MAX_ASIDS		(1 << 16)
672 	unsigned int			asid_bits;
673 
674 #define ARM_SMMU_MAX_VMIDS		(1 << 16)
675 	unsigned int			vmid_bits;
676 	struct ida			vmid_map;
677 
678 	unsigned int			ssid_bits;
679 	unsigned int			sid_bits;
680 
681 	struct arm_smmu_strtab_cfg	strtab_cfg;
682 
683 	/* IOMMU core code handle */
684 	struct iommu_device		iommu;
685 
686 	struct rb_root			streams;
687 	struct mutex			streams_mutex;
688 };
689 
690 struct arm_smmu_stream {
691 	u32				id;
692 	struct arm_smmu_master		*master;
693 	struct rb_node			node;
694 };
695 
696 /* SMMU private data for each master */
697 struct arm_smmu_master {
698 	struct arm_smmu_device		*smmu;
699 	struct device			*dev;
700 	struct arm_smmu_domain		*domain;
701 	struct list_head		domain_head;
702 	struct arm_smmu_stream		*streams;
703 	/* Locked by the iommu core using the group mutex */
704 	struct arm_smmu_ctx_desc_cfg	cd_table;
705 	unsigned int			num_streams;
706 	bool				ats_enabled;
707 	bool				stall_enabled;
708 	bool				sva_enabled;
709 	bool				iopf_enabled;
710 	struct list_head		bonds;
711 	unsigned int			ssid_bits;
712 };
713 
714 /* SMMU private data for an IOMMU domain */
715 enum arm_smmu_domain_stage {
716 	ARM_SMMU_DOMAIN_S1 = 0,
717 	ARM_SMMU_DOMAIN_S2,
718 	ARM_SMMU_DOMAIN_BYPASS,
719 };
720 
721 struct arm_smmu_domain {
722 	struct arm_smmu_device		*smmu;
723 	struct mutex			init_mutex; /* Protects smmu pointer */
724 
725 	struct io_pgtable_ops		*pgtbl_ops;
726 	atomic_t			nr_ats_masters;
727 
728 	enum arm_smmu_domain_stage	stage;
729 	union {
730 		struct arm_smmu_ctx_desc	cd;
731 		struct arm_smmu_s2_cfg		s2_cfg;
732 	};
733 
734 	struct iommu_domain		domain;
735 
736 	struct list_head		devices;
737 	spinlock_t			devices_lock;
738 
739 	struct list_head		mmu_notifiers;
740 };
741 
742 static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
743 {
744 	return container_of(dom, struct arm_smmu_domain, domain);
745 }
746 
747 extern struct xarray arm_smmu_asid_xa;
748 extern struct mutex arm_smmu_asid_lock;
749 extern struct arm_smmu_ctx_desc quiet_cd;
750 
751 int arm_smmu_write_ctx_desc(struct arm_smmu_master *smmu_master, int ssid,
752 			    struct arm_smmu_ctx_desc *cd);
753 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
754 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
755 				 size_t granule, bool leaf,
756 				 struct arm_smmu_domain *smmu_domain);
757 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
758 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
759 			    unsigned long iova, size_t size);
760 
761 #ifdef CONFIG_ARM_SMMU_V3_SVA
762 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
763 bool arm_smmu_master_sva_supported(struct arm_smmu_master *master);
764 bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);
765 int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
766 int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
767 bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master);
768 void arm_smmu_sva_notifier_synchronize(void);
769 struct iommu_domain *arm_smmu_sva_domain_alloc(void);
770 void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
771 				   struct device *dev, ioasid_t id);
772 #else /* CONFIG_ARM_SMMU_V3_SVA */
773 static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
774 {
775 	return false;
776 }
777 
778 static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
779 {
780 	return false;
781 }
782 
783 static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
784 {
785 	return false;
786 }
787 
788 static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
789 {
790 	return -ENODEV;
791 }
792 
793 static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
794 {
795 	return -ENODEV;
796 }
797 
798 static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master)
799 {
800 	return false;
801 }
802 
803 static inline void arm_smmu_sva_notifier_synchronize(void) {}
804 
805 static inline struct iommu_domain *arm_smmu_sva_domain_alloc(void)
806 {
807 	return NULL;
808 }
809 
810 static inline void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
811 						 struct device *dev,
812 						 ioasid_t id)
813 {
814 }
815 #endif /* CONFIG_ARM_SMMU_V3_SVA */
816 #endif /* _ARM_SMMU_V3_H */
817