1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 * Leo Duran <leo.duran@amd.com>
6 */
7
8 #ifndef _ASM_X86_AMD_IOMMU_TYPES_H
9 #define _ASM_X86_AMD_IOMMU_TYPES_H
10
11 #include <linux/bitfield.h>
12 #include <linux/iommu.h>
13 #include <linux/types.h>
14 #include <linux/mmu_notifier.h>
15 #include <linux/mutex.h>
16 #include <linux/msi.h>
17 #include <linux/list.h>
18 #include <linux/spinlock.h>
19 #include <linux/pci.h>
20 #include <linux/irqreturn.h>
21 #include <linux/generic_pt/iommu.h>
22
23 /*
24 * Maximum number of IOMMUs supported
25 */
26 #define MAX_IOMMUS 32
27
28 /*
29 * some size calculation constants
30 */
31 #define DEV_TABLE_ENTRY_SIZE 32
32
33 /* Capability offsets used by the driver */
34 #define MMIO_CAP_HDR_OFFSET 0x00
35 #define MMIO_RANGE_OFFSET 0x0c
36 #define MMIO_MISC_OFFSET 0x10
37
38 /* Masks, shifts and macros to parse the device range capability */
39 #define MMIO_RANGE_LD_MASK 0xff000000
40 #define MMIO_RANGE_FD_MASK 0x00ff0000
41 #define MMIO_RANGE_BUS_MASK 0x0000ff00
42 #define MMIO_RANGE_LD_SHIFT 24
43 #define MMIO_RANGE_FD_SHIFT 16
44 #define MMIO_RANGE_BUS_SHIFT 8
45 #define MMIO_GET_LD(x) (((x) & MMIO_RANGE_LD_MASK) >> MMIO_RANGE_LD_SHIFT)
46 #define MMIO_GET_FD(x) (((x) & MMIO_RANGE_FD_MASK) >> MMIO_RANGE_FD_SHIFT)
47 #define MMIO_GET_BUS(x) (((x) & MMIO_RANGE_BUS_MASK) >> MMIO_RANGE_BUS_SHIFT)
48 #define MMIO_MSI_NUM(x) ((x) & 0x1f)
49
50 /* Flag masks for the AMD IOMMU exclusion range */
51 #define MMIO_EXCL_ENABLE_MASK 0x01ULL
52 #define MMIO_EXCL_ALLOW_MASK 0x02ULL
53
54 /* Used offsets into the MMIO space */
55 #define MMIO_DEV_TABLE_OFFSET 0x0000
56 #define MMIO_CMD_BUF_OFFSET 0x0008
57 #define MMIO_EVT_BUF_OFFSET 0x0010
58 #define MMIO_CONTROL_OFFSET 0x0018
59 #define MMIO_EXCL_BASE_OFFSET 0x0020
60 #define MMIO_EXCL_LIMIT_OFFSET 0x0028
61 #define MMIO_EXT_FEATURES 0x0030
62 #define MMIO_PPR_LOG_OFFSET 0x0038
63 #define MMIO_GA_LOG_BASE_OFFSET 0x00e0
64 #define MMIO_GA_LOG_TAIL_OFFSET 0x00e8
65 #define MMIO_MSI_ADDR_LO_OFFSET 0x015C
66 #define MMIO_MSI_ADDR_HI_OFFSET 0x0160
67 #define MMIO_MSI_DATA_OFFSET 0x0164
68 #define MMIO_INTCAPXT_EVT_OFFSET 0x0170
69 #define MMIO_INTCAPXT_PPR_OFFSET 0x0178
70 #define MMIO_INTCAPXT_GALOG_OFFSET 0x0180
71 #define MMIO_EXT_FEATURES2 0x01A0
72 #define MMIO_CMD_HEAD_OFFSET 0x2000
73 #define MMIO_CMD_TAIL_OFFSET 0x2008
74 #define MMIO_EVT_HEAD_OFFSET 0x2010
75 #define MMIO_EVT_TAIL_OFFSET 0x2018
76 #define MMIO_STATUS_OFFSET 0x2020
77 #define MMIO_PPR_HEAD_OFFSET 0x2030
78 #define MMIO_PPR_TAIL_OFFSET 0x2038
79 #define MMIO_GA_HEAD_OFFSET 0x2040
80 #define MMIO_GA_TAIL_OFFSET 0x2048
81 #define MMIO_CNTR_CONF_OFFSET 0x4000
82 #define MMIO_CNTR_REG_OFFSET 0x40000
83 #define MMIO_REG_END_OFFSET 0x80000
84
85
86
87 /* Extended Feature Bits */
88 #define FEATURE_PREFETCH BIT_ULL(0)
89 #define FEATURE_PPR BIT_ULL(1)
90 #define FEATURE_X2APIC BIT_ULL(2)
91 #define FEATURE_NX BIT_ULL(3)
92 #define FEATURE_GT BIT_ULL(4)
93 #define FEATURE_IA BIT_ULL(6)
94 #define FEATURE_GA BIT_ULL(7)
95 #define FEATURE_HE BIT_ULL(8)
96 #define FEATURE_PC BIT_ULL(9)
97 #define FEATURE_HATS GENMASK_ULL(11, 10)
98 #define FEATURE_GATS GENMASK_ULL(13, 12)
99 #define FEATURE_GLX GENMASK_ULL(15, 14)
100 #define FEATURE_GAM_VAPIC BIT_ULL(21)
101 #define FEATURE_PASMAX GENMASK_ULL(36, 32)
102 #define FEATURE_GIOSUP BIT_ULL(48)
103 #define FEATURE_HASUP BIT_ULL(49)
104 #define FEATURE_EPHSUP BIT_ULL(50)
105 #define FEATURE_HDSUP BIT_ULL(52)
106 #define FEATURE_SNP BIT_ULL(63)
107
108
109 /* Extended Feature 2 Bits */
110 #define FEATURE_SEVSNPIO_SUP BIT_ULL(1)
111 #define FEATURE_SNPAVICSUP GENMASK_ULL(7, 5)
112 #define FEATURE_SNPAVICSUP_GAM(x) \
113 (FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1)
114 #define FEATURE_HT_RANGE_IGNORE BIT_ULL(11)
115
116 #define FEATURE_NUM_INT_REMAP_SUP GENMASK_ULL(9, 8)
117 #define FEATURE_NUM_INT_REMAP_SUP_2K(x) \
118 (FIELD_GET(FEATURE_NUM_INT_REMAP_SUP, x) == 0x1)
119
120 /* Note:
121 * The current driver only support 16-bit PASID.
122 * Currently, hardware only implement upto 16-bit PASID
123 * even though the spec says it could have upto 20 bits.
124 */
125 #define PASID_MASK 0x0000ffff
126
127 /* MMIO status bits */
128 #define MMIO_STATUS_EVT_OVERFLOW_MASK BIT(0)
129 #define MMIO_STATUS_EVT_INT_MASK BIT(1)
130 #define MMIO_STATUS_COM_WAIT_INT_MASK BIT(2)
131 #define MMIO_STATUS_EVT_RUN_MASK BIT(3)
132 #define MMIO_STATUS_PPR_OVERFLOW_MASK BIT(5)
133 #define MMIO_STATUS_PPR_INT_MASK BIT(6)
134 #define MMIO_STATUS_PPR_RUN_MASK BIT(7)
135 #define MMIO_STATUS_GALOG_RUN_MASK BIT(8)
136 #define MMIO_STATUS_GALOG_OVERFLOW_MASK BIT(9)
137 #define MMIO_STATUS_GALOG_INT_MASK BIT(10)
138
139 /* event logging constants */
140 #define EVENT_ENTRY_SIZE 0x10
141 #define EVENT_TYPE_SHIFT 28
142 #define EVENT_TYPE_MASK 0xf
143 #define EVENT_TYPE_ILL_DEV 0x1
144 #define EVENT_TYPE_IO_FAULT 0x2
145 #define EVENT_TYPE_DEV_TAB_ERR 0x3
146 #define EVENT_TYPE_PAGE_TAB_ERR 0x4
147 #define EVENT_TYPE_ILL_CMD 0x5
148 #define EVENT_TYPE_CMD_HARD_ERR 0x6
149 #define EVENT_TYPE_IOTLB_INV_TO 0x7
150 #define EVENT_TYPE_INV_DEV_REQ 0x8
151 #define EVENT_TYPE_INV_PPR_REQ 0x9
152 #define EVENT_TYPE_RMP_FAULT 0xd
153 #define EVENT_TYPE_RMP_HW_ERR 0xe
154 #define EVENT_DEVID_MASK 0xffff
155 #define EVENT_DEVID_SHIFT 0
156 #define EVENT_DOMID_MASK_LO 0xffff
157 #define EVENT_DOMID_MASK_HI 0xf0000
158 #define EVENT_FLAGS_MASK 0xfff
159 #define EVENT_FLAGS_SHIFT 0x10
160 #define EVENT_FLAG_RW 0x020
161 #define EVENT_FLAG_I 0x008
162
163 /* feature control bits */
164 #define CONTROL_IOMMU_EN 0
165 #define CONTROL_HT_TUN_EN 1
166 #define CONTROL_EVT_LOG_EN 2
167 #define CONTROL_EVT_INT_EN 3
168 #define CONTROL_COMWAIT_EN 4
169 #define CONTROL_INV_TIMEOUT 5
170 #define CONTROL_PASSPW_EN 8
171 #define CONTROL_RESPASSPW_EN 9
172 #define CONTROL_COHERENT_EN 10
173 #define CONTROL_ISOC_EN 11
174 #define CONTROL_CMDBUF_EN 12
175 #define CONTROL_PPRLOG_EN 13
176 #define CONTROL_PPRINT_EN 14
177 #define CONTROL_PPR_EN 15
178 #define CONTROL_GT_EN 16
179 #define CONTROL_GA_EN 17
180 #define CONTROL_GAM_EN 25
181 #define CONTROL_GALOG_EN 28
182 #define CONTROL_GAINT_EN 29
183 #define CONTROL_NUM_INT_REMAP_MODE 43
184 #define CONTROL_NUM_INT_REMAP_MODE_MASK 0x03
185 #define CONTROL_NUM_INT_REMAP_MODE_2K 0x01
186 #define CONTROL_EPH_EN 45
187 #define CONTROL_XT_EN 50
188 #define CONTROL_INTCAPXT_EN 51
189 #define CONTROL_IRTCACHEDIS 59
190 #define CONTROL_SNPAVIC_EN 61
191
192 #define CTRL_INV_TO_MASK 7
193 #define CTRL_INV_TO_NONE 0
194 #define CTRL_INV_TO_1MS 1
195 #define CTRL_INV_TO_10MS 2
196 #define CTRL_INV_TO_100MS 3
197 #define CTRL_INV_TO_1S 4
198 #define CTRL_INV_TO_10S 5
199 #define CTRL_INV_TO_100S 6
200
201 /* command specific defines */
202 #define CMD_COMPL_WAIT 0x01
203 #define CMD_INV_DEV_ENTRY 0x02
204 #define CMD_INV_IOMMU_PAGES 0x03
205 #define CMD_INV_IOTLB_PAGES 0x04
206 #define CMD_INV_IRT 0x05
207 #define CMD_COMPLETE_PPR 0x07
208 #define CMD_INV_ALL 0x08
209
210 #define CMD_COMPL_WAIT_STORE_MASK 0x01
211 #define CMD_COMPL_WAIT_INT_MASK 0x02
212 #define CMD_INV_IOMMU_PAGES_SIZE_MASK 0x01
213 #define CMD_INV_IOMMU_PAGES_PDE_MASK 0x02
214 #define CMD_INV_IOMMU_PAGES_GN_MASK 0x04
215
216 #define PPR_STATUS_MASK 0xf
217 #define PPR_STATUS_SHIFT 12
218
219 #define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7fffffffffffffffULL
220
221 /* macros and definitions for device table entries */
222 #define DEV_ENTRY_VALID 0x00
223 #define DEV_ENTRY_TRANSLATION 0x01
224 #define DEV_ENTRY_HAD 0x07
225 #define DEV_ENTRY_PPR 0x34
226 #define DEV_ENTRY_IR 0x3d
227 #define DEV_ENTRY_IW 0x3e
228 #define DEV_ENTRY_NO_PAGE_FAULT 0x62
229 #define DEV_ENTRY_EX 0x67
230 #define DEV_ENTRY_SYSMGT1 0x68
231 #define DEV_ENTRY_SYSMGT2 0x69
232 #define DTE_DATA1_SYSMGT_MASK GENMASK_ULL(41, 40)
233
234 #define DEV_ENTRY_IRQ_TBL_EN 0x80
235 #define DEV_ENTRY_INIT_PASS 0xb8
236 #define DEV_ENTRY_EINT_PASS 0xb9
237 #define DEV_ENTRY_NMI_PASS 0xba
238 #define DEV_ENTRY_LINT0_PASS 0xbe
239 #define DEV_ENTRY_LINT1_PASS 0xbf
240 #define DEV_ENTRY_MODE_MASK 0x07
241 #define DEV_ENTRY_MODE_SHIFT 0x09
242
243 #define MAX_DEV_TABLE_ENTRIES 0xffff
244
245 /* constants to configure the command buffer */
246 #define CMD_BUFFER_SIZE 8192
247 #define CMD_BUFFER_UNINITIALIZED 1
248 #define CMD_BUFFER_ENTRIES 512
249 #define MMIO_CMD_SIZE_SHIFT 56
250 #define MMIO_CMD_SIZE_512 (0x9ULL << MMIO_CMD_SIZE_SHIFT)
251 #define MMIO_CMD_HEAD_MASK GENMASK_ULL(18, 4) /* Command buffer head ptr field [18:4] */
252 #define MMIO_CMD_BUFFER_HEAD(x) FIELD_GET(MMIO_CMD_HEAD_MASK, (x))
253 #define MMIO_CMD_TAIL_MASK GENMASK_ULL(18, 4) /* Command buffer tail ptr field [18:4] */
254 #define MMIO_CMD_BUFFER_TAIL(x) FIELD_GET(MMIO_CMD_TAIL_MASK, (x))
255
256 /* constants for event buffer handling */
257 #define EVT_BUFFER_SIZE 8192 /* 512 entries */
258 #define EVT_LEN_MASK (0x9ULL << 56)
259
260 /* Constants for PPR Log handling */
261 #define PPR_LOG_ENTRIES 512
262 #define PPR_LOG_SIZE_SHIFT 56
263 #define PPR_LOG_SIZE_512 (0x9ULL << PPR_LOG_SIZE_SHIFT)
264 #define PPR_ENTRY_SIZE 16
265 #define PPR_LOG_SIZE (PPR_ENTRY_SIZE * PPR_LOG_ENTRIES)
266
267 /* PAGE_SERVICE_REQUEST PPR Log Buffer Entry flags */
268 #define PPR_FLAG_EXEC 0x002 /* Execute permission requested */
269 #define PPR_FLAG_READ 0x004 /* Read permission requested */
270 #define PPR_FLAG_WRITE 0x020 /* Write permission requested */
271 #define PPR_FLAG_US 0x040 /* 1: User, 0: Supervisor */
272 #define PPR_FLAG_RVSD 0x080 /* Reserved bit not zero */
273 #define PPR_FLAG_GN 0x100 /* GVA and PASID is valid */
274
275 #define PPR_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
276 #define PPR_FLAGS(x) (((x) >> 48) & 0xfffULL)
277 #define PPR_DEVID(x) ((x) & 0xffffULL)
278 #define PPR_TAG(x) (((x) >> 32) & 0x3ffULL)
279 #define PPR_PASID1(x) (((x) >> 16) & 0xffffULL)
280 #define PPR_PASID2(x) (((x) >> 42) & 0xfULL)
281 #define PPR_PASID(x) ((PPR_PASID2(x) << 16) | PPR_PASID1(x))
282
283 #define PPR_REQ_FAULT 0x01
284
285 /* Constants for GA Log handling */
286 #define GA_LOG_ENTRIES 512
287 #define GA_LOG_SIZE_SHIFT 56
288 #define GA_LOG_SIZE_512 (0x8ULL << GA_LOG_SIZE_SHIFT)
289 #define GA_ENTRY_SIZE 8
290 #define GA_LOG_SIZE (GA_ENTRY_SIZE * GA_LOG_ENTRIES)
291
292 #define GA_TAG(x) (u32)(x & 0xffffffffULL)
293 #define GA_DEVID(x) (u16)(((x) >> 32) & 0xffffULL)
294 #define GA_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
295
296 #define GA_GUEST_NR 0x1
297
298 #define IOMMU_IN_ADDR_BIT_SIZE 52
299 #define IOMMU_OUT_ADDR_BIT_SIZE 52
300
301 /*
302 * This bitmap is used to advertise the page sizes our hardware support
303 * to the IOMMU core, which will then use this information to split
304 * physically contiguous memory regions it is mapping into page sizes
305 * that we support.
306 *
307 * 512GB Pages are not supported due to a hardware bug
308 * Page sizes >= the 52 bit max physical address of the CPU are not supported.
309 */
310 #define AMD_IOMMU_PGSIZES (GENMASK_ULL(51, 12) ^ SZ_512G)
311
312 /* Special mode where page-sizes are limited to 4 KiB */
313 #define AMD_IOMMU_PGSIZES_4K (PAGE_SIZE)
314
315 /* 4K, 2MB, 1G page sizes are supported */
316 #define AMD_IOMMU_PGSIZES_V2 (PAGE_SIZE | (1ULL << 21) | (1ULL << 30))
317
318 /* Bit value definition for dte irq remapping fields*/
319 #define DTE_IRQ_PHYS_ADDR_MASK GENMASK_ULL(51, 6)
320 #define DTE_IRQ_REMAP_INTCTL_MASK (0x3ULL << 60)
321 #define DTE_IRQ_REMAP_INTCTL (2ULL << 60)
322 #define DTE_IRQ_REMAP_ENABLE 1ULL
323
324 #define DTE_INTTAB_ALIGNMENT 128
325 #define DTE_INTTABLEN_MASK (0xfULL << 1)
326 #define DTE_INTTABLEN_VALUE_512 9ULL
327 #define DTE_INTTABLEN_512 (DTE_INTTABLEN_VALUE_512 << 1)
328 #define MAX_IRQS_PER_TABLE_512 BIT(DTE_INTTABLEN_VALUE_512)
329 #define DTE_INTTABLEN_VALUE_2K 11ULL
330 #define DTE_INTTABLEN_2K (DTE_INTTABLEN_VALUE_2K << 1)
331 #define MAX_IRQS_PER_TABLE_2K BIT(DTE_INTTABLEN_VALUE_2K)
332
333 #define PAGE_MODE_NONE 0x00
334 #define PAGE_MODE_1_LEVEL 0x01
335 #define PAGE_MODE_2_LEVEL 0x02
336 #define PAGE_MODE_3_LEVEL 0x03
337 #define PAGE_MODE_4_LEVEL 0x04
338 #define PAGE_MODE_5_LEVEL 0x05
339 #define PAGE_MODE_6_LEVEL 0x06
340 #define PAGE_MODE_7_LEVEL 0x07
341
342 #define GUEST_PGTABLE_4_LEVEL 0x00
343 #define GUEST_PGTABLE_5_LEVEL 0x01
344
345 #define PM_ADDR_MASK 0x000ffffffffff000ULL
346
347 /*
348 * Bit value definition for DTE fields
349 */
350 #define DTE_FLAG_V BIT_ULL(0)
351 #define DTE_FLAG_TV BIT_ULL(1)
352 #define DTE_FLAG_HAD (3ULL << 7)
353 #define DTE_FLAG_GIOV BIT_ULL(54)
354 #define DTE_FLAG_GV BIT_ULL(55)
355 #define DTE_GLX GENMASK_ULL(57, 56)
356 #define DTE_FLAG_IR BIT_ULL(61)
357 #define DTE_FLAG_IW BIT_ULL(62)
358
359 #define DTE_FLAG_IOTLB BIT_ULL(32)
360 #define DTE_FLAG_MASK (0x3ffULL << 32)
361 #define DEV_DOMID_MASK 0xffffULL
362
363 #define DTE_GCR3_14_12 GENMASK_ULL(60, 58)
364 #define DTE_GCR3_30_15 GENMASK_ULL(31, 16)
365 #define DTE_GCR3_51_31 GENMASK_ULL(63, 43)
366
367 #define DTE_GPT_LEVEL_SHIFT 54
368 #define DTE_GPT_LEVEL_MASK GENMASK_ULL(55, 54)
369
370 #define GCR3_VALID 0x01ULL
371
372 /* DTE[128:179] | DTE[184:191] */
373 #define DTE_DATA2_INTR_MASK ~GENMASK_ULL(55, 52)
374
375 #define IOMMU_PROT_MASK 0x03
376 #define IOMMU_PROT_IR 0x01
377 #define IOMMU_PROT_IW 0x02
378
379 #define IOMMU_UNITY_MAP_FLAG_EXCL_RANGE (1 << 2)
380
381 /* IOMMU capabilities */
382 #define IOMMU_CAP_IOTLB 24
383 #define IOMMU_CAP_NPCACHE 26
384 #define IOMMU_CAP_EFR 27
385
386 /* IOMMU IVINFO */
387 #define IOMMU_IVINFO_OFFSET 36
388 #define IOMMU_IVINFO_EFRSUP BIT(0)
389 #define IOMMU_IVINFO_DMA_REMAP BIT(1)
390
391 /* IOMMU Feature Reporting Field (for IVHD type 10h */
392 #define IOMMU_FEAT_GASUP_SHIFT 6
393
394 /* IOMMU HATDIS for IVHD type 11h and 40h */
395 #define IOMMU_IVHD_ATTR_HATDIS_SHIFT 0
396
397 /* IOMMU Extended Feature Register (EFR) */
398 #define IOMMU_EFR_XTSUP_SHIFT 2
399 #define IOMMU_EFR_GASUP_SHIFT 7
400 #define IOMMU_EFR_MSICAPMMIOSUP_SHIFT 46
401
402 #define MAX_DOMAIN_ID 65536
403
404 /* Timeout stuff */
405 #define LOOP_TIMEOUT 100000
406 #define MMIO_STATUS_TIMEOUT 2000000
407
408 extern bool amd_iommu_dump;
409 #define DUMP_printk(format, arg...) \
410 do { \
411 if (amd_iommu_dump) \
412 pr_info(format, ## arg); \
413 } while(0);
414
415 /* global flag if IOMMUs cache non-present entries */
416 extern bool amd_iommu_np_cache;
417 /* Only true if all IOMMUs support device IOTLBs */
418 extern bool amd_iommu_iotlb_sup;
419
420 struct irq_remap_table {
421 raw_spinlock_t lock;
422 unsigned min_index;
423 u32 *table;
424 };
425
426 /* Interrupt remapping feature used? */
427 extern bool amd_iommu_irq_remap;
428
429 extern const struct iommu_ops amd_iommu_ops;
430
431 /* IVRS indicates that pre-boot remapping was enabled */
432 extern bool amdr_ivrs_remap_support;
433
434 #define PCI_SBDF_TO_SEGID(sbdf) (((sbdf) >> 16) & 0xffff)
435 #define PCI_SBDF_TO_DEVID(sbdf) ((sbdf) & 0xffff)
436 #define PCI_SEG_DEVID_TO_SBDF(seg, devid) ((((u32)(seg) & 0xffff) << 16) | \
437 ((devid) & 0xffff))
438
439 /* Make iterating over all pci segment easier */
440 #define for_each_pci_segment(pci_seg) \
441 list_for_each_entry((pci_seg), &amd_iommu_pci_seg_list, list)
442 #define for_each_pci_segment_safe(pci_seg, next) \
443 list_for_each_entry_safe((pci_seg), (next), &amd_iommu_pci_seg_list, list)
444 /*
445 * Make iterating over all IOMMUs easier
446 */
447 #define for_each_iommu(iommu) \
448 list_for_each_entry((iommu), &amd_iommu_list, list)
449 #define for_each_iommu_safe(iommu, next) \
450 list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list)
451 /* Making iterating over protection_domain->dev_data_list easier */
452 #define for_each_pdom_dev_data(pdom_dev_data, pdom) \
453 list_for_each_entry(pdom_dev_data, &pdom->dev_data_list, list)
454 #define for_each_pdom_dev_data_safe(pdom_dev_data, next, pdom) \
455 list_for_each_entry_safe((pdom_dev_data), (next), &pdom->dev_data_list, list)
456
457 #define for_each_ivhd_dte_flags(entry) \
458 list_for_each_entry((entry), &amd_ivhd_dev_flags_list, list)
459
460 struct amd_iommu;
461 struct iommu_domain;
462 struct irq_domain;
463 struct amd_irte_ops;
464
465 #define AMD_IOMMU_FLAG_TRANS_PRE_ENABLED (1 << 0)
466
467 struct gcr3_tbl_info {
468 u64 *gcr3_tbl; /* Guest CR3 table */
469 int glx; /* Number of levels for GCR3 table */
470 u32 pasid_cnt; /* Track attached PASIDs */
471 u16 domid; /* Per device domain ID */
472 };
473
474 enum protection_domain_mode {
475 PD_MODE_NONE,
476 PD_MODE_V1,
477 PD_MODE_V2,
478 };
479
480 /* Track dev_data/PASID list for the protection domain */
481 struct pdom_dev_data {
482 /* Points to attached device data */
483 struct iommu_dev_data *dev_data;
484 /* PASID attached to the protection domain */
485 ioasid_t pasid;
486 /* For protection_domain->dev_data_list */
487 struct list_head list;
488 };
489
490 /* Keeps track of the IOMMUs attached to protection domain */
491 struct pdom_iommu_info {
492 struct amd_iommu *iommu; /* IOMMUs attach to protection domain */
493 u32 refcnt; /* Count of attached dev/pasid per domain/IOMMU */
494 };
495
496 /*
497 * This structure contains generic data for IOMMU protection domains
498 * independent of their use.
499 */
500 struct protection_domain {
501 union {
502 struct iommu_domain domain;
503 struct pt_iommu iommu;
504 struct pt_iommu_amdv1 amdv1;
505 struct pt_iommu_x86_64 amdv2;
506 };
507 struct list_head dev_list; /* List of all devices in this domain */
508 spinlock_t lock; /* mostly used to lock the page table*/
509 u16 id; /* the domain id written to the device table */
510 enum protection_domain_mode pd_mode; /* Track page table type */
511 bool dirty_tracking; /* dirty tracking is enabled in the domain */
512 struct xarray iommu_array; /* per-IOMMU reference count */
513
514 struct mmu_notifier mn; /* mmu notifier for the SVA domain */
515 struct list_head dev_data_list; /* List of pdom_dev_data */
516 };
517 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, iommu, domain);
518 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv1.iommu, domain);
519 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv2.iommu, domain);
520
521 /*
522 * This structure contains information about one PCI segment in the system.
523 */
524 struct amd_iommu_pci_seg {
525 /* List with all PCI segments in the system */
526 struct list_head list;
527
528 /* List of all available dev_data structures */
529 struct llist_head dev_data_list;
530
531 /* PCI segment number */
532 u16 id;
533
534 /* Largest PCI device id we expect translation requests for */
535 u16 last_bdf;
536
537 /* Size of the device table */
538 u32 dev_table_size;
539
540 /*
541 * device table virtual address
542 *
543 * Pointer to the per PCI segment device table.
544 * It is indexed by the PCI device id or the HT unit id and contains
545 * information about the domain the device belongs to as well as the
546 * page table root pointer.
547 */
548 struct dev_table_entry *dev_table;
549
550 /*
551 * The rlookup iommu table is used to find the IOMMU which is
552 * responsible for a specific device. It is indexed by the PCI
553 * device id.
554 */
555 struct amd_iommu **rlookup_table;
556
557 /*
558 * This table is used to find the irq remapping table for a given
559 * device id quickly.
560 */
561 struct irq_remap_table **irq_lookup_table;
562
563 /*
564 * Pointer to a device table which the content of old device table
565 * will be copied to. It's only be used in kdump kernel.
566 */
567 struct dev_table_entry *old_dev_tbl_cpy;
568
569 /*
570 * The alias table is a driver specific data structure which contains the
571 * mappings of the PCI device ids to the actual requestor ids on the IOMMU.
572 * More than one device can share the same requestor id.
573 */
574 u16 *alias_table;
575
576 /*
577 * A list of required unity mappings we find in ACPI. It is not locked
578 * because as runtime it is only read. It is created at ACPI table
579 * parsing time.
580 */
581 struct list_head unity_map;
582 };
583
584 /*
585 * Structure where we save information about one hardware AMD IOMMU in the
586 * system.
587 */
588 struct amd_iommu {
589 struct list_head list;
590
591 /* Index within the IOMMU array */
592 int index;
593
594 /* locks the accesses to the hardware */
595 raw_spinlock_t lock;
596
597 /* Pointer to PCI device of this IOMMU */
598 struct pci_dev *dev;
599
600 /* Cache pdev to root device for resume quirks */
601 struct pci_dev *root_pdev;
602
603 /* physical address of MMIO space */
604 u64 mmio_phys;
605
606 /* physical end address of MMIO space */
607 u64 mmio_phys_end;
608
609 /* virtual address of MMIO space */
610 u8 __iomem *mmio_base;
611
612 /* capabilities of that IOMMU read from ACPI */
613 u32 cap;
614
615 /* flags read from acpi table */
616 u8 acpi_flags;
617
618 /* Extended features */
619 u64 features;
620
621 /* Extended features 2 */
622 u64 features2;
623
624 /* PCI device id of the IOMMU device */
625 u16 devid;
626
627 /*
628 * Capability pointer. There could be more than one IOMMU per PCI
629 * device function if there are more than one AMD IOMMU capability
630 * pointers.
631 */
632 u16 cap_ptr;
633
634 /* pci domain of this IOMMU */
635 struct amd_iommu_pci_seg *pci_seg;
636
637 /* start of exclusion range of that IOMMU */
638 u64 exclusion_start;
639 /* length of exclusion range of that IOMMU */
640 u64 exclusion_length;
641
642 /* command buffer virtual address */
643 u8 *cmd_buf;
644 u32 cmd_buf_head;
645 u32 cmd_buf_tail;
646
647 /* event buffer virtual address */
648 u8 *evt_buf;
649
650 /* Name for event log interrupt */
651 unsigned char evt_irq_name[16];
652
653 /* Base of the PPR log, if present */
654 u8 *ppr_log;
655
656 /* Name for PPR log interrupt */
657 unsigned char ppr_irq_name[16];
658
659 /* Base of the GA log, if present */
660 u8 *ga_log;
661
662 /* Name for GA log interrupt */
663 unsigned char ga_irq_name[16];
664
665 /* Tail of the GA log, if present */
666 u8 *ga_log_tail;
667
668 /* true if interrupts for this IOMMU are already enabled */
669 bool int_enabled;
670
671 /* if one, we need to send a completion wait command */
672 bool need_sync;
673
674 /* true if disable irte caching */
675 bool irtcachedis_enabled;
676
677 /* Handle for IOMMU core code */
678 struct iommu_device iommu;
679
680 /*
681 * We can't rely on the BIOS to restore all values on reinit, so we
682 * need to stash them
683 */
684
685 /* The iommu BAR */
686 u32 stored_addr_lo;
687 u32 stored_addr_hi;
688
689 /*
690 * Each iommu has 6 l1s, each of which is documented as having 0x12
691 * registers
692 */
693 u32 stored_l1[6][0x12];
694
695 /* The l2 indirect registers */
696 u32 stored_l2[0x83];
697
698 /* The maximum PC banks and counters/bank (PCSup=1) */
699 u8 max_banks;
700 u8 max_counters;
701 #ifdef CONFIG_IRQ_REMAP
702 struct irq_domain *ir_domain;
703
704 struct amd_irte_ops *irte_ops;
705 #endif
706
707 u32 flags;
708 volatile u64 *cmd_sem;
709 atomic64_t cmd_sem_val;
710 /*
711 * Track physical address to directly use it in build_completion_wait()
712 * and avoid adding any special checks and handling for kdump.
713 */
714 u64 cmd_sem_paddr;
715
716 #ifdef CONFIG_AMD_IOMMU_DEBUGFS
717 /* DebugFS Info */
718 struct dentry *debugfs;
719 int dbg_mmio_offset;
720 int dbg_cap_offset;
721 #endif
722
723 /* IOPF support */
724 struct iopf_queue *iopf_queue;
725 unsigned char iopfq_name[32];
726 };
727
dev_to_amd_iommu(struct device * dev)728 static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev)
729 {
730 struct iommu_device *iommu = dev_to_iommu_device(dev);
731
732 return container_of(iommu, struct amd_iommu, iommu);
733 }
734
735 #define ACPIHID_UID_LEN 256
736 #define ACPIHID_HID_LEN 9
737
738 struct acpihid_map_entry {
739 struct list_head list;
740 u8 uid[ACPIHID_UID_LEN];
741 u8 hid[ACPIHID_HID_LEN];
742 u32 devid;
743 u32 root_devid;
744 bool cmd_line;
745 struct iommu_group *group;
746 };
747
748 struct devid_map {
749 struct list_head list;
750 u8 id;
751 u32 devid;
752 bool cmd_line;
753 };
754
755 #define AMD_IOMMU_DEVICE_FLAG_ATS_SUP 0x1 /* ATS feature supported */
756 #define AMD_IOMMU_DEVICE_FLAG_PRI_SUP 0x2 /* PRI feature supported */
757 #define AMD_IOMMU_DEVICE_FLAG_PASID_SUP 0x4 /* PASID context supported */
758 /* Device may request execution on memory pages */
759 #define AMD_IOMMU_DEVICE_FLAG_EXEC_SUP 0x8
760 /* Device may request super-user privileges */
761 #define AMD_IOMMU_DEVICE_FLAG_PRIV_SUP 0x10
762
763 /*
764 * This struct contains device specific data for the IOMMU
765 */
766 struct iommu_dev_data {
767 /*Protect against attach/detach races */
768 struct mutex mutex;
769 spinlock_t dte_lock; /* DTE lock for 256-bit access */
770
771 struct list_head list; /* For domain->dev_list */
772 struct llist_node dev_data_list; /* For global dev_data_list */
773 struct protection_domain *domain; /* Domain the device is bound to */
774 struct gcr3_tbl_info gcr3_info; /* Per-device GCR3 table */
775 struct device *dev;
776 u16 devid; /* PCI Device ID */
777
778 unsigned int max_irqs; /* Maximum IRQs supported by device */
779 u32 max_pasids; /* Max supported PASIDs */
780 u32 flags; /* Holds AMD_IOMMU_DEVICE_FLAG_<*> */
781 int ats_qdep;
782 u8 ats_enabled :1; /* ATS state */
783 u8 pri_enabled :1; /* PRI state */
784 u8 pasid_enabled:1; /* PASID state */
785 u8 pri_tlp :1; /* PASID TLB required for
786 PPR completions */
787 u8 ppr :1; /* Enable device PPR support */
788 bool use_vapic; /* Enable device to use vapic mode */
789 bool defer_attach;
790
791 struct ratelimit_state rs; /* Ratelimit IOPF messages */
792 };
793
794 /* Map HPET and IOAPIC ids to the devid used by the IOMMU */
795 extern struct list_head ioapic_map;
796 extern struct list_head hpet_map;
797 extern struct list_head acpihid_map;
798
799 /*
800 * List with all PCI segments in the system. This list is not locked because
801 * it is only written at driver initialization time
802 */
803 extern struct list_head amd_iommu_pci_seg_list;
804
805 /*
806 * List with all IOMMUs in the system. This list is not locked because it is
807 * only written and read at driver initialization or suspend time
808 */
809 extern struct list_head amd_iommu_list;
810
811 /*
812 * Structure defining one entry in the device table
813 */
814 struct dev_table_entry {
815 union {
816 u64 data[4];
817 u128 data128[2];
818 };
819 };
820
821 /*
822 * Structure defining one entry in the command buffer
823 */
824 struct iommu_cmd {
825 u32 data[4];
826 };
827
828 /*
829 * Structure to sture persistent DTE flags from IVHD
830 */
831 struct ivhd_dte_flags {
832 struct list_head list;
833 u16 segid;
834 u16 devid_first;
835 u16 devid_last;
836 struct dev_table_entry dte;
837 };
838
839 /*
840 * One entry for unity mappings parsed out of the ACPI table.
841 */
842 struct unity_map_entry {
843 struct list_head list;
844
845 /* starting device id this entry is used for (including) */
846 u16 devid_start;
847 /* end device id this entry is used for (including) */
848 u16 devid_end;
849
850 /* start address to unity map (including) */
851 u64 address_start;
852 /* end address to unity map (including) */
853 u64 address_end;
854
855 /* required protection */
856 int prot;
857 };
858
859 /*
860 * Data structures for device handling
861 */
862
863 extern bool amd_iommu_force_isolation;
864
865 /* Max levels of glxval supported */
866 extern int amd_iommu_max_glx_val;
867
868 /* IDA to track protection domain IDs */
869 extern struct ida pdom_ids;
870
871 /* Global EFR and EFR2 registers */
872 extern u64 amd_iommu_efr;
873 extern u64 amd_iommu_efr2;
874
get_ioapic_devid(int id)875 static inline int get_ioapic_devid(int id)
876 {
877 struct devid_map *entry;
878
879 list_for_each_entry(entry, &ioapic_map, list) {
880 if (entry->id == id)
881 return entry->devid;
882 }
883
884 return -EINVAL;
885 }
886
get_hpet_devid(int id)887 static inline int get_hpet_devid(int id)
888 {
889 struct devid_map *entry;
890
891 list_for_each_entry(entry, &hpet_map, list) {
892 if (entry->id == id)
893 return entry->devid;
894 }
895
896 return -EINVAL;
897 }
898
899 enum amd_iommu_intr_mode_type {
900 AMD_IOMMU_GUEST_IR_LEGACY,
901
902 /* This mode is not visible to users. It is used when
903 * we cannot fully enable vAPIC and fallback to only support
904 * legacy interrupt remapping via 128-bit IRTE.
905 */
906 AMD_IOMMU_GUEST_IR_LEGACY_GA,
907 AMD_IOMMU_GUEST_IR_VAPIC,
908 };
909
910 #define AMD_IOMMU_GUEST_IR_GA(x) (x == AMD_IOMMU_GUEST_IR_VAPIC || \
911 x == AMD_IOMMU_GUEST_IR_LEGACY_GA)
912
913 #define AMD_IOMMU_GUEST_IR_VAPIC(x) (x == AMD_IOMMU_GUEST_IR_VAPIC)
914
915 union irte {
916 u32 val;
917 struct {
918 u32 valid : 1,
919 no_fault : 1,
920 int_type : 3,
921 rq_eoi : 1,
922 dm : 1,
923 rsvd_1 : 1,
924 destination : 8,
925 vector : 8,
926 rsvd_2 : 8;
927 } fields;
928 };
929
930 #define APICID_TO_IRTE_DEST_LO(x) (x & 0xffffff)
931 #define APICID_TO_IRTE_DEST_HI(x) ((x >> 24) & 0xff)
932
933 union irte_ga_lo {
934 u64 val;
935
936 /* For int remapping */
937 struct {
938 u64 valid : 1,
939 no_fault : 1,
940 /* ------ */
941 int_type : 3,
942 rq_eoi : 1,
943 dm : 1,
944 /* ------ */
945 guest_mode : 1,
946 destination : 24,
947 ga_tag : 32;
948 } fields_remap;
949
950 /* For guest vAPIC */
951 struct {
952 u64 valid : 1,
953 no_fault : 1,
954 /* ------ */
955 ga_log_intr : 1,
956 rsvd1 : 3,
957 is_run : 1,
958 /* ------ */
959 guest_mode : 1,
960 destination : 24,
961 ga_tag : 32;
962 } fields_vapic;
963 };
964
965 union irte_ga_hi {
966 u64 val;
967 struct {
968 u64 vector : 8,
969 rsvd_1 : 4,
970 ga_root_ptr : 40,
971 rsvd_2 : 4,
972 destination : 8;
973 } fields;
974 };
975
976 struct irte_ga {
977 union {
978 struct {
979 union irte_ga_lo lo;
980 union irte_ga_hi hi;
981 };
982 u128 irte;
983 };
984 };
985
986 struct irq_2_irte {
987 u16 devid; /* Device ID for IRTE table */
988 u16 index; /* Index into IRTE table*/
989 };
990
991 struct amd_ir_data {
992 struct amd_iommu *iommu;
993 struct irq_2_irte irq_2_irte;
994 struct msi_msg msi_entry;
995 void *entry; /* Pointer to union irte or struct irte_ga */
996
997 /**
998 * Store information for activate/de-activate
999 * Guest virtual APIC mode during runtime.
1000 */
1001 struct irq_cfg *cfg;
1002 int ga_vector;
1003 u64 ga_root_ptr;
1004 u32 ga_tag;
1005 };
1006
1007 struct amd_irte_ops {
1008 void (*prepare)(void *, u32, bool, u8, u32, int);
1009 void (*activate)(struct amd_iommu *iommu, void *, u16, u16);
1010 void (*deactivate)(struct amd_iommu *iommu, void *, u16, u16);
1011 void (*set_affinity)(struct amd_iommu *iommu, void *, u16, u16, u8, u32);
1012 void *(*get)(struct irq_remap_table *, int);
1013 void (*set_allocated)(struct irq_remap_table *, int);
1014 bool (*is_allocated)(struct irq_remap_table *, int);
1015 void (*clear_allocated)(struct irq_remap_table *, int);
1016 };
1017
1018 #ifdef CONFIG_IRQ_REMAP
1019 extern struct amd_irte_ops irte_32_ops;
1020 extern struct amd_irte_ops irte_128_ops;
1021 #endif
1022
1023 #endif /* _ASM_X86_AMD_IOMMU_TYPES_H */
1024