1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 * Leo Duran <leo.duran@amd.com>
6 */
7
8 #ifndef _ASM_X86_AMD_IOMMU_TYPES_H
9 #define _ASM_X86_AMD_IOMMU_TYPES_H
10
11 #include <linux/bitfield.h>
12 #include <linux/iommu.h>
13 #include <linux/types.h>
14 #include <linux/mmu_notifier.h>
15 #include <linux/mutex.h>
16 #include <linux/msi.h>
17 #include <linux/list.h>
18 #include <linux/sizes.h>
19 #include <linux/spinlock.h>
20 #include <linux/pci.h>
21 #include <linux/iommufd.h>
22 #include <linux/irqreturn.h>
23 #include <linux/generic_pt/iommu.h>
24
25 #include <uapi/linux/iommufd.h>
26
27 /*
28 * Maximum number of IOMMUs supported
29 */
30 #define MAX_IOMMUS 32
31
32 /*
33 * some size calculation constants
34 */
35 #define DEV_TABLE_ENTRY_SIZE 32
36
37 /* Capability offsets used by the driver */
38 #define MMIO_CAP_HDR_OFFSET 0x00
39 #define MMIO_RANGE_OFFSET 0x0c
40 #define MMIO_MISC_OFFSET 0x10
41
42 /* Used offsets into the MMIO space */
43 #define MMIO_DEV_TABLE_OFFSET 0x0000
44 #define MMIO_CMD_BUF_OFFSET 0x0008
45 #define MMIO_EVT_BUF_OFFSET 0x0010
46 #define MMIO_CONTROL_OFFSET 0x0018
47 #define MMIO_EXCL_BASE_OFFSET 0x0020
48 #define MMIO_EXCL_LIMIT_OFFSET 0x0028
49 #define MMIO_EXT_FEATURES 0x0030
50 #define MMIO_PPR_LOG_OFFSET 0x0038
51 #define MMIO_GA_LOG_BASE_OFFSET 0x00e0
52 #define MMIO_GA_LOG_TAIL_OFFSET 0x00e8
53 #define MMIO_MSI_ADDR_LO_OFFSET 0x015C
54 #define MMIO_MSI_ADDR_HI_OFFSET 0x0160
55 #define MMIO_MSI_DATA_OFFSET 0x0164
56 #define MMIO_INTCAPXT_EVT_OFFSET 0x0170
57 #define MMIO_INTCAPXT_PPR_OFFSET 0x0178
58 #define MMIO_INTCAPXT_GALOG_OFFSET 0x0180
59 #define MMIO_EXT_FEATURES2 0x01A0
60 #define MMIO_CMD_HEAD_OFFSET 0x2000
61 #define MMIO_CMD_TAIL_OFFSET 0x2008
62 #define MMIO_EVT_HEAD_OFFSET 0x2010
63 #define MMIO_EVT_TAIL_OFFSET 0x2018
64 #define MMIO_STATUS_OFFSET 0x2020
65 #define MMIO_PPR_HEAD_OFFSET 0x2030
66 #define MMIO_PPR_TAIL_OFFSET 0x2038
67 #define MMIO_GA_HEAD_OFFSET 0x2040
68 #define MMIO_GA_TAIL_OFFSET 0x2048
69 #define MMIO_CNTR_CONF_OFFSET 0x4000
70 #define MMIO_CNTR_REG_OFFSET 0x40000
71 #define MMIO_REG_END_OFFSET 0x80000
72
73
74
75 /* Extended Feature Bits */
76 #define FEATURE_PREFETCH BIT_ULL(0)
77 #define FEATURE_PPR BIT_ULL(1)
78 #define FEATURE_X2APIC BIT_ULL(2)
79 #define FEATURE_NX BIT_ULL(3)
80 #define FEATURE_GT BIT_ULL(4)
81 #define FEATURE_IA BIT_ULL(6)
82 #define FEATURE_GA BIT_ULL(7)
83 #define FEATURE_HE BIT_ULL(8)
84 #define FEATURE_PC BIT_ULL(9)
85 #define FEATURE_HATS GENMASK_ULL(11, 10)
86 #define FEATURE_GATS GENMASK_ULL(13, 12)
87 #define FEATURE_GLX GENMASK_ULL(15, 14)
88 #define FEATURE_GAM_VAPIC BIT_ULL(21)
89 #define FEATURE_PASMAX GENMASK_ULL(36, 32)
90 #define FEATURE_GIOSUP BIT_ULL(48)
91 #define FEATURE_HASUP BIT_ULL(49)
92 #define FEATURE_EPHSUP BIT_ULL(50)
93 #define FEATURE_HDSUP BIT_ULL(52)
94 #define FEATURE_SNP BIT_ULL(63)
95
96
97 /* Extended Feature 2 Bits */
98 #define FEATURE_SEVSNPIO_SUP BIT_ULL(1)
99 #define FEATURE_GCR3TRPMODE BIT_ULL(3)
100 #define FEATURE_SNPAVICSUP GENMASK_ULL(7, 5)
101 #define FEATURE_SNPAVICSUP_GAM(x) \
102 (FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1)
103 #define FEATURE_HT_RANGE_IGNORE BIT_ULL(11)
104 #define FEATURE_SNP_PAGE_MODE0_SUP BIT_ULL(13)
105
106 #define FEATURE_NUM_INT_REMAP_SUP GENMASK_ULL(9, 8)
107 #define FEATURE_NUM_INT_REMAP_SUP_2K(x) \
108 (FIELD_GET(FEATURE_NUM_INT_REMAP_SUP, x) == 0x1)
109
110 /* Note:
111 * The current driver only support 16-bit PASID.
112 * Currently, hardware only implement upto 16-bit PASID
113 * even though the spec says it could have upto 20 bits.
114 */
115 #define PASID_MASK 0x0000ffff
116
117 /* MMIO status bits */
118 #define MMIO_STATUS_EVT_OVERFLOW_MASK BIT(0)
119 #define MMIO_STATUS_EVT_INT_MASK BIT(1)
120 #define MMIO_STATUS_COM_WAIT_INT_MASK BIT(2)
121 #define MMIO_STATUS_EVT_RUN_MASK BIT(3)
122 #define MMIO_STATUS_PPR_OVERFLOW_MASK BIT(5)
123 #define MMIO_STATUS_PPR_INT_MASK BIT(6)
124 #define MMIO_STATUS_PPR_RUN_MASK BIT(7)
125 #define MMIO_STATUS_GALOG_RUN_MASK BIT(8)
126 #define MMIO_STATUS_GALOG_OVERFLOW_MASK BIT(9)
127 #define MMIO_STATUS_GALOG_INT_MASK BIT(10)
128
129 /* event logging constants */
130 #define EVENT_TYPE_SHIFT 28
131 #define EVENT_TYPE_MASK 0xf
132 #define EVENT_TYPE_ILL_DEV 0x1
133 #define EVENT_TYPE_IO_FAULT 0x2
134 #define EVENT_TYPE_DEV_TAB_ERR 0x3
135 #define EVENT_TYPE_PAGE_TAB_ERR 0x4
136 #define EVENT_TYPE_ILL_CMD 0x5
137 #define EVENT_TYPE_CMD_HARD_ERR 0x6
138 #define EVENT_TYPE_IOTLB_INV_TO 0x7
139 #define EVENT_TYPE_INV_DEV_REQ 0x8
140 #define EVENT_TYPE_INV_PPR_REQ 0x9
141 #define EVENT_TYPE_RMP_FAULT 0xd
142 #define EVENT_TYPE_RMP_HW_ERR 0xe
143 #define EVENT_DEVID_MASK 0xffff
144 #define EVENT_DEVID_SHIFT 0
145 #define EVENT_DOMID_MASK_LO 0xffff
146 #define EVENT_DOMID_MASK_HI 0xf0000
147 #define EVENT_FLAGS_MASK 0xfff
148 #define EVENT_FLAGS_SHIFT 0x10
149 #define EVENT_FLAG_RW 0x020
150 #define EVENT_FLAG_I 0x008
151 #define EVENT_FLAG_PPR_RX 0x001
152 #define EVENT_FLAG_PPR_GN 0x200
153
154 /* feature control bits */
155 #define CONTROL_IOMMU_EN 0
156 #define CONTROL_HT_TUN_EN 1
157 #define CONTROL_EVT_LOG_EN 2
158 #define CONTROL_EVT_INT_EN 3
159 #define CONTROL_COMWAIT_EN 4
160 #define CONTROL_INV_TIMEOUT 5
161 #define CONTROL_PASSPW_EN 8
162 #define CONTROL_RESPASSPW_EN 9
163 #define CONTROL_COHERENT_EN 10
164 #define CONTROL_ISOC_EN 11
165 #define CONTROL_CMDBUF_EN 12
166 #define CONTROL_PPRLOG_EN 13
167 #define CONTROL_PPRINT_EN 14
168 #define CONTROL_PPR_EN 15
169 #define CONTROL_GT_EN 16
170 #define CONTROL_GA_EN 17
171 #define CONTROL_GAM_EN 25
172 #define CONTROL_GALOG_EN 28
173 #define CONTROL_GAINT_EN 29
174 #define CONTROL_NUM_INT_REMAP_MODE 43
175 #define CONTROL_NUM_INT_REMAP_MODE_MASK 0x03
176 #define CONTROL_NUM_INT_REMAP_MODE_2K 0x01
177 #define CONTROL_EPH_EN 45
178 #define CONTROL_XT_EN 50
179 #define CONTROL_INTCAPXT_EN 51
180 #define CONTROL_GCR3TRPMODE 58
181 #define CONTROL_IRTCACHEDIS 59
182 #define CONTROL_SNPAVIC_EN 61
183
184 #define CTRL_INV_TO_MASK 7
185 #define CTRL_INV_TO_NONE 0
186 #define CTRL_INV_TO_1MS 1
187 #define CTRL_INV_TO_10MS 2
188 #define CTRL_INV_TO_100MS 3
189 #define CTRL_INV_TO_1S 4
190 #define CTRL_INV_TO_10S 5
191 #define CTRL_INV_TO_100S 6
192
193 /* command specific defines */
194 #define CMD_COMPL_WAIT 0x01
195 #define CMD_INV_DEV_ENTRY 0x02
196 #define CMD_INV_IOMMU_PAGES 0x03
197 #define CMD_INV_IOTLB_PAGES 0x04
198 #define CMD_INV_IRT 0x05
199 #define CMD_COMPLETE_PPR 0x07
200 #define CMD_INV_ALL 0x08
201
202 #define CMD_COMPL_WAIT_STORE_MASK 0x01
203 #define CMD_COMPL_WAIT_INT_MASK 0x02
204 #define CMD_INV_IOMMU_PAGES_SIZE_MASK 0x01
205 #define CMD_INV_IOMMU_PAGES_PDE_MASK 0x02
206 #define CMD_INV_IOMMU_PAGES_GN_MASK 0x04
207
208 #define PPR_STATUS_MASK 0xf
209 #define PPR_STATUS_SHIFT 12
210
211 #define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7ffffffffffff000ULL
212
213 /* macros and definitions for device table entries */
214 #define DEV_ENTRY_VALID 0x00
215 #define DEV_ENTRY_TRANSLATION 0x01
216 #define DEV_ENTRY_HAD 0x07
217 #define DEV_ENTRY_PPR 0x34
218 #define DEV_ENTRY_IR 0x3d
219 #define DEV_ENTRY_IW 0x3e
220 #define DEV_ENTRY_NO_PAGE_FAULT 0x62
221 #define DEV_ENTRY_SYSMGT1 0x68
222 #define DEV_ENTRY_SYSMGT2 0x69
223 #define DTE_DATA1_SYSMGT_MASK GENMASK_ULL(41, 40)
224
225 #define DEV_ENTRY_IRQ_TBL_EN 0x80
226 #define DEV_ENTRY_INIT_PASS 0xb8
227 #define DEV_ENTRY_EINT_PASS 0xb9
228 #define DEV_ENTRY_NMI_PASS 0xba
229 #define DEV_ENTRY_LINT0_PASS 0xbe
230 #define DEV_ENTRY_LINT1_PASS 0xbf
231 #define DEV_ENTRY_MODE_MASK 0x07
232 #define DEV_ENTRY_MODE_SHIFT 0x09
233
234 #define MAX_DEV_TABLE_ENTRIES 0xffff
235
236 /* constants to configure the command buffer */
237 #define CMD_BUFFER_SIZE 8192
238 #define CMD_BUFFER_ENTRIES 512
239 #define MMIO_CMD_SIZE_SHIFT 56
240 #define MMIO_CMD_SIZE_512 (0x9ULL << MMIO_CMD_SIZE_SHIFT)
241 #define MMIO_CMD_HEAD_MASK GENMASK_ULL(18, 4) /* Command buffer head ptr field [18:4] */
242 #define MMIO_CMD_BUFFER_HEAD(x) FIELD_GET(MMIO_CMD_HEAD_MASK, (x))
243 #define MMIO_CMD_TAIL_MASK GENMASK_ULL(18, 4) /* Command buffer tail ptr field [18:4] */
244 #define MMIO_CMD_BUFFER_TAIL(x) FIELD_GET(MMIO_CMD_TAIL_MASK, (x))
245
246 /* constants for event buffer handling */
247 #define EVTLOG_ENTRY_SIZE 0x10
248 #define EVTLOG_SIZE_SHIFT 56
249 #define EVTLOG_SIZE_DEF SZ_8K /* 512 entries */
250 #define EVTLOG_LEN_MASK_DEF (0x9ULL << EVTLOG_SIZE_SHIFT)
251 #define EVTLOG_SIZE_MAX SZ_512K /* 32K entries */
252 #define EVTLOG_LEN_MASK_MAX (0xFULL << EVTLOG_SIZE_SHIFT)
253
254 /* Constants for PPR Log handling */
255 #define PPRLOG_ENTRY_SIZE 0x10
256 #define PPRLOG_SIZE_SHIFT 56
257 #define PPRLOG_SIZE_DEF SZ_8K /* 512 entries */
258 #define PPRLOG_LEN_MASK_DEF (0x9ULL << PPRLOG_SIZE_SHIFT)
259 #define PPRLOG_SIZE_MAX SZ_512K /* 32K entries */
260 #define PPRLOG_LEN_MASK_MAX (0xFULL << PPRLOG_SIZE_SHIFT)
261
262 /* PAGE_SERVICE_REQUEST PPR Log Buffer Entry flags */
263 #define PPR_FLAG_EXEC 0x002 /* Execute permission requested */
264 #define PPR_FLAG_READ 0x004 /* Read permission requested */
265 #define PPR_FLAG_WRITE 0x020 /* Write permission requested */
266 #define PPR_FLAG_US 0x040 /* 1: User, 0: Supervisor */
267 #define PPR_FLAG_RVSD 0x080 /* Reserved bit not zero */
268 #define PPR_FLAG_GN 0x100 /* GVA and PASID is valid */
269
270 #define PPR_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
271 #define PPR_FLAGS(x) (((x) >> 48) & 0xfffULL)
272 #define PPR_DEVID(x) ((x) & 0xffffULL)
273 #define PPR_TAG(x) (((x) >> 32) & 0x1ffULL)
274 #define PPR_TAG_LAST_PAGE(x) (((x) >> 32) & 0x200ULL)
275 #define PPR_PASID1(x) (((x) >> 16) & 0xffffULL)
276 #define PPR_PASID2(x) (((x) >> 42) & 0xfULL)
277 #define PPR_PASID(x) ((PPR_PASID2(x) << 16) | PPR_PASID1(x))
278
279 #define PPR_REQ_FAULT 0x01
280
281 /* Constants for GA Log handling */
282 #define GA_LOG_ENTRIES 512
283 #define GA_LOG_SIZE_SHIFT 56
284 #define GA_LOG_SIZE_512 (0x8ULL << GA_LOG_SIZE_SHIFT)
285 #define GA_ENTRY_SIZE 8
286 #define GA_LOG_SIZE (GA_ENTRY_SIZE * GA_LOG_ENTRIES)
287
288 #define GA_TAG(x) (u32)(x & 0xffffffffULL)
289 #define GA_DEVID(x) (u16)(((x) >> 32) & 0xffffULL)
290 #define GA_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
291
292 #define GA_GUEST_NR 0x1
293
294 /*
295 * This bitmap is used to advertise the page sizes our hardware support
296 * to the IOMMU core, which will then use this information to split
297 * physically contiguous memory regions it is mapping into page sizes
298 * that we support.
299 *
300 * 512GB Pages are not supported due to a hardware bug
301 * Page sizes >= the 52 bit max physical address of the CPU are not supported.
302 */
303 #define AMD_IOMMU_PGSIZES (GENMASK_ULL(51, 12) ^ SZ_512G)
304
305 /* Special mode where page-sizes are limited to 4 KiB */
306 #define AMD_IOMMU_PGSIZES_4K (PAGE_SIZE)
307
308 /* 4K, 2MB, 1G page sizes are supported */
309 #define AMD_IOMMU_PGSIZES_V2 (PAGE_SIZE | (1ULL << 21) | (1ULL << 30))
310
311 /* Bit value definition for dte irq remapping fields*/
312 #define DTE_IRQ_PHYS_ADDR_MASK GENMASK_ULL(51, 6)
313 #define DTE_IRQ_REMAP_INTCTL_MASK (0x3ULL << 60)
314 #define DTE_IRQ_REMAP_INTCTL (2ULL << 60)
315 #define DTE_IRQ_REMAP_ENABLE 1ULL
316
317 #define DTE_INTTAB_ALIGNMENT 128
318 #define DTE_INTTABLEN_MASK (0xfULL << 1)
319 #define DTE_INTTABLEN_VALUE_512 9ULL
320 #define DTE_INTTABLEN_512 (DTE_INTTABLEN_VALUE_512 << 1)
321 #define MAX_IRQS_PER_TABLE_512 BIT(DTE_INTTABLEN_VALUE_512)
322 #define DTE_INTTABLEN_VALUE_2K 11ULL
323 #define DTE_INTTABLEN_2K (DTE_INTTABLEN_VALUE_2K << 1)
324 #define MAX_IRQS_PER_TABLE_2K BIT(DTE_INTTABLEN_VALUE_2K)
325
326 #define PAGE_MODE_NONE 0x00
327 #define PAGE_MODE_1_LEVEL 0x01
328 #define PAGE_MODE_2_LEVEL 0x02
329 #define PAGE_MODE_3_LEVEL 0x03
330 #define PAGE_MODE_4_LEVEL 0x04
331 #define PAGE_MODE_5_LEVEL 0x05
332 #define PAGE_MODE_6_LEVEL 0x06
333 #define PAGE_MODE_7_LEVEL 0x07
334
335 #define GUEST_PGTABLE_4_LEVEL 0x00
336 #define GUEST_PGTABLE_5_LEVEL 0x01
337
338 #define PM_ADDR_MASK 0x000ffffffffff000ULL
339
340 /*
341 * Bit value definition for DTE fields
342 */
343 #define DTE_FLAG_V BIT_ULL(0)
344 #define DTE_FLAG_TV BIT_ULL(1)
345 #define DTE_FLAG_HAD (3ULL << 7)
346 #define DTE_MODE_MASK GENMASK_ULL(11, 9)
347 #define DTE_HOST_TRP GENMASK_ULL(51, 12)
348 #define DTE_FLAG_PPR BIT_ULL(52)
349 #define DTE_FLAG_GIOV BIT_ULL(54)
350 #define DTE_FLAG_GV BIT_ULL(55)
351 #define DTE_GLX GENMASK_ULL(57, 56)
352 #define DTE_FLAG_IR BIT_ULL(61)
353 #define DTE_FLAG_IW BIT_ULL(62)
354
355 #define DTE_FLAG_IOTLB BIT_ULL(32)
356 #define DTE_FLAG_MASK (0x3ffULL << 32)
357 #define DTE_DOMID_MASK GENMASK_ULL(15, 0)
358
359 #define DTE_GCR3_14_12 GENMASK_ULL(60, 58)
360 #define DTE_GCR3_30_15 GENMASK_ULL(31, 16)
361 #define DTE_GCR3_51_31 GENMASK_ULL(63, 43)
362
363 #define DTE_GPT_LEVEL_SHIFT 54
364 #define DTE_GPT_LEVEL_MASK GENMASK_ULL(55, 54)
365
366 #define GCR3_VALID 0x01ULL
367
368 /* DTE[128:179] | DTE[184:191] */
369 #define DTE_DATA2_INTR_MASK ~GENMASK_ULL(55, 52)
370
371 #define IOMMU_PROT_MASK 0x03
372 #define IOMMU_PROT_IR 0x01
373 #define IOMMU_PROT_IW 0x02
374
375 /* IOMMU capabilities */
376 #define IOMMU_CAP_IOTLB 24
377 #define IOMMU_CAP_NPCACHE 26
378 #define IOMMU_CAP_EFR 27
379
380 /* IOMMU IVINFO */
381 #define IOMMU_IVINFO_OFFSET 36
382 #define IOMMU_IVINFO_EFRSUP BIT(0)
383 #define IOMMU_IVINFO_DMA_REMAP BIT(1)
384 #define IOMMU_IVINFO_VASIZE GENMASK_ULL(21, 15)
385
386 /* IOMMU Feature Reporting Field (for IVHD type 10h */
387 #define IOMMU_FEAT_GASUP_SHIFT 6
388
389 /* IOMMU HATDIS for IVHD type 11h and 40h */
390 #define IOMMU_IVHD_ATTR_HATDIS_SHIFT 0
391
392 /* IOMMU Extended Feature Register (EFR) */
393 #define IOMMU_EFR_XTSUP_SHIFT 2
394 #define IOMMU_EFR_GASUP_SHIFT 7
395 #define IOMMU_EFR_MSICAPMMIOSUP_SHIFT 46
396
397 #define MAX_DOMAIN_ID 65536
398
399 /* Timeout stuff */
400 #define LOOP_TIMEOUT 100000
401 #define MMIO_STATUS_TIMEOUT 2000000
402
403 extern bool amd_iommu_dump;
404 #define DUMP_printk(format, arg...) \
405 do { \
406 if (amd_iommu_dump) \
407 pr_info(format, ## arg); \
408 } while(0);
409
410 /* SNP page mode 0 support */
411 extern bool amd_iommu_snp_mode0_sup;
412
413 /* global flag if IOMMUs cache non-present entries */
414 extern bool amd_iommu_np_cache;
415 /* Only true if all IOMMUs support device IOTLBs */
416 extern bool amd_iommu_iotlb_sup;
417
418 struct irq_remap_table {
419 raw_spinlock_t lock;
420 unsigned min_index;
421 u32 *table;
422 };
423
424 /* Interrupt remapping feature used? */
425 extern bool amd_iommu_irq_remap;
426
427 extern const struct iommu_ops amd_iommu_ops;
428
429 /* IVRS indicates that pre-boot remapping was enabled */
430 extern bool amdr_ivrs_remap_support;
431
432 #define PCI_SBDF_TO_SEGID(sbdf) (((sbdf) >> 16) & 0xffff)
433 #define PCI_SBDF_TO_DEVID(sbdf) ((sbdf) & 0xffff)
434 #define PCI_SEG_DEVID_TO_SBDF(seg, devid) ((((u32)(seg) & 0xffff) << 16) | \
435 ((devid) & 0xffff))
436
437 /* Make iterating over all pci segment easier */
438 #define for_each_pci_segment(pci_seg) \
439 list_for_each_entry((pci_seg), &amd_iommu_pci_seg_list, list)
440 #define for_each_pci_segment_safe(pci_seg, next) \
441 list_for_each_entry_safe((pci_seg), (next), &amd_iommu_pci_seg_list, list)
442 /*
443 * Make iterating over all IOMMUs easier
444 */
445 #define for_each_iommu(iommu) \
446 list_for_each_entry((iommu), &amd_iommu_list, list)
447 #define for_each_iommu_safe(iommu, next) \
448 list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list)
449 /* Making iterating over protection_domain->dev_data_list easier */
450 #define for_each_pdom_dev_data(pdom_dev_data, pdom) \
451 list_for_each_entry(pdom_dev_data, &pdom->dev_data_list, list)
452 #define for_each_pdom_dev_data_safe(pdom_dev_data, next, pdom) \
453 list_for_each_entry_safe((pdom_dev_data), (next), &pdom->dev_data_list, list)
454
455 #define for_each_ivhd_dte_flags(entry) \
456 list_for_each_entry((entry), &amd_ivhd_dev_flags_list, list)
457
458 struct amd_iommu;
459 struct iommu_domain;
460 struct irq_domain;
461 struct amd_irte_ops;
462
463 #define AMD_IOMMU_FLAG_TRANS_PRE_ENABLED (1 << 0)
464
465 struct gcr3_tbl_info {
466 u64 *gcr3_tbl; /* Guest CR3 table */
467 int glx; /* Number of levels for GCR3 table */
468 u32 pasid_cnt; /* Track attached PASIDs */
469 u16 domid; /* Per device domain ID */
470 };
471
472 enum protection_domain_mode {
473 PD_MODE_NONE,
474 PD_MODE_V1,
475 PD_MODE_V2,
476 };
477
478 /* Track dev_data/PASID list for the protection domain */
479 struct pdom_dev_data {
480 /* Points to attached device data */
481 struct iommu_dev_data *dev_data;
482 /* PASID attached to the protection domain */
483 ioasid_t pasid;
484 /* For protection_domain->dev_data_list */
485 struct list_head list;
486 };
487
488 /* Keeps track of the IOMMUs attached to protection domain */
489 struct pdom_iommu_info {
490 struct amd_iommu *iommu; /* IOMMUs attach to protection domain */
491 u32 refcnt; /* Count of attached dev/pasid per domain/IOMMU */
492 };
493
494 struct amd_iommu_viommu {
495 struct iommufd_viommu core;
496 struct protection_domain *parent; /* nest parent domain for this viommu */
497 struct list_head pdom_list; /* For protection_domain->viommu_list */
498
499 /*
500 * Per-vIOMMU guest domain ID to host domain ID mapping.
501 * Indexed by guest domain ID.
502 */
503 struct xarray gdomid_array;
504 };
505
506 /*
507 * Contains guest domain ID mapping info,
508 * which is stored in the struct xarray gdomid_array.
509 */
510 struct guest_domain_mapping_info {
511 refcount_t users;
512 u32 hdom_id; /* Host domain ID */
513 };
514
515 /*
516 * Nested domain is specifically used for nested translation
517 */
518 struct nested_domain {
519 struct iommu_domain domain; /* generic domain handle used by iommu core code */
520 u16 gdom_id; /* domain ID from gDTE */
521 struct guest_domain_mapping_info *gdom_info;
522 struct iommu_hwpt_amd_guest gdte; /* Guest vIOMMU DTE */
523 struct amd_iommu_viommu *viommu; /* AMD hw-viommu this nested domain belong to */
524 };
525
526 /*
527 * This structure contains generic data for IOMMU protection domains
528 * independent of their use.
529 */
530 struct protection_domain {
531 union {
532 struct iommu_domain domain;
533 struct pt_iommu iommu;
534 struct pt_iommu_amdv1 amdv1;
535 struct pt_iommu_x86_64 amdv2;
536 };
537 struct list_head dev_list; /* List of all devices in this domain */
538 spinlock_t lock; /* mostly used to lock the page table*/
539 u16 id; /* the domain id written to the device table */
540 enum protection_domain_mode pd_mode; /* Track page table type */
541 bool dirty_tracking; /* dirty tracking is enabled in the domain */
542 struct xarray iommu_array; /* per-IOMMU reference count */
543
544 struct mmu_notifier mn; /* mmu notifier for the SVA domain */
545 struct list_head dev_data_list; /* List of pdom_dev_data */
546
547 /*
548 * Store reference to list of vIOMMUs, which use this protection domain.
549 * This will be used to look up host domain ID when flushing this domain.
550 */
551 struct list_head viommu_list;
552 };
553 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, iommu, domain);
554 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv1.iommu, domain);
555 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv2.iommu, domain);
556
557 /*
558 * This structure contains information about one PCI segment in the system.
559 */
560 struct amd_iommu_pci_seg {
561 /* List with all PCI segments in the system */
562 struct list_head list;
563
564 /* List of all available dev_data structures */
565 struct llist_head dev_data_list;
566
567 /* PCI segment number */
568 u16 id;
569
570 /* Largest PCI device id we expect translation requests for */
571 u16 last_bdf;
572
573 /* Size of the device table */
574 u32 dev_table_size;
575
576 /*
577 * device table virtual address
578 *
579 * Pointer to the per PCI segment device table.
580 * It is indexed by the PCI device id or the HT unit id and contains
581 * information about the domain the device belongs to as well as the
582 * page table root pointer.
583 */
584 struct dev_table_entry *dev_table;
585
586 /*
587 * The rlookup iommu table is used to find the IOMMU which is
588 * responsible for a specific device. It is indexed by the PCI
589 * device id.
590 */
591 struct amd_iommu **rlookup_table;
592
593 /*
594 * This table is used to find the irq remapping table for a given
595 * device id quickly.
596 */
597 struct irq_remap_table **irq_lookup_table;
598
599 /*
600 * Pointer to a device table which the content of old device table
601 * will be copied to. It's only be used in kdump kernel.
602 */
603 struct dev_table_entry *old_dev_tbl_cpy;
604
605 /*
606 * The alias table is a driver specific data structure which contains the
607 * mappings of the PCI device ids to the actual requestor ids on the IOMMU.
608 * More than one device can share the same requestor id.
609 */
610 u16 *alias_table;
611
612 /*
613 * A list of required unity mappings we find in ACPI. It is not locked
614 * because as runtime it is only read. It is created at ACPI table
615 * parsing time.
616 */
617 struct list_head unity_map;
618 };
619
620 /*
621 * Structure where we save information about one hardware AMD IOMMU in the
622 * system.
623 */
624 struct amd_iommu {
625 struct list_head list;
626
627 /* Index within the IOMMU array */
628 int index;
629
630 /* locks the accesses to the hardware */
631 raw_spinlock_t lock;
632
633 /* Pointer to PCI device of this IOMMU */
634 struct pci_dev *dev;
635
636 /* Cache pdev to root device for resume quirks */
637 struct pci_dev *root_pdev;
638
639 /* physical address of MMIO space */
640 u64 mmio_phys;
641
642 /* physical end address of MMIO space */
643 u64 mmio_phys_end;
644
645 /* virtual address of MMIO space */
646 u8 __iomem *mmio_base;
647
648 /* capabilities of that IOMMU read from ACPI */
649 u32 cap;
650
651 /* flags read from acpi table */
652 u8 acpi_flags;
653
654 /* Extended features */
655 u64 features;
656
657 /* Extended features 2 */
658 u64 features2;
659
660 /* PCI device id of the IOMMU device */
661 u16 devid;
662
663 /*
664 * Capability pointer. There could be more than one IOMMU per PCI
665 * device function if there are more than one AMD IOMMU capability
666 * pointers.
667 */
668 u16 cap_ptr;
669
670 /* pci domain of this IOMMU */
671 struct amd_iommu_pci_seg *pci_seg;
672
673 /* command buffer virtual address */
674 u8 *cmd_buf;
675 u32 cmd_buf_head;
676 u32 cmd_buf_tail;
677
678 /* event buffer virtual address */
679 u8 *evt_buf;
680
681 /* Name for event log interrupt */
682 unsigned char evt_irq_name[16];
683
684 /* Base of the PPR log, if present */
685 u8 *ppr_log;
686
687 /* Name for PPR log interrupt */
688 unsigned char ppr_irq_name[16];
689
690 /* Base of the GA log, if present */
691 u8 *ga_log;
692
693 /* Name for GA log interrupt */
694 unsigned char ga_irq_name[16];
695
696 /* Tail of the GA log, if present */
697 u8 *ga_log_tail;
698
699 /* true if interrupts for this IOMMU are already enabled */
700 bool int_enabled;
701
702 /* if one, we need to send a completion wait command */
703 bool need_sync;
704
705 /* true if disable irte caching */
706 bool irtcachedis_enabled;
707
708 /* Handle for IOMMU core code */
709 struct iommu_device iommu;
710
711 /*
712 * We can't rely on the BIOS to restore all values on reinit, so we
713 * need to stash them
714 */
715
716 /* The iommu BAR */
717 u32 stored_addr_lo;
718 u32 stored_addr_hi;
719
720 /*
721 * Each iommu has 6 l1s, each of which is documented as having 0x12
722 * registers
723 */
724 u32 stored_l1[6][0x12];
725
726 /* The l2 indirect registers */
727 u32 stored_l2[0x83];
728
729 /* The maximum PC banks and counters/bank (PCSup=1) */
730 u8 max_banks;
731 u8 max_counters;
732 #ifdef CONFIG_IRQ_REMAP
733 struct irq_domain *ir_domain;
734
735 struct amd_irte_ops *irte_ops;
736 #endif
737
738 u32 flags;
739 volatile u64 *cmd_sem;
740 u64 cmd_sem_val;
741 /*
742 * Track physical address to directly use it in build_completion_wait()
743 * and avoid adding any special checks and handling for kdump.
744 */
745 u64 cmd_sem_paddr;
746
747 #ifdef CONFIG_AMD_IOMMU_DEBUGFS
748 /* DebugFS Info */
749 struct dentry *debugfs;
750 int dbg_mmio_offset;
751 int dbg_cap_offset;
752 #endif
753
754 /* IOPF support */
755 struct iopf_queue *iopf_queue;
756 unsigned char iopfq_name[32];
757 };
758
dev_to_amd_iommu(struct device * dev)759 static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev)
760 {
761 struct iommu_device *iommu = dev_to_iommu_device(dev);
762
763 return container_of(iommu, struct amd_iommu, iommu);
764 }
765
766 #define ACPIHID_UID_LEN 256
767 #define ACPIHID_HID_LEN 9
768
769 struct acpihid_map_entry {
770 struct list_head list;
771 u8 uid[ACPIHID_UID_LEN];
772 u8 hid[ACPIHID_HID_LEN];
773 u32 devid;
774 u32 root_devid;
775 bool cmd_line;
776 struct iommu_group *group;
777 };
778
779 struct devid_map {
780 struct list_head list;
781 u8 id;
782 u32 devid;
783 bool cmd_line;
784 };
785
786 #define AMD_IOMMU_DEVICE_FLAG_ATS_SUP 0x1 /* ATS feature supported */
787 #define AMD_IOMMU_DEVICE_FLAG_PRI_SUP 0x2 /* PRI feature supported */
788 #define AMD_IOMMU_DEVICE_FLAG_PASID_SUP 0x4 /* PASID context supported */
789 /* Device may request execution on memory pages */
790 #define AMD_IOMMU_DEVICE_FLAG_EXEC_SUP 0x8
791 /* Device may request super-user privileges */
792 #define AMD_IOMMU_DEVICE_FLAG_PRIV_SUP 0x10
793
794 /*
795 * This struct contains device specific data for the IOMMU
796 */
797 struct iommu_dev_data {
798 /*Protect against attach/detach races */
799 struct mutex mutex;
800 spinlock_t dte_lock; /* DTE lock for 256-bit access */
801
802 struct list_head list; /* For domain->dev_list */
803 struct llist_node dev_data_list; /* For global dev_data_list */
804 struct protection_domain *domain; /* Domain the device is bound to */
805 struct gcr3_tbl_info gcr3_info; /* Per-device GCR3 table */
806 struct device *dev;
807 u16 devid; /* PCI Device ID */
808
809 unsigned int max_irqs; /* Maximum IRQs supported by device */
810 u32 max_pasids; /* Max supported PASIDs */
811 u32 flags; /* Holds AMD_IOMMU_DEVICE_FLAG_<*> */
812 int ats_qdep;
813 u8 ats_enabled :1; /* ATS state */
814 u8 pri_enabled :1; /* PRI state */
815 u8 pasid_enabled:1; /* PASID state */
816 u8 pri_tlp :1; /* PASID TLB required for
817 PPR completions */
818 u8 ppr :1; /* Enable device PPR support */
819 bool use_vapic; /* Enable device to use vapic mode */
820 bool defer_attach;
821
822 struct ratelimit_state rs; /* Ratelimit IOPF messages */
823 };
824
825 /* Map HPET and IOAPIC ids to the devid used by the IOMMU */
826 extern struct list_head ioapic_map;
827 extern struct list_head hpet_map;
828 extern struct list_head acpihid_map;
829
830 /*
831 * List with all PCI segments in the system. This list is not locked because
832 * it is only written at driver initialization time
833 */
834 extern struct list_head amd_iommu_pci_seg_list;
835
836 /*
837 * List with all IOMMUs in the system. This list is not locked because it is
838 * only written and read at driver initialization or suspend time
839 */
840 extern struct list_head amd_iommu_list;
841
842 /*
843 * Structure defining one entry in the device table
844 */
845 struct dev_table_entry {
846 union {
847 u64 data[4];
848 u128 data128[2];
849 };
850 };
851
852 /*
853 * Structure defining one entry in the command buffer
854 */
855 struct iommu_cmd {
856 u32 data[4];
857 };
858
859 /*
860 * Structure to sture persistent DTE flags from IVHD
861 */
862 struct ivhd_dte_flags {
863 struct list_head list;
864 u16 segid;
865 u16 devid_first;
866 u16 devid_last;
867 struct dev_table_entry dte;
868 };
869
870 /*
871 * One entry for unity mappings parsed out of the ACPI table.
872 */
873 struct unity_map_entry {
874 struct list_head list;
875
876 /* starting device id this entry is used for (including) */
877 u16 devid_start;
878 /* end device id this entry is used for (including) */
879 u16 devid_end;
880
881 /* start address to unity map (including) */
882 u64 address_start;
883 /* end address to unity map (including) */
884 u64 address_end;
885
886 /* required protection */
887 int prot;
888 };
889
890 /*
891 * Data structures for device handling
892 */
893
894 extern bool amd_iommu_force_isolation;
895
896 /* Max levels of glxval supported */
897 extern int amd_iommu_max_glx_val;
898
899 /* IDA to track protection domain IDs */
900 extern struct ida pdom_ids;
901
902 /* Global EFR and EFR2 registers */
903 extern u64 amd_iommu_efr;
904 extern u64 amd_iommu_efr2;
905
get_ioapic_devid(int id)906 static inline int get_ioapic_devid(int id)
907 {
908 struct devid_map *entry;
909
910 list_for_each_entry(entry, &ioapic_map, list) {
911 if (entry->id == id)
912 return entry->devid;
913 }
914
915 return -EINVAL;
916 }
917
get_hpet_devid(int id)918 static inline int get_hpet_devid(int id)
919 {
920 struct devid_map *entry;
921
922 list_for_each_entry(entry, &hpet_map, list) {
923 if (entry->id == id)
924 return entry->devid;
925 }
926
927 return -EINVAL;
928 }
929
930 enum amd_iommu_intr_mode_type {
931 /*
932 * The legacy format mode is not visible to users to prevent the user
933 * from crashing x2APIC systems, which for all intents and purposes
934 * require 128-bit IRTEs. The legacy format will be forced as needed
935 * when hardware doesn't support 128-bit IRTEs.
936 */
937 AMD_IOMMU_GUEST_IR_LEGACY,
938 AMD_IOMMU_GUEST_IR_LEGACY_GA,
939 AMD_IOMMU_GUEST_IR_VAPIC,
940 };
941
942 #define AMD_IOMMU_GUEST_IR_GA(x) (x == AMD_IOMMU_GUEST_IR_VAPIC || \
943 x == AMD_IOMMU_GUEST_IR_LEGACY_GA)
944
945 #define AMD_IOMMU_GUEST_IR_VAPIC(x) (x == AMD_IOMMU_GUEST_IR_VAPIC)
946
947 union irte {
948 u32 val;
949 struct {
950 u32 valid : 1,
951 no_fault : 1,
952 int_type : 3,
953 rq_eoi : 1,
954 dm : 1,
955 rsvd_1 : 1,
956 destination : 8,
957 vector : 8,
958 rsvd_2 : 8;
959 } fields;
960 };
961
962 #define APICID_TO_IRTE_DEST_LO(x) (x & 0xffffff)
963 #define APICID_TO_IRTE_DEST_HI(x) ((x >> 24) & 0xff)
964
965 union irte_ga_lo {
966 u64 val;
967
968 /* For int remapping */
969 struct {
970 u64 valid : 1,
971 no_fault : 1,
972 /* ------ */
973 int_type : 3,
974 rq_eoi : 1,
975 dm : 1,
976 /* ------ */
977 guest_mode : 1,
978 destination : 24,
979 ga_tag : 32;
980 } fields_remap;
981
982 /* For guest vAPIC */
983 struct {
984 u64 valid : 1,
985 no_fault : 1,
986 /* ------ */
987 ga_log_intr : 1,
988 rsvd1 : 3,
989 is_run : 1,
990 /* ------ */
991 guest_mode : 1,
992 destination : 24,
993 ga_tag : 32;
994 } fields_vapic;
995 };
996
997 union irte_ga_hi {
998 u64 val;
999 struct {
1000 u64 vector : 8,
1001 rsvd_1 : 4,
1002 ga_root_ptr : 40,
1003 rsvd_2 : 4,
1004 destination : 8;
1005 } fields;
1006 };
1007
1008 struct irte_ga {
1009 union {
1010 struct {
1011 union irte_ga_lo lo;
1012 union irte_ga_hi hi;
1013 };
1014 u128 irte;
1015 };
1016 };
1017
1018 struct irq_2_irte {
1019 u16 devid; /* Device ID for IRTE table */
1020 u16 index; /* Index into IRTE table*/
1021 };
1022
1023 struct amd_ir_data {
1024 struct amd_iommu *iommu;
1025 struct irq_2_irte irq_2_irte;
1026 struct msi_msg msi_entry;
1027 void *entry; /* Pointer to union irte or struct irte_ga */
1028
1029 /**
1030 * Store information for activate/de-activate
1031 * Guest virtual APIC mode during runtime.
1032 */
1033 struct irq_cfg *cfg;
1034 int ga_vector;
1035 u64 ga_root_ptr;
1036 u32 ga_tag;
1037 };
1038
1039 struct amd_irte_ops {
1040 void (*prepare)(void *, u32, bool, u8, u32, int);
1041 void (*activate)(struct amd_iommu *iommu, void *, u16, u16);
1042 void (*deactivate)(struct amd_iommu *iommu, void *, u16, u16);
1043 void (*set_affinity)(struct amd_iommu *iommu, void *, u16, u16, u8, u32);
1044 void *(*get)(struct irq_remap_table *, int);
1045 void (*set_allocated)(struct irq_remap_table *, int);
1046 bool (*is_allocated)(struct irq_remap_table *, int);
1047 void (*clear_allocated)(struct irq_remap_table *, int);
1048 };
1049
1050 #ifdef CONFIG_IRQ_REMAP
1051 extern struct amd_irte_ops irte_32_ops;
1052 extern struct amd_irte_ops irte_128_ops;
1053 #endif
1054
1055 #endif /* _ASM_X86_AMD_IOMMU_TYPES_H */
1056