1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 * Leo Duran <leo.duran@amd.com>
6 */
7
8 #ifndef _ASM_X86_AMD_IOMMU_TYPES_H
9 #define _ASM_X86_AMD_IOMMU_TYPES_H
10
11 #include <linux/bitfield.h>
12 #include <linux/iommu.h>
13 #include <linux/types.h>
14 #include <linux/mmu_notifier.h>
15 #include <linux/mutex.h>
16 #include <linux/msi.h>
17 #include <linux/list.h>
18 #include <linux/sizes.h>
19 #include <linux/spinlock.h>
20 #include <linux/pci.h>
21 #include <linux/iommufd.h>
22 #include <linux/irqreturn.h>
23 #include <linux/generic_pt/iommu.h>
24
25 #include <uapi/linux/iommufd.h>
26
27 /*
28 * Maximum number of IOMMUs supported
29 */
30 #define MAX_IOMMUS 32
31
32 /*
33 * some size calculation constants
34 */
35 #define DEV_TABLE_ENTRY_SIZE 32
36
37 /* Capability offsets used by the driver */
38 #define MMIO_CAP_HDR_OFFSET 0x00
39 #define MMIO_RANGE_OFFSET 0x0c
40 #define MMIO_MISC_OFFSET 0x10
41
42 /* Used offsets into the MMIO space */
43 #define MMIO_DEV_TABLE_OFFSET 0x0000
44 #define MMIO_CMD_BUF_OFFSET 0x0008
45 #define MMIO_EVT_BUF_OFFSET 0x0010
46 #define MMIO_CONTROL_OFFSET 0x0018
47 #define MMIO_EXCL_BASE_OFFSET 0x0020
48 #define MMIO_EXCL_LIMIT_OFFSET 0x0028
49 #define MMIO_EXT_FEATURES 0x0030
50 #define MMIO_PPR_LOG_OFFSET 0x0038
51 #define MMIO_GA_LOG_BASE_OFFSET 0x00e0
52 #define MMIO_GA_LOG_TAIL_OFFSET 0x00e8
53 #define MMIO_MSI_ADDR_LO_OFFSET 0x015C
54 #define MMIO_MSI_ADDR_HI_OFFSET 0x0160
55 #define MMIO_MSI_DATA_OFFSET 0x0164
56 #define MMIO_INTCAPXT_EVT_OFFSET 0x0170
57 #define MMIO_INTCAPXT_PPR_OFFSET 0x0178
58 #define MMIO_INTCAPXT_GALOG_OFFSET 0x0180
59 #define MMIO_EXT_FEATURES2 0x01A0
60 #define MMIO_CMD_HEAD_OFFSET 0x2000
61 #define MMIO_CMD_TAIL_OFFSET 0x2008
62 #define MMIO_EVT_HEAD_OFFSET 0x2010
63 #define MMIO_EVT_TAIL_OFFSET 0x2018
64 #define MMIO_STATUS_OFFSET 0x2020
65 #define MMIO_PPR_HEAD_OFFSET 0x2030
66 #define MMIO_PPR_TAIL_OFFSET 0x2038
67 #define MMIO_GA_HEAD_OFFSET 0x2040
68 #define MMIO_GA_TAIL_OFFSET 0x2048
69 #define MMIO_CNTR_CONF_OFFSET 0x4000
70 #define MMIO_CNTR_REG_OFFSET 0x40000
71 #define MMIO_REG_END_OFFSET 0x80000
72
73
74
75 /* Extended Feature Bits */
76 #define FEATURE_PREFETCH BIT_ULL(0)
77 #define FEATURE_PPR BIT_ULL(1)
78 #define FEATURE_X2APIC BIT_ULL(2)
79 #define FEATURE_NX BIT_ULL(3)
80 #define FEATURE_GT BIT_ULL(4)
81 #define FEATURE_IA BIT_ULL(6)
82 #define FEATURE_GA BIT_ULL(7)
83 #define FEATURE_HE BIT_ULL(8)
84 #define FEATURE_PC BIT_ULL(9)
85 #define FEATURE_HATS GENMASK_ULL(11, 10)
86 #define FEATURE_GATS GENMASK_ULL(13, 12)
87 #define FEATURE_GLX GENMASK_ULL(15, 14)
88 #define FEATURE_GAM_VAPIC BIT_ULL(21)
89 #define FEATURE_PASMAX GENMASK_ULL(36, 32)
90 #define FEATURE_GIOSUP BIT_ULL(48)
91 #define FEATURE_HASUP BIT_ULL(49)
92 #define FEATURE_EPHSUP BIT_ULL(50)
93 #define FEATURE_HDSUP BIT_ULL(52)
94 #define FEATURE_SNP BIT_ULL(63)
95
96
97 /* Extended Feature 2 Bits */
98 #define FEATURE_SEVSNPIO_SUP BIT_ULL(1)
99 #define FEATURE_GCR3TRPMODE BIT_ULL(3)
100 #define FEATURE_SNPAVICSUP GENMASK_ULL(7, 5)
101 #define FEATURE_SNPAVICSUP_GAM(x) \
102 (FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1)
103 #define FEATURE_HT_RANGE_IGNORE BIT_ULL(11)
104 #define FEATURE_SNP_PAGE_MODE0_SUP BIT_ULL(13)
105
106 #define FEATURE_NUM_INT_REMAP_SUP GENMASK_ULL(9, 8)
107 #define FEATURE_NUM_INT_REMAP_SUP_2K(x) \
108 (FIELD_GET(FEATURE_NUM_INT_REMAP_SUP, x) == 0x1)
109
110 /* Note:
111 * The current driver only support 16-bit PASID.
112 * Currently, hardware only implement upto 16-bit PASID
113 * even though the spec says it could have upto 20 bits.
114 */
115 #define PASID_MASK 0x0000ffff
116
117 /* MMIO status bits */
118 #define MMIO_STATUS_EVT_OVERFLOW_MASK BIT(0)
119 #define MMIO_STATUS_EVT_INT_MASK BIT(1)
120 #define MMIO_STATUS_COM_WAIT_INT_MASK BIT(2)
121 #define MMIO_STATUS_EVT_RUN_MASK BIT(3)
122 #define MMIO_STATUS_PPR_OVERFLOW_MASK BIT(5)
123 #define MMIO_STATUS_PPR_INT_MASK BIT(6)
124 #define MMIO_STATUS_PPR_RUN_MASK BIT(7)
125 #define MMIO_STATUS_GALOG_RUN_MASK BIT(8)
126 #define MMIO_STATUS_GALOG_OVERFLOW_MASK BIT(9)
127 #define MMIO_STATUS_GALOG_INT_MASK BIT(10)
128
129 /* event logging constants */
130 #define EVENT_TYPE_SHIFT 28
131 #define EVENT_TYPE_MASK 0xf
132 #define EVENT_TYPE_ILL_DEV 0x1
133 #define EVENT_TYPE_IO_FAULT 0x2
134 #define EVENT_TYPE_DEV_TAB_ERR 0x3
135 #define EVENT_TYPE_PAGE_TAB_ERR 0x4
136 #define EVENT_TYPE_ILL_CMD 0x5
137 #define EVENT_TYPE_CMD_HARD_ERR 0x6
138 #define EVENT_TYPE_IOTLB_INV_TO 0x7
139 #define EVENT_TYPE_INV_DEV_REQ 0x8
140 #define EVENT_TYPE_INV_PPR_REQ 0x9
141 #define EVENT_TYPE_RMP_FAULT 0xd
142 #define EVENT_TYPE_RMP_HW_ERR 0xe
143 #define EVENT_DEVID_MASK 0xffff
144 #define EVENT_DEVID_SHIFT 0
145 #define EVENT_DOMID_MASK_LO 0xffff
146 #define EVENT_DOMID_MASK_HI 0xf0000
147 #define EVENT_FLAGS_MASK 0xfff
148 #define EVENT_FLAGS_SHIFT 0x10
149 #define EVENT_FLAG_RW 0x020
150 #define EVENT_FLAG_I 0x008
151 #define EVENT_FLAG_PPR_RX 0x001
152 #define EVENT_FLAG_PPR_GN 0x200
153
154 /* feature control bits */
155 #define CONTROL_IOMMU_EN 0
156 #define CONTROL_HT_TUN_EN 1
157 #define CONTROL_EVT_LOG_EN 2
158 #define CONTROL_EVT_INT_EN 3
159 #define CONTROL_COMWAIT_EN 4
160 #define CONTROL_INV_TIMEOUT 5
161 #define CONTROL_PASSPW_EN 8
162 #define CONTROL_RESPASSPW_EN 9
163 #define CONTROL_COHERENT_EN 10
164 #define CONTROL_ISOC_EN 11
165 #define CONTROL_CMDBUF_EN 12
166 #define CONTROL_PPRLOG_EN 13
167 #define CONTROL_PPRINT_EN 14
168 #define CONTROL_PPR_EN 15
169 #define CONTROL_GT_EN 16
170 #define CONTROL_GA_EN 17
171 #define CONTROL_GAM_EN 25
172 #define CONTROL_GALOG_EN 28
173 #define CONTROL_GAINT_EN 29
174 #define CONTROL_NUM_INT_REMAP_MODE 43
175 #define CONTROL_NUM_INT_REMAP_MODE_MASK 0x03
176 #define CONTROL_NUM_INT_REMAP_MODE_2K 0x01
177 #define CONTROL_EPH_EN 45
178 #define CONTROL_XT_EN 50
179 #define CONTROL_INTCAPXT_EN 51
180 #define CONTROL_GCR3TRPMODE 58
181 #define CONTROL_IRTCACHEDIS 59
182 #define CONTROL_SNPAVIC_EN 61
183
184 #define CTRL_INV_TO_MASK 7
185 #define CTRL_INV_TO_NONE 0
186 #define CTRL_INV_TO_1MS 1
187 #define CTRL_INV_TO_10MS 2
188 #define CTRL_INV_TO_100MS 3
189 #define CTRL_INV_TO_1S 4
190 #define CTRL_INV_TO_10S 5
191 #define CTRL_INV_TO_100S 6
192
193 /* command specific defines */
194 #define CMD_COMPL_WAIT 0x01
195 #define CMD_INV_DEV_ENTRY 0x02
196 #define CMD_INV_IOMMU_PAGES 0x03
197 #define CMD_INV_IOTLB_PAGES 0x04
198 #define CMD_INV_IRT 0x05
199 #define CMD_COMPLETE_PPR 0x07
200 #define CMD_INV_ALL 0x08
201
202 #define CMD_COMPL_WAIT_STORE_MASK 0x01
203 #define CMD_COMPL_WAIT_INT_MASK 0x02
204 #define CMD_INV_IOMMU_PAGES_SIZE_MASK 0x01
205 #define CMD_INV_IOMMU_PAGES_PDE_MASK 0x02
206 #define CMD_INV_IOMMU_PAGES_GN_MASK 0x04
207
208 #define PPR_STATUS_MASK 0xf
209 #define PPR_STATUS_SHIFT 12
210
211 #define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7ffffffffffff000ULL
212
213 /* macros and definitions for device table entries */
214 #define DEV_ENTRY_VALID 0x00
215 #define DEV_ENTRY_TRANSLATION 0x01
216 #define DEV_ENTRY_HAD 0x07
217 #define DEV_ENTRY_PPR 0x34
218 #define DEV_ENTRY_IR 0x3d
219 #define DEV_ENTRY_IW 0x3e
220 #define DEV_ENTRY_NO_PAGE_FAULT 0x62
221 #define DEV_ENTRY_SYSMGT1 0x68
222 #define DEV_ENTRY_SYSMGT2 0x69
223 #define DTE_DATA1_SYSMGT_MASK GENMASK_ULL(41, 40)
224
225 #define DEV_ENTRY_IRQ_TBL_EN 0x80
226 #define DEV_ENTRY_INIT_PASS 0xb8
227 #define DEV_ENTRY_EINT_PASS 0xb9
228 #define DEV_ENTRY_NMI_PASS 0xba
229 #define DEV_ENTRY_LINT0_PASS 0xbe
230 #define DEV_ENTRY_LINT1_PASS 0xbf
231 #define DEV_ENTRY_MODE_MASK 0x07
232 #define DEV_ENTRY_MODE_SHIFT 0x09
233
234 #define MAX_DEV_TABLE_ENTRIES 0xffff
235
236 /* constants to configure the command buffer */
237 #define CMD_BUFFER_SIZE 8192
238 #define CMD_BUFFER_ENTRIES 512
239 #define MMIO_CMD_SIZE_SHIFT 56
240 #define MMIO_CMD_SIZE_512 (0x9ULL << MMIO_CMD_SIZE_SHIFT)
241 #define MMIO_CMD_HEAD_MASK GENMASK_ULL(18, 4) /* Command buffer head ptr field [18:4] */
242 #define MMIO_CMD_BUFFER_HEAD(x) FIELD_GET(MMIO_CMD_HEAD_MASK, (x))
243 #define MMIO_CMD_TAIL_MASK GENMASK_ULL(18, 4) /* Command buffer tail ptr field [18:4] */
244 #define MMIO_CMD_BUFFER_TAIL(x) FIELD_GET(MMIO_CMD_TAIL_MASK, (x))
245
246 /* constants for event buffer handling */
247 #define EVTLOG_ENTRY_SIZE 0x10
248 #define EVTLOG_SIZE_SHIFT 56
249 #define EVTLOG_SIZE_DEF SZ_8K /* 512 entries */
250 #define EVTLOG_LEN_MASK_DEF (0x9ULL << EVTLOG_SIZE_SHIFT)
251 #define EVTLOG_SIZE_MAX SZ_512K /* 32K entries */
252 #define EVTLOG_LEN_MASK_MAX (0xFULL << EVTLOG_SIZE_SHIFT)
253
254 /* Constants for PPR Log handling */
255 #define PPRLOG_ENTRY_SIZE 0x10
256 #define PPRLOG_SIZE_SHIFT 56
257 #define PPRLOG_SIZE_DEF SZ_8K /* 512 entries */
258 #define PPRLOG_LEN_MASK_DEF (0x9ULL << PPRLOG_SIZE_SHIFT)
259 #define PPRLOG_SIZE_MAX SZ_512K /* 32K entries */
260 #define PPRLOG_LEN_MASK_MAX (0xFULL << PPRLOG_SIZE_SHIFT)
261
262 /* PAGE_SERVICE_REQUEST PPR Log Buffer Entry flags */
263 #define PPR_FLAG_EXEC 0x002 /* Execute permission requested */
264 #define PPR_FLAG_READ 0x004 /* Read permission requested */
265 #define PPR_FLAG_WRITE 0x020 /* Write permission requested */
266 #define PPR_FLAG_US 0x040 /* 1: User, 0: Supervisor */
267 #define PPR_FLAG_RVSD 0x080 /* Reserved bit not zero */
268 #define PPR_FLAG_GN 0x100 /* GVA and PASID is valid */
269
270 #define PPR_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
271 #define PPR_FLAGS(x) (((x) >> 48) & 0xfffULL)
272 #define PPR_DEVID(x) ((x) & 0xffffULL)
273 #define PPR_TAG(x) (((x) >> 32) & 0x1ffULL)
274 #define PPR_TAG_LAST_PAGE(x) (((x) >> 32) & 0x200ULL)
275 #define PPR_PASID1(x) (((x) >> 16) & 0xffffULL)
276 #define PPR_PASID2(x) (((x) >> 42) & 0xfULL)
277 #define PPR_PASID(x) ((PPR_PASID2(x) << 16) | PPR_PASID1(x))
278
279 #define PPR_REQ_FAULT 0x01
280
281 /* Constants for GA Log handling */
282 #define GA_LOG_ENTRIES 512
283 #define GA_LOG_SIZE_SHIFT 56
284 #define GA_LOG_SIZE_512 (0x8ULL << GA_LOG_SIZE_SHIFT)
285 #define GA_ENTRY_SIZE 8
286 #define GA_LOG_SIZE (GA_ENTRY_SIZE * GA_LOG_ENTRIES)
287
288 #define GA_TAG(x) (u32)(x & 0xffffffffULL)
289 #define GA_DEVID(x) (u16)(((x) >> 32) & 0xffffULL)
290 #define GA_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
291
292 #define GA_GUEST_NR 0x1
293
294 /*
295 * This bitmap is used to advertise the page sizes our hardware support
296 * to the IOMMU core, which will then use this information to split
297 * physically contiguous memory regions it is mapping into page sizes
298 * that we support.
299 *
300 * 512GB Pages are not supported due to a hardware bug
301 * Page sizes >= the 52 bit max physical address of the CPU are not supported.
302 */
303 #define AMD_IOMMU_PGSIZES (GENMASK_ULL(51, 12) ^ SZ_512G)
304
305 /* Special mode where page-sizes are limited to 4 KiB */
306 #define AMD_IOMMU_PGSIZES_4K (PAGE_SIZE)
307
308 /* 4K, 2MB, 1G page sizes are supported */
309 #define AMD_IOMMU_PGSIZES_V2 (PAGE_SIZE | (1ULL << 21) | (1ULL << 30))
310
311 /* Bit value definition for dte irq remapping fields*/
312 #define DTE_IRQ_PHYS_ADDR_MASK GENMASK_ULL(51, 6)
313 #define DTE_IRQ_REMAP_INTCTL_MASK (0x3ULL << 60)
314 #define DTE_IRQ_REMAP_INTCTL (2ULL << 60)
315 #define DTE_IRQ_REMAP_ENABLE 1ULL
316
317 #define DTE_INTTAB_ALIGNMENT 128
318 #define DTE_INTTABLEN_MASK (0xfULL << 1)
319 #define DTE_INTTABLEN_VALUE_512 9ULL
320 #define DTE_INTTABLEN_512 (DTE_INTTABLEN_VALUE_512 << 1)
321 #define MAX_IRQS_PER_TABLE_512 BIT(DTE_INTTABLEN_VALUE_512)
322 #define DTE_INTTABLEN_VALUE_2K 11ULL
323 #define DTE_INTTABLEN_2K (DTE_INTTABLEN_VALUE_2K << 1)
324 #define MAX_IRQS_PER_TABLE_2K BIT(DTE_INTTABLEN_VALUE_2K)
325
326 #define PAGE_MODE_NONE 0x00
327 #define PAGE_MODE_1_LEVEL 0x01
328 #define PAGE_MODE_2_LEVEL 0x02
329 #define PAGE_MODE_3_LEVEL 0x03
330 #define PAGE_MODE_4_LEVEL 0x04
331 #define PAGE_MODE_5_LEVEL 0x05
332 #define PAGE_MODE_6_LEVEL 0x06
333 #define PAGE_MODE_7_LEVEL 0x07
334
335 #define GUEST_PGTABLE_4_LEVEL 0x00
336 #define GUEST_PGTABLE_5_LEVEL 0x01
337
338 #define PM_ADDR_MASK 0x000ffffffffff000ULL
339
340 /*
341 * Bit value definition for DTE fields
342 */
343 #define DTE_FLAG_V BIT_ULL(0)
344 #define DTE_FLAG_TV BIT_ULL(1)
345 #define DTE_FLAG_HAD (3ULL << 7)
346 #define DTE_MODE_MASK GENMASK_ULL(11, 9)
347 #define DTE_HOST_TRP GENMASK_ULL(51, 12)
348 #define DTE_FLAG_PPR BIT_ULL(52)
349 #define DTE_FLAG_GIOV BIT_ULL(54)
350 #define DTE_FLAG_GV BIT_ULL(55)
351 #define DTE_GLX GENMASK_ULL(57, 56)
352 #define DTE_FLAG_IR BIT_ULL(61)
353 #define DTE_FLAG_IW BIT_ULL(62)
354
355 #define DTE_FLAG_IOTLB BIT_ULL(32)
356 #define DTE_FLAG_MASK (0x3ffULL << 32)
357 #define DTE_DOMID_MASK GENMASK_ULL(15, 0)
358
359 #define DTE_GCR3_14_12 GENMASK_ULL(60, 58)
360 #define DTE_GCR3_30_15 GENMASK_ULL(31, 16)
361 #define DTE_GCR3_51_31 GENMASK_ULL(63, 43)
362
363 #define DTE_GPT_LEVEL_SHIFT 54
364 #define DTE_GPT_LEVEL_MASK GENMASK_ULL(55, 54)
365
366 #define GCR3_VALID 0x01ULL
367
368 /* DTE[128:179] | DTE[184:191] */
369 #define DTE_DATA2_INTR_MASK ~GENMASK_ULL(55, 52)
370
371 #define IOMMU_PROT_MASK 0x03
372 #define IOMMU_PROT_IR 0x01
373 #define IOMMU_PROT_IW 0x02
374
375 /* IOMMU capabilities */
376 #define IOMMU_CAP_IOTLB 24
377 #define IOMMU_CAP_NPCACHE 26
378 #define IOMMU_CAP_EFR 27
379
380 /* IOMMU IVINFO */
381 #define IOMMU_IVINFO_OFFSET 36
382 #define IOMMU_IVINFO_EFRSUP BIT(0)
383 #define IOMMU_IVINFO_DMA_REMAP BIT(1)
384 #define IOMMU_IVINFO_VASIZE GENMASK_ULL(21, 15)
385
386 /* IOMMU Feature Reporting Field (for IVHD type 10h */
387 #define IOMMU_FEAT_GASUP_SHIFT 6
388
389 /* IOMMU HATDIS for IVHD type 11h and 40h */
390 #define IOMMU_IVHD_ATTR_HATDIS_SHIFT 0
391
392 /* IOMMU Extended Feature Register (EFR) */
393 #define IOMMU_EFR_XTSUP_SHIFT 2
394 #define IOMMU_EFR_GASUP_SHIFT 7
395 #define IOMMU_EFR_MSICAPMMIOSUP_SHIFT 46
396
397 #define MAX_DOMAIN_ID 65536
398
399 /* Timeout stuff */
400 #define LOOP_TIMEOUT 100000
401 #define MMIO_STATUS_TIMEOUT 2000000
402
403 extern bool amd_iommu_dump;
404 #define DUMP_printk(format, arg...) \
405 do { \
406 if (amd_iommu_dump) \
407 pr_info(format, ## arg); \
408 } while(0);
409
410 /* SNP page mode 0 support */
411 extern bool amd_iommu_snp_mode0_sup;
412
413 /* global flag if IOMMUs cache non-present entries */
414 extern bool amd_iommu_np_cache;
415 /* Only true if all IOMMUs support device IOTLBs */
416 extern bool amd_iommu_iotlb_sup;
417
418 struct irq_remap_table {
419 raw_spinlock_t lock;
420 unsigned min_index;
421 u32 *table;
422 };
423
424 extern const struct iommu_ops amd_iommu_ops;
425
426 /* IVRS indicates that pre-boot remapping was enabled */
427 extern bool amdr_ivrs_remap_support;
428
429 #define PCI_SBDF_TO_SEGID(sbdf) (((sbdf) >> 16) & 0xffff)
430 #define PCI_SBDF_TO_DEVID(sbdf) ((sbdf) & 0xffff)
431 #define PCI_SEG_DEVID_TO_SBDF(seg, devid) ((((u32)(seg) & 0xffff) << 16) | \
432 ((devid) & 0xffff))
433
434 /* Make iterating over all pci segment easier */
435 #define for_each_pci_segment(pci_seg) \
436 list_for_each_entry((pci_seg), &amd_iommu_pci_seg_list, list)
437 #define for_each_pci_segment_safe(pci_seg, next) \
438 list_for_each_entry_safe((pci_seg), (next), &amd_iommu_pci_seg_list, list)
439 /*
440 * Make iterating over all IOMMUs easier
441 */
442 #define for_each_iommu(iommu) \
443 list_for_each_entry((iommu), &amd_iommu_list, list)
444 #define for_each_iommu_safe(iommu, next) \
445 list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list)
446 /* Making iterating over protection_domain->dev_data_list easier */
447 #define for_each_pdom_dev_data(pdom_dev_data, pdom) \
448 list_for_each_entry(pdom_dev_data, &pdom->dev_data_list, list)
449 #define for_each_pdom_dev_data_safe(pdom_dev_data, next, pdom) \
450 list_for_each_entry_safe((pdom_dev_data), (next), &pdom->dev_data_list, list)
451
452 #define for_each_ivhd_dte_flags(entry) \
453 list_for_each_entry((entry), &amd_ivhd_dev_flags_list, list)
454
455 struct amd_iommu;
456 struct iommu_domain;
457 struct irq_domain;
458 struct amd_irte_ops;
459
460 #define AMD_IOMMU_FLAG_TRANS_PRE_ENABLED (1 << 0)
461
462 struct gcr3_tbl_info {
463 u64 *gcr3_tbl; /* Guest CR3 table */
464 int glx; /* Number of levels for GCR3 table */
465 u32 pasid_cnt; /* Track attached PASIDs */
466 u16 domid; /* Per device domain ID */
467 };
468
469 enum protection_domain_mode {
470 PD_MODE_NONE,
471 PD_MODE_V1,
472 PD_MODE_V2,
473 };
474
475 /* Track dev_data/PASID list for the protection domain */
476 struct pdom_dev_data {
477 /* Points to attached device data */
478 struct iommu_dev_data *dev_data;
479 /* PASID attached to the protection domain */
480 ioasid_t pasid;
481 /* For protection_domain->dev_data_list */
482 struct list_head list;
483 };
484
485 /* Keeps track of the IOMMUs attached to protection domain */
486 struct pdom_iommu_info {
487 struct amd_iommu *iommu; /* IOMMUs attach to protection domain */
488 u32 refcnt; /* Count of attached dev/pasid per domain/IOMMU */
489 };
490
491 struct amd_iommu_viommu {
492 struct iommufd_viommu core;
493 struct protection_domain *parent; /* nest parent domain for this viommu */
494 struct list_head pdom_list; /* For protection_domain->viommu_list */
495
496 /*
497 * Per-vIOMMU guest domain ID to host domain ID mapping.
498 * Indexed by guest domain ID.
499 */
500 struct xarray gdomid_array;
501 };
502
503 /*
504 * Contains guest domain ID mapping info,
505 * which is stored in the struct xarray gdomid_array.
506 */
507 struct guest_domain_mapping_info {
508 refcount_t users;
509 u32 hdom_id; /* Host domain ID */
510 };
511
512 /*
513 * Nested domain is specifically used for nested translation
514 */
515 struct nested_domain {
516 struct iommu_domain domain; /* generic domain handle used by iommu core code */
517 u16 gdom_id; /* domain ID from gDTE */
518 struct guest_domain_mapping_info *gdom_info;
519 struct iommu_hwpt_amd_guest gdte; /* Guest vIOMMU DTE */
520 struct amd_iommu_viommu *viommu; /* AMD hw-viommu this nested domain belong to */
521 };
522
523 /*
524 * This structure contains generic data for IOMMU protection domains
525 * independent of their use.
526 */
527 struct protection_domain {
528 union {
529 struct iommu_domain domain;
530 struct pt_iommu iommu;
531 struct pt_iommu_amdv1 amdv1;
532 struct pt_iommu_x86_64 amdv2;
533 };
534 struct list_head dev_list; /* List of all devices in this domain */
535 spinlock_t lock; /* mostly used to lock the page table*/
536 u16 id; /* the domain id written to the device table */
537 enum protection_domain_mode pd_mode; /* Track page table type */
538 bool dirty_tracking; /* dirty tracking is enabled in the domain */
539 struct xarray iommu_array; /* per-IOMMU reference count */
540
541 struct mmu_notifier mn; /* mmu notifier for the SVA domain */
542 struct list_head dev_data_list; /* List of pdom_dev_data */
543
544 /*
545 * Store reference to list of vIOMMUs, which use this protection domain.
546 * This will be used to look up host domain ID when flushing this domain.
547 */
548 struct list_head viommu_list;
549 };
550 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, iommu, domain);
551 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv1.iommu, domain);
552 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv2.iommu, domain);
553
554 /*
555 * This structure contains information about one PCI segment in the system.
556 */
557 struct amd_iommu_pci_seg {
558 /* List with all PCI segments in the system */
559 struct list_head list;
560
561 /* List of all available dev_data structures */
562 struct llist_head dev_data_list;
563
564 /* PCI segment number */
565 u16 id;
566
567 /* Largest PCI device id we expect translation requests for */
568 u16 last_bdf;
569
570 /* Size of the device table */
571 u32 dev_table_size;
572
573 /*
574 * device table virtual address
575 *
576 * Pointer to the per PCI segment device table.
577 * It is indexed by the PCI device id or the HT unit id and contains
578 * information about the domain the device belongs to as well as the
579 * page table root pointer.
580 */
581 struct dev_table_entry *dev_table;
582
583 /*
584 * The rlookup iommu table is used to find the IOMMU which is
585 * responsible for a specific device. It is indexed by the PCI
586 * device id.
587 */
588 struct amd_iommu **rlookup_table;
589
590 /*
591 * This table is used to find the irq remapping table for a given
592 * device id quickly.
593 */
594 struct irq_remap_table **irq_lookup_table;
595
596 /*
597 * Pointer to a device table which the content of old device table
598 * will be copied to. It's only be used in kdump kernel.
599 */
600 struct dev_table_entry *old_dev_tbl_cpy;
601
602 /*
603 * The alias table is a driver specific data structure which contains the
604 * mappings of the PCI device ids to the actual requestor ids on the IOMMU.
605 * More than one device can share the same requestor id.
606 */
607 u16 *alias_table;
608
609 /*
610 * A list of required unity mappings we find in ACPI. It is not locked
611 * because as runtime it is only read. It is created at ACPI table
612 * parsing time.
613 */
614 struct list_head unity_map;
615 };
616
617 /*
618 * Structure where we save information about one hardware AMD IOMMU in the
619 * system.
620 */
621 struct amd_iommu {
622 struct list_head list;
623
624 /* Index within the IOMMU array */
625 int index;
626
627 /* locks the accesses to the hardware */
628 raw_spinlock_t lock;
629
630 /* Pointer to PCI device of this IOMMU */
631 struct pci_dev *dev;
632
633 /* Cache pdev to root device for resume quirks */
634 struct pci_dev *root_pdev;
635
636 /* physical address of MMIO space */
637 u64 mmio_phys;
638
639 /* physical end address of MMIO space */
640 u64 mmio_phys_end;
641
642 /* virtual address of MMIO space */
643 u8 __iomem *mmio_base;
644
645 /* capabilities of that IOMMU read from ACPI */
646 u32 cap;
647
648 /* flags read from acpi table */
649 u8 acpi_flags;
650
651 /* Extended features */
652 u64 features;
653
654 /* Extended features 2 */
655 u64 features2;
656
657 /* PCI device id of the IOMMU device */
658 u16 devid;
659
660 /*
661 * Capability pointer. There could be more than one IOMMU per PCI
662 * device function if there are more than one AMD IOMMU capability
663 * pointers.
664 */
665 u16 cap_ptr;
666
667 /* pci domain of this IOMMU */
668 struct amd_iommu_pci_seg *pci_seg;
669
670 /* command buffer virtual address */
671 u8 *cmd_buf;
672 u32 cmd_buf_head;
673 u32 cmd_buf_tail;
674
675 /* event buffer virtual address */
676 u8 *evt_buf;
677
678 /* Name for event log interrupt */
679 unsigned char evt_irq_name[16];
680
681 /* Base of the PPR log, if present */
682 u8 *ppr_log;
683
684 /* Name for PPR log interrupt */
685 unsigned char ppr_irq_name[16];
686
687 /* Base of the GA log, if present */
688 u8 *ga_log;
689
690 /* Name for GA log interrupt */
691 unsigned char ga_irq_name[16];
692
693 /* Tail of the GA log, if present */
694 u8 *ga_log_tail;
695
696 /* true if interrupts for this IOMMU are already enabled */
697 bool int_enabled;
698
699 /* if one, we need to send a completion wait command */
700 bool need_sync;
701
702 /* true if disable irte caching */
703 bool irtcachedis_enabled;
704
705 /* Handle for IOMMU core code */
706 struct iommu_device iommu;
707
708 /*
709 * We can't rely on the BIOS to restore all values on reinit, so we
710 * need to stash them
711 */
712
713 /* The iommu BAR */
714 u32 stored_addr_lo;
715 u32 stored_addr_hi;
716
717 /*
718 * Each iommu has 6 l1s, each of which is documented as having 0x12
719 * registers
720 */
721 u32 stored_l1[6][0x12];
722
723 /* The l2 indirect registers */
724 u32 stored_l2[0x83];
725
726 /* The maximum PC banks and counters/bank (PCSup=1) */
727 u8 max_banks;
728 u8 max_counters;
729 #ifdef CONFIG_IRQ_REMAP
730 struct irq_domain *ir_domain;
731
732 struct amd_irte_ops *irte_ops;
733 #endif
734
735 u32 flags;
736 volatile u64 *cmd_sem;
737 u64 cmd_sem_val;
738 /*
739 * Track physical address to directly use it in build_completion_wait()
740 * and avoid adding any special checks and handling for kdump.
741 */
742 u64 cmd_sem_paddr;
743
744 #ifdef CONFIG_AMD_IOMMU_DEBUGFS
745 /* DebugFS Info */
746 struct dentry *debugfs;
747 int dbg_mmio_offset;
748 int dbg_cap_offset;
749 #endif
750
751 /* IOPF support */
752 struct iopf_queue *iopf_queue;
753 unsigned char iopfq_name[32];
754 };
755
dev_to_amd_iommu(struct device * dev)756 static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev)
757 {
758 struct iommu_device *iommu = dev_to_iommu_device(dev);
759
760 return container_of(iommu, struct amd_iommu, iommu);
761 }
762
763 #define ACPIHID_UID_LEN 256
764 #define ACPIHID_HID_LEN 9
765
766 struct acpihid_map_entry {
767 struct list_head list;
768 u8 uid[ACPIHID_UID_LEN];
769 u8 hid[ACPIHID_HID_LEN];
770 u32 devid;
771 u32 root_devid;
772 bool cmd_line;
773 struct iommu_group *group;
774 };
775
776 struct devid_map {
777 struct list_head list;
778 u8 id;
779 u32 devid;
780 bool cmd_line;
781 };
782
783 #define AMD_IOMMU_DEVICE_FLAG_ATS_SUP 0x1 /* ATS feature supported */
784 #define AMD_IOMMU_DEVICE_FLAG_PRI_SUP 0x2 /* PRI feature supported */
785 #define AMD_IOMMU_DEVICE_FLAG_PASID_SUP 0x4 /* PASID context supported */
786 /* Device may request execution on memory pages */
787 #define AMD_IOMMU_DEVICE_FLAG_EXEC_SUP 0x8
788 /* Device may request super-user privileges */
789 #define AMD_IOMMU_DEVICE_FLAG_PRIV_SUP 0x10
790
791 /*
792 * This struct contains device specific data for the IOMMU
793 */
794 struct iommu_dev_data {
795 /*Protect against attach/detach races */
796 struct mutex mutex;
797 spinlock_t dte_lock; /* DTE lock for 256-bit access */
798
799 struct list_head list; /* For domain->dev_list */
800 struct llist_node dev_data_list; /* For global dev_data_list */
801 struct protection_domain *domain; /* Domain the device is bound to */
802 struct gcr3_tbl_info gcr3_info; /* Per-device GCR3 table */
803 struct device *dev;
804 u16 devid; /* PCI Device ID */
805
806 unsigned int max_irqs; /* Maximum IRQs supported by device */
807 u32 max_pasids; /* Max supported PASIDs */
808 u32 flags; /* Holds AMD_IOMMU_DEVICE_FLAG_<*> */
809 int ats_qdep;
810 u8 ats_enabled :1; /* ATS state */
811 u8 pri_enabled :1; /* PRI state */
812 u8 pasid_enabled:1; /* PASID state */
813 u8 pri_tlp :1; /* PASID TLB required for
814 PPR completions */
815 u8 ppr :1; /* Enable device PPR support */
816 bool use_vapic; /* Enable device to use vapic mode */
817 bool defer_attach;
818
819 struct ratelimit_state rs; /* Ratelimit IOPF messages */
820 };
821
822 /* Map HPET and IOAPIC ids to the devid used by the IOMMU */
823 extern struct list_head ioapic_map;
824 extern struct list_head hpet_map;
825 extern struct list_head acpihid_map;
826
827 /*
828 * List with all PCI segments in the system. This list is not locked because
829 * it is only written at driver initialization time
830 */
831 extern struct list_head amd_iommu_pci_seg_list;
832
833 /*
834 * List with all IOMMUs in the system. This list is not locked because it is
835 * only written and read at driver initialization or suspend time
836 */
837 extern struct list_head amd_iommu_list;
838
839 /*
840 * Structure defining one entry in the device table
841 */
842 struct dev_table_entry {
843 union {
844 u64 data[4];
845 u128 data128[2];
846 };
847 };
848
849 /*
850 * Structure defining one entry in the command buffer
851 */
852 struct iommu_cmd {
853 u32 data[4];
854 };
855
856 /*
857 * Structure to sture persistent DTE flags from IVHD
858 */
859 struct ivhd_dte_flags {
860 struct list_head list;
861 u16 segid;
862 u16 devid_first;
863 u16 devid_last;
864 struct dev_table_entry dte;
865 };
866
867 /*
868 * One entry for unity mappings parsed out of the ACPI table.
869 */
870 struct unity_map_entry {
871 struct list_head list;
872
873 /* starting device id this entry is used for (including) */
874 u16 devid_start;
875 /* end device id this entry is used for (including) */
876 u16 devid_end;
877
878 /* start address to unity map (including) */
879 u64 address_start;
880 /* end address to unity map (including) */
881 u64 address_end;
882
883 /* required protection */
884 int prot;
885 };
886
887 /*
888 * Data structures for device handling
889 */
890
891 extern bool amd_iommu_force_isolation;
892
893 /* Max levels of glxval supported */
894 extern int amd_iommu_max_glx_val;
895
896 /* IDA to track protection domain IDs */
897 extern struct ida pdom_ids;
898
899 /* Global EFR and EFR2 registers */
900 extern u64 amd_iommu_efr;
901 extern u64 amd_iommu_efr2;
902
get_ioapic_devid(int id)903 static inline int get_ioapic_devid(int id)
904 {
905 struct devid_map *entry;
906
907 list_for_each_entry(entry, &ioapic_map, list) {
908 if (entry->id == id)
909 return entry->devid;
910 }
911
912 return -EINVAL;
913 }
914
get_hpet_devid(int id)915 static inline int get_hpet_devid(int id)
916 {
917 struct devid_map *entry;
918
919 list_for_each_entry(entry, &hpet_map, list) {
920 if (entry->id == id)
921 return entry->devid;
922 }
923
924 return -EINVAL;
925 }
926
927 enum amd_iommu_intr_mode_type {
928 /*
929 * The legacy format mode is not visible to users to prevent the user
930 * from crashing x2APIC systems, which for all intents and purposes
931 * require 128-bit IRTEs. The legacy format will be forced as needed
932 * when hardware doesn't support 128-bit IRTEs.
933 */
934 AMD_IOMMU_GUEST_IR_LEGACY,
935 AMD_IOMMU_GUEST_IR_LEGACY_GA,
936 AMD_IOMMU_GUEST_IR_VAPIC,
937 };
938
939 #define AMD_IOMMU_GUEST_IR_GA(x) (x == AMD_IOMMU_GUEST_IR_VAPIC || \
940 x == AMD_IOMMU_GUEST_IR_LEGACY_GA)
941
942 #define AMD_IOMMU_GUEST_IR_VAPIC(x) (x == AMD_IOMMU_GUEST_IR_VAPIC)
943
944 union irte {
945 u32 val;
946 struct {
947 u32 valid : 1,
948 no_fault : 1,
949 int_type : 3,
950 rq_eoi : 1,
951 dm : 1,
952 rsvd_1 : 1,
953 destination : 8,
954 vector : 8,
955 rsvd_2 : 8;
956 } fields;
957 };
958
959 #define APICID_TO_IRTE_DEST_LO(x) (x & 0xffffff)
960 #define APICID_TO_IRTE_DEST_HI(x) ((x >> 24) & 0xff)
961
962 union irte_ga_lo {
963 u64 val;
964
965 /* For int remapping */
966 struct {
967 u64 valid : 1,
968 no_fault : 1,
969 /* ------ */
970 int_type : 3,
971 rq_eoi : 1,
972 dm : 1,
973 /* ------ */
974 guest_mode : 1,
975 destination : 24,
976 ga_tag : 32;
977 } fields_remap;
978
979 /* For guest vAPIC */
980 struct {
981 u64 valid : 1,
982 no_fault : 1,
983 /* ------ */
984 ga_log_intr : 1,
985 rsvd1 : 3,
986 is_run : 1,
987 /* ------ */
988 guest_mode : 1,
989 destination : 24,
990 ga_tag : 32;
991 } fields_vapic;
992 };
993
994 union irte_ga_hi {
995 u64 val;
996 struct {
997 u64 vector : 8,
998 rsvd_1 : 4,
999 ga_root_ptr : 40,
1000 rsvd_2 : 4,
1001 destination : 8;
1002 } fields;
1003 };
1004
1005 struct irte_ga {
1006 union {
1007 struct {
1008 union irte_ga_lo lo;
1009 union irte_ga_hi hi;
1010 };
1011 u128 irte;
1012 };
1013 };
1014
1015 struct irq_2_irte {
1016 u16 devid; /* Device ID for IRTE table */
1017 u16 index; /* Index into IRTE table*/
1018 };
1019
1020 struct amd_ir_data {
1021 struct amd_iommu *iommu;
1022 struct irq_2_irte irq_2_irte;
1023 struct msi_msg msi_entry;
1024 void *entry; /* Pointer to union irte or struct irte_ga */
1025
1026 /**
1027 * Store information for activate/de-activate
1028 * Guest virtual APIC mode during runtime.
1029 */
1030 struct irq_cfg *cfg;
1031 int ga_vector;
1032 u64 ga_root_ptr;
1033 u32 ga_tag;
1034 };
1035
1036 struct amd_irte_ops {
1037 void (*prepare)(void *, u32, bool, u8, u32, int);
1038 void (*activate)(struct amd_iommu *iommu, void *, u16, u16);
1039 void (*deactivate)(struct amd_iommu *iommu, void *, u16, u16);
1040 void (*set_affinity)(struct amd_iommu *iommu, void *, u16, u16, u8, u32);
1041 void *(*get)(struct irq_remap_table *, int);
1042 void (*set_allocated)(struct irq_remap_table *, int);
1043 bool (*is_allocated)(struct irq_remap_table *, int);
1044 void (*clear_allocated)(struct irq_remap_table *, int);
1045 };
1046
1047 #ifdef CONFIG_IRQ_REMAP
1048 extern struct amd_irte_ops irte_32_ops;
1049 extern struct amd_irte_ops irte_128_ops;
1050 #endif
1051
1052 #endif /* _ASM_X86_AMD_IOMMU_TYPES_H */
1053