1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 * Leo Duran <leo.duran@amd.com>
6 */
7
8 #define pr_fmt(fmt) "AMD-Vi: " fmt
9 #define dev_fmt(fmt) pr_fmt(fmt)
10
11 #include <linux/pci.h>
12 #include <linux/acpi.h>
13 #include <linux/list.h>
14 #include <linux/bitmap.h>
15 #include <linux/slab.h>
16 #include <linux/syscore_ops.h>
17 #include <linux/interrupt.h>
18 #include <linux/msi.h>
19 #include <linux/irq.h>
20 #include <linux/amd-iommu.h>
21 #include <linux/export.h>
22 #include <linux/kmemleak.h>
23 #include <linux/cc_platform.h>
24 #include <linux/iopoll.h>
25 #include <asm/pci-direct.h>
26 #include <asm/iommu.h>
27 #include <asm/apic.h>
28 #include <asm/gart.h>
29 #include <asm/x86_init.h>
30 #include <asm/io_apic.h>
31 #include <asm/irq_remapping.h>
32 #include <asm/set_memory.h>
33 #include <asm/sev.h>
34
35 #include <linux/crash_dump.h>
36
37 #include "amd_iommu.h"
38 #include "../irq_remapping.h"
39 #include "../iommu-pages.h"
40
41 /*
42 * definitions for the ACPI scanning code
43 */
44 #define IVRS_HEADER_LENGTH 48
45
46 #define ACPI_IVHD_TYPE_MAX_SUPPORTED 0x40
47 #define ACPI_IVMD_TYPE_ALL 0x20
48 #define ACPI_IVMD_TYPE 0x21
49 #define ACPI_IVMD_TYPE_RANGE 0x22
50
51 #define IVHD_DEV_ALL 0x01
52 #define IVHD_DEV_SELECT 0x02
53 #define IVHD_DEV_SELECT_RANGE_START 0x03
54 #define IVHD_DEV_RANGE_END 0x04
55 #define IVHD_DEV_ALIAS 0x42
56 #define IVHD_DEV_ALIAS_RANGE 0x43
57 #define IVHD_DEV_EXT_SELECT 0x46
58 #define IVHD_DEV_EXT_SELECT_RANGE 0x47
59 #define IVHD_DEV_SPECIAL 0x48
60 #define IVHD_DEV_ACPI_HID 0xf0
61
62 #define UID_NOT_PRESENT 0
63 #define UID_IS_INTEGER 1
64 #define UID_IS_CHARACTER 2
65
66 #define IVHD_SPECIAL_IOAPIC 1
67 #define IVHD_SPECIAL_HPET 2
68
69 #define IVHD_FLAG_HT_TUN_EN_MASK 0x01
70 #define IVHD_FLAG_PASSPW_EN_MASK 0x02
71 #define IVHD_FLAG_RESPASSPW_EN_MASK 0x04
72 #define IVHD_FLAG_ISOC_EN_MASK 0x08
73
74 #define IVMD_FLAG_EXCL_RANGE 0x08
75 #define IVMD_FLAG_IW 0x04
76 #define IVMD_FLAG_IR 0x02
77 #define IVMD_FLAG_UNITY_MAP 0x01
78
79 #define ACPI_DEVFLAG_INITPASS 0x01
80 #define ACPI_DEVFLAG_EXTINT 0x02
81 #define ACPI_DEVFLAG_NMI 0x04
82 #define ACPI_DEVFLAG_SYSMGT1 0x10
83 #define ACPI_DEVFLAG_SYSMGT2 0x20
84 #define ACPI_DEVFLAG_LINT0 0x40
85 #define ACPI_DEVFLAG_LINT1 0x80
86 #define ACPI_DEVFLAG_ATSDIS 0x10000000
87
88 #define IVRS_GET_SBDF_ID(seg, bus, dev, fn) (((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \
89 | ((dev & 0x1f) << 3) | (fn & 0x7))
90
91 /*
92 * ACPI table definitions
93 *
94 * These data structures are laid over the table to parse the important values
95 * out of it.
96 */
97
98 /*
99 * structure describing one IOMMU in the ACPI table. Typically followed by one
100 * or more ivhd_entrys.
101 */
102 struct ivhd_header {
103 u8 type;
104 u8 flags;
105 u16 length;
106 u16 devid;
107 u16 cap_ptr;
108 u64 mmio_phys;
109 u16 pci_seg;
110 u16 info;
111 u32 efr_attr;
112
113 /* Following only valid on IVHD type 11h and 40h */
114 u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */
115 u64 efr_reg2;
116 } __attribute__((packed));
117
118 /*
119 * A device entry describing which devices a specific IOMMU translates and
120 * which requestor ids they use.
121 */
122 struct ivhd_entry {
123 u8 type;
124 u16 devid;
125 u8 flags;
126 struct_group(ext_hid,
127 u32 ext;
128 u32 hidh;
129 );
130 u64 cid;
131 u8 uidf;
132 u8 uidl;
133 u8 uid;
134 } __attribute__((packed));
135
136 /*
137 * An AMD IOMMU memory definition structure. It defines things like exclusion
138 * ranges for devices and regions that should be unity mapped.
139 */
140 struct ivmd_header {
141 u8 type;
142 u8 flags;
143 u16 length;
144 u16 devid;
145 u16 aux;
146 u16 pci_seg;
147 u8 resv[6];
148 u64 range_start;
149 u64 range_length;
150 } __attribute__((packed));
151
152 bool amd_iommu_dump;
153 bool amd_iommu_irq_remap __read_mostly;
154
155 enum io_pgtable_fmt amd_iommu_pgtable = AMD_IOMMU_V1;
156 /* Guest page table level */
157 int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL;
158
159 int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
160 static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
161
162 static bool amd_iommu_detected;
163 static bool amd_iommu_disabled __initdata;
164 static bool amd_iommu_force_enable __initdata;
165 static bool amd_iommu_irtcachedis;
166 static int amd_iommu_target_ivhd_type;
167
168 /* Global EFR and EFR2 registers */
169 u64 amd_iommu_efr;
170 u64 amd_iommu_efr2;
171
172 /* SNP is enabled on the system? */
173 bool amd_iommu_snp_en;
174 EXPORT_SYMBOL(amd_iommu_snp_en);
175
176 LIST_HEAD(amd_iommu_pci_seg_list); /* list of all PCI segments */
177 LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the
178 system */
179
180 /* Array to assign indices to IOMMUs*/
181 struct amd_iommu *amd_iommus[MAX_IOMMUS];
182
183 /* Number of IOMMUs present in the system */
184 static int amd_iommus_present;
185
186 /* IOMMUs have a non-present cache? */
187 bool amd_iommu_np_cache __read_mostly;
188 bool amd_iommu_iotlb_sup __read_mostly = true;
189
190 static bool amd_iommu_pc_present __read_mostly;
191 bool amdr_ivrs_remap_support __read_mostly;
192
193 bool amd_iommu_force_isolation __read_mostly;
194
195 unsigned long amd_iommu_pgsize_bitmap __ro_after_init = AMD_IOMMU_PGSIZES;
196
197 /*
198 * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap
199 * to know which ones are already in use.
200 */
201 unsigned long *amd_iommu_pd_alloc_bitmap;
202
203 enum iommu_init_state {
204 IOMMU_START_STATE,
205 IOMMU_IVRS_DETECTED,
206 IOMMU_ACPI_FINISHED,
207 IOMMU_ENABLED,
208 IOMMU_PCI_INIT,
209 IOMMU_INTERRUPTS_EN,
210 IOMMU_INITIALIZED,
211 IOMMU_NOT_FOUND,
212 IOMMU_INIT_ERROR,
213 IOMMU_CMDLINE_DISABLED,
214 };
215
216 /* Early ioapic and hpet maps from kernel command line */
217 #define EARLY_MAP_SIZE 4
218 static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE];
219 static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE];
220 static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE];
221
222 static int __initdata early_ioapic_map_size;
223 static int __initdata early_hpet_map_size;
224 static int __initdata early_acpihid_map_size;
225
226 static bool __initdata cmdline_maps;
227
228 static enum iommu_init_state init_state = IOMMU_START_STATE;
229
230 static int amd_iommu_enable_interrupts(void);
231 static int __init iommu_go_to_state(enum iommu_init_state state);
232 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg);
233
234 static bool amd_iommu_pre_enabled = true;
235
236 static u32 amd_iommu_ivinfo __initdata;
237
translation_pre_enabled(struct amd_iommu * iommu)238 bool translation_pre_enabled(struct amd_iommu *iommu)
239 {
240 return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED);
241 }
242
clear_translation_pre_enabled(struct amd_iommu * iommu)243 static void clear_translation_pre_enabled(struct amd_iommu *iommu)
244 {
245 iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
246 }
247
init_translation_status(struct amd_iommu * iommu)248 static void init_translation_status(struct amd_iommu *iommu)
249 {
250 u64 ctrl;
251
252 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
253 if (ctrl & (1<<CONTROL_IOMMU_EN))
254 iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
255 }
256
tbl_size(int entry_size,int last_bdf)257 static inline unsigned long tbl_size(int entry_size, int last_bdf)
258 {
259 unsigned shift = PAGE_SHIFT +
260 get_order((last_bdf + 1) * entry_size);
261
262 return 1UL << shift;
263 }
264
amd_iommu_get_num_iommus(void)265 int amd_iommu_get_num_iommus(void)
266 {
267 return amd_iommus_present;
268 }
269
270 /*
271 * Iterate through all the IOMMUs to get common EFR
272 * masks among all IOMMUs and warn if found inconsistency.
273 */
get_global_efr(void)274 static __init void get_global_efr(void)
275 {
276 struct amd_iommu *iommu;
277
278 for_each_iommu(iommu) {
279 u64 tmp = iommu->features;
280 u64 tmp2 = iommu->features2;
281
282 if (list_is_first(&iommu->list, &amd_iommu_list)) {
283 amd_iommu_efr = tmp;
284 amd_iommu_efr2 = tmp2;
285 continue;
286 }
287
288 if (amd_iommu_efr == tmp &&
289 amd_iommu_efr2 == tmp2)
290 continue;
291
292 pr_err(FW_BUG
293 "Found inconsistent EFR/EFR2 %#llx,%#llx (global %#llx,%#llx) on iommu%d (%04x:%02x:%02x.%01x).\n",
294 tmp, tmp2, amd_iommu_efr, amd_iommu_efr2,
295 iommu->index, iommu->pci_seg->id,
296 PCI_BUS_NUM(iommu->devid), PCI_SLOT(iommu->devid),
297 PCI_FUNC(iommu->devid));
298
299 amd_iommu_efr &= tmp;
300 amd_iommu_efr2 &= tmp2;
301 }
302
303 pr_info("Using global IVHD EFR:%#llx, EFR2:%#llx\n", amd_iommu_efr, amd_iommu_efr2);
304 }
305
306 /*
307 * For IVHD type 0x11/0x40, EFR is also available via IVHD.
308 * Default to IVHD EFR since it is available sooner
309 * (i.e. before PCI init).
310 */
early_iommu_features_init(struct amd_iommu * iommu,struct ivhd_header * h)311 static void __init early_iommu_features_init(struct amd_iommu *iommu,
312 struct ivhd_header *h)
313 {
314 if (amd_iommu_ivinfo & IOMMU_IVINFO_EFRSUP) {
315 iommu->features = h->efr_reg;
316 iommu->features2 = h->efr_reg2;
317 }
318 if (amd_iommu_ivinfo & IOMMU_IVINFO_DMA_REMAP)
319 amdr_ivrs_remap_support = true;
320 }
321
322 /* Access to l1 and l2 indexed register spaces */
323
iommu_read_l1(struct amd_iommu * iommu,u16 l1,u8 address)324 static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)
325 {
326 u32 val;
327
328 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
329 pci_read_config_dword(iommu->dev, 0xfc, &val);
330 return val;
331 }
332
iommu_write_l1(struct amd_iommu * iommu,u16 l1,u8 address,u32 val)333 static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)
334 {
335 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));
336 pci_write_config_dword(iommu->dev, 0xfc, val);
337 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
338 }
339
iommu_read_l2(struct amd_iommu * iommu,u8 address)340 static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)
341 {
342 u32 val;
343
344 pci_write_config_dword(iommu->dev, 0xf0, address);
345 pci_read_config_dword(iommu->dev, 0xf4, &val);
346 return val;
347 }
348
iommu_write_l2(struct amd_iommu * iommu,u8 address,u32 val)349 static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
350 {
351 pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));
352 pci_write_config_dword(iommu->dev, 0xf4, val);
353 }
354
355 /****************************************************************************
356 *
357 * AMD IOMMU MMIO register space handling functions
358 *
359 * These functions are used to program the IOMMU device registers in
360 * MMIO space required for that driver.
361 *
362 ****************************************************************************/
363
364 /*
365 * This function set the exclusion range in the IOMMU. DMA accesses to the
366 * exclusion range are passed through untranslated
367 */
iommu_set_exclusion_range(struct amd_iommu * iommu)368 static void iommu_set_exclusion_range(struct amd_iommu *iommu)
369 {
370 u64 start = iommu->exclusion_start & PAGE_MASK;
371 u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK;
372 u64 entry;
373
374 if (!iommu->exclusion_start)
375 return;
376
377 entry = start | MMIO_EXCL_ENABLE_MASK;
378 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
379 &entry, sizeof(entry));
380
381 entry = limit;
382 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
383 &entry, sizeof(entry));
384 }
385
iommu_set_cwwb_range(struct amd_iommu * iommu)386 static void iommu_set_cwwb_range(struct amd_iommu *iommu)
387 {
388 u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem);
389 u64 entry = start & PM_ADDR_MASK;
390
391 if (!check_feature(FEATURE_SNP))
392 return;
393
394 /* Note:
395 * Re-purpose Exclusion base/limit registers for Completion wait
396 * write-back base/limit.
397 */
398 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
399 &entry, sizeof(entry));
400
401 /* Note:
402 * Default to 4 Kbytes, which can be specified by setting base
403 * address equal to the limit address.
404 */
405 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
406 &entry, sizeof(entry));
407 }
408
409 /* Programs the physical address of the device table into the IOMMU hardware */
iommu_set_device_table(struct amd_iommu * iommu)410 static void iommu_set_device_table(struct amd_iommu *iommu)
411 {
412 u64 entry;
413 u32 dev_table_size = iommu->pci_seg->dev_table_size;
414 void *dev_table = (void *)get_dev_table(iommu);
415
416 BUG_ON(iommu->mmio_base == NULL);
417
418 entry = iommu_virt_to_phys(dev_table);
419 entry |= (dev_table_size >> 12) - 1;
420 memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
421 &entry, sizeof(entry));
422 }
423
424 /* Generic functions to enable/disable certain features of the IOMMU. */
iommu_feature_enable(struct amd_iommu * iommu,u8 bit)425 void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
426 {
427 u64 ctrl;
428
429 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
430 ctrl |= (1ULL << bit);
431 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
432 }
433
iommu_feature_disable(struct amd_iommu * iommu,u8 bit)434 static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
435 {
436 u64 ctrl;
437
438 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
439 ctrl &= ~(1ULL << bit);
440 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
441 }
442
iommu_set_inv_tlb_timeout(struct amd_iommu * iommu,int timeout)443 static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)
444 {
445 u64 ctrl;
446
447 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
448 ctrl &= ~CTRL_INV_TO_MASK;
449 ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;
450 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
451 }
452
453 /* Function to enable the hardware */
iommu_enable(struct amd_iommu * iommu)454 static void iommu_enable(struct amd_iommu *iommu)
455 {
456 iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
457 }
458
iommu_disable(struct amd_iommu * iommu)459 static void iommu_disable(struct amd_iommu *iommu)
460 {
461 if (!iommu->mmio_base)
462 return;
463
464 /* Disable command buffer */
465 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
466
467 /* Disable event logging and event interrupts */
468 iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);
469 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
470
471 /* Disable IOMMU GA_LOG */
472 iommu_feature_disable(iommu, CONTROL_GALOG_EN);
473 iommu_feature_disable(iommu, CONTROL_GAINT_EN);
474
475 /* Disable IOMMU PPR logging */
476 iommu_feature_disable(iommu, CONTROL_PPRLOG_EN);
477 iommu_feature_disable(iommu, CONTROL_PPRINT_EN);
478
479 /* Disable IOMMU hardware itself */
480 iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
481
482 /* Clear IRTE cache disabling bit */
483 iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);
484 }
485
486 /*
487 * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in
488 * the system has one.
489 */
iommu_map_mmio_space(u64 address,u64 end)490 static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end)
491 {
492 if (!request_mem_region(address, end, "amd_iommu")) {
493 pr_err("Can not reserve memory region %llx-%llx for mmio\n",
494 address, end);
495 pr_err("This is a BIOS bug. Please contact your hardware vendor\n");
496 return NULL;
497 }
498
499 return (u8 __iomem *)ioremap(address, end);
500 }
501
iommu_unmap_mmio_space(struct amd_iommu * iommu)502 static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)
503 {
504 if (iommu->mmio_base)
505 iounmap(iommu->mmio_base);
506 release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end);
507 }
508
get_ivhd_header_size(struct ivhd_header * h)509 static inline u32 get_ivhd_header_size(struct ivhd_header *h)
510 {
511 u32 size = 0;
512
513 switch (h->type) {
514 case 0x10:
515 size = 24;
516 break;
517 case 0x11:
518 case 0x40:
519 size = 40;
520 break;
521 }
522 return size;
523 }
524
525 /****************************************************************************
526 *
527 * The functions below belong to the first pass of AMD IOMMU ACPI table
528 * parsing. In this pass we try to find out the highest device id this
529 * code has to handle. Upon this information the size of the shared data
530 * structures is determined later.
531 *
532 ****************************************************************************/
533
534 /*
535 * This function calculates the length of a given IVHD entry
536 */
ivhd_entry_length(u8 * ivhd)537 static inline int ivhd_entry_length(u8 *ivhd)
538 {
539 u32 type = ((struct ivhd_entry *)ivhd)->type;
540
541 if (type < 0x80) {
542 return 0x04 << (*ivhd >> 6);
543 } else if (type == IVHD_DEV_ACPI_HID) {
544 /* For ACPI_HID, offset 21 is uid len */
545 return *((u8 *)ivhd + 21) + 22;
546 }
547 return 0;
548 }
549
550 /*
551 * After reading the highest device id from the IOMMU PCI capability header
552 * this function looks if there is a higher device id defined in the ACPI table
553 */
find_last_devid_from_ivhd(struct ivhd_header * h)554 static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
555 {
556 u8 *p = (void *)h, *end = (void *)h;
557 struct ivhd_entry *dev;
558 int last_devid = -EINVAL;
559
560 u32 ivhd_size = get_ivhd_header_size(h);
561
562 if (!ivhd_size) {
563 pr_err("Unsupported IVHD type %#x\n", h->type);
564 return -EINVAL;
565 }
566
567 p += ivhd_size;
568 end += h->length;
569
570 while (p < end) {
571 dev = (struct ivhd_entry *)p;
572 switch (dev->type) {
573 case IVHD_DEV_ALL:
574 /* Use maximum BDF value for DEV_ALL */
575 return 0xffff;
576 case IVHD_DEV_SELECT:
577 case IVHD_DEV_RANGE_END:
578 case IVHD_DEV_ALIAS:
579 case IVHD_DEV_EXT_SELECT:
580 /* all the above subfield types refer to device ids */
581 if (dev->devid > last_devid)
582 last_devid = dev->devid;
583 break;
584 default:
585 break;
586 }
587 p += ivhd_entry_length(p);
588 }
589
590 WARN_ON(p != end);
591
592 return last_devid;
593 }
594
check_ivrs_checksum(struct acpi_table_header * table)595 static int __init check_ivrs_checksum(struct acpi_table_header *table)
596 {
597 int i;
598 u8 checksum = 0, *p = (u8 *)table;
599
600 for (i = 0; i < table->length; ++i)
601 checksum += p[i];
602 if (checksum != 0) {
603 /* ACPI table corrupt */
604 pr_err(FW_BUG "IVRS invalid checksum\n");
605 return -ENODEV;
606 }
607
608 return 0;
609 }
610
611 /*
612 * Iterate over all IVHD entries in the ACPI table and find the highest device
613 * id which we need to handle. This is the first of three functions which parse
614 * the ACPI table. So we check the checksum here.
615 */
find_last_devid_acpi(struct acpi_table_header * table,u16 pci_seg)616 static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_seg)
617 {
618 u8 *p = (u8 *)table, *end = (u8 *)table;
619 struct ivhd_header *h;
620 int last_devid, last_bdf = 0;
621
622 p += IVRS_HEADER_LENGTH;
623
624 end += table->length;
625 while (p < end) {
626 h = (struct ivhd_header *)p;
627 if (h->pci_seg == pci_seg &&
628 h->type == amd_iommu_target_ivhd_type) {
629 last_devid = find_last_devid_from_ivhd(h);
630
631 if (last_devid < 0)
632 return -EINVAL;
633 if (last_devid > last_bdf)
634 last_bdf = last_devid;
635 }
636 p += h->length;
637 }
638 WARN_ON(p != end);
639
640 return last_bdf;
641 }
642
643 /****************************************************************************
644 *
645 * The following functions belong to the code path which parses the ACPI table
646 * the second time. In this ACPI parsing iteration we allocate IOMMU specific
647 * data structures, initialize the per PCI segment device/alias/rlookup table
648 * and also basically initialize the hardware.
649 *
650 ****************************************************************************/
651
652 /* Allocate per PCI segment device table */
alloc_dev_table(struct amd_iommu_pci_seg * pci_seg)653 static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg)
654 {
655 pci_seg->dev_table = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32,
656 get_order(pci_seg->dev_table_size));
657 if (!pci_seg->dev_table)
658 return -ENOMEM;
659
660 return 0;
661 }
662
free_dev_table(struct amd_iommu_pci_seg * pci_seg)663 static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg)
664 {
665 iommu_free_pages(pci_seg->dev_table,
666 get_order(pci_seg->dev_table_size));
667 pci_seg->dev_table = NULL;
668 }
669
670 /* Allocate per PCI segment IOMMU rlookup table. */
alloc_rlookup_table(struct amd_iommu_pci_seg * pci_seg)671 static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg)
672 {
673 pci_seg->rlookup_table = iommu_alloc_pages(GFP_KERNEL,
674 get_order(pci_seg->rlookup_table_size));
675 if (pci_seg->rlookup_table == NULL)
676 return -ENOMEM;
677
678 return 0;
679 }
680
free_rlookup_table(struct amd_iommu_pci_seg * pci_seg)681 static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg)
682 {
683 iommu_free_pages(pci_seg->rlookup_table,
684 get_order(pci_seg->rlookup_table_size));
685 pci_seg->rlookup_table = NULL;
686 }
687
alloc_irq_lookup_table(struct amd_iommu_pci_seg * pci_seg)688 static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)
689 {
690 pci_seg->irq_lookup_table = iommu_alloc_pages(GFP_KERNEL,
691 get_order(pci_seg->rlookup_table_size));
692 kmemleak_alloc(pci_seg->irq_lookup_table,
693 pci_seg->rlookup_table_size, 1, GFP_KERNEL);
694 if (pci_seg->irq_lookup_table == NULL)
695 return -ENOMEM;
696
697 return 0;
698 }
699
free_irq_lookup_table(struct amd_iommu_pci_seg * pci_seg)700 static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)
701 {
702 kmemleak_free(pci_seg->irq_lookup_table);
703 iommu_free_pages(pci_seg->irq_lookup_table,
704 get_order(pci_seg->rlookup_table_size));
705 pci_seg->irq_lookup_table = NULL;
706 }
707
alloc_alias_table(struct amd_iommu_pci_seg * pci_seg)708 static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg)
709 {
710 int i;
711
712 pci_seg->alias_table = iommu_alloc_pages(GFP_KERNEL,
713 get_order(pci_seg->alias_table_size));
714 if (!pci_seg->alias_table)
715 return -ENOMEM;
716
717 /*
718 * let all alias entries point to itself
719 */
720 for (i = 0; i <= pci_seg->last_bdf; ++i)
721 pci_seg->alias_table[i] = i;
722
723 return 0;
724 }
725
free_alias_table(struct amd_iommu_pci_seg * pci_seg)726 static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg)
727 {
728 iommu_free_pages(pci_seg->alias_table,
729 get_order(pci_seg->alias_table_size));
730 pci_seg->alias_table = NULL;
731 }
732
733 /*
734 * Allocates the command buffer. This buffer is per AMD IOMMU. We can
735 * write commands to that buffer later and the IOMMU will execute them
736 * asynchronously
737 */
alloc_command_buffer(struct amd_iommu * iommu)738 static int __init alloc_command_buffer(struct amd_iommu *iommu)
739 {
740 iommu->cmd_buf = iommu_alloc_pages(GFP_KERNEL,
741 get_order(CMD_BUFFER_SIZE));
742
743 return iommu->cmd_buf ? 0 : -ENOMEM;
744 }
745
746 /*
747 * Interrupt handler has processed all pending events and adjusted head
748 * and tail pointer. Reset overflow mask and restart logging again.
749 */
amd_iommu_restart_log(struct amd_iommu * iommu,const char * evt_type,u8 cntrl_intr,u8 cntrl_log,u32 status_run_mask,u32 status_overflow_mask)750 void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type,
751 u8 cntrl_intr, u8 cntrl_log,
752 u32 status_run_mask, u32 status_overflow_mask)
753 {
754 u32 status;
755
756 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
757 if (status & status_run_mask)
758 return;
759
760 pr_info_ratelimited("IOMMU %s log restarting\n", evt_type);
761
762 iommu_feature_disable(iommu, cntrl_log);
763 iommu_feature_disable(iommu, cntrl_intr);
764
765 writel(status_overflow_mask, iommu->mmio_base + MMIO_STATUS_OFFSET);
766
767 iommu_feature_enable(iommu, cntrl_intr);
768 iommu_feature_enable(iommu, cntrl_log);
769 }
770
771 /*
772 * This function restarts event logging in case the IOMMU experienced
773 * an event log buffer overflow.
774 */
amd_iommu_restart_event_logging(struct amd_iommu * iommu)775 void amd_iommu_restart_event_logging(struct amd_iommu *iommu)
776 {
777 amd_iommu_restart_log(iommu, "Event", CONTROL_EVT_INT_EN,
778 CONTROL_EVT_LOG_EN, MMIO_STATUS_EVT_RUN_MASK,
779 MMIO_STATUS_EVT_OVERFLOW_MASK);
780 }
781
782 /*
783 * This function restarts event logging in case the IOMMU experienced
784 * GA log overflow.
785 */
amd_iommu_restart_ga_log(struct amd_iommu * iommu)786 void amd_iommu_restart_ga_log(struct amd_iommu *iommu)
787 {
788 amd_iommu_restart_log(iommu, "GA", CONTROL_GAINT_EN,
789 CONTROL_GALOG_EN, MMIO_STATUS_GALOG_RUN_MASK,
790 MMIO_STATUS_GALOG_OVERFLOW_MASK);
791 }
792
793 /*
794 * This function resets the command buffer if the IOMMU stopped fetching
795 * commands from it.
796 */
amd_iommu_reset_cmd_buffer(struct amd_iommu * iommu)797 static void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
798 {
799 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
800
801 writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
802 writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
803 iommu->cmd_buf_head = 0;
804 iommu->cmd_buf_tail = 0;
805
806 iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
807 }
808
809 /*
810 * This function writes the command buffer address to the hardware and
811 * enables it.
812 */
iommu_enable_command_buffer(struct amd_iommu * iommu)813 static void iommu_enable_command_buffer(struct amd_iommu *iommu)
814 {
815 u64 entry;
816
817 BUG_ON(iommu->cmd_buf == NULL);
818
819 entry = iommu_virt_to_phys(iommu->cmd_buf);
820 entry |= MMIO_CMD_SIZE_512;
821
822 memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
823 &entry, sizeof(entry));
824
825 amd_iommu_reset_cmd_buffer(iommu);
826 }
827
828 /*
829 * This function disables the command buffer
830 */
iommu_disable_command_buffer(struct amd_iommu * iommu)831 static void iommu_disable_command_buffer(struct amd_iommu *iommu)
832 {
833 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
834 }
835
free_command_buffer(struct amd_iommu * iommu)836 static void __init free_command_buffer(struct amd_iommu *iommu)
837 {
838 iommu_free_pages(iommu->cmd_buf, get_order(CMD_BUFFER_SIZE));
839 }
840
iommu_alloc_4k_pages(struct amd_iommu * iommu,gfp_t gfp,size_t size)841 void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp,
842 size_t size)
843 {
844 int order = get_order(size);
845 void *buf = iommu_alloc_pages(gfp, order);
846
847 if (buf &&
848 check_feature(FEATURE_SNP) &&
849 set_memory_4k((unsigned long)buf, (1 << order))) {
850 iommu_free_pages(buf, order);
851 buf = NULL;
852 }
853
854 return buf;
855 }
856
857 /* allocates the memory where the IOMMU will log its events to */
alloc_event_buffer(struct amd_iommu * iommu)858 static int __init alloc_event_buffer(struct amd_iommu *iommu)
859 {
860 iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL,
861 EVT_BUFFER_SIZE);
862
863 return iommu->evt_buf ? 0 : -ENOMEM;
864 }
865
iommu_enable_event_buffer(struct amd_iommu * iommu)866 static void iommu_enable_event_buffer(struct amd_iommu *iommu)
867 {
868 u64 entry;
869
870 BUG_ON(iommu->evt_buf == NULL);
871
872 entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
873
874 memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
875 &entry, sizeof(entry));
876
877 /* set head and tail to zero manually */
878 writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
879 writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
880
881 iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
882 }
883
884 /*
885 * This function disables the event log buffer
886 */
iommu_disable_event_buffer(struct amd_iommu * iommu)887 static void iommu_disable_event_buffer(struct amd_iommu *iommu)
888 {
889 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
890 }
891
free_event_buffer(struct amd_iommu * iommu)892 static void __init free_event_buffer(struct amd_iommu *iommu)
893 {
894 iommu_free_pages(iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
895 }
896
free_ga_log(struct amd_iommu * iommu)897 static void free_ga_log(struct amd_iommu *iommu)
898 {
899 #ifdef CONFIG_IRQ_REMAP
900 iommu_free_pages(iommu->ga_log, get_order(GA_LOG_SIZE));
901 iommu_free_pages(iommu->ga_log_tail, get_order(8));
902 #endif
903 }
904
905 #ifdef CONFIG_IRQ_REMAP
iommu_ga_log_enable(struct amd_iommu * iommu)906 static int iommu_ga_log_enable(struct amd_iommu *iommu)
907 {
908 u32 status, i;
909 u64 entry;
910
911 if (!iommu->ga_log)
912 return -EINVAL;
913
914 entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
915 memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
916 &entry, sizeof(entry));
917 entry = (iommu_virt_to_phys(iommu->ga_log_tail) &
918 (BIT_ULL(52)-1)) & ~7ULL;
919 memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,
920 &entry, sizeof(entry));
921 writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
922 writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
923
924
925 iommu_feature_enable(iommu, CONTROL_GAINT_EN);
926 iommu_feature_enable(iommu, CONTROL_GALOG_EN);
927
928 for (i = 0; i < MMIO_STATUS_TIMEOUT; ++i) {
929 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
930 if (status & (MMIO_STATUS_GALOG_RUN_MASK))
931 break;
932 udelay(10);
933 }
934
935 if (WARN_ON(i >= MMIO_STATUS_TIMEOUT))
936 return -EINVAL;
937
938 return 0;
939 }
940
iommu_init_ga_log(struct amd_iommu * iommu)941 static int iommu_init_ga_log(struct amd_iommu *iommu)
942 {
943 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
944 return 0;
945
946 iommu->ga_log = iommu_alloc_pages(GFP_KERNEL, get_order(GA_LOG_SIZE));
947 if (!iommu->ga_log)
948 goto err_out;
949
950 iommu->ga_log_tail = iommu_alloc_pages(GFP_KERNEL, get_order(8));
951 if (!iommu->ga_log_tail)
952 goto err_out;
953
954 return 0;
955 err_out:
956 free_ga_log(iommu);
957 return -EINVAL;
958 }
959 #endif /* CONFIG_IRQ_REMAP */
960
alloc_cwwb_sem(struct amd_iommu * iommu)961 static int __init alloc_cwwb_sem(struct amd_iommu *iommu)
962 {
963 iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL, 1);
964
965 return iommu->cmd_sem ? 0 : -ENOMEM;
966 }
967
free_cwwb_sem(struct amd_iommu * iommu)968 static void __init free_cwwb_sem(struct amd_iommu *iommu)
969 {
970 if (iommu->cmd_sem)
971 iommu_free_page((void *)iommu->cmd_sem);
972 }
973
iommu_enable_xt(struct amd_iommu * iommu)974 static void iommu_enable_xt(struct amd_iommu *iommu)
975 {
976 #ifdef CONFIG_IRQ_REMAP
977 /*
978 * XT mode (32-bit APIC destination ID) requires
979 * GA mode (128-bit IRTE support) as a prerequisite.
980 */
981 if (AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir) &&
982 amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
983 iommu_feature_enable(iommu, CONTROL_XT_EN);
984 #endif /* CONFIG_IRQ_REMAP */
985 }
986
iommu_enable_gt(struct amd_iommu * iommu)987 static void iommu_enable_gt(struct amd_iommu *iommu)
988 {
989 if (!check_feature(FEATURE_GT))
990 return;
991
992 iommu_feature_enable(iommu, CONTROL_GT_EN);
993 }
994
995 /* sets a specific bit in the device table entry. */
__set_dev_entry_bit(struct dev_table_entry * dev_table,u16 devid,u8 bit)996 static void __set_dev_entry_bit(struct dev_table_entry *dev_table,
997 u16 devid, u8 bit)
998 {
999 int i = (bit >> 6) & 0x03;
1000 int _bit = bit & 0x3f;
1001
1002 dev_table[devid].data[i] |= (1UL << _bit);
1003 }
1004
set_dev_entry_bit(struct amd_iommu * iommu,u16 devid,u8 bit)1005 static void set_dev_entry_bit(struct amd_iommu *iommu, u16 devid, u8 bit)
1006 {
1007 struct dev_table_entry *dev_table = get_dev_table(iommu);
1008
1009 return __set_dev_entry_bit(dev_table, devid, bit);
1010 }
1011
__get_dev_entry_bit(struct dev_table_entry * dev_table,u16 devid,u8 bit)1012 static int __get_dev_entry_bit(struct dev_table_entry *dev_table,
1013 u16 devid, u8 bit)
1014 {
1015 int i = (bit >> 6) & 0x03;
1016 int _bit = bit & 0x3f;
1017
1018 return (dev_table[devid].data[i] & (1UL << _bit)) >> _bit;
1019 }
1020
get_dev_entry_bit(struct amd_iommu * iommu,u16 devid,u8 bit)1021 static int get_dev_entry_bit(struct amd_iommu *iommu, u16 devid, u8 bit)
1022 {
1023 struct dev_table_entry *dev_table = get_dev_table(iommu);
1024
1025 return __get_dev_entry_bit(dev_table, devid, bit);
1026 }
1027
__copy_device_table(struct amd_iommu * iommu)1028 static bool __copy_device_table(struct amd_iommu *iommu)
1029 {
1030 u64 int_ctl, int_tab_len, entry = 0;
1031 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
1032 struct dev_table_entry *old_devtb = NULL;
1033 u32 lo, hi, devid, old_devtb_size;
1034 phys_addr_t old_devtb_phys;
1035 u16 dom_id, dte_v, irq_v;
1036 u64 tmp;
1037
1038 /* Each IOMMU use separate device table with the same size */
1039 lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET);
1040 hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4);
1041 entry = (((u64) hi) << 32) + lo;
1042
1043 old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12;
1044 if (old_devtb_size != pci_seg->dev_table_size) {
1045 pr_err("The device table size of IOMMU:%d is not expected!\n",
1046 iommu->index);
1047 return false;
1048 }
1049
1050 /*
1051 * When SME is enabled in the first kernel, the entry includes the
1052 * memory encryption mask(sme_me_mask), we must remove the memory
1053 * encryption mask to obtain the true physical address in kdump kernel.
1054 */
1055 old_devtb_phys = __sme_clr(entry) & PAGE_MASK;
1056
1057 if (old_devtb_phys >= 0x100000000ULL) {
1058 pr_err("The address of old device table is above 4G, not trustworthy!\n");
1059 return false;
1060 }
1061 old_devtb = (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) && is_kdump_kernel())
1062 ? (__force void *)ioremap_encrypted(old_devtb_phys,
1063 pci_seg->dev_table_size)
1064 : memremap(old_devtb_phys, pci_seg->dev_table_size, MEMREMAP_WB);
1065
1066 if (!old_devtb)
1067 return false;
1068
1069 pci_seg->old_dev_tbl_cpy = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32,
1070 get_order(pci_seg->dev_table_size));
1071 if (pci_seg->old_dev_tbl_cpy == NULL) {
1072 pr_err("Failed to allocate memory for copying old device table!\n");
1073 memunmap(old_devtb);
1074 return false;
1075 }
1076
1077 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
1078 pci_seg->old_dev_tbl_cpy[devid] = old_devtb[devid];
1079 dom_id = old_devtb[devid].data[1] & DEV_DOMID_MASK;
1080 dte_v = old_devtb[devid].data[0] & DTE_FLAG_V;
1081
1082 if (dte_v && dom_id) {
1083 pci_seg->old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0];
1084 pci_seg->old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1];
1085 __set_bit(dom_id, amd_iommu_pd_alloc_bitmap);
1086 /* If gcr3 table existed, mask it out */
1087 if (old_devtb[devid].data[0] & DTE_FLAG_GV) {
1088 tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
1089 tmp |= DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
1090 pci_seg->old_dev_tbl_cpy[devid].data[1] &= ~tmp;
1091 tmp = DTE_GCR3_VAL_A(~0ULL) << DTE_GCR3_SHIFT_A;
1092 tmp |= DTE_FLAG_GV;
1093 pci_seg->old_dev_tbl_cpy[devid].data[0] &= ~tmp;
1094 }
1095 }
1096
1097 irq_v = old_devtb[devid].data[2] & DTE_IRQ_REMAP_ENABLE;
1098 int_ctl = old_devtb[devid].data[2] & DTE_IRQ_REMAP_INTCTL_MASK;
1099 int_tab_len = old_devtb[devid].data[2] & DTE_INTTABLEN_MASK;
1100 if (irq_v && (int_ctl || int_tab_len)) {
1101 if ((int_ctl != DTE_IRQ_REMAP_INTCTL) ||
1102 (int_tab_len != DTE_INTTABLEN)) {
1103 pr_err("Wrong old irq remapping flag: %#x\n", devid);
1104 memunmap(old_devtb);
1105 return false;
1106 }
1107
1108 pci_seg->old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2];
1109 }
1110 }
1111 memunmap(old_devtb);
1112
1113 return true;
1114 }
1115
copy_device_table(void)1116 static bool copy_device_table(void)
1117 {
1118 struct amd_iommu *iommu;
1119 struct amd_iommu_pci_seg *pci_seg;
1120
1121 if (!amd_iommu_pre_enabled)
1122 return false;
1123
1124 pr_warn("Translation is already enabled - trying to copy translation structures\n");
1125
1126 /*
1127 * All IOMMUs within PCI segment shares common device table.
1128 * Hence copy device table only once per PCI segment.
1129 */
1130 for_each_pci_segment(pci_seg) {
1131 for_each_iommu(iommu) {
1132 if (pci_seg->id != iommu->pci_seg->id)
1133 continue;
1134 if (!__copy_device_table(iommu))
1135 return false;
1136 break;
1137 }
1138 }
1139
1140 return true;
1141 }
1142
amd_iommu_apply_erratum_63(struct amd_iommu * iommu,u16 devid)1143 void amd_iommu_apply_erratum_63(struct amd_iommu *iommu, u16 devid)
1144 {
1145 int sysmgt;
1146
1147 sysmgt = get_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT1) |
1148 (get_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT2) << 1);
1149
1150 if (sysmgt == 0x01)
1151 set_dev_entry_bit(iommu, devid, DEV_ENTRY_IW);
1152 }
1153
1154 /*
1155 * This function takes the device specific flags read from the ACPI
1156 * table and sets up the device table entry with that information
1157 */
set_dev_entry_from_acpi(struct amd_iommu * iommu,u16 devid,u32 flags,u32 ext_flags)1158 static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
1159 u16 devid, u32 flags, u32 ext_flags)
1160 {
1161 if (flags & ACPI_DEVFLAG_INITPASS)
1162 set_dev_entry_bit(iommu, devid, DEV_ENTRY_INIT_PASS);
1163 if (flags & ACPI_DEVFLAG_EXTINT)
1164 set_dev_entry_bit(iommu, devid, DEV_ENTRY_EINT_PASS);
1165 if (flags & ACPI_DEVFLAG_NMI)
1166 set_dev_entry_bit(iommu, devid, DEV_ENTRY_NMI_PASS);
1167 if (flags & ACPI_DEVFLAG_SYSMGT1)
1168 set_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT1);
1169 if (flags & ACPI_DEVFLAG_SYSMGT2)
1170 set_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT2);
1171 if (flags & ACPI_DEVFLAG_LINT0)
1172 set_dev_entry_bit(iommu, devid, DEV_ENTRY_LINT0_PASS);
1173 if (flags & ACPI_DEVFLAG_LINT1)
1174 set_dev_entry_bit(iommu, devid, DEV_ENTRY_LINT1_PASS);
1175
1176 amd_iommu_apply_erratum_63(iommu, devid);
1177
1178 amd_iommu_set_rlookup_table(iommu, devid);
1179 }
1180
add_special_device(u8 type,u8 id,u32 * devid,bool cmd_line)1181 int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line)
1182 {
1183 struct devid_map *entry;
1184 struct list_head *list;
1185
1186 if (type == IVHD_SPECIAL_IOAPIC)
1187 list = &ioapic_map;
1188 else if (type == IVHD_SPECIAL_HPET)
1189 list = &hpet_map;
1190 else
1191 return -EINVAL;
1192
1193 list_for_each_entry(entry, list, list) {
1194 if (!(entry->id == id && entry->cmd_line))
1195 continue;
1196
1197 pr_info("Command-line override present for %s id %d - ignoring\n",
1198 type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id);
1199
1200 *devid = entry->devid;
1201
1202 return 0;
1203 }
1204
1205 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1206 if (!entry)
1207 return -ENOMEM;
1208
1209 entry->id = id;
1210 entry->devid = *devid;
1211 entry->cmd_line = cmd_line;
1212
1213 list_add_tail(&entry->list, list);
1214
1215 return 0;
1216 }
1217
add_acpi_hid_device(u8 * hid,u8 * uid,u32 * devid,bool cmd_line)1218 static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u32 *devid,
1219 bool cmd_line)
1220 {
1221 struct acpihid_map_entry *entry;
1222 struct list_head *list = &acpihid_map;
1223
1224 list_for_each_entry(entry, list, list) {
1225 if (strcmp(entry->hid, hid) ||
1226 (*uid && *entry->uid && strcmp(entry->uid, uid)) ||
1227 !entry->cmd_line)
1228 continue;
1229
1230 pr_info("Command-line override for hid:%s uid:%s\n",
1231 hid, uid);
1232 *devid = entry->devid;
1233 return 0;
1234 }
1235
1236 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1237 if (!entry)
1238 return -ENOMEM;
1239
1240 memcpy(entry->uid, uid, strlen(uid));
1241 memcpy(entry->hid, hid, strlen(hid));
1242 entry->devid = *devid;
1243 entry->cmd_line = cmd_line;
1244 entry->root_devid = (entry->devid & (~0x7));
1245
1246 pr_info("%s, add hid:%s, uid:%s, rdevid:%d\n",
1247 entry->cmd_line ? "cmd" : "ivrs",
1248 entry->hid, entry->uid, entry->root_devid);
1249
1250 list_add_tail(&entry->list, list);
1251 return 0;
1252 }
1253
add_early_maps(void)1254 static int __init add_early_maps(void)
1255 {
1256 int i, ret;
1257
1258 for (i = 0; i < early_ioapic_map_size; ++i) {
1259 ret = add_special_device(IVHD_SPECIAL_IOAPIC,
1260 early_ioapic_map[i].id,
1261 &early_ioapic_map[i].devid,
1262 early_ioapic_map[i].cmd_line);
1263 if (ret)
1264 return ret;
1265 }
1266
1267 for (i = 0; i < early_hpet_map_size; ++i) {
1268 ret = add_special_device(IVHD_SPECIAL_HPET,
1269 early_hpet_map[i].id,
1270 &early_hpet_map[i].devid,
1271 early_hpet_map[i].cmd_line);
1272 if (ret)
1273 return ret;
1274 }
1275
1276 for (i = 0; i < early_acpihid_map_size; ++i) {
1277 ret = add_acpi_hid_device(early_acpihid_map[i].hid,
1278 early_acpihid_map[i].uid,
1279 &early_acpihid_map[i].devid,
1280 early_acpihid_map[i].cmd_line);
1281 if (ret)
1282 return ret;
1283 }
1284
1285 return 0;
1286 }
1287
1288 /*
1289 * Takes a pointer to an AMD IOMMU entry in the ACPI table and
1290 * initializes the hardware and our data structures with it.
1291 */
init_iommu_from_acpi(struct amd_iommu * iommu,struct ivhd_header * h)1292 static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
1293 struct ivhd_header *h)
1294 {
1295 u8 *p = (u8 *)h;
1296 u8 *end = p, flags = 0;
1297 u16 devid = 0, devid_start = 0, devid_to = 0, seg_id;
1298 u32 dev_i, ext_flags = 0;
1299 bool alias = false;
1300 struct ivhd_entry *e;
1301 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
1302 u32 ivhd_size;
1303 int ret;
1304
1305
1306 ret = add_early_maps();
1307 if (ret)
1308 return ret;
1309
1310 amd_iommu_apply_ivrs_quirks();
1311
1312 /*
1313 * First save the recommended feature enable bits from ACPI
1314 */
1315 iommu->acpi_flags = h->flags;
1316
1317 /*
1318 * Done. Now parse the device entries
1319 */
1320 ivhd_size = get_ivhd_header_size(h);
1321 if (!ivhd_size) {
1322 pr_err("Unsupported IVHD type %#x\n", h->type);
1323 return -EINVAL;
1324 }
1325
1326 p += ivhd_size;
1327
1328 end += h->length;
1329
1330
1331 while (p < end) {
1332 e = (struct ivhd_entry *)p;
1333 seg_id = pci_seg->id;
1334
1335 switch (e->type) {
1336 case IVHD_DEV_ALL:
1337
1338 DUMP_printk(" DEV_ALL\t\t\tflags: %02x\n", e->flags);
1339
1340 for (dev_i = 0; dev_i <= pci_seg->last_bdf; ++dev_i)
1341 set_dev_entry_from_acpi(iommu, dev_i, e->flags, 0);
1342 break;
1343 case IVHD_DEV_SELECT:
1344
1345 DUMP_printk(" DEV_SELECT\t\t\t devid: %04x:%02x:%02x.%x "
1346 "flags: %02x\n",
1347 seg_id, PCI_BUS_NUM(e->devid),
1348 PCI_SLOT(e->devid),
1349 PCI_FUNC(e->devid),
1350 e->flags);
1351
1352 devid = e->devid;
1353 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1354 break;
1355 case IVHD_DEV_SELECT_RANGE_START:
1356
1357 DUMP_printk(" DEV_SELECT_RANGE_START\t "
1358 "devid: %04x:%02x:%02x.%x flags: %02x\n",
1359 seg_id, PCI_BUS_NUM(e->devid),
1360 PCI_SLOT(e->devid),
1361 PCI_FUNC(e->devid),
1362 e->flags);
1363
1364 devid_start = e->devid;
1365 flags = e->flags;
1366 ext_flags = 0;
1367 alias = false;
1368 break;
1369 case IVHD_DEV_ALIAS:
1370
1371 DUMP_printk(" DEV_ALIAS\t\t\t devid: %04x:%02x:%02x.%x "
1372 "flags: %02x devid_to: %02x:%02x.%x\n",
1373 seg_id, PCI_BUS_NUM(e->devid),
1374 PCI_SLOT(e->devid),
1375 PCI_FUNC(e->devid),
1376 e->flags,
1377 PCI_BUS_NUM(e->ext >> 8),
1378 PCI_SLOT(e->ext >> 8),
1379 PCI_FUNC(e->ext >> 8));
1380
1381 devid = e->devid;
1382 devid_to = e->ext >> 8;
1383 set_dev_entry_from_acpi(iommu, devid , e->flags, 0);
1384 set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);
1385 pci_seg->alias_table[devid] = devid_to;
1386 break;
1387 case IVHD_DEV_ALIAS_RANGE:
1388
1389 DUMP_printk(" DEV_ALIAS_RANGE\t\t "
1390 "devid: %04x:%02x:%02x.%x flags: %02x "
1391 "devid_to: %04x:%02x:%02x.%x\n",
1392 seg_id, PCI_BUS_NUM(e->devid),
1393 PCI_SLOT(e->devid),
1394 PCI_FUNC(e->devid),
1395 e->flags,
1396 seg_id, PCI_BUS_NUM(e->ext >> 8),
1397 PCI_SLOT(e->ext >> 8),
1398 PCI_FUNC(e->ext >> 8));
1399
1400 devid_start = e->devid;
1401 flags = e->flags;
1402 devid_to = e->ext >> 8;
1403 ext_flags = 0;
1404 alias = true;
1405 break;
1406 case IVHD_DEV_EXT_SELECT:
1407
1408 DUMP_printk(" DEV_EXT_SELECT\t\t devid: %04x:%02x:%02x.%x "
1409 "flags: %02x ext: %08x\n",
1410 seg_id, PCI_BUS_NUM(e->devid),
1411 PCI_SLOT(e->devid),
1412 PCI_FUNC(e->devid),
1413 e->flags, e->ext);
1414
1415 devid = e->devid;
1416 set_dev_entry_from_acpi(iommu, devid, e->flags,
1417 e->ext);
1418 break;
1419 case IVHD_DEV_EXT_SELECT_RANGE:
1420
1421 DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: "
1422 "%04x:%02x:%02x.%x flags: %02x ext: %08x\n",
1423 seg_id, PCI_BUS_NUM(e->devid),
1424 PCI_SLOT(e->devid),
1425 PCI_FUNC(e->devid),
1426 e->flags, e->ext);
1427
1428 devid_start = e->devid;
1429 flags = e->flags;
1430 ext_flags = e->ext;
1431 alias = false;
1432 break;
1433 case IVHD_DEV_RANGE_END:
1434
1435 DUMP_printk(" DEV_RANGE_END\t\t devid: %04x:%02x:%02x.%x\n",
1436 seg_id, PCI_BUS_NUM(e->devid),
1437 PCI_SLOT(e->devid),
1438 PCI_FUNC(e->devid));
1439
1440 devid = e->devid;
1441 for (dev_i = devid_start; dev_i <= devid; ++dev_i) {
1442 if (alias) {
1443 pci_seg->alias_table[dev_i] = devid_to;
1444 set_dev_entry_from_acpi(iommu,
1445 devid_to, flags, ext_flags);
1446 }
1447 set_dev_entry_from_acpi(iommu, dev_i,
1448 flags, ext_flags);
1449 }
1450 break;
1451 case IVHD_DEV_SPECIAL: {
1452 u8 handle, type;
1453 const char *var;
1454 u32 devid;
1455 int ret;
1456
1457 handle = e->ext & 0xff;
1458 devid = PCI_SEG_DEVID_TO_SBDF(seg_id, (e->ext >> 8));
1459 type = (e->ext >> 24) & 0xff;
1460
1461 if (type == IVHD_SPECIAL_IOAPIC)
1462 var = "IOAPIC";
1463 else if (type == IVHD_SPECIAL_HPET)
1464 var = "HPET";
1465 else
1466 var = "UNKNOWN";
1467
1468 DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %04x:%02x:%02x.%x\n",
1469 var, (int)handle,
1470 seg_id, PCI_BUS_NUM(devid),
1471 PCI_SLOT(devid),
1472 PCI_FUNC(devid));
1473
1474 ret = add_special_device(type, handle, &devid, false);
1475 if (ret)
1476 return ret;
1477
1478 /*
1479 * add_special_device might update the devid in case a
1480 * command-line override is present. So call
1481 * set_dev_entry_from_acpi after add_special_device.
1482 */
1483 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1484
1485 break;
1486 }
1487 case IVHD_DEV_ACPI_HID: {
1488 u32 devid;
1489 u8 hid[ACPIHID_HID_LEN];
1490 u8 uid[ACPIHID_UID_LEN];
1491 int ret;
1492
1493 if (h->type != 0x40) {
1494 pr_err(FW_BUG "Invalid IVHD device type %#x\n",
1495 e->type);
1496 break;
1497 }
1498
1499 BUILD_BUG_ON(sizeof(e->ext_hid) != ACPIHID_HID_LEN - 1);
1500 memcpy(hid, &e->ext_hid, ACPIHID_HID_LEN - 1);
1501 hid[ACPIHID_HID_LEN - 1] = '\0';
1502
1503 if (!(*hid)) {
1504 pr_err(FW_BUG "Invalid HID.\n");
1505 break;
1506 }
1507
1508 uid[0] = '\0';
1509 switch (e->uidf) {
1510 case UID_NOT_PRESENT:
1511
1512 if (e->uidl != 0)
1513 pr_warn(FW_BUG "Invalid UID length.\n");
1514
1515 break;
1516 case UID_IS_INTEGER:
1517
1518 sprintf(uid, "%d", e->uid);
1519
1520 break;
1521 case UID_IS_CHARACTER:
1522
1523 memcpy(uid, &e->uid, e->uidl);
1524 uid[e->uidl] = '\0';
1525
1526 break;
1527 default:
1528 break;
1529 }
1530
1531 devid = PCI_SEG_DEVID_TO_SBDF(seg_id, e->devid);
1532 DUMP_printk(" DEV_ACPI_HID(%s[%s])\t\tdevid: %04x:%02x:%02x.%x\n",
1533 hid, uid, seg_id,
1534 PCI_BUS_NUM(devid),
1535 PCI_SLOT(devid),
1536 PCI_FUNC(devid));
1537
1538 flags = e->flags;
1539
1540 ret = add_acpi_hid_device(hid, uid, &devid, false);
1541 if (ret)
1542 return ret;
1543
1544 /*
1545 * add_special_device might update the devid in case a
1546 * command-line override is present. So call
1547 * set_dev_entry_from_acpi after add_special_device.
1548 */
1549 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1550
1551 break;
1552 }
1553 default:
1554 break;
1555 }
1556
1557 p += ivhd_entry_length(p);
1558 }
1559
1560 return 0;
1561 }
1562
1563 /* Allocate PCI segment data structure */
alloc_pci_segment(u16 id,struct acpi_table_header * ivrs_base)1564 static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id,
1565 struct acpi_table_header *ivrs_base)
1566 {
1567 struct amd_iommu_pci_seg *pci_seg;
1568 int last_bdf;
1569
1570 /*
1571 * First parse ACPI tables to find the largest Bus/Dev/Func we need to
1572 * handle in this PCI segment. Upon this information the shared data
1573 * structures for the PCI segments in the system will be allocated.
1574 */
1575 last_bdf = find_last_devid_acpi(ivrs_base, id);
1576 if (last_bdf < 0)
1577 return NULL;
1578
1579 pci_seg = kzalloc(sizeof(struct amd_iommu_pci_seg), GFP_KERNEL);
1580 if (pci_seg == NULL)
1581 return NULL;
1582
1583 pci_seg->last_bdf = last_bdf;
1584 DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf);
1585 pci_seg->dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE, last_bdf);
1586 pci_seg->alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE, last_bdf);
1587 pci_seg->rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE, last_bdf);
1588
1589 pci_seg->id = id;
1590 init_llist_head(&pci_seg->dev_data_list);
1591 INIT_LIST_HEAD(&pci_seg->unity_map);
1592 list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list);
1593
1594 if (alloc_dev_table(pci_seg))
1595 return NULL;
1596 if (alloc_alias_table(pci_seg))
1597 return NULL;
1598 if (alloc_rlookup_table(pci_seg))
1599 return NULL;
1600
1601 return pci_seg;
1602 }
1603
get_pci_segment(u16 id,struct acpi_table_header * ivrs_base)1604 static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id,
1605 struct acpi_table_header *ivrs_base)
1606 {
1607 struct amd_iommu_pci_seg *pci_seg;
1608
1609 for_each_pci_segment(pci_seg) {
1610 if (pci_seg->id == id)
1611 return pci_seg;
1612 }
1613
1614 return alloc_pci_segment(id, ivrs_base);
1615 }
1616
free_pci_segments(void)1617 static void __init free_pci_segments(void)
1618 {
1619 struct amd_iommu_pci_seg *pci_seg, *next;
1620
1621 for_each_pci_segment_safe(pci_seg, next) {
1622 list_del(&pci_seg->list);
1623 free_irq_lookup_table(pci_seg);
1624 free_rlookup_table(pci_seg);
1625 free_alias_table(pci_seg);
1626 free_dev_table(pci_seg);
1627 kfree(pci_seg);
1628 }
1629 }
1630
free_sysfs(struct amd_iommu * iommu)1631 static void __init free_sysfs(struct amd_iommu *iommu)
1632 {
1633 if (iommu->iommu.dev) {
1634 iommu_device_unregister(&iommu->iommu);
1635 iommu_device_sysfs_remove(&iommu->iommu);
1636 }
1637 }
1638
free_iommu_one(struct amd_iommu * iommu)1639 static void __init free_iommu_one(struct amd_iommu *iommu)
1640 {
1641 free_sysfs(iommu);
1642 free_cwwb_sem(iommu);
1643 free_command_buffer(iommu);
1644 free_event_buffer(iommu);
1645 amd_iommu_free_ppr_log(iommu);
1646 free_ga_log(iommu);
1647 iommu_unmap_mmio_space(iommu);
1648 amd_iommu_iopf_uninit(iommu);
1649 }
1650
free_iommu_all(void)1651 static void __init free_iommu_all(void)
1652 {
1653 struct amd_iommu *iommu, *next;
1654
1655 for_each_iommu_safe(iommu, next) {
1656 list_del(&iommu->list);
1657 free_iommu_one(iommu);
1658 kfree(iommu);
1659 }
1660 }
1661
1662 /*
1663 * Family15h Model 10h-1fh erratum 746 (IOMMU Logging May Stall Translations)
1664 * Workaround:
1665 * BIOS should disable L2B micellaneous clock gating by setting
1666 * L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b
1667 */
amd_iommu_erratum_746_workaround(struct amd_iommu * iommu)1668 static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu)
1669 {
1670 u32 value;
1671
1672 if ((boot_cpu_data.x86 != 0x15) ||
1673 (boot_cpu_data.x86_model < 0x10) ||
1674 (boot_cpu_data.x86_model > 0x1f))
1675 return;
1676
1677 pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1678 pci_read_config_dword(iommu->dev, 0xf4, &value);
1679
1680 if (value & BIT(2))
1681 return;
1682
1683 /* Select NB indirect register 0x90 and enable writing */
1684 pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8));
1685
1686 pci_write_config_dword(iommu->dev, 0xf4, value | 0x4);
1687 pci_info(iommu->dev, "Applying erratum 746 workaround\n");
1688
1689 /* Clear the enable writing bit */
1690 pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1691 }
1692
1693 /*
1694 * Family15h Model 30h-3fh (IOMMU Mishandles ATS Write Permission)
1695 * Workaround:
1696 * BIOS should enable ATS write permission check by setting
1697 * L2_DEBUG_3[AtsIgnoreIWDis](D0F2xF4_x47[0]) = 1b
1698 */
amd_iommu_ats_write_check_workaround(struct amd_iommu * iommu)1699 static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu)
1700 {
1701 u32 value;
1702
1703 if ((boot_cpu_data.x86 != 0x15) ||
1704 (boot_cpu_data.x86_model < 0x30) ||
1705 (boot_cpu_data.x86_model > 0x3f))
1706 return;
1707
1708 /* Test L2_DEBUG_3[AtsIgnoreIWDis] == 1 */
1709 value = iommu_read_l2(iommu, 0x47);
1710
1711 if (value & BIT(0))
1712 return;
1713
1714 /* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */
1715 iommu_write_l2(iommu, 0x47, value | BIT(0));
1716
1717 pci_info(iommu->dev, "Applying ATS write check workaround\n");
1718 }
1719
1720 /*
1721 * This function glues the initialization function for one IOMMU
1722 * together and also allocates the command buffer and programs the
1723 * hardware. It does NOT enable the IOMMU. This is done afterwards.
1724 */
init_iommu_one(struct amd_iommu * iommu,struct ivhd_header * h,struct acpi_table_header * ivrs_base)1725 static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
1726 struct acpi_table_header *ivrs_base)
1727 {
1728 struct amd_iommu_pci_seg *pci_seg;
1729
1730 pci_seg = get_pci_segment(h->pci_seg, ivrs_base);
1731 if (pci_seg == NULL)
1732 return -ENOMEM;
1733 iommu->pci_seg = pci_seg;
1734
1735 raw_spin_lock_init(&iommu->lock);
1736 atomic64_set(&iommu->cmd_sem_val, 0);
1737
1738 /* Add IOMMU to internal data structures */
1739 list_add_tail(&iommu->list, &amd_iommu_list);
1740 iommu->index = amd_iommus_present++;
1741
1742 if (unlikely(iommu->index >= MAX_IOMMUS)) {
1743 WARN(1, "System has more IOMMUs than supported by this driver\n");
1744 return -ENOSYS;
1745 }
1746
1747 /* Index is fine - add IOMMU to the array */
1748 amd_iommus[iommu->index] = iommu;
1749
1750 /*
1751 * Copy data from ACPI table entry to the iommu struct
1752 */
1753 iommu->devid = h->devid;
1754 iommu->cap_ptr = h->cap_ptr;
1755 iommu->mmio_phys = h->mmio_phys;
1756
1757 switch (h->type) {
1758 case 0x10:
1759 /* Check if IVHD EFR contains proper max banks/counters */
1760 if ((h->efr_attr != 0) &&
1761 ((h->efr_attr & (0xF << 13)) != 0) &&
1762 ((h->efr_attr & (0x3F << 17)) != 0))
1763 iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
1764 else
1765 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1766
1767 /*
1768 * Note: GA (128-bit IRTE) mode requires cmpxchg16b supports.
1769 * GAM also requires GA mode. Therefore, we need to
1770 * check cmpxchg16b support before enabling it.
1771 */
1772 if (!boot_cpu_has(X86_FEATURE_CX16) ||
1773 ((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0))
1774 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1775 break;
1776 case 0x11:
1777 case 0x40:
1778 if (h->efr_reg & (1 << 9))
1779 iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
1780 else
1781 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1782
1783 /*
1784 * Note: GA (128-bit IRTE) mode requires cmpxchg16b supports.
1785 * XT, GAM also requires GA mode. Therefore, we need to
1786 * check cmpxchg16b support before enabling them.
1787 */
1788 if (!boot_cpu_has(X86_FEATURE_CX16) ||
1789 ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0)) {
1790 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1791 break;
1792 }
1793
1794 if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT))
1795 amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE;
1796
1797 early_iommu_features_init(iommu, h);
1798
1799 break;
1800 default:
1801 return -EINVAL;
1802 }
1803
1804 iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys,
1805 iommu->mmio_phys_end);
1806 if (!iommu->mmio_base)
1807 return -ENOMEM;
1808
1809 return init_iommu_from_acpi(iommu, h);
1810 }
1811
init_iommu_one_late(struct amd_iommu * iommu)1812 static int __init init_iommu_one_late(struct amd_iommu *iommu)
1813 {
1814 int ret;
1815
1816 if (alloc_cwwb_sem(iommu))
1817 return -ENOMEM;
1818
1819 if (alloc_command_buffer(iommu))
1820 return -ENOMEM;
1821
1822 if (alloc_event_buffer(iommu))
1823 return -ENOMEM;
1824
1825 iommu->int_enabled = false;
1826
1827 init_translation_status(iommu);
1828 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
1829 iommu_disable(iommu);
1830 clear_translation_pre_enabled(iommu);
1831 pr_warn("Translation was enabled for IOMMU:%d but we are not in kdump mode\n",
1832 iommu->index);
1833 }
1834 if (amd_iommu_pre_enabled)
1835 amd_iommu_pre_enabled = translation_pre_enabled(iommu);
1836
1837 if (amd_iommu_irq_remap) {
1838 ret = amd_iommu_create_irq_domain(iommu);
1839 if (ret)
1840 return ret;
1841 }
1842
1843 /*
1844 * Make sure IOMMU is not considered to translate itself. The IVRS
1845 * table tells us so, but this is a lie!
1846 */
1847 iommu->pci_seg->rlookup_table[iommu->devid] = NULL;
1848
1849 return 0;
1850 }
1851
1852 /**
1853 * get_highest_supported_ivhd_type - Look up the appropriate IVHD type
1854 * @ivrs: Pointer to the IVRS header
1855 *
1856 * This function search through all IVDB of the maximum supported IVHD
1857 */
get_highest_supported_ivhd_type(struct acpi_table_header * ivrs)1858 static u8 get_highest_supported_ivhd_type(struct acpi_table_header *ivrs)
1859 {
1860 u8 *base = (u8 *)ivrs;
1861 struct ivhd_header *ivhd = (struct ivhd_header *)
1862 (base + IVRS_HEADER_LENGTH);
1863 u8 last_type = ivhd->type;
1864 u16 devid = ivhd->devid;
1865
1866 while (((u8 *)ivhd - base < ivrs->length) &&
1867 (ivhd->type <= ACPI_IVHD_TYPE_MAX_SUPPORTED)) {
1868 u8 *p = (u8 *) ivhd;
1869
1870 if (ivhd->devid == devid)
1871 last_type = ivhd->type;
1872 ivhd = (struct ivhd_header *)(p + ivhd->length);
1873 }
1874
1875 return last_type;
1876 }
1877
1878 /*
1879 * Iterates over all IOMMU entries in the ACPI table, allocates the
1880 * IOMMU structure and initializes it with init_iommu_one()
1881 */
init_iommu_all(struct acpi_table_header * table)1882 static int __init init_iommu_all(struct acpi_table_header *table)
1883 {
1884 u8 *p = (u8 *)table, *end = (u8 *)table;
1885 struct ivhd_header *h;
1886 struct amd_iommu *iommu;
1887 int ret;
1888
1889 end += table->length;
1890 p += IVRS_HEADER_LENGTH;
1891
1892 /* Phase 1: Process all IVHD blocks */
1893 while (p < end) {
1894 h = (struct ivhd_header *)p;
1895 if (*p == amd_iommu_target_ivhd_type) {
1896
1897 DUMP_printk("device: %04x:%02x:%02x.%01x cap: %04x "
1898 "flags: %01x info %04x\n",
1899 h->pci_seg, PCI_BUS_NUM(h->devid),
1900 PCI_SLOT(h->devid), PCI_FUNC(h->devid),
1901 h->cap_ptr, h->flags, h->info);
1902 DUMP_printk(" mmio-addr: %016llx\n",
1903 h->mmio_phys);
1904
1905 iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
1906 if (iommu == NULL)
1907 return -ENOMEM;
1908
1909 ret = init_iommu_one(iommu, h, table);
1910 if (ret)
1911 return ret;
1912 }
1913 p += h->length;
1914
1915 }
1916 WARN_ON(p != end);
1917
1918 /* Phase 2 : Early feature support check */
1919 get_global_efr();
1920
1921 /* Phase 3 : Enabling IOMMU features */
1922 for_each_iommu(iommu) {
1923 ret = init_iommu_one_late(iommu);
1924 if (ret)
1925 return ret;
1926 }
1927
1928 return 0;
1929 }
1930
init_iommu_perf_ctr(struct amd_iommu * iommu)1931 static void init_iommu_perf_ctr(struct amd_iommu *iommu)
1932 {
1933 u64 val;
1934 struct pci_dev *pdev = iommu->dev;
1935
1936 if (!check_feature(FEATURE_PC))
1937 return;
1938
1939 amd_iommu_pc_present = true;
1940
1941 pci_info(pdev, "IOMMU performance counters supported\n");
1942
1943 val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET);
1944 iommu->max_banks = (u8) ((val >> 12) & 0x3f);
1945 iommu->max_counters = (u8) ((val >> 7) & 0xf);
1946
1947 return;
1948 }
1949
amd_iommu_show_cap(struct device * dev,struct device_attribute * attr,char * buf)1950 static ssize_t amd_iommu_show_cap(struct device *dev,
1951 struct device_attribute *attr,
1952 char *buf)
1953 {
1954 struct amd_iommu *iommu = dev_to_amd_iommu(dev);
1955 return sysfs_emit(buf, "%x\n", iommu->cap);
1956 }
1957 static DEVICE_ATTR(cap, S_IRUGO, amd_iommu_show_cap, NULL);
1958
amd_iommu_show_features(struct device * dev,struct device_attribute * attr,char * buf)1959 static ssize_t amd_iommu_show_features(struct device *dev,
1960 struct device_attribute *attr,
1961 char *buf)
1962 {
1963 return sysfs_emit(buf, "%llx:%llx\n", amd_iommu_efr, amd_iommu_efr2);
1964 }
1965 static DEVICE_ATTR(features, S_IRUGO, amd_iommu_show_features, NULL);
1966
1967 static struct attribute *amd_iommu_attrs[] = {
1968 &dev_attr_cap.attr,
1969 &dev_attr_features.attr,
1970 NULL,
1971 };
1972
1973 static struct attribute_group amd_iommu_group = {
1974 .name = "amd-iommu",
1975 .attrs = amd_iommu_attrs,
1976 };
1977
1978 static const struct attribute_group *amd_iommu_groups[] = {
1979 &amd_iommu_group,
1980 NULL,
1981 };
1982
1983 /*
1984 * Note: IVHD 0x11 and 0x40 also contains exact copy
1985 * of the IOMMU Extended Feature Register [MMIO Offset 0030h].
1986 * Default to EFR in IVHD since it is available sooner (i.e. before PCI init).
1987 */
late_iommu_features_init(struct amd_iommu * iommu)1988 static void __init late_iommu_features_init(struct amd_iommu *iommu)
1989 {
1990 u64 features, features2;
1991
1992 if (!(iommu->cap & (1 << IOMMU_CAP_EFR)))
1993 return;
1994
1995 /* read extended feature bits */
1996 features = readq(iommu->mmio_base + MMIO_EXT_FEATURES);
1997 features2 = readq(iommu->mmio_base + MMIO_EXT_FEATURES2);
1998
1999 if (!amd_iommu_efr) {
2000 amd_iommu_efr = features;
2001 amd_iommu_efr2 = features2;
2002 return;
2003 }
2004
2005 /*
2006 * Sanity check and warn if EFR values from
2007 * IVHD and MMIO conflict.
2008 */
2009 if (features != amd_iommu_efr ||
2010 features2 != amd_iommu_efr2) {
2011 pr_warn(FW_WARN
2012 "EFR mismatch. Use IVHD EFR (%#llx : %#llx), EFR2 (%#llx : %#llx).\n",
2013 features, amd_iommu_efr,
2014 features2, amd_iommu_efr2);
2015 }
2016 }
2017
iommu_init_pci(struct amd_iommu * iommu)2018 static int __init iommu_init_pci(struct amd_iommu *iommu)
2019 {
2020 int cap_ptr = iommu->cap_ptr;
2021 int ret;
2022
2023 iommu->dev = pci_get_domain_bus_and_slot(iommu->pci_seg->id,
2024 PCI_BUS_NUM(iommu->devid),
2025 iommu->devid & 0xff);
2026 if (!iommu->dev)
2027 return -ENODEV;
2028
2029 /* Prevent binding other PCI device drivers to IOMMU devices */
2030 iommu->dev->match_driver = false;
2031
2032 /* ACPI _PRT won't have an IRQ for IOMMU */
2033 iommu->dev->irq_managed = 1;
2034
2035 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
2036 &iommu->cap);
2037
2038 if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
2039 amd_iommu_iotlb_sup = false;
2040
2041 late_iommu_features_init(iommu);
2042
2043 if (check_feature(FEATURE_GT)) {
2044 int glxval;
2045 u64 pasmax;
2046
2047 pasmax = FIELD_GET(FEATURE_PASMAX, amd_iommu_efr);
2048 iommu->iommu.max_pasids = (1 << (pasmax + 1)) - 1;
2049
2050 BUG_ON(iommu->iommu.max_pasids & ~PASID_MASK);
2051
2052 glxval = FIELD_GET(FEATURE_GLX, amd_iommu_efr);
2053
2054 if (amd_iommu_max_glx_val == -1)
2055 amd_iommu_max_glx_val = glxval;
2056 else
2057 amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
2058
2059 iommu_enable_gt(iommu);
2060 }
2061
2062 if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu))
2063 return -ENOMEM;
2064
2065 if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) {
2066 pr_info("Using strict mode due to virtualization\n");
2067 iommu_set_dma_strict();
2068 amd_iommu_np_cache = true;
2069 }
2070
2071 init_iommu_perf_ctr(iommu);
2072
2073 if (amd_iommu_pgtable == AMD_IOMMU_V2) {
2074 if (!check_feature(FEATURE_GIOSUP) ||
2075 !check_feature(FEATURE_GT)) {
2076 pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n");
2077 amd_iommu_pgtable = AMD_IOMMU_V1;
2078 }
2079 }
2080
2081 if (is_rd890_iommu(iommu->dev)) {
2082 int i, j;
2083
2084 iommu->root_pdev =
2085 pci_get_domain_bus_and_slot(iommu->pci_seg->id,
2086 iommu->dev->bus->number,
2087 PCI_DEVFN(0, 0));
2088
2089 /*
2090 * Some rd890 systems may not be fully reconfigured by the
2091 * BIOS, so it's necessary for us to store this information so
2092 * it can be reprogrammed on resume
2093 */
2094 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,
2095 &iommu->stored_addr_lo);
2096 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,
2097 &iommu->stored_addr_hi);
2098
2099 /* Low bit locks writes to configuration space */
2100 iommu->stored_addr_lo &= ~1;
2101
2102 for (i = 0; i < 6; i++)
2103 for (j = 0; j < 0x12; j++)
2104 iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);
2105
2106 for (i = 0; i < 0x83; i++)
2107 iommu->stored_l2[i] = iommu_read_l2(iommu, i);
2108 }
2109
2110 amd_iommu_erratum_746_workaround(iommu);
2111 amd_iommu_ats_write_check_workaround(iommu);
2112
2113 ret = iommu_device_sysfs_add(&iommu->iommu, &iommu->dev->dev,
2114 amd_iommu_groups, "ivhd%d", iommu->index);
2115 if (ret)
2116 return ret;
2117
2118 /*
2119 * Allocate per IOMMU IOPF queue here so that in attach device path,
2120 * PRI capable device can be added to IOPF queue
2121 */
2122 if (amd_iommu_gt_ppr_supported()) {
2123 ret = amd_iommu_iopf_init(iommu);
2124 if (ret)
2125 return ret;
2126 }
2127
2128 iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL);
2129
2130 return pci_enable_device(iommu->dev);
2131 }
2132
print_iommu_info(void)2133 static void print_iommu_info(void)
2134 {
2135 int i;
2136 static const char * const feat_str[] = {
2137 "PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
2138 "IA", "GA", "HE", "PC"
2139 };
2140
2141 if (amd_iommu_efr) {
2142 pr_info("Extended features (%#llx, %#llx):", amd_iommu_efr, amd_iommu_efr2);
2143
2144 for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
2145 if (check_feature(1ULL << i))
2146 pr_cont(" %s", feat_str[i]);
2147 }
2148
2149 if (check_feature(FEATURE_GAM_VAPIC))
2150 pr_cont(" GA_vAPIC");
2151
2152 if (check_feature(FEATURE_SNP))
2153 pr_cont(" SNP");
2154
2155 pr_cont("\n");
2156 }
2157
2158 if (irq_remapping_enabled) {
2159 pr_info("Interrupt remapping enabled\n");
2160 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2161 pr_info("X2APIC enabled\n");
2162 }
2163 if (amd_iommu_pgtable == AMD_IOMMU_V2) {
2164 pr_info("V2 page table enabled (Paging mode : %d level)\n",
2165 amd_iommu_gpt_level);
2166 }
2167 }
2168
amd_iommu_init_pci(void)2169 static int __init amd_iommu_init_pci(void)
2170 {
2171 struct amd_iommu *iommu;
2172 struct amd_iommu_pci_seg *pci_seg;
2173 int ret;
2174
2175 for_each_iommu(iommu) {
2176 ret = iommu_init_pci(iommu);
2177 if (ret) {
2178 pr_err("IOMMU%d: Failed to initialize IOMMU Hardware (error=%d)!\n",
2179 iommu->index, ret);
2180 goto out;
2181 }
2182 /* Need to setup range after PCI init */
2183 iommu_set_cwwb_range(iommu);
2184 }
2185
2186 /*
2187 * Order is important here to make sure any unity map requirements are
2188 * fulfilled. The unity mappings are created and written to the device
2189 * table during the iommu_init_pci() call.
2190 *
2191 * After that we call init_device_table_dma() to make sure any
2192 * uninitialized DTE will block DMA, and in the end we flush the caches
2193 * of all IOMMUs to make sure the changes to the device table are
2194 * active.
2195 */
2196 for_each_pci_segment(pci_seg)
2197 init_device_table_dma(pci_seg);
2198
2199 for_each_iommu(iommu)
2200 amd_iommu_flush_all_caches(iommu);
2201
2202 print_iommu_info();
2203
2204 out:
2205 return ret;
2206 }
2207
2208 /****************************************************************************
2209 *
2210 * The following functions initialize the MSI interrupts for all IOMMUs
2211 * in the system. It's a bit challenging because there could be multiple
2212 * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per
2213 * pci_dev.
2214 *
2215 ****************************************************************************/
2216
iommu_setup_msi(struct amd_iommu * iommu)2217 static int iommu_setup_msi(struct amd_iommu *iommu)
2218 {
2219 int r;
2220
2221 r = pci_enable_msi(iommu->dev);
2222 if (r)
2223 return r;
2224
2225 r = request_threaded_irq(iommu->dev->irq,
2226 amd_iommu_int_handler,
2227 amd_iommu_int_thread,
2228 0, "AMD-Vi",
2229 iommu);
2230
2231 if (r) {
2232 pci_disable_msi(iommu->dev);
2233 return r;
2234 }
2235
2236 return 0;
2237 }
2238
2239 union intcapxt {
2240 u64 capxt;
2241 struct {
2242 u64 reserved_0 : 2,
2243 dest_mode_logical : 1,
2244 reserved_1 : 5,
2245 destid_0_23 : 24,
2246 vector : 8,
2247 reserved_2 : 16,
2248 destid_24_31 : 8;
2249 };
2250 } __attribute__ ((packed));
2251
2252
2253 static struct irq_chip intcapxt_controller;
2254
intcapxt_irqdomain_activate(struct irq_domain * domain,struct irq_data * irqd,bool reserve)2255 static int intcapxt_irqdomain_activate(struct irq_domain *domain,
2256 struct irq_data *irqd, bool reserve)
2257 {
2258 return 0;
2259 }
2260
intcapxt_irqdomain_deactivate(struct irq_domain * domain,struct irq_data * irqd)2261 static void intcapxt_irqdomain_deactivate(struct irq_domain *domain,
2262 struct irq_data *irqd)
2263 {
2264 }
2265
2266
intcapxt_irqdomain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)2267 static int intcapxt_irqdomain_alloc(struct irq_domain *domain, unsigned int virq,
2268 unsigned int nr_irqs, void *arg)
2269 {
2270 struct irq_alloc_info *info = arg;
2271 int i, ret;
2272
2273 if (!info || info->type != X86_IRQ_ALLOC_TYPE_AMDVI)
2274 return -EINVAL;
2275
2276 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
2277 if (ret < 0)
2278 return ret;
2279
2280 for (i = virq; i < virq + nr_irqs; i++) {
2281 struct irq_data *irqd = irq_domain_get_irq_data(domain, i);
2282
2283 irqd->chip = &intcapxt_controller;
2284 irqd->hwirq = info->hwirq;
2285 irqd->chip_data = info->data;
2286 __irq_set_handler(i, handle_edge_irq, 0, "edge");
2287 }
2288
2289 return ret;
2290 }
2291
intcapxt_irqdomain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)2292 static void intcapxt_irqdomain_free(struct irq_domain *domain, unsigned int virq,
2293 unsigned int nr_irqs)
2294 {
2295 irq_domain_free_irqs_top(domain, virq, nr_irqs);
2296 }
2297
2298
intcapxt_unmask_irq(struct irq_data * irqd)2299 static void intcapxt_unmask_irq(struct irq_data *irqd)
2300 {
2301 struct amd_iommu *iommu = irqd->chip_data;
2302 struct irq_cfg *cfg = irqd_cfg(irqd);
2303 union intcapxt xt;
2304
2305 xt.capxt = 0ULL;
2306 xt.dest_mode_logical = apic->dest_mode_logical;
2307 xt.vector = cfg->vector;
2308 xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0);
2309 xt.destid_24_31 = cfg->dest_apicid >> 24;
2310
2311 writeq(xt.capxt, iommu->mmio_base + irqd->hwirq);
2312 }
2313
intcapxt_mask_irq(struct irq_data * irqd)2314 static void intcapxt_mask_irq(struct irq_data *irqd)
2315 {
2316 struct amd_iommu *iommu = irqd->chip_data;
2317
2318 writeq(0, iommu->mmio_base + irqd->hwirq);
2319 }
2320
2321
intcapxt_set_affinity(struct irq_data * irqd,const struct cpumask * mask,bool force)2322 static int intcapxt_set_affinity(struct irq_data *irqd,
2323 const struct cpumask *mask, bool force)
2324 {
2325 struct irq_data *parent = irqd->parent_data;
2326 int ret;
2327
2328 ret = parent->chip->irq_set_affinity(parent, mask, force);
2329 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
2330 return ret;
2331 return 0;
2332 }
2333
intcapxt_set_wake(struct irq_data * irqd,unsigned int on)2334 static int intcapxt_set_wake(struct irq_data *irqd, unsigned int on)
2335 {
2336 return on ? -EOPNOTSUPP : 0;
2337 }
2338
2339 static struct irq_chip intcapxt_controller = {
2340 .name = "IOMMU-MSI",
2341 .irq_unmask = intcapxt_unmask_irq,
2342 .irq_mask = intcapxt_mask_irq,
2343 .irq_ack = irq_chip_ack_parent,
2344 .irq_retrigger = irq_chip_retrigger_hierarchy,
2345 .irq_set_affinity = intcapxt_set_affinity,
2346 .irq_set_wake = intcapxt_set_wake,
2347 .flags = IRQCHIP_MASK_ON_SUSPEND,
2348 };
2349
2350 static const struct irq_domain_ops intcapxt_domain_ops = {
2351 .alloc = intcapxt_irqdomain_alloc,
2352 .free = intcapxt_irqdomain_free,
2353 .activate = intcapxt_irqdomain_activate,
2354 .deactivate = intcapxt_irqdomain_deactivate,
2355 };
2356
2357
2358 static struct irq_domain *iommu_irqdomain;
2359
iommu_get_irqdomain(void)2360 static struct irq_domain *iommu_get_irqdomain(void)
2361 {
2362 struct fwnode_handle *fn;
2363
2364 /* No need for locking here (yet) as the init is single-threaded */
2365 if (iommu_irqdomain)
2366 return iommu_irqdomain;
2367
2368 fn = irq_domain_alloc_named_fwnode("AMD-Vi-MSI");
2369 if (!fn)
2370 return NULL;
2371
2372 iommu_irqdomain = irq_domain_create_hierarchy(x86_vector_domain, 0, 0,
2373 fn, &intcapxt_domain_ops,
2374 NULL);
2375 if (!iommu_irqdomain)
2376 irq_domain_free_fwnode(fn);
2377
2378 return iommu_irqdomain;
2379 }
2380
__iommu_setup_intcapxt(struct amd_iommu * iommu,const char * devname,int hwirq,irq_handler_t thread_fn)2381 static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname,
2382 int hwirq, irq_handler_t thread_fn)
2383 {
2384 struct irq_domain *domain;
2385 struct irq_alloc_info info;
2386 int irq, ret;
2387 int node = dev_to_node(&iommu->dev->dev);
2388
2389 domain = iommu_get_irqdomain();
2390 if (!domain)
2391 return -ENXIO;
2392
2393 init_irq_alloc_info(&info, NULL);
2394 info.type = X86_IRQ_ALLOC_TYPE_AMDVI;
2395 info.data = iommu;
2396 info.hwirq = hwirq;
2397
2398 irq = irq_domain_alloc_irqs(domain, 1, node, &info);
2399 if (irq < 0) {
2400 irq_domain_remove(domain);
2401 return irq;
2402 }
2403
2404 ret = request_threaded_irq(irq, amd_iommu_int_handler,
2405 thread_fn, 0, devname, iommu);
2406 if (ret) {
2407 irq_domain_free_irqs(irq, 1);
2408 irq_domain_remove(domain);
2409 return ret;
2410 }
2411
2412 return 0;
2413 }
2414
iommu_setup_intcapxt(struct amd_iommu * iommu)2415 static int iommu_setup_intcapxt(struct amd_iommu *iommu)
2416 {
2417 int ret;
2418
2419 snprintf(iommu->evt_irq_name, sizeof(iommu->evt_irq_name),
2420 "AMD-Vi%d-Evt", iommu->index);
2421 ret = __iommu_setup_intcapxt(iommu, iommu->evt_irq_name,
2422 MMIO_INTCAPXT_EVT_OFFSET,
2423 amd_iommu_int_thread_evtlog);
2424 if (ret)
2425 return ret;
2426
2427 snprintf(iommu->ppr_irq_name, sizeof(iommu->ppr_irq_name),
2428 "AMD-Vi%d-PPR", iommu->index);
2429 ret = __iommu_setup_intcapxt(iommu, iommu->ppr_irq_name,
2430 MMIO_INTCAPXT_PPR_OFFSET,
2431 amd_iommu_int_thread_pprlog);
2432 if (ret)
2433 return ret;
2434
2435 #ifdef CONFIG_IRQ_REMAP
2436 snprintf(iommu->ga_irq_name, sizeof(iommu->ga_irq_name),
2437 "AMD-Vi%d-GA", iommu->index);
2438 ret = __iommu_setup_intcapxt(iommu, iommu->ga_irq_name,
2439 MMIO_INTCAPXT_GALOG_OFFSET,
2440 amd_iommu_int_thread_galog);
2441 #endif
2442
2443 return ret;
2444 }
2445
iommu_init_irq(struct amd_iommu * iommu)2446 static int iommu_init_irq(struct amd_iommu *iommu)
2447 {
2448 int ret;
2449
2450 if (iommu->int_enabled)
2451 goto enable_faults;
2452
2453 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2454 ret = iommu_setup_intcapxt(iommu);
2455 else if (iommu->dev->msi_cap)
2456 ret = iommu_setup_msi(iommu);
2457 else
2458 ret = -ENODEV;
2459
2460 if (ret)
2461 return ret;
2462
2463 iommu->int_enabled = true;
2464 enable_faults:
2465
2466 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2467 iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);
2468
2469 iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
2470
2471 return 0;
2472 }
2473
2474 /****************************************************************************
2475 *
2476 * The next functions belong to the third pass of parsing the ACPI
2477 * table. In this last pass the memory mapping requirements are
2478 * gathered (like exclusion and unity mapping ranges).
2479 *
2480 ****************************************************************************/
2481
free_unity_maps(void)2482 static void __init free_unity_maps(void)
2483 {
2484 struct unity_map_entry *entry, *next;
2485 struct amd_iommu_pci_seg *p, *pci_seg;
2486
2487 for_each_pci_segment_safe(pci_seg, p) {
2488 list_for_each_entry_safe(entry, next, &pci_seg->unity_map, list) {
2489 list_del(&entry->list);
2490 kfree(entry);
2491 }
2492 }
2493 }
2494
2495 /* called for unity map ACPI definition */
init_unity_map_range(struct ivmd_header * m,struct acpi_table_header * ivrs_base)2496 static int __init init_unity_map_range(struct ivmd_header *m,
2497 struct acpi_table_header *ivrs_base)
2498 {
2499 struct unity_map_entry *e = NULL;
2500 struct amd_iommu_pci_seg *pci_seg;
2501 char *s;
2502
2503 pci_seg = get_pci_segment(m->pci_seg, ivrs_base);
2504 if (pci_seg == NULL)
2505 return -ENOMEM;
2506
2507 e = kzalloc(sizeof(*e), GFP_KERNEL);
2508 if (e == NULL)
2509 return -ENOMEM;
2510
2511 switch (m->type) {
2512 default:
2513 kfree(e);
2514 return 0;
2515 case ACPI_IVMD_TYPE:
2516 s = "IVMD_TYPEi\t\t\t";
2517 e->devid_start = e->devid_end = m->devid;
2518 break;
2519 case ACPI_IVMD_TYPE_ALL:
2520 s = "IVMD_TYPE_ALL\t\t";
2521 e->devid_start = 0;
2522 e->devid_end = pci_seg->last_bdf;
2523 break;
2524 case ACPI_IVMD_TYPE_RANGE:
2525 s = "IVMD_TYPE_RANGE\t\t";
2526 e->devid_start = m->devid;
2527 e->devid_end = m->aux;
2528 break;
2529 }
2530 e->address_start = PAGE_ALIGN(m->range_start);
2531 e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
2532 e->prot = m->flags >> 1;
2533
2534 /*
2535 * Treat per-device exclusion ranges as r/w unity-mapped regions
2536 * since some buggy BIOSes might lead to the overwritten exclusion
2537 * range (exclusion_start and exclusion_length members). This
2538 * happens when there are multiple exclusion ranges (IVMD entries)
2539 * defined in ACPI table.
2540 */
2541 if (m->flags & IVMD_FLAG_EXCL_RANGE)
2542 e->prot = (IVMD_FLAG_IW | IVMD_FLAG_IR) >> 1;
2543
2544 DUMP_printk("%s devid_start: %04x:%02x:%02x.%x devid_end: "
2545 "%04x:%02x:%02x.%x range_start: %016llx range_end: %016llx"
2546 " flags: %x\n", s, m->pci_seg,
2547 PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start),
2548 PCI_FUNC(e->devid_start), m->pci_seg,
2549 PCI_BUS_NUM(e->devid_end),
2550 PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
2551 e->address_start, e->address_end, m->flags);
2552
2553 list_add_tail(&e->list, &pci_seg->unity_map);
2554
2555 return 0;
2556 }
2557
2558 /* iterates over all memory definitions we find in the ACPI table */
init_memory_definitions(struct acpi_table_header * table)2559 static int __init init_memory_definitions(struct acpi_table_header *table)
2560 {
2561 u8 *p = (u8 *)table, *end = (u8 *)table;
2562 struct ivmd_header *m;
2563
2564 end += table->length;
2565 p += IVRS_HEADER_LENGTH;
2566
2567 while (p < end) {
2568 m = (struct ivmd_header *)p;
2569 if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE))
2570 init_unity_map_range(m, table);
2571
2572 p += m->length;
2573 }
2574
2575 return 0;
2576 }
2577
2578 /*
2579 * Init the device table to not allow DMA access for devices
2580 */
init_device_table_dma(struct amd_iommu_pci_seg * pci_seg)2581 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg)
2582 {
2583 u32 devid;
2584 struct dev_table_entry *dev_table = pci_seg->dev_table;
2585
2586 if (dev_table == NULL)
2587 return;
2588
2589 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
2590 __set_dev_entry_bit(dev_table, devid, DEV_ENTRY_VALID);
2591 if (!amd_iommu_snp_en)
2592 __set_dev_entry_bit(dev_table, devid, DEV_ENTRY_TRANSLATION);
2593 }
2594 }
2595
uninit_device_table_dma(struct amd_iommu_pci_seg * pci_seg)2596 static void __init uninit_device_table_dma(struct amd_iommu_pci_seg *pci_seg)
2597 {
2598 u32 devid;
2599 struct dev_table_entry *dev_table = pci_seg->dev_table;
2600
2601 if (dev_table == NULL)
2602 return;
2603
2604 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
2605 dev_table[devid].data[0] = 0ULL;
2606 dev_table[devid].data[1] = 0ULL;
2607 }
2608 }
2609
init_device_table(void)2610 static void init_device_table(void)
2611 {
2612 struct amd_iommu_pci_seg *pci_seg;
2613 u32 devid;
2614
2615 if (!amd_iommu_irq_remap)
2616 return;
2617
2618 for_each_pci_segment(pci_seg) {
2619 for (devid = 0; devid <= pci_seg->last_bdf; ++devid)
2620 __set_dev_entry_bit(pci_seg->dev_table,
2621 devid, DEV_ENTRY_IRQ_TBL_EN);
2622 }
2623 }
2624
iommu_init_flags(struct amd_iommu * iommu)2625 static void iommu_init_flags(struct amd_iommu *iommu)
2626 {
2627 iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?
2628 iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :
2629 iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);
2630
2631 iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?
2632 iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :
2633 iommu_feature_disable(iommu, CONTROL_PASSPW_EN);
2634
2635 iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?
2636 iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :
2637 iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);
2638
2639 iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?
2640 iommu_feature_enable(iommu, CONTROL_ISOC_EN) :
2641 iommu_feature_disable(iommu, CONTROL_ISOC_EN);
2642
2643 /*
2644 * make IOMMU memory accesses cache coherent
2645 */
2646 iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
2647
2648 /* Set IOTLB invalidation timeout to 1s */
2649 iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);
2650 }
2651
iommu_apply_resume_quirks(struct amd_iommu * iommu)2652 static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
2653 {
2654 int i, j;
2655 u32 ioc_feature_control;
2656 struct pci_dev *pdev = iommu->root_pdev;
2657
2658 /* RD890 BIOSes may not have completely reconfigured the iommu */
2659 if (!is_rd890_iommu(iommu->dev) || !pdev)
2660 return;
2661
2662 /*
2663 * First, we need to ensure that the iommu is enabled. This is
2664 * controlled by a register in the northbridge
2665 */
2666
2667 /* Select Northbridge indirect register 0x75 and enable writing */
2668 pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));
2669 pci_read_config_dword(pdev, 0x64, &ioc_feature_control);
2670
2671 /* Enable the iommu */
2672 if (!(ioc_feature_control & 0x1))
2673 pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);
2674
2675 /* Restore the iommu BAR */
2676 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
2677 iommu->stored_addr_lo);
2678 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,
2679 iommu->stored_addr_hi);
2680
2681 /* Restore the l1 indirect regs for each of the 6 l1s */
2682 for (i = 0; i < 6; i++)
2683 for (j = 0; j < 0x12; j++)
2684 iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);
2685
2686 /* Restore the l2 indirect regs */
2687 for (i = 0; i < 0x83; i++)
2688 iommu_write_l2(iommu, i, iommu->stored_l2[i]);
2689
2690 /* Lock PCI setup registers */
2691 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
2692 iommu->stored_addr_lo | 1);
2693 }
2694
iommu_enable_ga(struct amd_iommu * iommu)2695 static void iommu_enable_ga(struct amd_iommu *iommu)
2696 {
2697 #ifdef CONFIG_IRQ_REMAP
2698 switch (amd_iommu_guest_ir) {
2699 case AMD_IOMMU_GUEST_IR_VAPIC:
2700 case AMD_IOMMU_GUEST_IR_LEGACY_GA:
2701 iommu_feature_enable(iommu, CONTROL_GA_EN);
2702 iommu->irte_ops = &irte_128_ops;
2703 break;
2704 default:
2705 iommu->irte_ops = &irte_32_ops;
2706 break;
2707 }
2708 #endif
2709 }
2710
iommu_disable_irtcachedis(struct amd_iommu * iommu)2711 static void iommu_disable_irtcachedis(struct amd_iommu *iommu)
2712 {
2713 iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);
2714 }
2715
iommu_enable_irtcachedis(struct amd_iommu * iommu)2716 static void iommu_enable_irtcachedis(struct amd_iommu *iommu)
2717 {
2718 u64 ctrl;
2719
2720 if (!amd_iommu_irtcachedis)
2721 return;
2722
2723 /*
2724 * Note:
2725 * The support for IRTCacheDis feature is dertermined by
2726 * checking if the bit is writable.
2727 */
2728 iommu_feature_enable(iommu, CONTROL_IRTCACHEDIS);
2729 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
2730 ctrl &= (1ULL << CONTROL_IRTCACHEDIS);
2731 if (ctrl)
2732 iommu->irtcachedis_enabled = true;
2733 pr_info("iommu%d (%#06x) : IRT cache is %s\n",
2734 iommu->index, iommu->devid,
2735 iommu->irtcachedis_enabled ? "disabled" : "enabled");
2736 }
2737
early_enable_iommu(struct amd_iommu * iommu)2738 static void early_enable_iommu(struct amd_iommu *iommu)
2739 {
2740 iommu_disable(iommu);
2741 iommu_init_flags(iommu);
2742 iommu_set_device_table(iommu);
2743 iommu_enable_command_buffer(iommu);
2744 iommu_enable_event_buffer(iommu);
2745 iommu_set_exclusion_range(iommu);
2746 iommu_enable_gt(iommu);
2747 iommu_enable_ga(iommu);
2748 iommu_enable_xt(iommu);
2749 iommu_enable_irtcachedis(iommu);
2750 iommu_enable(iommu);
2751 amd_iommu_flush_all_caches(iommu);
2752 }
2753
2754 /*
2755 * This function finally enables all IOMMUs found in the system after
2756 * they have been initialized.
2757 *
2758 * Or if in kdump kernel and IOMMUs are all pre-enabled, try to copy
2759 * the old content of device table entries. Not this case or copy failed,
2760 * just continue as normal kernel does.
2761 */
early_enable_iommus(void)2762 static void early_enable_iommus(void)
2763 {
2764 struct amd_iommu *iommu;
2765 struct amd_iommu_pci_seg *pci_seg;
2766
2767 if (!copy_device_table()) {
2768 /*
2769 * If come here because of failure in copying device table from old
2770 * kernel with all IOMMUs enabled, print error message and try to
2771 * free allocated old_dev_tbl_cpy.
2772 */
2773 if (amd_iommu_pre_enabled)
2774 pr_err("Failed to copy DEV table from previous kernel.\n");
2775
2776 for_each_pci_segment(pci_seg) {
2777 if (pci_seg->old_dev_tbl_cpy != NULL) {
2778 iommu_free_pages(pci_seg->old_dev_tbl_cpy,
2779 get_order(pci_seg->dev_table_size));
2780 pci_seg->old_dev_tbl_cpy = NULL;
2781 }
2782 }
2783
2784 for_each_iommu(iommu) {
2785 clear_translation_pre_enabled(iommu);
2786 early_enable_iommu(iommu);
2787 }
2788 } else {
2789 pr_info("Copied DEV table from previous kernel.\n");
2790
2791 for_each_pci_segment(pci_seg) {
2792 iommu_free_pages(pci_seg->dev_table,
2793 get_order(pci_seg->dev_table_size));
2794 pci_seg->dev_table = pci_seg->old_dev_tbl_cpy;
2795 }
2796
2797 for_each_iommu(iommu) {
2798 iommu_disable_command_buffer(iommu);
2799 iommu_disable_event_buffer(iommu);
2800 iommu_disable_irtcachedis(iommu);
2801 iommu_enable_command_buffer(iommu);
2802 iommu_enable_event_buffer(iommu);
2803 iommu_enable_ga(iommu);
2804 iommu_enable_xt(iommu);
2805 iommu_enable_irtcachedis(iommu);
2806 iommu_set_device_table(iommu);
2807 amd_iommu_flush_all_caches(iommu);
2808 }
2809 }
2810 }
2811
enable_iommus_ppr(void)2812 static void enable_iommus_ppr(void)
2813 {
2814 struct amd_iommu *iommu;
2815
2816 if (!amd_iommu_gt_ppr_supported())
2817 return;
2818
2819 for_each_iommu(iommu)
2820 amd_iommu_enable_ppr_log(iommu);
2821 }
2822
enable_iommus_vapic(void)2823 static void enable_iommus_vapic(void)
2824 {
2825 #ifdef CONFIG_IRQ_REMAP
2826 u32 status, i;
2827 struct amd_iommu *iommu;
2828
2829 for_each_iommu(iommu) {
2830 /*
2831 * Disable GALog if already running. It could have been enabled
2832 * in the previous boot before kdump.
2833 */
2834 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
2835 if (!(status & MMIO_STATUS_GALOG_RUN_MASK))
2836 continue;
2837
2838 iommu_feature_disable(iommu, CONTROL_GALOG_EN);
2839 iommu_feature_disable(iommu, CONTROL_GAINT_EN);
2840
2841 /*
2842 * Need to set and poll check the GALOGRun bit to zero before
2843 * we can set/ modify GA Log registers safely.
2844 */
2845 for (i = 0; i < MMIO_STATUS_TIMEOUT; ++i) {
2846 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
2847 if (!(status & MMIO_STATUS_GALOG_RUN_MASK))
2848 break;
2849 udelay(10);
2850 }
2851
2852 if (WARN_ON(i >= MMIO_STATUS_TIMEOUT))
2853 return;
2854 }
2855
2856 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) &&
2857 !check_feature(FEATURE_GAM_VAPIC)) {
2858 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
2859 return;
2860 }
2861
2862 if (amd_iommu_snp_en &&
2863 !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) {
2864 pr_warn("Force to disable Virtual APIC due to SNP\n");
2865 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
2866 return;
2867 }
2868
2869 /* Enabling GAM and SNPAVIC support */
2870 for_each_iommu(iommu) {
2871 if (iommu_init_ga_log(iommu) ||
2872 iommu_ga_log_enable(iommu))
2873 return;
2874
2875 iommu_feature_enable(iommu, CONTROL_GAM_EN);
2876 if (amd_iommu_snp_en)
2877 iommu_feature_enable(iommu, CONTROL_SNPAVIC_EN);
2878 }
2879
2880 amd_iommu_irq_ops.capability |= (1 << IRQ_POSTING_CAP);
2881 pr_info("Virtual APIC enabled\n");
2882 #endif
2883 }
2884
enable_iommus(void)2885 static void enable_iommus(void)
2886 {
2887 early_enable_iommus();
2888 }
2889
disable_iommus(void)2890 static void disable_iommus(void)
2891 {
2892 struct amd_iommu *iommu;
2893
2894 for_each_iommu(iommu)
2895 iommu_disable(iommu);
2896
2897 #ifdef CONFIG_IRQ_REMAP
2898 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
2899 amd_iommu_irq_ops.capability &= ~(1 << IRQ_POSTING_CAP);
2900 #endif
2901 }
2902
2903 /*
2904 * Suspend/Resume support
2905 * disable suspend until real resume implemented
2906 */
2907
amd_iommu_resume(void)2908 static void amd_iommu_resume(void)
2909 {
2910 struct amd_iommu *iommu;
2911
2912 for_each_iommu(iommu)
2913 iommu_apply_resume_quirks(iommu);
2914
2915 /* re-load the hardware */
2916 enable_iommus();
2917
2918 amd_iommu_enable_interrupts();
2919 }
2920
amd_iommu_suspend(void)2921 static int amd_iommu_suspend(void)
2922 {
2923 /* disable IOMMUs to go out of the way for BIOS */
2924 disable_iommus();
2925
2926 return 0;
2927 }
2928
2929 static struct syscore_ops amd_iommu_syscore_ops = {
2930 .suspend = amd_iommu_suspend,
2931 .resume = amd_iommu_resume,
2932 };
2933
free_iommu_resources(void)2934 static void __init free_iommu_resources(void)
2935 {
2936 kmem_cache_destroy(amd_iommu_irq_cache);
2937 amd_iommu_irq_cache = NULL;
2938
2939 free_iommu_all();
2940 free_pci_segments();
2941 }
2942
2943 /* SB IOAPIC is always on this device in AMD systems */
2944 #define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0))
2945
check_ioapic_information(void)2946 static bool __init check_ioapic_information(void)
2947 {
2948 const char *fw_bug = FW_BUG;
2949 bool ret, has_sb_ioapic;
2950 int idx;
2951
2952 has_sb_ioapic = false;
2953 ret = false;
2954
2955 /*
2956 * If we have map overrides on the kernel command line the
2957 * messages in this function might not describe firmware bugs
2958 * anymore - so be careful
2959 */
2960 if (cmdline_maps)
2961 fw_bug = "";
2962
2963 for (idx = 0; idx < nr_ioapics; idx++) {
2964 int devid, id = mpc_ioapic_id(idx);
2965
2966 devid = get_ioapic_devid(id);
2967 if (devid < 0) {
2968 pr_err("%s: IOAPIC[%d] not in IVRS table\n",
2969 fw_bug, id);
2970 ret = false;
2971 } else if (devid == IOAPIC_SB_DEVID) {
2972 has_sb_ioapic = true;
2973 ret = true;
2974 }
2975 }
2976
2977 if (!has_sb_ioapic) {
2978 /*
2979 * We expect the SB IOAPIC to be listed in the IVRS
2980 * table. The system timer is connected to the SB IOAPIC
2981 * and if we don't have it in the list the system will
2982 * panic at boot time. This situation usually happens
2983 * when the BIOS is buggy and provides us the wrong
2984 * device id for the IOAPIC in the system.
2985 */
2986 pr_err("%s: No southbridge IOAPIC found\n", fw_bug);
2987 }
2988
2989 if (!ret)
2990 pr_err("Disabling interrupt remapping\n");
2991
2992 return ret;
2993 }
2994
free_dma_resources(void)2995 static void __init free_dma_resources(void)
2996 {
2997 iommu_free_pages(amd_iommu_pd_alloc_bitmap,
2998 get_order(MAX_DOMAIN_ID / 8));
2999 amd_iommu_pd_alloc_bitmap = NULL;
3000
3001 free_unity_maps();
3002 }
3003
ivinfo_init(void * ivrs)3004 static void __init ivinfo_init(void *ivrs)
3005 {
3006 amd_iommu_ivinfo = *((u32 *)(ivrs + IOMMU_IVINFO_OFFSET));
3007 }
3008
3009 /*
3010 * This is the hardware init function for AMD IOMMU in the system.
3011 * This function is called either from amd_iommu_init or from the interrupt
3012 * remapping setup code.
3013 *
3014 * This function basically parses the ACPI table for AMD IOMMU (IVRS)
3015 * four times:
3016 *
3017 * 1 pass) Discover the most comprehensive IVHD type to use.
3018 *
3019 * 2 pass) Find the highest PCI device id the driver has to handle.
3020 * Upon this information the size of the data structures is
3021 * determined that needs to be allocated.
3022 *
3023 * 3 pass) Initialize the data structures just allocated with the
3024 * information in the ACPI table about available AMD IOMMUs
3025 * in the system. It also maps the PCI devices in the
3026 * system to specific IOMMUs
3027 *
3028 * 4 pass) After the basic data structures are allocated and
3029 * initialized we update them with information about memory
3030 * remapping requirements parsed out of the ACPI table in
3031 * this last pass.
3032 *
3033 * After everything is set up the IOMMUs are enabled and the necessary
3034 * hotplug and suspend notifiers are registered.
3035 */
early_amd_iommu_init(void)3036 static int __init early_amd_iommu_init(void)
3037 {
3038 struct acpi_table_header *ivrs_base;
3039 int remap_cache_sz, ret;
3040 acpi_status status;
3041
3042 if (!amd_iommu_detected)
3043 return -ENODEV;
3044
3045 status = acpi_get_table("IVRS", 0, &ivrs_base);
3046 if (status == AE_NOT_FOUND)
3047 return -ENODEV;
3048 else if (ACPI_FAILURE(status)) {
3049 const char *err = acpi_format_exception(status);
3050 pr_err("IVRS table error: %s\n", err);
3051 return -EINVAL;
3052 }
3053
3054 /*
3055 * Validate checksum here so we don't need to do it when
3056 * we actually parse the table
3057 */
3058 ret = check_ivrs_checksum(ivrs_base);
3059 if (ret)
3060 goto out;
3061
3062 ivinfo_init(ivrs_base);
3063
3064 amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base);
3065 DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type);
3066
3067 /* Device table - directly used by all IOMMUs */
3068 ret = -ENOMEM;
3069
3070 amd_iommu_pd_alloc_bitmap = iommu_alloc_pages(GFP_KERNEL,
3071 get_order(MAX_DOMAIN_ID / 8));
3072 if (amd_iommu_pd_alloc_bitmap == NULL)
3073 goto out;
3074
3075 /*
3076 * never allocate domain 0 because its used as the non-allocated and
3077 * error value placeholder
3078 */
3079 __set_bit(0, amd_iommu_pd_alloc_bitmap);
3080
3081 /*
3082 * now the data structures are allocated and basically initialized
3083 * start the real acpi table scan
3084 */
3085 ret = init_iommu_all(ivrs_base);
3086 if (ret)
3087 goto out;
3088
3089 /* 5 level guest page table */
3090 if (cpu_feature_enabled(X86_FEATURE_LA57) &&
3091 FIELD_GET(FEATURE_GATS, amd_iommu_efr) == GUEST_PGTABLE_5_LEVEL)
3092 amd_iommu_gpt_level = PAGE_MODE_5_LEVEL;
3093
3094 /* Disable any previously enabled IOMMUs */
3095 if (!is_kdump_kernel() || amd_iommu_disabled)
3096 disable_iommus();
3097
3098 if (amd_iommu_irq_remap)
3099 amd_iommu_irq_remap = check_ioapic_information();
3100
3101 if (amd_iommu_irq_remap) {
3102 struct amd_iommu_pci_seg *pci_seg;
3103 /*
3104 * Interrupt remapping enabled, create kmem_cache for the
3105 * remapping tables.
3106 */
3107 ret = -ENOMEM;
3108 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
3109 remap_cache_sz = MAX_IRQS_PER_TABLE * sizeof(u32);
3110 else
3111 remap_cache_sz = MAX_IRQS_PER_TABLE * (sizeof(u64) * 2);
3112 amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache",
3113 remap_cache_sz,
3114 DTE_INTTAB_ALIGNMENT,
3115 0, NULL);
3116 if (!amd_iommu_irq_cache)
3117 goto out;
3118
3119 for_each_pci_segment(pci_seg) {
3120 if (alloc_irq_lookup_table(pci_seg))
3121 goto out;
3122 }
3123 }
3124
3125 ret = init_memory_definitions(ivrs_base);
3126 if (ret)
3127 goto out;
3128
3129 /* init the device table */
3130 init_device_table();
3131
3132 out:
3133 /* Don't leak any ACPI memory */
3134 acpi_put_table(ivrs_base);
3135
3136 return ret;
3137 }
3138
amd_iommu_enable_interrupts(void)3139 static int amd_iommu_enable_interrupts(void)
3140 {
3141 struct amd_iommu *iommu;
3142 int ret = 0;
3143
3144 for_each_iommu(iommu) {
3145 ret = iommu_init_irq(iommu);
3146 if (ret)
3147 goto out;
3148 }
3149
3150 /*
3151 * Interrupt handler is ready to process interrupts. Enable
3152 * PPR and GA log interrupt for all IOMMUs.
3153 */
3154 enable_iommus_vapic();
3155 enable_iommus_ppr();
3156
3157 out:
3158 return ret;
3159 }
3160
detect_ivrs(void)3161 static bool __init detect_ivrs(void)
3162 {
3163 struct acpi_table_header *ivrs_base;
3164 acpi_status status;
3165 int i;
3166
3167 status = acpi_get_table("IVRS", 0, &ivrs_base);
3168 if (status == AE_NOT_FOUND)
3169 return false;
3170 else if (ACPI_FAILURE(status)) {
3171 const char *err = acpi_format_exception(status);
3172 pr_err("IVRS table error: %s\n", err);
3173 return false;
3174 }
3175
3176 acpi_put_table(ivrs_base);
3177
3178 if (amd_iommu_force_enable)
3179 goto out;
3180
3181 /* Don't use IOMMU if there is Stoney Ridge graphics */
3182 for (i = 0; i < 32; i++) {
3183 u32 pci_id;
3184
3185 pci_id = read_pci_config(0, i, 0, 0);
3186 if ((pci_id & 0xffff) == 0x1002 && (pci_id >> 16) == 0x98e4) {
3187 pr_info("Disable IOMMU on Stoney Ridge\n");
3188 return false;
3189 }
3190 }
3191
3192 out:
3193 /* Make sure ACS will be enabled during PCI probe */
3194 pci_request_acs();
3195
3196 return true;
3197 }
3198
iommu_snp_enable(void)3199 static void iommu_snp_enable(void)
3200 {
3201 #ifdef CONFIG_KVM_AMD_SEV
3202 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
3203 return;
3204 /*
3205 * The SNP support requires that IOMMU must be enabled, and is
3206 * configured with V1 page table (DTE[Mode] = 0 is not supported).
3207 */
3208 if (no_iommu || iommu_default_passthrough()) {
3209 pr_warn("SNP: IOMMU disabled or configured in passthrough mode, SNP cannot be supported.\n");
3210 goto disable_snp;
3211 }
3212
3213 if (amd_iommu_pgtable != AMD_IOMMU_V1) {
3214 pr_warn("SNP: IOMMU is configured with V2 page table mode, SNP cannot be supported.\n");
3215 goto disable_snp;
3216 }
3217
3218 amd_iommu_snp_en = check_feature(FEATURE_SNP);
3219 if (!amd_iommu_snp_en) {
3220 pr_warn("SNP: IOMMU SNP feature not enabled, SNP cannot be supported.\n");
3221 goto disable_snp;
3222 }
3223
3224 pr_info("IOMMU SNP support enabled.\n");
3225 return;
3226
3227 disable_snp:
3228 cc_platform_clear(CC_ATTR_HOST_SEV_SNP);
3229 #endif
3230 }
3231
3232 /****************************************************************************
3233 *
3234 * AMD IOMMU Initialization State Machine
3235 *
3236 ****************************************************************************/
3237
state_next(void)3238 static int __init state_next(void)
3239 {
3240 int ret = 0;
3241
3242 switch (init_state) {
3243 case IOMMU_START_STATE:
3244 if (!detect_ivrs()) {
3245 init_state = IOMMU_NOT_FOUND;
3246 ret = -ENODEV;
3247 } else {
3248 init_state = IOMMU_IVRS_DETECTED;
3249 }
3250 break;
3251 case IOMMU_IVRS_DETECTED:
3252 if (amd_iommu_disabled) {
3253 init_state = IOMMU_CMDLINE_DISABLED;
3254 ret = -EINVAL;
3255 } else {
3256 ret = early_amd_iommu_init();
3257 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
3258 }
3259 break;
3260 case IOMMU_ACPI_FINISHED:
3261 early_enable_iommus();
3262 x86_platform.iommu_shutdown = disable_iommus;
3263 init_state = IOMMU_ENABLED;
3264 break;
3265 case IOMMU_ENABLED:
3266 register_syscore_ops(&amd_iommu_syscore_ops);
3267 iommu_snp_enable();
3268 ret = amd_iommu_init_pci();
3269 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT;
3270 break;
3271 case IOMMU_PCI_INIT:
3272 ret = amd_iommu_enable_interrupts();
3273 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN;
3274 break;
3275 case IOMMU_INTERRUPTS_EN:
3276 init_state = IOMMU_INITIALIZED;
3277 break;
3278 case IOMMU_INITIALIZED:
3279 /* Nothing to do */
3280 break;
3281 case IOMMU_NOT_FOUND:
3282 case IOMMU_INIT_ERROR:
3283 case IOMMU_CMDLINE_DISABLED:
3284 /* Error states => do nothing */
3285 ret = -EINVAL;
3286 break;
3287 default:
3288 /* Unknown state */
3289 BUG();
3290 }
3291
3292 if (ret) {
3293 free_dma_resources();
3294 if (!irq_remapping_enabled) {
3295 disable_iommus();
3296 free_iommu_resources();
3297 } else {
3298 struct amd_iommu *iommu;
3299 struct amd_iommu_pci_seg *pci_seg;
3300
3301 for_each_pci_segment(pci_seg)
3302 uninit_device_table_dma(pci_seg);
3303
3304 for_each_iommu(iommu)
3305 amd_iommu_flush_all_caches(iommu);
3306 }
3307 }
3308 return ret;
3309 }
3310
iommu_go_to_state(enum iommu_init_state state)3311 static int __init iommu_go_to_state(enum iommu_init_state state)
3312 {
3313 int ret = -EINVAL;
3314
3315 while (init_state != state) {
3316 if (init_state == IOMMU_NOT_FOUND ||
3317 init_state == IOMMU_INIT_ERROR ||
3318 init_state == IOMMU_CMDLINE_DISABLED)
3319 break;
3320 ret = state_next();
3321 }
3322
3323 return ret;
3324 }
3325
3326 #ifdef CONFIG_IRQ_REMAP
amd_iommu_prepare(void)3327 int __init amd_iommu_prepare(void)
3328 {
3329 int ret;
3330
3331 amd_iommu_irq_remap = true;
3332
3333 ret = iommu_go_to_state(IOMMU_ACPI_FINISHED);
3334 if (ret) {
3335 amd_iommu_irq_remap = false;
3336 return ret;
3337 }
3338
3339 return amd_iommu_irq_remap ? 0 : -ENODEV;
3340 }
3341
amd_iommu_enable(void)3342 int __init amd_iommu_enable(void)
3343 {
3344 int ret;
3345
3346 ret = iommu_go_to_state(IOMMU_ENABLED);
3347 if (ret)
3348 return ret;
3349
3350 irq_remapping_enabled = 1;
3351 return amd_iommu_xt_mode;
3352 }
3353
amd_iommu_disable(void)3354 void amd_iommu_disable(void)
3355 {
3356 amd_iommu_suspend();
3357 }
3358
amd_iommu_reenable(int mode)3359 int amd_iommu_reenable(int mode)
3360 {
3361 amd_iommu_resume();
3362
3363 return 0;
3364 }
3365
amd_iommu_enable_faulting(unsigned int cpu)3366 int amd_iommu_enable_faulting(unsigned int cpu)
3367 {
3368 /* We enable MSI later when PCI is initialized */
3369 return 0;
3370 }
3371 #endif
3372
3373 /*
3374 * This is the core init function for AMD IOMMU hardware in the system.
3375 * This function is called from the generic x86 DMA layer initialization
3376 * code.
3377 */
amd_iommu_init(void)3378 static int __init amd_iommu_init(void)
3379 {
3380 struct amd_iommu *iommu;
3381 int ret;
3382
3383 ret = iommu_go_to_state(IOMMU_INITIALIZED);
3384 #ifdef CONFIG_GART_IOMMU
3385 if (ret && list_empty(&amd_iommu_list)) {
3386 /*
3387 * We failed to initialize the AMD IOMMU - try fallback
3388 * to GART if possible.
3389 */
3390 gart_iommu_init();
3391 }
3392 #endif
3393
3394 for_each_iommu(iommu)
3395 amd_iommu_debugfs_setup(iommu);
3396
3397 return ret;
3398 }
3399
amd_iommu_sme_check(void)3400 static bool amd_iommu_sme_check(void)
3401 {
3402 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) ||
3403 (boot_cpu_data.x86 != 0x17))
3404 return true;
3405
3406 /* For Fam17h, a specific level of support is required */
3407 if (boot_cpu_data.microcode >= 0x08001205)
3408 return true;
3409
3410 if ((boot_cpu_data.microcode >= 0x08001126) &&
3411 (boot_cpu_data.microcode <= 0x080011ff))
3412 return true;
3413
3414 pr_notice("IOMMU not currently supported when SME is active\n");
3415
3416 return false;
3417 }
3418
3419 /****************************************************************************
3420 *
3421 * Early detect code. This code runs at IOMMU detection time in the DMA
3422 * layer. It just looks if there is an IVRS ACPI table to detect AMD
3423 * IOMMUs
3424 *
3425 ****************************************************************************/
amd_iommu_detect(void)3426 int __init amd_iommu_detect(void)
3427 {
3428 int ret;
3429
3430 if (no_iommu || (iommu_detected && !gart_iommu_aperture))
3431 return -ENODEV;
3432
3433 if (!amd_iommu_sme_check())
3434 return -ENODEV;
3435
3436 ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
3437 if (ret)
3438 return ret;
3439
3440 amd_iommu_detected = true;
3441 iommu_detected = 1;
3442 x86_init.iommu.iommu_init = amd_iommu_init;
3443
3444 return 1;
3445 }
3446
3447 /****************************************************************************
3448 *
3449 * Parsing functions for the AMD IOMMU specific kernel command line
3450 * options.
3451 *
3452 ****************************************************************************/
3453
parse_amd_iommu_dump(char * str)3454 static int __init parse_amd_iommu_dump(char *str)
3455 {
3456 amd_iommu_dump = true;
3457
3458 return 1;
3459 }
3460
parse_amd_iommu_intr(char * str)3461 static int __init parse_amd_iommu_intr(char *str)
3462 {
3463 for (; *str; ++str) {
3464 if (strncmp(str, "legacy", 6) == 0) {
3465 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
3466 break;
3467 }
3468 if (strncmp(str, "vapic", 5) == 0) {
3469 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
3470 break;
3471 }
3472 }
3473 return 1;
3474 }
3475
parse_amd_iommu_options(char * str)3476 static int __init parse_amd_iommu_options(char *str)
3477 {
3478 if (!str)
3479 return -EINVAL;
3480
3481 while (*str) {
3482 if (strncmp(str, "fullflush", 9) == 0) {
3483 pr_warn("amd_iommu=fullflush deprecated; use iommu.strict=1 instead\n");
3484 iommu_set_dma_strict();
3485 } else if (strncmp(str, "force_enable", 12) == 0) {
3486 amd_iommu_force_enable = true;
3487 } else if (strncmp(str, "off", 3) == 0) {
3488 amd_iommu_disabled = true;
3489 } else if (strncmp(str, "force_isolation", 15) == 0) {
3490 amd_iommu_force_isolation = true;
3491 } else if (strncmp(str, "pgtbl_v1", 8) == 0) {
3492 amd_iommu_pgtable = AMD_IOMMU_V1;
3493 } else if (strncmp(str, "pgtbl_v2", 8) == 0) {
3494 amd_iommu_pgtable = AMD_IOMMU_V2;
3495 } else if (strncmp(str, "irtcachedis", 11) == 0) {
3496 amd_iommu_irtcachedis = true;
3497 } else if (strncmp(str, "nohugepages", 11) == 0) {
3498 pr_info("Restricting V1 page-sizes to 4KiB");
3499 amd_iommu_pgsize_bitmap = AMD_IOMMU_PGSIZES_4K;
3500 } else if (strncmp(str, "v2_pgsizes_only", 15) == 0) {
3501 pr_info("Restricting V1 page-sizes to 4KiB/2MiB/1GiB");
3502 amd_iommu_pgsize_bitmap = AMD_IOMMU_PGSIZES_V2;
3503 } else {
3504 pr_notice("Unknown option - '%s'\n", str);
3505 }
3506
3507 str += strcspn(str, ",");
3508 while (*str == ',')
3509 str++;
3510 }
3511
3512 return 1;
3513 }
3514
parse_ivrs_ioapic(char * str)3515 static int __init parse_ivrs_ioapic(char *str)
3516 {
3517 u32 seg = 0, bus, dev, fn;
3518 int id, i;
3519 u32 devid;
3520
3521 if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3522 sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
3523 goto found;
3524
3525 if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3526 sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
3527 pr_warn("ivrs_ioapic%s option format deprecated; use ivrs_ioapic=%d@%04x:%02x:%02x.%d instead\n",
3528 str, id, seg, bus, dev, fn);
3529 goto found;
3530 }
3531
3532 pr_err("Invalid command line: ivrs_ioapic%s\n", str);
3533 return 1;
3534
3535 found:
3536 if (early_ioapic_map_size == EARLY_MAP_SIZE) {
3537 pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n",
3538 str);
3539 return 1;
3540 }
3541
3542 devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3543
3544 cmdline_maps = true;
3545 i = early_ioapic_map_size++;
3546 early_ioapic_map[i].id = id;
3547 early_ioapic_map[i].devid = devid;
3548 early_ioapic_map[i].cmd_line = true;
3549
3550 return 1;
3551 }
3552
parse_ivrs_hpet(char * str)3553 static int __init parse_ivrs_hpet(char *str)
3554 {
3555 u32 seg = 0, bus, dev, fn;
3556 int id, i;
3557 u32 devid;
3558
3559 if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3560 sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
3561 goto found;
3562
3563 if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3564 sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
3565 pr_warn("ivrs_hpet%s option format deprecated; use ivrs_hpet=%d@%04x:%02x:%02x.%d instead\n",
3566 str, id, seg, bus, dev, fn);
3567 goto found;
3568 }
3569
3570 pr_err("Invalid command line: ivrs_hpet%s\n", str);
3571 return 1;
3572
3573 found:
3574 if (early_hpet_map_size == EARLY_MAP_SIZE) {
3575 pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n",
3576 str);
3577 return 1;
3578 }
3579
3580 devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3581
3582 cmdline_maps = true;
3583 i = early_hpet_map_size++;
3584 early_hpet_map[i].id = id;
3585 early_hpet_map[i].devid = devid;
3586 early_hpet_map[i].cmd_line = true;
3587
3588 return 1;
3589 }
3590
3591 #define ACPIID_LEN (ACPIHID_UID_LEN + ACPIHID_HID_LEN)
3592
parse_ivrs_acpihid(char * str)3593 static int __init parse_ivrs_acpihid(char *str)
3594 {
3595 u32 seg = 0, bus, dev, fn;
3596 char *hid, *uid, *p, *addr;
3597 char acpiid[ACPIID_LEN] = {0};
3598 int i;
3599
3600 addr = strchr(str, '@');
3601 if (!addr) {
3602 addr = strchr(str, '=');
3603 if (!addr)
3604 goto not_found;
3605
3606 ++addr;
3607
3608 if (strlen(addr) > ACPIID_LEN)
3609 goto not_found;
3610
3611 if (sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid) == 4 ||
3612 sscanf(str, "[%x:%x:%x.%x]=%s", &seg, &bus, &dev, &fn, acpiid) == 5) {
3613 pr_warn("ivrs_acpihid%s option format deprecated; use ivrs_acpihid=%s@%04x:%02x:%02x.%d instead\n",
3614 str, acpiid, seg, bus, dev, fn);
3615 goto found;
3616 }
3617 goto not_found;
3618 }
3619
3620 /* We have the '@', make it the terminator to get just the acpiid */
3621 *addr++ = 0;
3622
3623 if (strlen(str) > ACPIID_LEN + 1)
3624 goto not_found;
3625
3626 if (sscanf(str, "=%s", acpiid) != 1)
3627 goto not_found;
3628
3629 if (sscanf(addr, "%x:%x.%x", &bus, &dev, &fn) == 3 ||
3630 sscanf(addr, "%x:%x:%x.%x", &seg, &bus, &dev, &fn) == 4)
3631 goto found;
3632
3633 not_found:
3634 pr_err("Invalid command line: ivrs_acpihid%s\n", str);
3635 return 1;
3636
3637 found:
3638 p = acpiid;
3639 hid = strsep(&p, ":");
3640 uid = p;
3641
3642 if (!hid || !(*hid) || !uid) {
3643 pr_err("Invalid command line: hid or uid\n");
3644 return 1;
3645 }
3646
3647 /*
3648 * Ignore leading zeroes after ':', so e.g., AMDI0095:00
3649 * will match AMDI0095:0 in the second strcmp in acpi_dev_hid_uid_match
3650 */
3651 while (*uid == '0' && *(uid + 1))
3652 uid++;
3653
3654 i = early_acpihid_map_size++;
3655 memcpy(early_acpihid_map[i].hid, hid, strlen(hid));
3656 memcpy(early_acpihid_map[i].uid, uid, strlen(uid));
3657 early_acpihid_map[i].devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3658 early_acpihid_map[i].cmd_line = true;
3659
3660 return 1;
3661 }
3662
3663 __setup("amd_iommu_dump", parse_amd_iommu_dump);
3664 __setup("amd_iommu=", parse_amd_iommu_options);
3665 __setup("amd_iommu_intr=", parse_amd_iommu_intr);
3666 __setup("ivrs_ioapic", parse_ivrs_ioapic);
3667 __setup("ivrs_hpet", parse_ivrs_hpet);
3668 __setup("ivrs_acpihid", parse_ivrs_acpihid);
3669
amd_iommu_pasid_supported(void)3670 bool amd_iommu_pasid_supported(void)
3671 {
3672 /* CPU page table size should match IOMMU guest page table size */
3673 if (cpu_feature_enabled(X86_FEATURE_LA57) &&
3674 amd_iommu_gpt_level != PAGE_MODE_5_LEVEL)
3675 return false;
3676
3677 /*
3678 * Since DTE[Mode]=0 is prohibited on SNP-enabled system
3679 * (i.e. EFR[SNPSup]=1), IOMMUv2 page table cannot be used without
3680 * setting up IOMMUv1 page table.
3681 */
3682 return amd_iommu_gt_ppr_supported() && !amd_iommu_snp_en;
3683 }
3684
get_amd_iommu(unsigned int idx)3685 struct amd_iommu *get_amd_iommu(unsigned int idx)
3686 {
3687 unsigned int i = 0;
3688 struct amd_iommu *iommu;
3689
3690 for_each_iommu(iommu)
3691 if (i++ == idx)
3692 return iommu;
3693 return NULL;
3694 }
3695
3696 /****************************************************************************
3697 *
3698 * IOMMU EFR Performance Counter support functionality. This code allows
3699 * access to the IOMMU PC functionality.
3700 *
3701 ****************************************************************************/
3702
amd_iommu_pc_get_max_banks(unsigned int idx)3703 u8 amd_iommu_pc_get_max_banks(unsigned int idx)
3704 {
3705 struct amd_iommu *iommu = get_amd_iommu(idx);
3706
3707 if (iommu)
3708 return iommu->max_banks;
3709
3710 return 0;
3711 }
3712
amd_iommu_pc_supported(void)3713 bool amd_iommu_pc_supported(void)
3714 {
3715 return amd_iommu_pc_present;
3716 }
3717
amd_iommu_pc_get_max_counters(unsigned int idx)3718 u8 amd_iommu_pc_get_max_counters(unsigned int idx)
3719 {
3720 struct amd_iommu *iommu = get_amd_iommu(idx);
3721
3722 if (iommu)
3723 return iommu->max_counters;
3724
3725 return 0;
3726 }
3727
iommu_pc_get_set_reg(struct amd_iommu * iommu,u8 bank,u8 cntr,u8 fxn,u64 * value,bool is_write)3728 static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr,
3729 u8 fxn, u64 *value, bool is_write)
3730 {
3731 u32 offset;
3732 u32 max_offset_lim;
3733
3734 /* Make sure the IOMMU PC resource is available */
3735 if (!amd_iommu_pc_present)
3736 return -ENODEV;
3737
3738 /* Check for valid iommu and pc register indexing */
3739 if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7)))
3740 return -ENODEV;
3741
3742 offset = (u32)(((0x40 | bank) << 12) | (cntr << 8) | fxn);
3743
3744 /* Limit the offset to the hw defined mmio region aperture */
3745 max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) |
3746 (iommu->max_counters << 8) | 0x28);
3747 if ((offset < MMIO_CNTR_REG_OFFSET) ||
3748 (offset > max_offset_lim))
3749 return -EINVAL;
3750
3751 if (is_write) {
3752 u64 val = *value & GENMASK_ULL(47, 0);
3753
3754 writel((u32)val, iommu->mmio_base + offset);
3755 writel((val >> 32), iommu->mmio_base + offset + 4);
3756 } else {
3757 *value = readl(iommu->mmio_base + offset + 4);
3758 *value <<= 32;
3759 *value |= readl(iommu->mmio_base + offset);
3760 *value &= GENMASK_ULL(47, 0);
3761 }
3762
3763 return 0;
3764 }
3765
amd_iommu_pc_get_reg(struct amd_iommu * iommu,u8 bank,u8 cntr,u8 fxn,u64 * value)3766 int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)
3767 {
3768 if (!iommu)
3769 return -EINVAL;
3770
3771 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false);
3772 }
3773
amd_iommu_pc_set_reg(struct amd_iommu * iommu,u8 bank,u8 cntr,u8 fxn,u64 * value)3774 int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)
3775 {
3776 if (!iommu)
3777 return -EINVAL;
3778
3779 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true);
3780 }
3781
3782 #ifdef CONFIG_KVM_AMD_SEV
iommu_page_make_shared(void * page)3783 static int iommu_page_make_shared(void *page)
3784 {
3785 unsigned long paddr, pfn;
3786
3787 paddr = iommu_virt_to_phys(page);
3788 /* Cbit maybe set in the paddr */
3789 pfn = __sme_clr(paddr) >> PAGE_SHIFT;
3790
3791 if (!(pfn % PTRS_PER_PMD)) {
3792 int ret, level;
3793 bool assigned;
3794
3795 ret = snp_lookup_rmpentry(pfn, &assigned, &level);
3796 if (ret) {
3797 pr_warn("IOMMU PFN %lx RMP lookup failed, ret %d\n", pfn, ret);
3798 return ret;
3799 }
3800
3801 if (!assigned) {
3802 pr_warn("IOMMU PFN %lx not assigned in RMP table\n", pfn);
3803 return -EINVAL;
3804 }
3805
3806 if (level > PG_LEVEL_4K) {
3807 ret = psmash(pfn);
3808 if (!ret)
3809 goto done;
3810
3811 pr_warn("PSMASH failed for IOMMU PFN %lx huge RMP entry, ret: %d, level: %d\n",
3812 pfn, ret, level);
3813 return ret;
3814 }
3815 }
3816
3817 done:
3818 return rmp_make_shared(pfn, PG_LEVEL_4K);
3819 }
3820
iommu_make_shared(void * va,size_t size)3821 static int iommu_make_shared(void *va, size_t size)
3822 {
3823 void *page;
3824 int ret;
3825
3826 if (!va)
3827 return 0;
3828
3829 for (page = va; page < (va + size); page += PAGE_SIZE) {
3830 ret = iommu_page_make_shared(page);
3831 if (ret)
3832 return ret;
3833 }
3834
3835 return 0;
3836 }
3837
amd_iommu_snp_disable(void)3838 int amd_iommu_snp_disable(void)
3839 {
3840 struct amd_iommu *iommu;
3841 int ret;
3842
3843 if (!amd_iommu_snp_en)
3844 return 0;
3845
3846 for_each_iommu(iommu) {
3847 ret = iommu_make_shared(iommu->evt_buf, EVT_BUFFER_SIZE);
3848 if (ret)
3849 return ret;
3850
3851 ret = iommu_make_shared(iommu->ppr_log, PPR_LOG_SIZE);
3852 if (ret)
3853 return ret;
3854
3855 ret = iommu_make_shared((void *)iommu->cmd_sem, PAGE_SIZE);
3856 if (ret)
3857 return ret;
3858 }
3859
3860 return 0;
3861 }
3862 EXPORT_SYMBOL_GPL(amd_iommu_snp_disable);
3863 #endif
3864