xref: /linux/drivers/iommu/amd/init.c (revision f49040c7aaa5532a1f94355ef5073c49e6b32349)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4  * Author: Joerg Roedel <jroedel@suse.de>
5  *         Leo Duran <leo.duran@amd.com>
6  */
7 
8 #define pr_fmt(fmt)     "AMD-Vi: " fmt
9 #define dev_fmt(fmt)    pr_fmt(fmt)
10 
11 #include <linux/pci.h>
12 #include <linux/acpi.h>
13 #include <linux/list.h>
14 #include <linux/bitmap.h>
15 #include <linux/syscore_ops.h>
16 #include <linux/interrupt.h>
17 #include <linux/msi.h>
18 #include <linux/irq.h>
19 #include <linux/amd-iommu.h>
20 #include <linux/export.h>
21 #include <linux/kmemleak.h>
22 #include <linux/cc_platform.h>
23 #include <linux/iopoll.h>
24 #include <asm/pci-direct.h>
25 #include <asm/iommu.h>
26 #include <asm/apic.h>
27 #include <asm/gart.h>
28 #include <asm/x86_init.h>
29 #include <asm/io_apic.h>
30 #include <asm/irq_remapping.h>
31 #include <asm/set_memory.h>
32 #include <asm/sev.h>
33 
34 #include <linux/crash_dump.h>
35 
36 #include "amd_iommu.h"
37 #include "../irq_remapping.h"
38 #include "../iommu-pages.h"
39 
40 /*
41  * definitions for the ACPI scanning code
42  */
43 #define IVRS_HEADER_LENGTH 48
44 
45 #define ACPI_IVHD_TYPE_MAX_SUPPORTED	0x40
46 #define ACPI_IVMD_TYPE_ALL              0x20
47 #define ACPI_IVMD_TYPE                  0x21
48 #define ACPI_IVMD_TYPE_RANGE            0x22
49 
50 #define IVHD_DEV_ALL                    0x01
51 #define IVHD_DEV_SELECT                 0x02
52 #define IVHD_DEV_SELECT_RANGE_START     0x03
53 #define IVHD_DEV_RANGE_END              0x04
54 #define IVHD_DEV_ALIAS                  0x42
55 #define IVHD_DEV_ALIAS_RANGE            0x43
56 #define IVHD_DEV_EXT_SELECT             0x46
57 #define IVHD_DEV_EXT_SELECT_RANGE       0x47
58 #define IVHD_DEV_SPECIAL		0x48
59 #define IVHD_DEV_ACPI_HID		0xf0
60 
61 #define UID_NOT_PRESENT                 0
62 #define UID_IS_INTEGER                  1
63 #define UID_IS_CHARACTER                2
64 
65 #define IVHD_SPECIAL_IOAPIC		1
66 #define IVHD_SPECIAL_HPET		2
67 
68 #define IVHD_FLAG_HT_TUN_EN_MASK        0x01
69 #define IVHD_FLAG_PASSPW_EN_MASK        0x02
70 #define IVHD_FLAG_RESPASSPW_EN_MASK     0x04
71 #define IVHD_FLAG_ISOC_EN_MASK          0x08
72 
73 #define IVMD_FLAG_EXCL_RANGE            0x08
74 #define IVMD_FLAG_IW                    0x04
75 #define IVMD_FLAG_IR                    0x02
76 #define IVMD_FLAG_UNITY_MAP             0x01
77 
78 #define ACPI_DEVFLAG_INITPASS           0x01
79 #define ACPI_DEVFLAG_EXTINT             0x02
80 #define ACPI_DEVFLAG_NMI                0x04
81 #define ACPI_DEVFLAG_SYSMGT1            0x10
82 #define ACPI_DEVFLAG_SYSMGT2            0x20
83 #define ACPI_DEVFLAG_LINT0              0x40
84 #define ACPI_DEVFLAG_LINT1              0x80
85 #define ACPI_DEVFLAG_ATSDIS             0x10000000
86 
87 #define IVRS_GET_SBDF_ID(seg, bus, dev, fn)	(((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \
88 						 | ((dev & 0x1f) << 3) | (fn & 0x7))
89 
90 /*
91  * ACPI table definitions
92  *
93  * These data structures are laid over the table to parse the important values
94  * out of it.
95  */
96 
97 /*
98  * structure describing one IOMMU in the ACPI table. Typically followed by one
99  * or more ivhd_entrys.
100  */
101 struct ivhd_header {
102 	u8 type;
103 	u8 flags;
104 	u16 length;
105 	u16 devid;
106 	u16 cap_ptr;
107 	u64 mmio_phys;
108 	u16 pci_seg;
109 	u16 info;
110 	u32 efr_attr;
111 
112 	/* Following only valid on IVHD type 11h and 40h */
113 	u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */
114 	u64 efr_reg2;
115 } __attribute__((packed));
116 
117 /*
118  * A device entry describing which devices a specific IOMMU translates and
119  * which requestor ids they use.
120  */
121 struct ivhd_entry {
122 	u8 type;
123 	u16 devid;
124 	u8 flags;
125 	struct_group(ext_hid,
126 		u32 ext;
127 		u32 hidh;
128 	);
129 	u64 cid;
130 	u8 uidf;
131 	u8 uidl;
132 	u8 uid;
133 } __attribute__((packed));
134 
135 /*
136  * An AMD IOMMU memory definition structure. It defines things like exclusion
137  * ranges for devices and regions that should be unity mapped.
138  */
139 struct ivmd_header {
140 	u8 type;
141 	u8 flags;
142 	u16 length;
143 	u16 devid;
144 	u16 aux;
145 	u16 pci_seg;
146 	u8  resv[6];
147 	u64 range_start;
148 	u64 range_length;
149 } __attribute__((packed));
150 
151 bool amd_iommu_dump;
152 bool amd_iommu_irq_remap __read_mostly;
153 
154 enum protection_domain_mode amd_iommu_pgtable = PD_MODE_V1;
155 /* Guest page table level */
156 int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL;
157 
158 int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
159 static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
160 
161 static bool amd_iommu_detected;
162 static bool amd_iommu_disabled __initdata;
163 static bool amd_iommu_force_enable __initdata;
164 static bool amd_iommu_irtcachedis;
165 static int amd_iommu_target_ivhd_type;
166 
167 /* Global EFR and EFR2 registers */
168 u64 amd_iommu_efr;
169 u64 amd_iommu_efr2;
170 
171 /* SNP is enabled on the system? */
172 bool amd_iommu_snp_en;
173 EXPORT_SYMBOL(amd_iommu_snp_en);
174 
175 LIST_HEAD(amd_iommu_pci_seg_list);	/* list of all PCI segments */
176 LIST_HEAD(amd_iommu_list);		/* list of all AMD IOMMUs in the system */
177 LIST_HEAD(amd_ivhd_dev_flags_list);	/* list of all IVHD device entry settings */
178 
179 /* Number of IOMMUs present in the system */
180 static int amd_iommus_present;
181 
182 /* IOMMUs have a non-present cache? */
183 bool amd_iommu_np_cache __read_mostly;
184 bool amd_iommu_iotlb_sup __read_mostly = true;
185 
186 static bool amd_iommu_pc_present __read_mostly;
187 bool amdr_ivrs_remap_support __read_mostly;
188 
189 bool amd_iommu_force_isolation __read_mostly;
190 
191 unsigned long amd_iommu_pgsize_bitmap __ro_after_init = AMD_IOMMU_PGSIZES;
192 
193 enum iommu_init_state {
194 	IOMMU_START_STATE,
195 	IOMMU_IVRS_DETECTED,
196 	IOMMU_ACPI_FINISHED,
197 	IOMMU_ENABLED,
198 	IOMMU_PCI_INIT,
199 	IOMMU_INTERRUPTS_EN,
200 	IOMMU_INITIALIZED,
201 	IOMMU_NOT_FOUND,
202 	IOMMU_INIT_ERROR,
203 	IOMMU_CMDLINE_DISABLED,
204 };
205 
206 /* Early ioapic and hpet maps from kernel command line */
207 #define EARLY_MAP_SIZE		4
208 static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE];
209 static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE];
210 static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE];
211 
212 static int __initdata early_ioapic_map_size;
213 static int __initdata early_hpet_map_size;
214 static int __initdata early_acpihid_map_size;
215 
216 static bool __initdata cmdline_maps;
217 
218 static enum iommu_init_state init_state = IOMMU_START_STATE;
219 
220 static int amd_iommu_enable_interrupts(void);
221 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg);
222 
223 static bool amd_iommu_pre_enabled = true;
224 
225 static u32 amd_iommu_ivinfo __initdata;
226 
translation_pre_enabled(struct amd_iommu * iommu)227 bool translation_pre_enabled(struct amd_iommu *iommu)
228 {
229 	return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED);
230 }
231 
clear_translation_pre_enabled(struct amd_iommu * iommu)232 static void clear_translation_pre_enabled(struct amd_iommu *iommu)
233 {
234 	iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
235 }
236 
init_translation_status(struct amd_iommu * iommu)237 static void init_translation_status(struct amd_iommu *iommu)
238 {
239 	u64 ctrl;
240 
241 	ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
242 	if (ctrl & (1<<CONTROL_IOMMU_EN))
243 		iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
244 }
245 
tbl_size(int entry_size,int last_bdf)246 static inline unsigned long tbl_size(int entry_size, int last_bdf)
247 {
248 	unsigned shift = PAGE_SHIFT +
249 			 get_order((last_bdf + 1) * entry_size);
250 
251 	return 1UL << shift;
252 }
253 
amd_iommu_get_num_iommus(void)254 int amd_iommu_get_num_iommus(void)
255 {
256 	return amd_iommus_present;
257 }
258 
259 /*
260  * Iterate through all the IOMMUs to get common EFR
261  * masks among all IOMMUs and warn if found inconsistency.
262  */
get_global_efr(void)263 static __init void get_global_efr(void)
264 {
265 	struct amd_iommu *iommu;
266 
267 	for_each_iommu(iommu) {
268 		u64 tmp = iommu->features;
269 		u64 tmp2 = iommu->features2;
270 
271 		if (list_is_first(&iommu->list, &amd_iommu_list)) {
272 			amd_iommu_efr = tmp;
273 			amd_iommu_efr2 = tmp2;
274 			continue;
275 		}
276 
277 		if (amd_iommu_efr == tmp &&
278 		    amd_iommu_efr2 == tmp2)
279 			continue;
280 
281 		pr_err(FW_BUG
282 		       "Found inconsistent EFR/EFR2 %#llx,%#llx (global %#llx,%#llx) on iommu%d (%04x:%02x:%02x.%01x).\n",
283 		       tmp, tmp2, amd_iommu_efr, amd_iommu_efr2,
284 		       iommu->index, iommu->pci_seg->id,
285 		       PCI_BUS_NUM(iommu->devid), PCI_SLOT(iommu->devid),
286 		       PCI_FUNC(iommu->devid));
287 
288 		amd_iommu_efr &= tmp;
289 		amd_iommu_efr2 &= tmp2;
290 	}
291 
292 	pr_info("Using global IVHD EFR:%#llx, EFR2:%#llx\n", amd_iommu_efr, amd_iommu_efr2);
293 }
294 
295 /*
296  * For IVHD type 0x11/0x40, EFR is also available via IVHD.
297  * Default to IVHD EFR since it is available sooner
298  * (i.e. before PCI init).
299  */
early_iommu_features_init(struct amd_iommu * iommu,struct ivhd_header * h)300 static void __init early_iommu_features_init(struct amd_iommu *iommu,
301 					     struct ivhd_header *h)
302 {
303 	if (amd_iommu_ivinfo & IOMMU_IVINFO_EFRSUP) {
304 		iommu->features = h->efr_reg;
305 		iommu->features2 = h->efr_reg2;
306 	}
307 	if (amd_iommu_ivinfo & IOMMU_IVINFO_DMA_REMAP)
308 		amdr_ivrs_remap_support = true;
309 }
310 
311 /* Access to l1 and l2 indexed register spaces */
312 
iommu_read_l1(struct amd_iommu * iommu,u16 l1,u8 address)313 static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)
314 {
315 	u32 val;
316 
317 	pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
318 	pci_read_config_dword(iommu->dev, 0xfc, &val);
319 	return val;
320 }
321 
iommu_write_l1(struct amd_iommu * iommu,u16 l1,u8 address,u32 val)322 static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)
323 {
324 	pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));
325 	pci_write_config_dword(iommu->dev, 0xfc, val);
326 	pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
327 }
328 
iommu_read_l2(struct amd_iommu * iommu,u8 address)329 static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)
330 {
331 	u32 val;
332 
333 	pci_write_config_dword(iommu->dev, 0xf0, address);
334 	pci_read_config_dword(iommu->dev, 0xf4, &val);
335 	return val;
336 }
337 
iommu_write_l2(struct amd_iommu * iommu,u8 address,u32 val)338 static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
339 {
340 	pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));
341 	pci_write_config_dword(iommu->dev, 0xf4, val);
342 }
343 
344 /****************************************************************************
345  *
346  * AMD IOMMU MMIO register space handling functions
347  *
348  * These functions are used to program the IOMMU device registers in
349  * MMIO space required for that driver.
350  *
351  ****************************************************************************/
352 
353 /*
354  * This function set the exclusion range in the IOMMU. DMA accesses to the
355  * exclusion range are passed through untranslated
356  */
iommu_set_exclusion_range(struct amd_iommu * iommu)357 static void iommu_set_exclusion_range(struct amd_iommu *iommu)
358 {
359 	u64 start = iommu->exclusion_start & PAGE_MASK;
360 	u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK;
361 	u64 entry;
362 
363 	if (!iommu->exclusion_start)
364 		return;
365 
366 	entry = start | MMIO_EXCL_ENABLE_MASK;
367 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
368 			&entry, sizeof(entry));
369 
370 	entry = limit;
371 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
372 			&entry, sizeof(entry));
373 }
374 
iommu_set_cwwb_range(struct amd_iommu * iommu)375 static void iommu_set_cwwb_range(struct amd_iommu *iommu)
376 {
377 	u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem);
378 	u64 entry = start & PM_ADDR_MASK;
379 
380 	if (!check_feature(FEATURE_SNP))
381 		return;
382 
383 	/* Note:
384 	 * Re-purpose Exclusion base/limit registers for Completion wait
385 	 * write-back base/limit.
386 	 */
387 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
388 		    &entry, sizeof(entry));
389 
390 	/* Note:
391 	 * Default to 4 Kbytes, which can be specified by setting base
392 	 * address equal to the limit address.
393 	 */
394 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
395 		    &entry, sizeof(entry));
396 }
397 
398 /* Programs the physical address of the device table into the IOMMU hardware */
iommu_set_device_table(struct amd_iommu * iommu)399 static void iommu_set_device_table(struct amd_iommu *iommu)
400 {
401 	u64 entry;
402 	u32 dev_table_size = iommu->pci_seg->dev_table_size;
403 	void *dev_table = (void *)get_dev_table(iommu);
404 
405 	BUG_ON(iommu->mmio_base == NULL);
406 
407 	entry = iommu_virt_to_phys(dev_table);
408 	entry |= (dev_table_size >> 12) - 1;
409 	memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
410 			&entry, sizeof(entry));
411 }
412 
iommu_feature_set(struct amd_iommu * iommu,u64 val,u64 mask,u8 shift)413 static void iommu_feature_set(struct amd_iommu *iommu, u64 val, u64 mask, u8 shift)
414 {
415 	u64 ctrl;
416 
417 	ctrl = readq(iommu->mmio_base +  MMIO_CONTROL_OFFSET);
418 	mask <<= shift;
419 	ctrl &= ~mask;
420 	ctrl |= (val << shift) & mask;
421 	writeq(ctrl, iommu->mmio_base +  MMIO_CONTROL_OFFSET);
422 }
423 
424 /* Generic functions to enable/disable certain features of the IOMMU. */
iommu_feature_enable(struct amd_iommu * iommu,u8 bit)425 void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
426 {
427 	iommu_feature_set(iommu, 1ULL, 1ULL, bit);
428 }
429 
iommu_feature_disable(struct amd_iommu * iommu,u8 bit)430 static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
431 {
432 	iommu_feature_set(iommu, 0ULL, 1ULL, bit);
433 }
434 
435 /* Function to enable the hardware */
iommu_enable(struct amd_iommu * iommu)436 static void iommu_enable(struct amd_iommu *iommu)
437 {
438 	iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
439 }
440 
iommu_disable(struct amd_iommu * iommu)441 static void iommu_disable(struct amd_iommu *iommu)
442 {
443 	if (!iommu->mmio_base)
444 		return;
445 
446 	/* Disable command buffer */
447 	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
448 
449 	/* Disable event logging and event interrupts */
450 	iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);
451 	iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
452 
453 	/* Disable IOMMU GA_LOG */
454 	iommu_feature_disable(iommu, CONTROL_GALOG_EN);
455 	iommu_feature_disable(iommu, CONTROL_GAINT_EN);
456 
457 	/* Disable IOMMU PPR logging */
458 	iommu_feature_disable(iommu, CONTROL_PPRLOG_EN);
459 	iommu_feature_disable(iommu, CONTROL_PPRINT_EN);
460 
461 	/* Disable IOMMU hardware itself */
462 	iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
463 
464 	/* Clear IRTE cache disabling bit */
465 	iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);
466 }
467 
468 /*
469  * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in
470  * the system has one.
471  */
iommu_map_mmio_space(u64 address,u64 end)472 static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end)
473 {
474 	if (!request_mem_region(address, end, "amd_iommu")) {
475 		pr_err("Can not reserve memory region %llx-%llx for mmio\n",
476 			address, end);
477 		pr_err("This is a BIOS bug. Please contact your hardware vendor\n");
478 		return NULL;
479 	}
480 
481 	return (u8 __iomem *)ioremap(address, end);
482 }
483 
iommu_unmap_mmio_space(struct amd_iommu * iommu)484 static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)
485 {
486 	if (iommu->mmio_base)
487 		iounmap(iommu->mmio_base);
488 	release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end);
489 }
490 
get_ivhd_header_size(struct ivhd_header * h)491 static inline u32 get_ivhd_header_size(struct ivhd_header *h)
492 {
493 	u32 size = 0;
494 
495 	switch (h->type) {
496 	case 0x10:
497 		size = 24;
498 		break;
499 	case 0x11:
500 	case 0x40:
501 		size = 40;
502 		break;
503 	}
504 	return size;
505 }
506 
507 /****************************************************************************
508  *
509  * The functions below belong to the first pass of AMD IOMMU ACPI table
510  * parsing. In this pass we try to find out the highest device id this
511  * code has to handle. Upon this information the size of the shared data
512  * structures is determined later.
513  *
514  ****************************************************************************/
515 
516 /*
517  * This function calculates the length of a given IVHD entry
518  */
ivhd_entry_length(u8 * ivhd)519 static inline int ivhd_entry_length(u8 *ivhd)
520 {
521 	u32 type = ((struct ivhd_entry *)ivhd)->type;
522 
523 	if (type < 0x80) {
524 		return 0x04 << (*ivhd >> 6);
525 	} else if (type == IVHD_DEV_ACPI_HID) {
526 		/* For ACPI_HID, offset 21 is uid len */
527 		return *((u8 *)ivhd + 21) + 22;
528 	}
529 	return 0;
530 }
531 
532 /*
533  * After reading the highest device id from the IOMMU PCI capability header
534  * this function looks if there is a higher device id defined in the ACPI table
535  */
find_last_devid_from_ivhd(struct ivhd_header * h)536 static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
537 {
538 	u8 *p = (void *)h, *end = (void *)h;
539 	struct ivhd_entry *dev;
540 	int last_devid = -EINVAL;
541 
542 	u32 ivhd_size = get_ivhd_header_size(h);
543 
544 	if (!ivhd_size) {
545 		pr_err("Unsupported IVHD type %#x\n", h->type);
546 		return -EINVAL;
547 	}
548 
549 	p += ivhd_size;
550 	end += h->length;
551 
552 	while (p < end) {
553 		dev = (struct ivhd_entry *)p;
554 		switch (dev->type) {
555 		case IVHD_DEV_ALL:
556 			/* Use maximum BDF value for DEV_ALL */
557 			return 0xffff;
558 		case IVHD_DEV_SELECT:
559 		case IVHD_DEV_RANGE_END:
560 		case IVHD_DEV_ALIAS:
561 		case IVHD_DEV_EXT_SELECT:
562 			/* all the above subfield types refer to device ids */
563 			if (dev->devid > last_devid)
564 				last_devid = dev->devid;
565 			break;
566 		default:
567 			break;
568 		}
569 		p += ivhd_entry_length(p);
570 	}
571 
572 	WARN_ON(p != end);
573 
574 	return last_devid;
575 }
576 
check_ivrs_checksum(struct acpi_table_header * table)577 static int __init check_ivrs_checksum(struct acpi_table_header *table)
578 {
579 	int i;
580 	u8 checksum = 0, *p = (u8 *)table;
581 
582 	for (i = 0; i < table->length; ++i)
583 		checksum += p[i];
584 	if (checksum != 0) {
585 		/* ACPI table corrupt */
586 		pr_err(FW_BUG "IVRS invalid checksum\n");
587 		return -ENODEV;
588 	}
589 
590 	return 0;
591 }
592 
593 /*
594  * Iterate over all IVHD entries in the ACPI table and find the highest device
595  * id which we need to handle. This is the first of three functions which parse
596  * the ACPI table. So we check the checksum here.
597  */
find_last_devid_acpi(struct acpi_table_header * table,u16 pci_seg)598 static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_seg)
599 {
600 	u8 *p = (u8 *)table, *end = (u8 *)table;
601 	struct ivhd_header *h;
602 	int last_devid, last_bdf = 0;
603 
604 	p += IVRS_HEADER_LENGTH;
605 
606 	end += table->length;
607 	while (p < end) {
608 		h = (struct ivhd_header *)p;
609 		if (h->pci_seg == pci_seg &&
610 		    h->type == amd_iommu_target_ivhd_type) {
611 			last_devid = find_last_devid_from_ivhd(h);
612 
613 			if (last_devid < 0)
614 				return -EINVAL;
615 			if (last_devid > last_bdf)
616 				last_bdf = last_devid;
617 		}
618 		p += h->length;
619 	}
620 	WARN_ON(p != end);
621 
622 	return last_bdf;
623 }
624 
625 /****************************************************************************
626  *
627  * The following functions belong to the code path which parses the ACPI table
628  * the second time. In this ACPI parsing iteration we allocate IOMMU specific
629  * data structures, initialize the per PCI segment device/alias/rlookup table
630  * and also basically initialize the hardware.
631  *
632  ****************************************************************************/
633 
634 /* Allocate per PCI segment device table */
alloc_dev_table(struct amd_iommu_pci_seg * pci_seg)635 static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg)
636 {
637 	pci_seg->dev_table = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32,
638 					       get_order(pci_seg->dev_table_size));
639 	if (!pci_seg->dev_table)
640 		return -ENOMEM;
641 
642 	return 0;
643 }
644 
free_dev_table(struct amd_iommu_pci_seg * pci_seg)645 static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg)
646 {
647 	iommu_free_pages(pci_seg->dev_table,
648 			 get_order(pci_seg->dev_table_size));
649 	pci_seg->dev_table = NULL;
650 }
651 
652 /* Allocate per PCI segment IOMMU rlookup table. */
alloc_rlookup_table(struct amd_iommu_pci_seg * pci_seg)653 static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg)
654 {
655 	pci_seg->rlookup_table = iommu_alloc_pages(GFP_KERNEL,
656 						   get_order(pci_seg->rlookup_table_size));
657 	if (pci_seg->rlookup_table == NULL)
658 		return -ENOMEM;
659 
660 	return 0;
661 }
662 
free_rlookup_table(struct amd_iommu_pci_seg * pci_seg)663 static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg)
664 {
665 	iommu_free_pages(pci_seg->rlookup_table,
666 			 get_order(pci_seg->rlookup_table_size));
667 	pci_seg->rlookup_table = NULL;
668 }
669 
alloc_irq_lookup_table(struct amd_iommu_pci_seg * pci_seg)670 static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)
671 {
672 	pci_seg->irq_lookup_table = iommu_alloc_pages(GFP_KERNEL,
673 						      get_order(pci_seg->rlookup_table_size));
674 	kmemleak_alloc(pci_seg->irq_lookup_table,
675 		       pci_seg->rlookup_table_size, 1, GFP_KERNEL);
676 	if (pci_seg->irq_lookup_table == NULL)
677 		return -ENOMEM;
678 
679 	return 0;
680 }
681 
free_irq_lookup_table(struct amd_iommu_pci_seg * pci_seg)682 static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)
683 {
684 	kmemleak_free(pci_seg->irq_lookup_table);
685 	iommu_free_pages(pci_seg->irq_lookup_table,
686 			 get_order(pci_seg->rlookup_table_size));
687 	pci_seg->irq_lookup_table = NULL;
688 }
689 
alloc_alias_table(struct amd_iommu_pci_seg * pci_seg)690 static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg)
691 {
692 	int i;
693 
694 	pci_seg->alias_table = iommu_alloc_pages(GFP_KERNEL,
695 						 get_order(pci_seg->alias_table_size));
696 	if (!pci_seg->alias_table)
697 		return -ENOMEM;
698 
699 	/*
700 	 * let all alias entries point to itself
701 	 */
702 	for (i = 0; i <= pci_seg->last_bdf; ++i)
703 		pci_seg->alias_table[i] = i;
704 
705 	return 0;
706 }
707 
free_alias_table(struct amd_iommu_pci_seg * pci_seg)708 static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg)
709 {
710 	iommu_free_pages(pci_seg->alias_table,
711 			 get_order(pci_seg->alias_table_size));
712 	pci_seg->alias_table = NULL;
713 }
714 
715 /*
716  * Allocates the command buffer. This buffer is per AMD IOMMU. We can
717  * write commands to that buffer later and the IOMMU will execute them
718  * asynchronously
719  */
alloc_command_buffer(struct amd_iommu * iommu)720 static int __init alloc_command_buffer(struct amd_iommu *iommu)
721 {
722 	iommu->cmd_buf = iommu_alloc_pages(GFP_KERNEL,
723 					   get_order(CMD_BUFFER_SIZE));
724 
725 	return iommu->cmd_buf ? 0 : -ENOMEM;
726 }
727 
728 /*
729  * Interrupt handler has processed all pending events and adjusted head
730  * and tail pointer. Reset overflow mask and restart logging again.
731  */
amd_iommu_restart_log(struct amd_iommu * iommu,const char * evt_type,u8 cntrl_intr,u8 cntrl_log,u32 status_run_mask,u32 status_overflow_mask)732 void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type,
733 			   u8 cntrl_intr, u8 cntrl_log,
734 			   u32 status_run_mask, u32 status_overflow_mask)
735 {
736 	u32 status;
737 
738 	status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
739 	if (status & status_run_mask)
740 		return;
741 
742 	pr_info_ratelimited("IOMMU %s log restarting\n", evt_type);
743 
744 	iommu_feature_disable(iommu, cntrl_log);
745 	iommu_feature_disable(iommu, cntrl_intr);
746 
747 	writel(status_overflow_mask, iommu->mmio_base + MMIO_STATUS_OFFSET);
748 
749 	iommu_feature_enable(iommu, cntrl_intr);
750 	iommu_feature_enable(iommu, cntrl_log);
751 }
752 
753 /*
754  * This function restarts event logging in case the IOMMU experienced
755  * an event log buffer overflow.
756  */
amd_iommu_restart_event_logging(struct amd_iommu * iommu)757 void amd_iommu_restart_event_logging(struct amd_iommu *iommu)
758 {
759 	amd_iommu_restart_log(iommu, "Event", CONTROL_EVT_INT_EN,
760 			      CONTROL_EVT_LOG_EN, MMIO_STATUS_EVT_RUN_MASK,
761 			      MMIO_STATUS_EVT_OVERFLOW_MASK);
762 }
763 
764 /*
765  * This function restarts event logging in case the IOMMU experienced
766  * GA log overflow.
767  */
amd_iommu_restart_ga_log(struct amd_iommu * iommu)768 void amd_iommu_restart_ga_log(struct amd_iommu *iommu)
769 {
770 	amd_iommu_restart_log(iommu, "GA", CONTROL_GAINT_EN,
771 			      CONTROL_GALOG_EN, MMIO_STATUS_GALOG_RUN_MASK,
772 			      MMIO_STATUS_GALOG_OVERFLOW_MASK);
773 }
774 
775 /*
776  * This function resets the command buffer if the IOMMU stopped fetching
777  * commands from it.
778  */
amd_iommu_reset_cmd_buffer(struct amd_iommu * iommu)779 static void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
780 {
781 	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
782 
783 	writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
784 	writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
785 	iommu->cmd_buf_head = 0;
786 	iommu->cmd_buf_tail = 0;
787 
788 	iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
789 }
790 
791 /*
792  * This function writes the command buffer address to the hardware and
793  * enables it.
794  */
iommu_enable_command_buffer(struct amd_iommu * iommu)795 static void iommu_enable_command_buffer(struct amd_iommu *iommu)
796 {
797 	u64 entry;
798 
799 	BUG_ON(iommu->cmd_buf == NULL);
800 
801 	entry = iommu_virt_to_phys(iommu->cmd_buf);
802 	entry |= MMIO_CMD_SIZE_512;
803 
804 	memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
805 		    &entry, sizeof(entry));
806 
807 	amd_iommu_reset_cmd_buffer(iommu);
808 }
809 
810 /*
811  * This function disables the command buffer
812  */
iommu_disable_command_buffer(struct amd_iommu * iommu)813 static void iommu_disable_command_buffer(struct amd_iommu *iommu)
814 {
815 	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
816 }
817 
free_command_buffer(struct amd_iommu * iommu)818 static void __init free_command_buffer(struct amd_iommu *iommu)
819 {
820 	iommu_free_pages(iommu->cmd_buf, get_order(CMD_BUFFER_SIZE));
821 }
822 
iommu_alloc_4k_pages(struct amd_iommu * iommu,gfp_t gfp,size_t size)823 void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp,
824 				  size_t size)
825 {
826 	int order = get_order(size);
827 	void *buf = iommu_alloc_pages(gfp, order);
828 
829 	if (buf &&
830 	    check_feature(FEATURE_SNP) &&
831 	    set_memory_4k((unsigned long)buf, (1 << order))) {
832 		iommu_free_pages(buf, order);
833 		buf = NULL;
834 	}
835 
836 	return buf;
837 }
838 
839 /* allocates the memory where the IOMMU will log its events to */
alloc_event_buffer(struct amd_iommu * iommu)840 static int __init alloc_event_buffer(struct amd_iommu *iommu)
841 {
842 	iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL,
843 					      EVT_BUFFER_SIZE);
844 
845 	return iommu->evt_buf ? 0 : -ENOMEM;
846 }
847 
iommu_enable_event_buffer(struct amd_iommu * iommu)848 static void iommu_enable_event_buffer(struct amd_iommu *iommu)
849 {
850 	u64 entry;
851 
852 	BUG_ON(iommu->evt_buf == NULL);
853 
854 	entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
855 
856 	memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
857 		    &entry, sizeof(entry));
858 
859 	/* set head and tail to zero manually */
860 	writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
861 	writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
862 
863 	iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
864 }
865 
866 /*
867  * This function disables the event log buffer
868  */
iommu_disable_event_buffer(struct amd_iommu * iommu)869 static void iommu_disable_event_buffer(struct amd_iommu *iommu)
870 {
871 	iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
872 }
873 
free_event_buffer(struct amd_iommu * iommu)874 static void __init free_event_buffer(struct amd_iommu *iommu)
875 {
876 	iommu_free_pages(iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
877 }
878 
free_ga_log(struct amd_iommu * iommu)879 static void free_ga_log(struct amd_iommu *iommu)
880 {
881 #ifdef CONFIG_IRQ_REMAP
882 	iommu_free_pages(iommu->ga_log, get_order(GA_LOG_SIZE));
883 	iommu_free_pages(iommu->ga_log_tail, get_order(8));
884 #endif
885 }
886 
887 #ifdef CONFIG_IRQ_REMAP
iommu_ga_log_enable(struct amd_iommu * iommu)888 static int iommu_ga_log_enable(struct amd_iommu *iommu)
889 {
890 	u32 status, i;
891 	u64 entry;
892 
893 	if (!iommu->ga_log)
894 		return -EINVAL;
895 
896 	entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
897 	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
898 		    &entry, sizeof(entry));
899 	entry = (iommu_virt_to_phys(iommu->ga_log_tail) &
900 		 (BIT_ULL(52)-1)) & ~7ULL;
901 	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,
902 		    &entry, sizeof(entry));
903 	writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
904 	writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
905 
906 
907 	iommu_feature_enable(iommu, CONTROL_GAINT_EN);
908 	iommu_feature_enable(iommu, CONTROL_GALOG_EN);
909 
910 	for (i = 0; i < MMIO_STATUS_TIMEOUT; ++i) {
911 		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
912 		if (status & (MMIO_STATUS_GALOG_RUN_MASK))
913 			break;
914 		udelay(10);
915 	}
916 
917 	if (WARN_ON(i >= MMIO_STATUS_TIMEOUT))
918 		return -EINVAL;
919 
920 	return 0;
921 }
922 
iommu_init_ga_log(struct amd_iommu * iommu)923 static int iommu_init_ga_log(struct amd_iommu *iommu)
924 {
925 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
926 		return 0;
927 
928 	iommu->ga_log = iommu_alloc_pages(GFP_KERNEL, get_order(GA_LOG_SIZE));
929 	if (!iommu->ga_log)
930 		goto err_out;
931 
932 	iommu->ga_log_tail = iommu_alloc_pages(GFP_KERNEL, get_order(8));
933 	if (!iommu->ga_log_tail)
934 		goto err_out;
935 
936 	return 0;
937 err_out:
938 	free_ga_log(iommu);
939 	return -EINVAL;
940 }
941 #endif /* CONFIG_IRQ_REMAP */
942 
alloc_cwwb_sem(struct amd_iommu * iommu)943 static int __init alloc_cwwb_sem(struct amd_iommu *iommu)
944 {
945 	iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL, 1);
946 
947 	return iommu->cmd_sem ? 0 : -ENOMEM;
948 }
949 
free_cwwb_sem(struct amd_iommu * iommu)950 static void __init free_cwwb_sem(struct amd_iommu *iommu)
951 {
952 	if (iommu->cmd_sem)
953 		iommu_free_page((void *)iommu->cmd_sem);
954 }
955 
iommu_enable_xt(struct amd_iommu * iommu)956 static void iommu_enable_xt(struct amd_iommu *iommu)
957 {
958 #ifdef CONFIG_IRQ_REMAP
959 	/*
960 	 * XT mode (32-bit APIC destination ID) requires
961 	 * GA mode (128-bit IRTE support) as a prerequisite.
962 	 */
963 	if (AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir) &&
964 	    amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
965 		iommu_feature_enable(iommu, CONTROL_XT_EN);
966 #endif /* CONFIG_IRQ_REMAP */
967 }
968 
iommu_enable_gt(struct amd_iommu * iommu)969 static void iommu_enable_gt(struct amd_iommu *iommu)
970 {
971 	if (!check_feature(FEATURE_GT))
972 		return;
973 
974 	iommu_feature_enable(iommu, CONTROL_GT_EN);
975 }
976 
977 /* sets a specific bit in the device table entry. */
set_dte_bit(struct dev_table_entry * dte,u8 bit)978 static void set_dte_bit(struct dev_table_entry *dte, u8 bit)
979 {
980 	int i = (bit >> 6) & 0x03;
981 	int _bit = bit & 0x3f;
982 
983 	dte->data[i] |= (1UL << _bit);
984 }
985 
__copy_device_table(struct amd_iommu * iommu)986 static bool __copy_device_table(struct amd_iommu *iommu)
987 {
988 	u64 int_ctl, int_tab_len, entry = 0;
989 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
990 	struct dev_table_entry *old_devtb = NULL;
991 	u32 lo, hi, devid, old_devtb_size;
992 	phys_addr_t old_devtb_phys;
993 	u16 dom_id, dte_v, irq_v;
994 	u64 tmp;
995 
996 	/* Each IOMMU use separate device table with the same size */
997 	lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET);
998 	hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4);
999 	entry = (((u64) hi) << 32) + lo;
1000 
1001 	old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12;
1002 	if (old_devtb_size != pci_seg->dev_table_size) {
1003 		pr_err("The device table size of IOMMU:%d is not expected!\n",
1004 			iommu->index);
1005 		return false;
1006 	}
1007 
1008 	/*
1009 	 * When SME is enabled in the first kernel, the entry includes the
1010 	 * memory encryption mask(sme_me_mask), we must remove the memory
1011 	 * encryption mask to obtain the true physical address in kdump kernel.
1012 	 */
1013 	old_devtb_phys = __sme_clr(entry) & PAGE_MASK;
1014 
1015 	if (old_devtb_phys >= 0x100000000ULL) {
1016 		pr_err("The address of old device table is above 4G, not trustworthy!\n");
1017 		return false;
1018 	}
1019 	old_devtb = (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) && is_kdump_kernel())
1020 		    ? (__force void *)ioremap_encrypted(old_devtb_phys,
1021 							pci_seg->dev_table_size)
1022 		    : memremap(old_devtb_phys, pci_seg->dev_table_size, MEMREMAP_WB);
1023 
1024 	if (!old_devtb)
1025 		return false;
1026 
1027 	pci_seg->old_dev_tbl_cpy = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32,
1028 						     get_order(pci_seg->dev_table_size));
1029 	if (pci_seg->old_dev_tbl_cpy == NULL) {
1030 		pr_err("Failed to allocate memory for copying old device table!\n");
1031 		memunmap(old_devtb);
1032 		return false;
1033 	}
1034 
1035 	for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
1036 		pci_seg->old_dev_tbl_cpy[devid] = old_devtb[devid];
1037 		dom_id = old_devtb[devid].data[1] & DEV_DOMID_MASK;
1038 		dte_v = old_devtb[devid].data[0] & DTE_FLAG_V;
1039 
1040 		if (dte_v && dom_id) {
1041 			pci_seg->old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0];
1042 			pci_seg->old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1];
1043 			/* Reserve the Domain IDs used by previous kernel */
1044 			if (ida_alloc_range(&pdom_ids, dom_id, dom_id, GFP_ATOMIC) != dom_id) {
1045 				pr_err("Failed to reserve domain ID 0x%x\n", dom_id);
1046 				memunmap(old_devtb);
1047 				return false;
1048 			}
1049 			/* If gcr3 table existed, mask it out */
1050 			if (old_devtb[devid].data[0] & DTE_FLAG_GV) {
1051 				tmp = (DTE_GCR3_30_15 | DTE_GCR3_51_31);
1052 				pci_seg->old_dev_tbl_cpy[devid].data[1] &= ~tmp;
1053 				tmp = (DTE_GCR3_14_12 | DTE_FLAG_GV);
1054 				pci_seg->old_dev_tbl_cpy[devid].data[0] &= ~tmp;
1055 			}
1056 		}
1057 
1058 		irq_v = old_devtb[devid].data[2] & DTE_IRQ_REMAP_ENABLE;
1059 		int_ctl = old_devtb[devid].data[2] & DTE_IRQ_REMAP_INTCTL_MASK;
1060 		int_tab_len = old_devtb[devid].data[2] & DTE_INTTABLEN_MASK;
1061 		if (irq_v && (int_ctl || int_tab_len)) {
1062 			if ((int_ctl != DTE_IRQ_REMAP_INTCTL) ||
1063 			    (int_tab_len != DTE_INTTABLEN_512 &&
1064 			     int_tab_len != DTE_INTTABLEN_2K)) {
1065 				pr_err("Wrong old irq remapping flag: %#x\n", devid);
1066 				memunmap(old_devtb);
1067 				return false;
1068 			}
1069 
1070 			pci_seg->old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2];
1071 		}
1072 	}
1073 	memunmap(old_devtb);
1074 
1075 	return true;
1076 }
1077 
copy_device_table(void)1078 static bool copy_device_table(void)
1079 {
1080 	struct amd_iommu *iommu;
1081 	struct amd_iommu_pci_seg *pci_seg;
1082 
1083 	if (!amd_iommu_pre_enabled)
1084 		return false;
1085 
1086 	pr_warn("Translation is already enabled - trying to copy translation structures\n");
1087 
1088 	/*
1089 	 * All IOMMUs within PCI segment shares common device table.
1090 	 * Hence copy device table only once per PCI segment.
1091 	 */
1092 	for_each_pci_segment(pci_seg) {
1093 		for_each_iommu(iommu) {
1094 			if (pci_seg->id != iommu->pci_seg->id)
1095 				continue;
1096 			if (!__copy_device_table(iommu))
1097 				return false;
1098 			break;
1099 		}
1100 	}
1101 
1102 	return true;
1103 }
1104 
amd_iommu_get_ivhd_dte_flags(u16 segid,u16 devid)1105 struct dev_table_entry *amd_iommu_get_ivhd_dte_flags(u16 segid, u16 devid)
1106 {
1107 	struct ivhd_dte_flags *e;
1108 	unsigned int best_len = UINT_MAX;
1109 	struct dev_table_entry *dte = NULL;
1110 
1111 	for_each_ivhd_dte_flags(e) {
1112 		/*
1113 		 * Need to go through the whole list to find the smallest range,
1114 		 * which contains the devid.
1115 		 */
1116 		if ((e->segid == segid) &&
1117 		    (e->devid_first <= devid) && (devid <= e->devid_last)) {
1118 			unsigned int len = e->devid_last - e->devid_first;
1119 
1120 			if (len < best_len) {
1121 				dte = &(e->dte);
1122 				best_len = len;
1123 			}
1124 		}
1125 	}
1126 	return dte;
1127 }
1128 
search_ivhd_dte_flags(u16 segid,u16 first,u16 last)1129 static bool search_ivhd_dte_flags(u16 segid, u16 first, u16 last)
1130 {
1131 	struct ivhd_dte_flags *e;
1132 
1133 	for_each_ivhd_dte_flags(e) {
1134 		if ((e->segid == segid) &&
1135 		    (e->devid_first == first) &&
1136 		    (e->devid_last == last))
1137 			return true;
1138 	}
1139 	return false;
1140 }
1141 
1142 /*
1143  * This function takes the device specific flags read from the ACPI
1144  * table and sets up the device table entry with that information
1145  */
1146 static void __init
set_dev_entry_from_acpi_range(struct amd_iommu * iommu,u16 first,u16 last,u32 flags,u32 ext_flags)1147 set_dev_entry_from_acpi_range(struct amd_iommu *iommu, u16 first, u16 last,
1148 			      u32 flags, u32 ext_flags)
1149 {
1150 	int i;
1151 	struct dev_table_entry dte = {};
1152 
1153 	/* Parse IVHD DTE setting flags and store information */
1154 	if (flags) {
1155 		struct ivhd_dte_flags *d;
1156 
1157 		if (search_ivhd_dte_flags(iommu->pci_seg->id, first, last))
1158 			return;
1159 
1160 		d = kzalloc(sizeof(struct ivhd_dte_flags), GFP_KERNEL);
1161 		if (!d)
1162 			return;
1163 
1164 		pr_debug("%s: devid range %#x:%#x\n", __func__, first, last);
1165 
1166 		if (flags & ACPI_DEVFLAG_INITPASS)
1167 			set_dte_bit(&dte, DEV_ENTRY_INIT_PASS);
1168 		if (flags & ACPI_DEVFLAG_EXTINT)
1169 			set_dte_bit(&dte, DEV_ENTRY_EINT_PASS);
1170 		if (flags & ACPI_DEVFLAG_NMI)
1171 			set_dte_bit(&dte, DEV_ENTRY_NMI_PASS);
1172 		if (flags & ACPI_DEVFLAG_SYSMGT1)
1173 			set_dte_bit(&dte, DEV_ENTRY_SYSMGT1);
1174 		if (flags & ACPI_DEVFLAG_SYSMGT2)
1175 			set_dte_bit(&dte, DEV_ENTRY_SYSMGT2);
1176 		if (flags & ACPI_DEVFLAG_LINT0)
1177 			set_dte_bit(&dte, DEV_ENTRY_LINT0_PASS);
1178 		if (flags & ACPI_DEVFLAG_LINT1)
1179 			set_dte_bit(&dte, DEV_ENTRY_LINT1_PASS);
1180 
1181 		/* Apply erratum 63, which needs info in initial_dte */
1182 		if (FIELD_GET(DTE_DATA1_SYSMGT_MASK, dte.data[1]) == 0x1)
1183 			dte.data[0] |= DTE_FLAG_IW;
1184 
1185 		memcpy(&d->dte, &dte, sizeof(dte));
1186 		d->segid = iommu->pci_seg->id;
1187 		d->devid_first = first;
1188 		d->devid_last = last;
1189 		list_add_tail(&d->list, &amd_ivhd_dev_flags_list);
1190 	}
1191 
1192 	for (i = first; i <= last; i++)  {
1193 		if (flags) {
1194 			struct dev_table_entry *dev_table = get_dev_table(iommu);
1195 
1196 			memcpy(&dev_table[i], &dte, sizeof(dte));
1197 		}
1198 		amd_iommu_set_rlookup_table(iommu, i);
1199 	}
1200 }
1201 
set_dev_entry_from_acpi(struct amd_iommu * iommu,u16 devid,u32 flags,u32 ext_flags)1202 static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
1203 					   u16 devid, u32 flags, u32 ext_flags)
1204 {
1205 	set_dev_entry_from_acpi_range(iommu, devid, devid, flags, ext_flags);
1206 }
1207 
add_special_device(u8 type,u8 id,u32 * devid,bool cmd_line)1208 int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line)
1209 {
1210 	struct devid_map *entry;
1211 	struct list_head *list;
1212 
1213 	if (type == IVHD_SPECIAL_IOAPIC)
1214 		list = &ioapic_map;
1215 	else if (type == IVHD_SPECIAL_HPET)
1216 		list = &hpet_map;
1217 	else
1218 		return -EINVAL;
1219 
1220 	list_for_each_entry(entry, list, list) {
1221 		if (!(entry->id == id && entry->cmd_line))
1222 			continue;
1223 
1224 		pr_info("Command-line override present for %s id %d - ignoring\n",
1225 			type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id);
1226 
1227 		*devid = entry->devid;
1228 
1229 		return 0;
1230 	}
1231 
1232 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1233 	if (!entry)
1234 		return -ENOMEM;
1235 
1236 	entry->id	= id;
1237 	entry->devid	= *devid;
1238 	entry->cmd_line	= cmd_line;
1239 
1240 	list_add_tail(&entry->list, list);
1241 
1242 	return 0;
1243 }
1244 
add_acpi_hid_device(u8 * hid,u8 * uid,u32 * devid,bool cmd_line)1245 static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u32 *devid,
1246 				      bool cmd_line)
1247 {
1248 	struct acpihid_map_entry *entry;
1249 	struct list_head *list = &acpihid_map;
1250 
1251 	list_for_each_entry(entry, list, list) {
1252 		if (strcmp(entry->hid, hid) ||
1253 		    (*uid && *entry->uid && strcmp(entry->uid, uid)) ||
1254 		    !entry->cmd_line)
1255 			continue;
1256 
1257 		pr_info("Command-line override for hid:%s uid:%s\n",
1258 			hid, uid);
1259 		*devid = entry->devid;
1260 		return 0;
1261 	}
1262 
1263 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1264 	if (!entry)
1265 		return -ENOMEM;
1266 
1267 	memcpy(entry->uid, uid, strlen(uid));
1268 	memcpy(entry->hid, hid, strlen(hid));
1269 	entry->devid = *devid;
1270 	entry->cmd_line	= cmd_line;
1271 	entry->root_devid = (entry->devid & (~0x7));
1272 
1273 	pr_info("%s, add hid:%s, uid:%s, rdevid:%#x\n",
1274 		entry->cmd_line ? "cmd" : "ivrs",
1275 		entry->hid, entry->uid, entry->root_devid);
1276 
1277 	list_add_tail(&entry->list, list);
1278 	return 0;
1279 }
1280 
add_early_maps(void)1281 static int __init add_early_maps(void)
1282 {
1283 	int i, ret;
1284 
1285 	for (i = 0; i < early_ioapic_map_size; ++i) {
1286 		ret = add_special_device(IVHD_SPECIAL_IOAPIC,
1287 					 early_ioapic_map[i].id,
1288 					 &early_ioapic_map[i].devid,
1289 					 early_ioapic_map[i].cmd_line);
1290 		if (ret)
1291 			return ret;
1292 	}
1293 
1294 	for (i = 0; i < early_hpet_map_size; ++i) {
1295 		ret = add_special_device(IVHD_SPECIAL_HPET,
1296 					 early_hpet_map[i].id,
1297 					 &early_hpet_map[i].devid,
1298 					 early_hpet_map[i].cmd_line);
1299 		if (ret)
1300 			return ret;
1301 	}
1302 
1303 	for (i = 0; i < early_acpihid_map_size; ++i) {
1304 		ret = add_acpi_hid_device(early_acpihid_map[i].hid,
1305 					  early_acpihid_map[i].uid,
1306 					  &early_acpihid_map[i].devid,
1307 					  early_acpihid_map[i].cmd_line);
1308 		if (ret)
1309 			return ret;
1310 	}
1311 
1312 	return 0;
1313 }
1314 
1315 /*
1316  * Takes a pointer to an AMD IOMMU entry in the ACPI table and
1317  * initializes the hardware and our data structures with it.
1318  */
init_iommu_from_acpi(struct amd_iommu * iommu,struct ivhd_header * h)1319 static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
1320 					struct ivhd_header *h)
1321 {
1322 	u8 *p = (u8 *)h;
1323 	u8 *end = p, flags = 0;
1324 	u16 devid = 0, devid_start = 0, devid_to = 0, seg_id;
1325 	u32 dev_i, ext_flags = 0;
1326 	bool alias = false;
1327 	struct ivhd_entry *e;
1328 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
1329 	u32 ivhd_size;
1330 	int ret;
1331 
1332 
1333 	ret = add_early_maps();
1334 	if (ret)
1335 		return ret;
1336 
1337 	amd_iommu_apply_ivrs_quirks();
1338 
1339 	/*
1340 	 * First save the recommended feature enable bits from ACPI
1341 	 */
1342 	iommu->acpi_flags = h->flags;
1343 
1344 	/*
1345 	 * Done. Now parse the device entries
1346 	 */
1347 	ivhd_size = get_ivhd_header_size(h);
1348 	if (!ivhd_size) {
1349 		pr_err("Unsupported IVHD type %#x\n", h->type);
1350 		return -EINVAL;
1351 	}
1352 
1353 	p += ivhd_size;
1354 
1355 	end += h->length;
1356 
1357 
1358 	while (p < end) {
1359 		e = (struct ivhd_entry *)p;
1360 		seg_id = pci_seg->id;
1361 
1362 		switch (e->type) {
1363 		case IVHD_DEV_ALL:
1364 
1365 			DUMP_printk("  DEV_ALL\t\t\tsetting: %#02x\n", e->flags);
1366 			set_dev_entry_from_acpi_range(iommu, 0, pci_seg->last_bdf, e->flags, 0);
1367 			break;
1368 		case IVHD_DEV_SELECT:
1369 
1370 			DUMP_printk("  DEV_SELECT\t\t\tdevid: %04x:%02x:%02x.%x flags: %#02x\n",
1371 				    seg_id, PCI_BUS_NUM(e->devid),
1372 				    PCI_SLOT(e->devid),
1373 				    PCI_FUNC(e->devid),
1374 				    e->flags);
1375 
1376 			devid = e->devid;
1377 			set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1378 			break;
1379 		case IVHD_DEV_SELECT_RANGE_START:
1380 
1381 			DUMP_printk("  DEV_SELECT_RANGE_START\tdevid: %04x:%02x:%02x.%x flags: %#02x\n",
1382 				    seg_id, PCI_BUS_NUM(e->devid),
1383 				    PCI_SLOT(e->devid),
1384 				    PCI_FUNC(e->devid),
1385 				    e->flags);
1386 
1387 			devid_start = e->devid;
1388 			flags = e->flags;
1389 			ext_flags = 0;
1390 			alias = false;
1391 			break;
1392 		case IVHD_DEV_ALIAS:
1393 
1394 			DUMP_printk("  DEV_ALIAS\t\t\tdevid: %04x:%02x:%02x.%x flags: %#02x devid_to: %02x:%02x.%x\n",
1395 				    seg_id, PCI_BUS_NUM(e->devid),
1396 				    PCI_SLOT(e->devid),
1397 				    PCI_FUNC(e->devid),
1398 				    e->flags,
1399 				    PCI_BUS_NUM(e->ext >> 8),
1400 				    PCI_SLOT(e->ext >> 8),
1401 				    PCI_FUNC(e->ext >> 8));
1402 
1403 			devid = e->devid;
1404 			devid_to = e->ext >> 8;
1405 			set_dev_entry_from_acpi(iommu, devid   , e->flags, 0);
1406 			set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);
1407 			pci_seg->alias_table[devid] = devid_to;
1408 			break;
1409 		case IVHD_DEV_ALIAS_RANGE:
1410 
1411 			DUMP_printk("  DEV_ALIAS_RANGE\t\tdevid: %04x:%02x:%02x.%x flags: %#02x devid_to: %04x:%02x:%02x.%x\n",
1412 				    seg_id, PCI_BUS_NUM(e->devid),
1413 				    PCI_SLOT(e->devid),
1414 				    PCI_FUNC(e->devid),
1415 				    e->flags,
1416 				    seg_id, PCI_BUS_NUM(e->ext >> 8),
1417 				    PCI_SLOT(e->ext >> 8),
1418 				    PCI_FUNC(e->ext >> 8));
1419 
1420 			devid_start = e->devid;
1421 			flags = e->flags;
1422 			devid_to = e->ext >> 8;
1423 			ext_flags = 0;
1424 			alias = true;
1425 			break;
1426 		case IVHD_DEV_EXT_SELECT:
1427 
1428 			DUMP_printk("  DEV_EXT_SELECT\t\tdevid: %04x:%02x:%02x.%x flags: %#02x ext: %08x\n",
1429 				    seg_id, PCI_BUS_NUM(e->devid),
1430 				    PCI_SLOT(e->devid),
1431 				    PCI_FUNC(e->devid),
1432 				    e->flags, e->ext);
1433 
1434 			devid = e->devid;
1435 			set_dev_entry_from_acpi(iommu, devid, e->flags,
1436 						e->ext);
1437 			break;
1438 		case IVHD_DEV_EXT_SELECT_RANGE:
1439 
1440 			DUMP_printk("  DEV_EXT_SELECT_RANGE\tdevid: %04x:%02x:%02x.%x flags: %#02x ext: %08x\n",
1441 				    seg_id, PCI_BUS_NUM(e->devid),
1442 				    PCI_SLOT(e->devid),
1443 				    PCI_FUNC(e->devid),
1444 				    e->flags, e->ext);
1445 
1446 			devid_start = e->devid;
1447 			flags = e->flags;
1448 			ext_flags = e->ext;
1449 			alias = false;
1450 			break;
1451 		case IVHD_DEV_RANGE_END:
1452 
1453 			DUMP_printk("  DEV_RANGE_END\t\tdevid: %04x:%02x:%02x.%x\n",
1454 				    seg_id, PCI_BUS_NUM(e->devid),
1455 				    PCI_SLOT(e->devid),
1456 				    PCI_FUNC(e->devid));
1457 
1458 			devid = e->devid;
1459 			for (dev_i = devid_start; dev_i <= devid; ++dev_i) {
1460 				if (alias)
1461 					pci_seg->alias_table[dev_i] = devid_to;
1462 			}
1463 			set_dev_entry_from_acpi_range(iommu, devid_start, devid, flags, ext_flags);
1464 			set_dev_entry_from_acpi(iommu, devid_to, flags, ext_flags);
1465 			break;
1466 		case IVHD_DEV_SPECIAL: {
1467 			u8 handle, type;
1468 			const char *var;
1469 			u32 devid;
1470 			int ret;
1471 
1472 			handle = e->ext & 0xff;
1473 			devid = PCI_SEG_DEVID_TO_SBDF(seg_id, (e->ext >> 8));
1474 			type   = (e->ext >> 24) & 0xff;
1475 
1476 			if (type == IVHD_SPECIAL_IOAPIC)
1477 				var = "IOAPIC";
1478 			else if (type == IVHD_SPECIAL_HPET)
1479 				var = "HPET";
1480 			else
1481 				var = "UNKNOWN";
1482 
1483 			DUMP_printk("  DEV_SPECIAL(%s[%d])\t\tdevid: %04x:%02x:%02x.%x, flags: %#02x\n",
1484 				    var, (int)handle,
1485 				    seg_id, PCI_BUS_NUM(devid),
1486 				    PCI_SLOT(devid),
1487 				    PCI_FUNC(devid),
1488 				    e->flags);
1489 
1490 			ret = add_special_device(type, handle, &devid, false);
1491 			if (ret)
1492 				return ret;
1493 
1494 			/*
1495 			 * add_special_device might update the devid in case a
1496 			 * command-line override is present. So call
1497 			 * set_dev_entry_from_acpi after add_special_device.
1498 			 */
1499 			set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1500 
1501 			break;
1502 		}
1503 		case IVHD_DEV_ACPI_HID: {
1504 			u32 devid;
1505 			u8 hid[ACPIHID_HID_LEN];
1506 			u8 uid[ACPIHID_UID_LEN];
1507 			int ret;
1508 
1509 			if (h->type != 0x40) {
1510 				pr_err(FW_BUG "Invalid IVHD device type %#x\n",
1511 				       e->type);
1512 				break;
1513 			}
1514 
1515 			BUILD_BUG_ON(sizeof(e->ext_hid) != ACPIHID_HID_LEN - 1);
1516 			memcpy(hid, &e->ext_hid, ACPIHID_HID_LEN - 1);
1517 			hid[ACPIHID_HID_LEN - 1] = '\0';
1518 
1519 			if (!(*hid)) {
1520 				pr_err(FW_BUG "Invalid HID.\n");
1521 				break;
1522 			}
1523 
1524 			uid[0] = '\0';
1525 			switch (e->uidf) {
1526 			case UID_NOT_PRESENT:
1527 
1528 				if (e->uidl != 0)
1529 					pr_warn(FW_BUG "Invalid UID length.\n");
1530 
1531 				break;
1532 			case UID_IS_INTEGER:
1533 
1534 				sprintf(uid, "%d", e->uid);
1535 
1536 				break;
1537 			case UID_IS_CHARACTER:
1538 
1539 				memcpy(uid, &e->uid, e->uidl);
1540 				uid[e->uidl] = '\0';
1541 
1542 				break;
1543 			default:
1544 				break;
1545 			}
1546 
1547 			devid = PCI_SEG_DEVID_TO_SBDF(seg_id, e->devid);
1548 			DUMP_printk("  DEV_ACPI_HID(%s[%s])\t\tdevid: %04x:%02x:%02x.%x, flags: %#02x\n",
1549 				    hid, uid, seg_id,
1550 				    PCI_BUS_NUM(devid),
1551 				    PCI_SLOT(devid),
1552 				    PCI_FUNC(devid),
1553 				    e->flags);
1554 
1555 			flags = e->flags;
1556 
1557 			ret = add_acpi_hid_device(hid, uid, &devid, false);
1558 			if (ret)
1559 				return ret;
1560 
1561 			/*
1562 			 * add_special_device might update the devid in case a
1563 			 * command-line override is present. So call
1564 			 * set_dev_entry_from_acpi after add_special_device.
1565 			 */
1566 			set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1567 
1568 			break;
1569 		}
1570 		default:
1571 			break;
1572 		}
1573 
1574 		p += ivhd_entry_length(p);
1575 	}
1576 
1577 	return 0;
1578 }
1579 
1580 /* Allocate PCI segment data structure */
alloc_pci_segment(u16 id,struct acpi_table_header * ivrs_base)1581 static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id,
1582 					  struct acpi_table_header *ivrs_base)
1583 {
1584 	struct amd_iommu_pci_seg *pci_seg;
1585 	int last_bdf;
1586 
1587 	/*
1588 	 * First parse ACPI tables to find the largest Bus/Dev/Func we need to
1589 	 * handle in this PCI segment. Upon this information the shared data
1590 	 * structures for the PCI segments in the system will be allocated.
1591 	 */
1592 	last_bdf = find_last_devid_acpi(ivrs_base, id);
1593 	if (last_bdf < 0)
1594 		return NULL;
1595 
1596 	pci_seg = kzalloc(sizeof(struct amd_iommu_pci_seg), GFP_KERNEL);
1597 	if (pci_seg == NULL)
1598 		return NULL;
1599 
1600 	pci_seg->last_bdf = last_bdf;
1601 	DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf);
1602 	pci_seg->dev_table_size     = tbl_size(DEV_TABLE_ENTRY_SIZE, last_bdf);
1603 	pci_seg->alias_table_size   = tbl_size(ALIAS_TABLE_ENTRY_SIZE, last_bdf);
1604 	pci_seg->rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE, last_bdf);
1605 
1606 	pci_seg->id = id;
1607 	init_llist_head(&pci_seg->dev_data_list);
1608 	INIT_LIST_HEAD(&pci_seg->unity_map);
1609 	list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list);
1610 
1611 	if (alloc_dev_table(pci_seg))
1612 		return NULL;
1613 	if (alloc_alias_table(pci_seg))
1614 		return NULL;
1615 	if (alloc_rlookup_table(pci_seg))
1616 		return NULL;
1617 
1618 	return pci_seg;
1619 }
1620 
get_pci_segment(u16 id,struct acpi_table_header * ivrs_base)1621 static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id,
1622 					struct acpi_table_header *ivrs_base)
1623 {
1624 	struct amd_iommu_pci_seg *pci_seg;
1625 
1626 	for_each_pci_segment(pci_seg) {
1627 		if (pci_seg->id == id)
1628 			return pci_seg;
1629 	}
1630 
1631 	return alloc_pci_segment(id, ivrs_base);
1632 }
1633 
free_pci_segments(void)1634 static void __init free_pci_segments(void)
1635 {
1636 	struct amd_iommu_pci_seg *pci_seg, *next;
1637 
1638 	for_each_pci_segment_safe(pci_seg, next) {
1639 		list_del(&pci_seg->list);
1640 		free_irq_lookup_table(pci_seg);
1641 		free_rlookup_table(pci_seg);
1642 		free_alias_table(pci_seg);
1643 		free_dev_table(pci_seg);
1644 		kfree(pci_seg);
1645 	}
1646 }
1647 
free_sysfs(struct amd_iommu * iommu)1648 static void __init free_sysfs(struct amd_iommu *iommu)
1649 {
1650 	if (iommu->iommu.dev) {
1651 		iommu_device_unregister(&iommu->iommu);
1652 		iommu_device_sysfs_remove(&iommu->iommu);
1653 	}
1654 }
1655 
free_iommu_one(struct amd_iommu * iommu)1656 static void __init free_iommu_one(struct amd_iommu *iommu)
1657 {
1658 	free_sysfs(iommu);
1659 	free_cwwb_sem(iommu);
1660 	free_command_buffer(iommu);
1661 	free_event_buffer(iommu);
1662 	amd_iommu_free_ppr_log(iommu);
1663 	free_ga_log(iommu);
1664 	iommu_unmap_mmio_space(iommu);
1665 	amd_iommu_iopf_uninit(iommu);
1666 }
1667 
free_iommu_all(void)1668 static void __init free_iommu_all(void)
1669 {
1670 	struct amd_iommu *iommu, *next;
1671 
1672 	for_each_iommu_safe(iommu, next) {
1673 		list_del(&iommu->list);
1674 		free_iommu_one(iommu);
1675 		kfree(iommu);
1676 	}
1677 }
1678 
1679 /*
1680  * Family15h Model 10h-1fh erratum 746 (IOMMU Logging May Stall Translations)
1681  * Workaround:
1682  *     BIOS should disable L2B micellaneous clock gating by setting
1683  *     L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b
1684  */
amd_iommu_erratum_746_workaround(struct amd_iommu * iommu)1685 static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu)
1686 {
1687 	u32 value;
1688 
1689 	if ((boot_cpu_data.x86 != 0x15) ||
1690 	    (boot_cpu_data.x86_model < 0x10) ||
1691 	    (boot_cpu_data.x86_model > 0x1f))
1692 		return;
1693 
1694 	pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1695 	pci_read_config_dword(iommu->dev, 0xf4, &value);
1696 
1697 	if (value & BIT(2))
1698 		return;
1699 
1700 	/* Select NB indirect register 0x90 and enable writing */
1701 	pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8));
1702 
1703 	pci_write_config_dword(iommu->dev, 0xf4, value | 0x4);
1704 	pci_info(iommu->dev, "Applying erratum 746 workaround\n");
1705 
1706 	/* Clear the enable writing bit */
1707 	pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1708 }
1709 
1710 /*
1711  * Family15h Model 30h-3fh (IOMMU Mishandles ATS Write Permission)
1712  * Workaround:
1713  *     BIOS should enable ATS write permission check by setting
1714  *     L2_DEBUG_3[AtsIgnoreIWDis](D0F2xF4_x47[0]) = 1b
1715  */
amd_iommu_ats_write_check_workaround(struct amd_iommu * iommu)1716 static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu)
1717 {
1718 	u32 value;
1719 
1720 	if ((boot_cpu_data.x86 != 0x15) ||
1721 	    (boot_cpu_data.x86_model < 0x30) ||
1722 	    (boot_cpu_data.x86_model > 0x3f))
1723 		return;
1724 
1725 	/* Test L2_DEBUG_3[AtsIgnoreIWDis] == 1 */
1726 	value = iommu_read_l2(iommu, 0x47);
1727 
1728 	if (value & BIT(0))
1729 		return;
1730 
1731 	/* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */
1732 	iommu_write_l2(iommu, 0x47, value | BIT(0));
1733 
1734 	pci_info(iommu->dev, "Applying ATS write check workaround\n");
1735 }
1736 
1737 /*
1738  * This function glues the initialization function for one IOMMU
1739  * together and also allocates the command buffer and programs the
1740  * hardware. It does NOT enable the IOMMU. This is done afterwards.
1741  */
init_iommu_one(struct amd_iommu * iommu,struct ivhd_header * h,struct acpi_table_header * ivrs_base)1742 static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
1743 				 struct acpi_table_header *ivrs_base)
1744 {
1745 	struct amd_iommu_pci_seg *pci_seg;
1746 
1747 	pci_seg = get_pci_segment(h->pci_seg, ivrs_base);
1748 	if (pci_seg == NULL)
1749 		return -ENOMEM;
1750 	iommu->pci_seg = pci_seg;
1751 
1752 	raw_spin_lock_init(&iommu->lock);
1753 	atomic64_set(&iommu->cmd_sem_val, 0);
1754 
1755 	/* Add IOMMU to internal data structures */
1756 	list_add_tail(&iommu->list, &amd_iommu_list);
1757 	iommu->index = amd_iommus_present++;
1758 
1759 	if (unlikely(iommu->index >= MAX_IOMMUS)) {
1760 		WARN(1, "System has more IOMMUs than supported by this driver\n");
1761 		return -ENOSYS;
1762 	}
1763 
1764 	/*
1765 	 * Copy data from ACPI table entry to the iommu struct
1766 	 */
1767 	iommu->devid   = h->devid;
1768 	iommu->cap_ptr = h->cap_ptr;
1769 	iommu->mmio_phys = h->mmio_phys;
1770 
1771 	switch (h->type) {
1772 	case 0x10:
1773 		/* Check if IVHD EFR contains proper max banks/counters */
1774 		if ((h->efr_attr != 0) &&
1775 		    ((h->efr_attr & (0xF << 13)) != 0) &&
1776 		    ((h->efr_attr & (0x3F << 17)) != 0))
1777 			iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
1778 		else
1779 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1780 
1781 		/* GAM requires GA mode. */
1782 		if ((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0)
1783 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1784 		break;
1785 	case 0x11:
1786 	case 0x40:
1787 		if (h->efr_reg & (1 << 9))
1788 			iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
1789 		else
1790 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1791 
1792 		/* XT and GAM require GA mode. */
1793 		if ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0) {
1794 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1795 			break;
1796 		}
1797 
1798 		if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT))
1799 			amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE;
1800 
1801 		early_iommu_features_init(iommu, h);
1802 
1803 		break;
1804 	default:
1805 		return -EINVAL;
1806 	}
1807 
1808 	iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys,
1809 						iommu->mmio_phys_end);
1810 	if (!iommu->mmio_base)
1811 		return -ENOMEM;
1812 
1813 	return init_iommu_from_acpi(iommu, h);
1814 }
1815 
init_iommu_one_late(struct amd_iommu * iommu)1816 static int __init init_iommu_one_late(struct amd_iommu *iommu)
1817 {
1818 	int ret;
1819 
1820 	if (alloc_cwwb_sem(iommu))
1821 		return -ENOMEM;
1822 
1823 	if (alloc_command_buffer(iommu))
1824 		return -ENOMEM;
1825 
1826 	if (alloc_event_buffer(iommu))
1827 		return -ENOMEM;
1828 
1829 	iommu->int_enabled = false;
1830 
1831 	init_translation_status(iommu);
1832 	if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
1833 		iommu_disable(iommu);
1834 		clear_translation_pre_enabled(iommu);
1835 		pr_warn("Translation was enabled for IOMMU:%d but we are not in kdump mode\n",
1836 			iommu->index);
1837 	}
1838 	if (amd_iommu_pre_enabled)
1839 		amd_iommu_pre_enabled = translation_pre_enabled(iommu);
1840 
1841 	if (amd_iommu_irq_remap) {
1842 		ret = amd_iommu_create_irq_domain(iommu);
1843 		if (ret)
1844 			return ret;
1845 	}
1846 
1847 	/*
1848 	 * Make sure IOMMU is not considered to translate itself. The IVRS
1849 	 * table tells us so, but this is a lie!
1850 	 */
1851 	iommu->pci_seg->rlookup_table[iommu->devid] = NULL;
1852 
1853 	return 0;
1854 }
1855 
1856 /**
1857  * get_highest_supported_ivhd_type - Look up the appropriate IVHD type
1858  * @ivrs: Pointer to the IVRS header
1859  *
1860  * This function search through all IVDB of the maximum supported IVHD
1861  */
get_highest_supported_ivhd_type(struct acpi_table_header * ivrs)1862 static u8 get_highest_supported_ivhd_type(struct acpi_table_header *ivrs)
1863 {
1864 	u8 *base = (u8 *)ivrs;
1865 	struct ivhd_header *ivhd = (struct ivhd_header *)
1866 					(base + IVRS_HEADER_LENGTH);
1867 	u8 last_type = ivhd->type;
1868 	u16 devid = ivhd->devid;
1869 
1870 	while (((u8 *)ivhd - base < ivrs->length) &&
1871 	       (ivhd->type <= ACPI_IVHD_TYPE_MAX_SUPPORTED)) {
1872 		u8 *p = (u8 *) ivhd;
1873 
1874 		if (ivhd->devid == devid)
1875 			last_type = ivhd->type;
1876 		ivhd = (struct ivhd_header *)(p + ivhd->length);
1877 	}
1878 
1879 	return last_type;
1880 }
1881 
1882 /*
1883  * Iterates over all IOMMU entries in the ACPI table, allocates the
1884  * IOMMU structure and initializes it with init_iommu_one()
1885  */
init_iommu_all(struct acpi_table_header * table)1886 static int __init init_iommu_all(struct acpi_table_header *table)
1887 {
1888 	u8 *p = (u8 *)table, *end = (u8 *)table;
1889 	struct ivhd_header *h;
1890 	struct amd_iommu *iommu;
1891 	int ret;
1892 
1893 	end += table->length;
1894 	p += IVRS_HEADER_LENGTH;
1895 
1896 	/* Phase 1: Process all IVHD blocks */
1897 	while (p < end) {
1898 		h = (struct ivhd_header *)p;
1899 		if (*p == amd_iommu_target_ivhd_type) {
1900 
1901 			DUMP_printk("device: %04x:%02x:%02x.%01x cap: %04x "
1902 				    "flags: %01x info %04x\n",
1903 				    h->pci_seg, PCI_BUS_NUM(h->devid),
1904 				    PCI_SLOT(h->devid), PCI_FUNC(h->devid),
1905 				    h->cap_ptr, h->flags, h->info);
1906 			DUMP_printk("       mmio-addr: %016llx\n",
1907 				    h->mmio_phys);
1908 
1909 			iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
1910 			if (iommu == NULL)
1911 				return -ENOMEM;
1912 
1913 			ret = init_iommu_one(iommu, h, table);
1914 			if (ret)
1915 				return ret;
1916 		}
1917 		p += h->length;
1918 
1919 	}
1920 	WARN_ON(p != end);
1921 
1922 	/* Phase 2 : Early feature support check */
1923 	get_global_efr();
1924 
1925 	/* Phase 3 : Enabling IOMMU features */
1926 	for_each_iommu(iommu) {
1927 		ret = init_iommu_one_late(iommu);
1928 		if (ret)
1929 			return ret;
1930 	}
1931 
1932 	return 0;
1933 }
1934 
init_iommu_perf_ctr(struct amd_iommu * iommu)1935 static void init_iommu_perf_ctr(struct amd_iommu *iommu)
1936 {
1937 	u64 val;
1938 	struct pci_dev *pdev = iommu->dev;
1939 
1940 	if (!check_feature(FEATURE_PC))
1941 		return;
1942 
1943 	amd_iommu_pc_present = true;
1944 
1945 	pci_info(pdev, "IOMMU performance counters supported\n");
1946 
1947 	val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET);
1948 	iommu->max_banks = (u8) ((val >> 12) & 0x3f);
1949 	iommu->max_counters = (u8) ((val >> 7) & 0xf);
1950 
1951 	return;
1952 }
1953 
amd_iommu_show_cap(struct device * dev,struct device_attribute * attr,char * buf)1954 static ssize_t amd_iommu_show_cap(struct device *dev,
1955 				  struct device_attribute *attr,
1956 				  char *buf)
1957 {
1958 	struct amd_iommu *iommu = dev_to_amd_iommu(dev);
1959 	return sysfs_emit(buf, "%x\n", iommu->cap);
1960 }
1961 static DEVICE_ATTR(cap, S_IRUGO, amd_iommu_show_cap, NULL);
1962 
amd_iommu_show_features(struct device * dev,struct device_attribute * attr,char * buf)1963 static ssize_t amd_iommu_show_features(struct device *dev,
1964 				       struct device_attribute *attr,
1965 				       char *buf)
1966 {
1967 	return sysfs_emit(buf, "%llx:%llx\n", amd_iommu_efr, amd_iommu_efr2);
1968 }
1969 static DEVICE_ATTR(features, S_IRUGO, amd_iommu_show_features, NULL);
1970 
1971 static struct attribute *amd_iommu_attrs[] = {
1972 	&dev_attr_cap.attr,
1973 	&dev_attr_features.attr,
1974 	NULL,
1975 };
1976 
1977 static struct attribute_group amd_iommu_group = {
1978 	.name = "amd-iommu",
1979 	.attrs = amd_iommu_attrs,
1980 };
1981 
1982 static const struct attribute_group *amd_iommu_groups[] = {
1983 	&amd_iommu_group,
1984 	NULL,
1985 };
1986 
1987 /*
1988  * Note: IVHD 0x11 and 0x40 also contains exact copy
1989  * of the IOMMU Extended Feature Register [MMIO Offset 0030h].
1990  * Default to EFR in IVHD since it is available sooner (i.e. before PCI init).
1991  */
late_iommu_features_init(struct amd_iommu * iommu)1992 static void __init late_iommu_features_init(struct amd_iommu *iommu)
1993 {
1994 	u64 features, features2;
1995 
1996 	if (!(iommu->cap & (1 << IOMMU_CAP_EFR)))
1997 		return;
1998 
1999 	/* read extended feature bits */
2000 	features = readq(iommu->mmio_base + MMIO_EXT_FEATURES);
2001 	features2 = readq(iommu->mmio_base + MMIO_EXT_FEATURES2);
2002 
2003 	if (!amd_iommu_efr) {
2004 		amd_iommu_efr = features;
2005 		amd_iommu_efr2 = features2;
2006 		return;
2007 	}
2008 
2009 	/*
2010 	 * Sanity check and warn if EFR values from
2011 	 * IVHD and MMIO conflict.
2012 	 */
2013 	if (features != amd_iommu_efr ||
2014 	    features2 != amd_iommu_efr2) {
2015 		pr_warn(FW_WARN
2016 			"EFR mismatch. Use IVHD EFR (%#llx : %#llx), EFR2 (%#llx : %#llx).\n",
2017 			features, amd_iommu_efr,
2018 			features2, amd_iommu_efr2);
2019 	}
2020 }
2021 
iommu_init_pci(struct amd_iommu * iommu)2022 static int __init iommu_init_pci(struct amd_iommu *iommu)
2023 {
2024 	int cap_ptr = iommu->cap_ptr;
2025 	int ret;
2026 
2027 	iommu->dev = pci_get_domain_bus_and_slot(iommu->pci_seg->id,
2028 						 PCI_BUS_NUM(iommu->devid),
2029 						 iommu->devid & 0xff);
2030 	if (!iommu->dev)
2031 		return -ENODEV;
2032 
2033 	/* Prevent binding other PCI device drivers to IOMMU devices */
2034 	iommu->dev->match_driver = false;
2035 
2036 	/* ACPI _PRT won't have an IRQ for IOMMU */
2037 	iommu->dev->irq_managed = 1;
2038 
2039 	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
2040 			      &iommu->cap);
2041 
2042 	if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
2043 		amd_iommu_iotlb_sup = false;
2044 
2045 	late_iommu_features_init(iommu);
2046 
2047 	if (check_feature(FEATURE_GT)) {
2048 		int glxval;
2049 		u64 pasmax;
2050 
2051 		pasmax = FIELD_GET(FEATURE_PASMAX, amd_iommu_efr);
2052 		iommu->iommu.max_pasids = (1 << (pasmax + 1)) - 1;
2053 
2054 		BUG_ON(iommu->iommu.max_pasids & ~PASID_MASK);
2055 
2056 		glxval = FIELD_GET(FEATURE_GLX, amd_iommu_efr);
2057 
2058 		if (amd_iommu_max_glx_val == -1)
2059 			amd_iommu_max_glx_val = glxval;
2060 		else
2061 			amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
2062 
2063 		iommu_enable_gt(iommu);
2064 	}
2065 
2066 	if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu))
2067 		return -ENOMEM;
2068 
2069 	if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) {
2070 		pr_info("Using strict mode due to virtualization\n");
2071 		iommu_set_dma_strict();
2072 		amd_iommu_np_cache = true;
2073 	}
2074 
2075 	init_iommu_perf_ctr(iommu);
2076 
2077 	if (is_rd890_iommu(iommu->dev)) {
2078 		int i, j;
2079 
2080 		iommu->root_pdev =
2081 			pci_get_domain_bus_and_slot(iommu->pci_seg->id,
2082 						    iommu->dev->bus->number,
2083 						    PCI_DEVFN(0, 0));
2084 
2085 		/*
2086 		 * Some rd890 systems may not be fully reconfigured by the
2087 		 * BIOS, so it's necessary for us to store this information so
2088 		 * it can be reprogrammed on resume
2089 		 */
2090 		pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,
2091 				&iommu->stored_addr_lo);
2092 		pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,
2093 				&iommu->stored_addr_hi);
2094 
2095 		/* Low bit locks writes to configuration space */
2096 		iommu->stored_addr_lo &= ~1;
2097 
2098 		for (i = 0; i < 6; i++)
2099 			for (j = 0; j < 0x12; j++)
2100 				iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);
2101 
2102 		for (i = 0; i < 0x83; i++)
2103 			iommu->stored_l2[i] = iommu_read_l2(iommu, i);
2104 	}
2105 
2106 	amd_iommu_erratum_746_workaround(iommu);
2107 	amd_iommu_ats_write_check_workaround(iommu);
2108 
2109 	ret = iommu_device_sysfs_add(&iommu->iommu, &iommu->dev->dev,
2110 			       amd_iommu_groups, "ivhd%d", iommu->index);
2111 	if (ret)
2112 		return ret;
2113 
2114 	/*
2115 	 * Allocate per IOMMU IOPF queue here so that in attach device path,
2116 	 * PRI capable device can be added to IOPF queue
2117 	 */
2118 	if (amd_iommu_gt_ppr_supported()) {
2119 		ret = amd_iommu_iopf_init(iommu);
2120 		if (ret)
2121 			return ret;
2122 	}
2123 
2124 	iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL);
2125 
2126 	return pci_enable_device(iommu->dev);
2127 }
2128 
print_iommu_info(void)2129 static void print_iommu_info(void)
2130 {
2131 	int i;
2132 	static const char * const feat_str[] = {
2133 		"PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
2134 		"IA", "GA", "HE", "PC"
2135 	};
2136 
2137 	if (amd_iommu_efr) {
2138 		pr_info("Extended features (%#llx, %#llx):", amd_iommu_efr, amd_iommu_efr2);
2139 
2140 		for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
2141 			if (check_feature(1ULL << i))
2142 				pr_cont(" %s", feat_str[i]);
2143 		}
2144 
2145 		if (check_feature(FEATURE_GAM_VAPIC))
2146 			pr_cont(" GA_vAPIC");
2147 
2148 		if (check_feature(FEATURE_SNP))
2149 			pr_cont(" SNP");
2150 
2151 		pr_cont("\n");
2152 	}
2153 
2154 	if (irq_remapping_enabled) {
2155 		pr_info("Interrupt remapping enabled\n");
2156 		if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2157 			pr_info("X2APIC enabled\n");
2158 	}
2159 	if (amd_iommu_pgtable == PD_MODE_V2) {
2160 		pr_info("V2 page table enabled (Paging mode : %d level)\n",
2161 			amd_iommu_gpt_level);
2162 	}
2163 }
2164 
amd_iommu_init_pci(void)2165 static int __init amd_iommu_init_pci(void)
2166 {
2167 	struct amd_iommu *iommu;
2168 	struct amd_iommu_pci_seg *pci_seg;
2169 	int ret;
2170 
2171 	/* Init global identity domain before registering IOMMU */
2172 	amd_iommu_init_identity_domain();
2173 
2174 	for_each_iommu(iommu) {
2175 		ret = iommu_init_pci(iommu);
2176 		if (ret) {
2177 			pr_err("IOMMU%d: Failed to initialize IOMMU Hardware (error=%d)!\n",
2178 			       iommu->index, ret);
2179 			goto out;
2180 		}
2181 		/* Need to setup range after PCI init */
2182 		iommu_set_cwwb_range(iommu);
2183 	}
2184 
2185 	/*
2186 	 * Order is important here to make sure any unity map requirements are
2187 	 * fulfilled. The unity mappings are created and written to the device
2188 	 * table during the iommu_init_pci() call.
2189 	 *
2190 	 * After that we call init_device_table_dma() to make sure any
2191 	 * uninitialized DTE will block DMA, and in the end we flush the caches
2192 	 * of all IOMMUs to make sure the changes to the device table are
2193 	 * active.
2194 	 */
2195 	for_each_pci_segment(pci_seg)
2196 		init_device_table_dma(pci_seg);
2197 
2198 	for_each_iommu(iommu)
2199 		amd_iommu_flush_all_caches(iommu);
2200 
2201 	print_iommu_info();
2202 
2203 out:
2204 	return ret;
2205 }
2206 
2207 /****************************************************************************
2208  *
2209  * The following functions initialize the MSI interrupts for all IOMMUs
2210  * in the system. It's a bit challenging because there could be multiple
2211  * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per
2212  * pci_dev.
2213  *
2214  ****************************************************************************/
2215 
iommu_setup_msi(struct amd_iommu * iommu)2216 static int iommu_setup_msi(struct amd_iommu *iommu)
2217 {
2218 	int r;
2219 
2220 	r = pci_enable_msi(iommu->dev);
2221 	if (r)
2222 		return r;
2223 
2224 	r = request_threaded_irq(iommu->dev->irq,
2225 				 amd_iommu_int_handler,
2226 				 amd_iommu_int_thread,
2227 				 0, "AMD-Vi",
2228 				 iommu);
2229 
2230 	if (r) {
2231 		pci_disable_msi(iommu->dev);
2232 		return r;
2233 	}
2234 
2235 	return 0;
2236 }
2237 
2238 union intcapxt {
2239 	u64	capxt;
2240 	struct {
2241 		u64	reserved_0		:  2,
2242 			dest_mode_logical	:  1,
2243 			reserved_1		:  5,
2244 			destid_0_23		: 24,
2245 			vector			:  8,
2246 			reserved_2		: 16,
2247 			destid_24_31		:  8;
2248 	};
2249 } __attribute__ ((packed));
2250 
2251 
2252 static struct irq_chip intcapxt_controller;
2253 
intcapxt_irqdomain_activate(struct irq_domain * domain,struct irq_data * irqd,bool reserve)2254 static int intcapxt_irqdomain_activate(struct irq_domain *domain,
2255 				       struct irq_data *irqd, bool reserve)
2256 {
2257 	return 0;
2258 }
2259 
intcapxt_irqdomain_deactivate(struct irq_domain * domain,struct irq_data * irqd)2260 static void intcapxt_irqdomain_deactivate(struct irq_domain *domain,
2261 					  struct irq_data *irqd)
2262 {
2263 }
2264 
2265 
intcapxt_irqdomain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)2266 static int intcapxt_irqdomain_alloc(struct irq_domain *domain, unsigned int virq,
2267 				    unsigned int nr_irqs, void *arg)
2268 {
2269 	struct irq_alloc_info *info = arg;
2270 	int i, ret;
2271 
2272 	if (!info || info->type != X86_IRQ_ALLOC_TYPE_AMDVI)
2273 		return -EINVAL;
2274 
2275 	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
2276 	if (ret < 0)
2277 		return ret;
2278 
2279 	for (i = virq; i < virq + nr_irqs; i++) {
2280 		struct irq_data *irqd = irq_domain_get_irq_data(domain, i);
2281 
2282 		irqd->chip = &intcapxt_controller;
2283 		irqd->hwirq = info->hwirq;
2284 		irqd->chip_data = info->data;
2285 		__irq_set_handler(i, handle_edge_irq, 0, "edge");
2286 	}
2287 
2288 	return ret;
2289 }
2290 
intcapxt_irqdomain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)2291 static void intcapxt_irqdomain_free(struct irq_domain *domain, unsigned int virq,
2292 				    unsigned int nr_irqs)
2293 {
2294 	irq_domain_free_irqs_top(domain, virq, nr_irqs);
2295 }
2296 
2297 
intcapxt_unmask_irq(struct irq_data * irqd)2298 static void intcapxt_unmask_irq(struct irq_data *irqd)
2299 {
2300 	struct amd_iommu *iommu = irqd->chip_data;
2301 	struct irq_cfg *cfg = irqd_cfg(irqd);
2302 	union intcapxt xt;
2303 
2304 	xt.capxt = 0ULL;
2305 	xt.dest_mode_logical = apic->dest_mode_logical;
2306 	xt.vector = cfg->vector;
2307 	xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0);
2308 	xt.destid_24_31 = cfg->dest_apicid >> 24;
2309 
2310 	writeq(xt.capxt, iommu->mmio_base + irqd->hwirq);
2311 }
2312 
intcapxt_mask_irq(struct irq_data * irqd)2313 static void intcapxt_mask_irq(struct irq_data *irqd)
2314 {
2315 	struct amd_iommu *iommu = irqd->chip_data;
2316 
2317 	writeq(0, iommu->mmio_base + irqd->hwirq);
2318 }
2319 
2320 
intcapxt_set_affinity(struct irq_data * irqd,const struct cpumask * mask,bool force)2321 static int intcapxt_set_affinity(struct irq_data *irqd,
2322 				 const struct cpumask *mask, bool force)
2323 {
2324 	struct irq_data *parent = irqd->parent_data;
2325 	int ret;
2326 
2327 	ret = parent->chip->irq_set_affinity(parent, mask, force);
2328 	if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
2329 		return ret;
2330 	return 0;
2331 }
2332 
intcapxt_set_wake(struct irq_data * irqd,unsigned int on)2333 static int intcapxt_set_wake(struct irq_data *irqd, unsigned int on)
2334 {
2335 	return on ? -EOPNOTSUPP : 0;
2336 }
2337 
2338 static struct irq_chip intcapxt_controller = {
2339 	.name			= "IOMMU-MSI",
2340 	.irq_unmask		= intcapxt_unmask_irq,
2341 	.irq_mask		= intcapxt_mask_irq,
2342 	.irq_ack		= irq_chip_ack_parent,
2343 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
2344 	.irq_set_affinity       = intcapxt_set_affinity,
2345 	.irq_set_wake		= intcapxt_set_wake,
2346 	.flags			= IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_MOVE_DEFERRED,
2347 };
2348 
2349 static const struct irq_domain_ops intcapxt_domain_ops = {
2350 	.alloc			= intcapxt_irqdomain_alloc,
2351 	.free			= intcapxt_irqdomain_free,
2352 	.activate		= intcapxt_irqdomain_activate,
2353 	.deactivate		= intcapxt_irqdomain_deactivate,
2354 };
2355 
2356 
2357 static struct irq_domain *iommu_irqdomain;
2358 
iommu_get_irqdomain(void)2359 static struct irq_domain *iommu_get_irqdomain(void)
2360 {
2361 	struct fwnode_handle *fn;
2362 
2363 	/* No need for locking here (yet) as the init is single-threaded */
2364 	if (iommu_irqdomain)
2365 		return iommu_irqdomain;
2366 
2367 	fn = irq_domain_alloc_named_fwnode("AMD-Vi-MSI");
2368 	if (!fn)
2369 		return NULL;
2370 
2371 	iommu_irqdomain = irq_domain_create_hierarchy(x86_vector_domain, 0, 0,
2372 						      fn, &intcapxt_domain_ops,
2373 						      NULL);
2374 	if (!iommu_irqdomain)
2375 		irq_domain_free_fwnode(fn);
2376 
2377 	return iommu_irqdomain;
2378 }
2379 
__iommu_setup_intcapxt(struct amd_iommu * iommu,const char * devname,int hwirq,irq_handler_t thread_fn)2380 static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname,
2381 				  int hwirq, irq_handler_t thread_fn)
2382 {
2383 	struct irq_domain *domain;
2384 	struct irq_alloc_info info;
2385 	int irq, ret;
2386 	int node = dev_to_node(&iommu->dev->dev);
2387 
2388 	domain = iommu_get_irqdomain();
2389 	if (!domain)
2390 		return -ENXIO;
2391 
2392 	init_irq_alloc_info(&info, NULL);
2393 	info.type = X86_IRQ_ALLOC_TYPE_AMDVI;
2394 	info.data = iommu;
2395 	info.hwirq = hwirq;
2396 
2397 	irq = irq_domain_alloc_irqs(domain, 1, node, &info);
2398 	if (irq < 0) {
2399 		irq_domain_remove(domain);
2400 		return irq;
2401 	}
2402 
2403 	ret = request_threaded_irq(irq, amd_iommu_int_handler,
2404 				   thread_fn, 0, devname, iommu);
2405 	if (ret) {
2406 		irq_domain_free_irqs(irq, 1);
2407 		irq_domain_remove(domain);
2408 		return ret;
2409 	}
2410 
2411 	return 0;
2412 }
2413 
iommu_setup_intcapxt(struct amd_iommu * iommu)2414 static int iommu_setup_intcapxt(struct amd_iommu *iommu)
2415 {
2416 	int ret;
2417 
2418 	snprintf(iommu->evt_irq_name, sizeof(iommu->evt_irq_name),
2419 		 "AMD-Vi%d-Evt", iommu->index);
2420 	ret = __iommu_setup_intcapxt(iommu, iommu->evt_irq_name,
2421 				     MMIO_INTCAPXT_EVT_OFFSET,
2422 				     amd_iommu_int_thread_evtlog);
2423 	if (ret)
2424 		return ret;
2425 
2426 	snprintf(iommu->ppr_irq_name, sizeof(iommu->ppr_irq_name),
2427 		 "AMD-Vi%d-PPR", iommu->index);
2428 	ret = __iommu_setup_intcapxt(iommu, iommu->ppr_irq_name,
2429 				     MMIO_INTCAPXT_PPR_OFFSET,
2430 				     amd_iommu_int_thread_pprlog);
2431 	if (ret)
2432 		return ret;
2433 
2434 #ifdef CONFIG_IRQ_REMAP
2435 	snprintf(iommu->ga_irq_name, sizeof(iommu->ga_irq_name),
2436 		 "AMD-Vi%d-GA", iommu->index);
2437 	ret = __iommu_setup_intcapxt(iommu, iommu->ga_irq_name,
2438 				     MMIO_INTCAPXT_GALOG_OFFSET,
2439 				     amd_iommu_int_thread_galog);
2440 #endif
2441 
2442 	return ret;
2443 }
2444 
iommu_init_irq(struct amd_iommu * iommu)2445 static int iommu_init_irq(struct amd_iommu *iommu)
2446 {
2447 	int ret;
2448 
2449 	if (iommu->int_enabled)
2450 		goto enable_faults;
2451 
2452 	if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2453 		ret = iommu_setup_intcapxt(iommu);
2454 	else if (iommu->dev->msi_cap)
2455 		ret = iommu_setup_msi(iommu);
2456 	else
2457 		ret = -ENODEV;
2458 
2459 	if (ret)
2460 		return ret;
2461 
2462 	iommu->int_enabled = true;
2463 enable_faults:
2464 
2465 	if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2466 		iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);
2467 
2468 	iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
2469 
2470 	return 0;
2471 }
2472 
2473 /****************************************************************************
2474  *
2475  * The next functions belong to the third pass of parsing the ACPI
2476  * table. In this last pass the memory mapping requirements are
2477  * gathered (like exclusion and unity mapping ranges).
2478  *
2479  ****************************************************************************/
2480 
free_unity_maps(void)2481 static void __init free_unity_maps(void)
2482 {
2483 	struct unity_map_entry *entry, *next;
2484 	struct amd_iommu_pci_seg *p, *pci_seg;
2485 
2486 	for_each_pci_segment_safe(pci_seg, p) {
2487 		list_for_each_entry_safe(entry, next, &pci_seg->unity_map, list) {
2488 			list_del(&entry->list);
2489 			kfree(entry);
2490 		}
2491 	}
2492 }
2493 
2494 /* called for unity map ACPI definition */
init_unity_map_range(struct ivmd_header * m,struct acpi_table_header * ivrs_base)2495 static int __init init_unity_map_range(struct ivmd_header *m,
2496 				       struct acpi_table_header *ivrs_base)
2497 {
2498 	struct unity_map_entry *e = NULL;
2499 	struct amd_iommu_pci_seg *pci_seg;
2500 	char *s;
2501 
2502 	pci_seg = get_pci_segment(m->pci_seg, ivrs_base);
2503 	if (pci_seg == NULL)
2504 		return -ENOMEM;
2505 
2506 	e = kzalloc(sizeof(*e), GFP_KERNEL);
2507 	if (e == NULL)
2508 		return -ENOMEM;
2509 
2510 	switch (m->type) {
2511 	default:
2512 		kfree(e);
2513 		return 0;
2514 	case ACPI_IVMD_TYPE:
2515 		s = "IVMD_TYPEi\t\t\t";
2516 		e->devid_start = e->devid_end = m->devid;
2517 		break;
2518 	case ACPI_IVMD_TYPE_ALL:
2519 		s = "IVMD_TYPE_ALL\t\t";
2520 		e->devid_start = 0;
2521 		e->devid_end = pci_seg->last_bdf;
2522 		break;
2523 	case ACPI_IVMD_TYPE_RANGE:
2524 		s = "IVMD_TYPE_RANGE\t\t";
2525 		e->devid_start = m->devid;
2526 		e->devid_end = m->aux;
2527 		break;
2528 	}
2529 	e->address_start = PAGE_ALIGN(m->range_start);
2530 	e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
2531 	e->prot = m->flags >> 1;
2532 
2533 	/*
2534 	 * Treat per-device exclusion ranges as r/w unity-mapped regions
2535 	 * since some buggy BIOSes might lead to the overwritten exclusion
2536 	 * range (exclusion_start and exclusion_length members). This
2537 	 * happens when there are multiple exclusion ranges (IVMD entries)
2538 	 * defined in ACPI table.
2539 	 */
2540 	if (m->flags & IVMD_FLAG_EXCL_RANGE)
2541 		e->prot = (IVMD_FLAG_IW | IVMD_FLAG_IR) >> 1;
2542 
2543 	DUMP_printk("%s devid_start: %04x:%02x:%02x.%x devid_end: "
2544 		    "%04x:%02x:%02x.%x range_start: %016llx range_end: %016llx"
2545 		    " flags: %x\n", s, m->pci_seg,
2546 		    PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start),
2547 		    PCI_FUNC(e->devid_start), m->pci_seg,
2548 		    PCI_BUS_NUM(e->devid_end),
2549 		    PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
2550 		    e->address_start, e->address_end, m->flags);
2551 
2552 	list_add_tail(&e->list, &pci_seg->unity_map);
2553 
2554 	return 0;
2555 }
2556 
2557 /* iterates over all memory definitions we find in the ACPI table */
init_memory_definitions(struct acpi_table_header * table)2558 static int __init init_memory_definitions(struct acpi_table_header *table)
2559 {
2560 	u8 *p = (u8 *)table, *end = (u8 *)table;
2561 	struct ivmd_header *m;
2562 
2563 	end += table->length;
2564 	p += IVRS_HEADER_LENGTH;
2565 
2566 	while (p < end) {
2567 		m = (struct ivmd_header *)p;
2568 		if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE))
2569 			init_unity_map_range(m, table);
2570 
2571 		p += m->length;
2572 	}
2573 
2574 	return 0;
2575 }
2576 
2577 /*
2578  * Init the device table to not allow DMA access for devices
2579  */
init_device_table_dma(struct amd_iommu_pci_seg * pci_seg)2580 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg)
2581 {
2582 	u32 devid;
2583 	struct dev_table_entry *dev_table = pci_seg->dev_table;
2584 
2585 	if (dev_table == NULL)
2586 		return;
2587 
2588 	for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
2589 		set_dte_bit(&dev_table[devid], DEV_ENTRY_VALID);
2590 		if (!amd_iommu_snp_en)
2591 			set_dte_bit(&dev_table[devid], DEV_ENTRY_TRANSLATION);
2592 	}
2593 }
2594 
uninit_device_table_dma(struct amd_iommu_pci_seg * pci_seg)2595 static void __init uninit_device_table_dma(struct amd_iommu_pci_seg *pci_seg)
2596 {
2597 	u32 devid;
2598 	struct dev_table_entry *dev_table = pci_seg->dev_table;
2599 
2600 	if (dev_table == NULL)
2601 		return;
2602 
2603 	for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
2604 		dev_table[devid].data[0] = 0ULL;
2605 		dev_table[devid].data[1] = 0ULL;
2606 	}
2607 }
2608 
init_device_table(void)2609 static void init_device_table(void)
2610 {
2611 	struct amd_iommu_pci_seg *pci_seg;
2612 	u32 devid;
2613 
2614 	if (!amd_iommu_irq_remap)
2615 		return;
2616 
2617 	for_each_pci_segment(pci_seg) {
2618 		for (devid = 0; devid <= pci_seg->last_bdf; ++devid)
2619 			set_dte_bit(&pci_seg->dev_table[devid], DEV_ENTRY_IRQ_TBL_EN);
2620 	}
2621 }
2622 
iommu_init_flags(struct amd_iommu * iommu)2623 static void iommu_init_flags(struct amd_iommu *iommu)
2624 {
2625 	iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?
2626 		iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :
2627 		iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);
2628 
2629 	iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?
2630 		iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :
2631 		iommu_feature_disable(iommu, CONTROL_PASSPW_EN);
2632 
2633 	iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?
2634 		iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :
2635 		iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);
2636 
2637 	iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?
2638 		iommu_feature_enable(iommu, CONTROL_ISOC_EN) :
2639 		iommu_feature_disable(iommu, CONTROL_ISOC_EN);
2640 
2641 	/*
2642 	 * make IOMMU memory accesses cache coherent
2643 	 */
2644 	iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
2645 
2646 	/* Set IOTLB invalidation timeout to 1s */
2647 	iommu_feature_set(iommu, CTRL_INV_TO_1S, CTRL_INV_TO_MASK, CONTROL_INV_TIMEOUT);
2648 
2649 	/* Enable Enhanced Peripheral Page Request Handling */
2650 	if (check_feature(FEATURE_EPHSUP))
2651 		iommu_feature_enable(iommu, CONTROL_EPH_EN);
2652 }
2653 
iommu_apply_resume_quirks(struct amd_iommu * iommu)2654 static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
2655 {
2656 	int i, j;
2657 	u32 ioc_feature_control;
2658 	struct pci_dev *pdev = iommu->root_pdev;
2659 
2660 	/* RD890 BIOSes may not have completely reconfigured the iommu */
2661 	if (!is_rd890_iommu(iommu->dev) || !pdev)
2662 		return;
2663 
2664 	/*
2665 	 * First, we need to ensure that the iommu is enabled. This is
2666 	 * controlled by a register in the northbridge
2667 	 */
2668 
2669 	/* Select Northbridge indirect register 0x75 and enable writing */
2670 	pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));
2671 	pci_read_config_dword(pdev, 0x64, &ioc_feature_control);
2672 
2673 	/* Enable the iommu */
2674 	if (!(ioc_feature_control & 0x1))
2675 		pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);
2676 
2677 	/* Restore the iommu BAR */
2678 	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
2679 			       iommu->stored_addr_lo);
2680 	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,
2681 			       iommu->stored_addr_hi);
2682 
2683 	/* Restore the l1 indirect regs for each of the 6 l1s */
2684 	for (i = 0; i < 6; i++)
2685 		for (j = 0; j < 0x12; j++)
2686 			iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);
2687 
2688 	/* Restore the l2 indirect regs */
2689 	for (i = 0; i < 0x83; i++)
2690 		iommu_write_l2(iommu, i, iommu->stored_l2[i]);
2691 
2692 	/* Lock PCI setup registers */
2693 	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
2694 			       iommu->stored_addr_lo | 1);
2695 }
2696 
iommu_enable_ga(struct amd_iommu * iommu)2697 static void iommu_enable_ga(struct amd_iommu *iommu)
2698 {
2699 #ifdef CONFIG_IRQ_REMAP
2700 	switch (amd_iommu_guest_ir) {
2701 	case AMD_IOMMU_GUEST_IR_VAPIC:
2702 	case AMD_IOMMU_GUEST_IR_LEGACY_GA:
2703 		iommu_feature_enable(iommu, CONTROL_GA_EN);
2704 		iommu->irte_ops = &irte_128_ops;
2705 		break;
2706 	default:
2707 		iommu->irte_ops = &irte_32_ops;
2708 		break;
2709 	}
2710 #endif
2711 }
2712 
iommu_disable_irtcachedis(struct amd_iommu * iommu)2713 static void iommu_disable_irtcachedis(struct amd_iommu *iommu)
2714 {
2715 	iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);
2716 }
2717 
iommu_enable_irtcachedis(struct amd_iommu * iommu)2718 static void iommu_enable_irtcachedis(struct amd_iommu *iommu)
2719 {
2720 	u64 ctrl;
2721 
2722 	if (!amd_iommu_irtcachedis)
2723 		return;
2724 
2725 	/*
2726 	 * Note:
2727 	 * The support for IRTCacheDis feature is dertermined by
2728 	 * checking if the bit is writable.
2729 	 */
2730 	iommu_feature_enable(iommu, CONTROL_IRTCACHEDIS);
2731 	ctrl = readq(iommu->mmio_base +  MMIO_CONTROL_OFFSET);
2732 	ctrl &= (1ULL << CONTROL_IRTCACHEDIS);
2733 	if (ctrl)
2734 		iommu->irtcachedis_enabled = true;
2735 	pr_info("iommu%d (%#06x) : IRT cache is %s\n",
2736 		iommu->index, iommu->devid,
2737 		iommu->irtcachedis_enabled ? "disabled" : "enabled");
2738 }
2739 
iommu_enable_2k_int(struct amd_iommu * iommu)2740 static void iommu_enable_2k_int(struct amd_iommu *iommu)
2741 {
2742 	if (!FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))
2743 		return;
2744 
2745 	iommu_feature_set(iommu,
2746 			  CONTROL_NUM_INT_REMAP_MODE_2K,
2747 			  CONTROL_NUM_INT_REMAP_MODE_MASK,
2748 			  CONTROL_NUM_INT_REMAP_MODE);
2749 }
2750 
early_enable_iommu(struct amd_iommu * iommu)2751 static void early_enable_iommu(struct amd_iommu *iommu)
2752 {
2753 	iommu_disable(iommu);
2754 	iommu_init_flags(iommu);
2755 	iommu_set_device_table(iommu);
2756 	iommu_enable_command_buffer(iommu);
2757 	iommu_enable_event_buffer(iommu);
2758 	iommu_set_exclusion_range(iommu);
2759 	iommu_enable_gt(iommu);
2760 	iommu_enable_ga(iommu);
2761 	iommu_enable_xt(iommu);
2762 	iommu_enable_irtcachedis(iommu);
2763 	iommu_enable_2k_int(iommu);
2764 	iommu_enable(iommu);
2765 	amd_iommu_flush_all_caches(iommu);
2766 }
2767 
2768 /*
2769  * This function finally enables all IOMMUs found in the system after
2770  * they have been initialized.
2771  *
2772  * Or if in kdump kernel and IOMMUs are all pre-enabled, try to copy
2773  * the old content of device table entries. Not this case or copy failed,
2774  * just continue as normal kernel does.
2775  */
early_enable_iommus(void)2776 static void early_enable_iommus(void)
2777 {
2778 	struct amd_iommu *iommu;
2779 	struct amd_iommu_pci_seg *pci_seg;
2780 
2781 	if (!copy_device_table()) {
2782 		/*
2783 		 * If come here because of failure in copying device table from old
2784 		 * kernel with all IOMMUs enabled, print error message and try to
2785 		 * free allocated old_dev_tbl_cpy.
2786 		 */
2787 		if (amd_iommu_pre_enabled)
2788 			pr_err("Failed to copy DEV table from previous kernel.\n");
2789 
2790 		for_each_pci_segment(pci_seg) {
2791 			if (pci_seg->old_dev_tbl_cpy != NULL) {
2792 				iommu_free_pages(pci_seg->old_dev_tbl_cpy,
2793 						 get_order(pci_seg->dev_table_size));
2794 				pci_seg->old_dev_tbl_cpy = NULL;
2795 			}
2796 		}
2797 
2798 		for_each_iommu(iommu) {
2799 			clear_translation_pre_enabled(iommu);
2800 			early_enable_iommu(iommu);
2801 		}
2802 	} else {
2803 		pr_info("Copied DEV table from previous kernel.\n");
2804 
2805 		for_each_pci_segment(pci_seg) {
2806 			iommu_free_pages(pci_seg->dev_table,
2807 					 get_order(pci_seg->dev_table_size));
2808 			pci_seg->dev_table = pci_seg->old_dev_tbl_cpy;
2809 		}
2810 
2811 		for_each_iommu(iommu) {
2812 			iommu_disable_command_buffer(iommu);
2813 			iommu_disable_event_buffer(iommu);
2814 			iommu_disable_irtcachedis(iommu);
2815 			iommu_enable_command_buffer(iommu);
2816 			iommu_enable_event_buffer(iommu);
2817 			iommu_enable_ga(iommu);
2818 			iommu_enable_xt(iommu);
2819 			iommu_enable_irtcachedis(iommu);
2820 			iommu_enable_2k_int(iommu);
2821 			iommu_set_device_table(iommu);
2822 			amd_iommu_flush_all_caches(iommu);
2823 		}
2824 	}
2825 }
2826 
enable_iommus_ppr(void)2827 static void enable_iommus_ppr(void)
2828 {
2829 	struct amd_iommu *iommu;
2830 
2831 	if (!amd_iommu_gt_ppr_supported())
2832 		return;
2833 
2834 	for_each_iommu(iommu)
2835 		amd_iommu_enable_ppr_log(iommu);
2836 }
2837 
enable_iommus_vapic(void)2838 static void enable_iommus_vapic(void)
2839 {
2840 #ifdef CONFIG_IRQ_REMAP
2841 	u32 status, i;
2842 	struct amd_iommu *iommu;
2843 
2844 	for_each_iommu(iommu) {
2845 		/*
2846 		 * Disable GALog if already running. It could have been enabled
2847 		 * in the previous boot before kdump.
2848 		 */
2849 		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
2850 		if (!(status & MMIO_STATUS_GALOG_RUN_MASK))
2851 			continue;
2852 
2853 		iommu_feature_disable(iommu, CONTROL_GALOG_EN);
2854 		iommu_feature_disable(iommu, CONTROL_GAINT_EN);
2855 
2856 		/*
2857 		 * Need to set and poll check the GALOGRun bit to zero before
2858 		 * we can set/ modify GA Log registers safely.
2859 		 */
2860 		for (i = 0; i < MMIO_STATUS_TIMEOUT; ++i) {
2861 			status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
2862 			if (!(status & MMIO_STATUS_GALOG_RUN_MASK))
2863 				break;
2864 			udelay(10);
2865 		}
2866 
2867 		if (WARN_ON(i >= MMIO_STATUS_TIMEOUT))
2868 			return;
2869 	}
2870 
2871 	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) &&
2872 	    !check_feature(FEATURE_GAM_VAPIC)) {
2873 		amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
2874 		return;
2875 	}
2876 
2877 	if (amd_iommu_snp_en &&
2878 	    !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) {
2879 		pr_warn("Force to disable Virtual APIC due to SNP\n");
2880 		amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
2881 		return;
2882 	}
2883 
2884 	/* Enabling GAM and SNPAVIC support */
2885 	for_each_iommu(iommu) {
2886 		if (iommu_init_ga_log(iommu) ||
2887 		    iommu_ga_log_enable(iommu))
2888 			return;
2889 
2890 		iommu_feature_enable(iommu, CONTROL_GAM_EN);
2891 		if (amd_iommu_snp_en)
2892 			iommu_feature_enable(iommu, CONTROL_SNPAVIC_EN);
2893 	}
2894 
2895 	amd_iommu_irq_ops.capability |= (1 << IRQ_POSTING_CAP);
2896 	pr_info("Virtual APIC enabled\n");
2897 #endif
2898 }
2899 
disable_iommus(void)2900 static void disable_iommus(void)
2901 {
2902 	struct amd_iommu *iommu;
2903 
2904 	for_each_iommu(iommu)
2905 		iommu_disable(iommu);
2906 
2907 #ifdef CONFIG_IRQ_REMAP
2908 	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
2909 		amd_iommu_irq_ops.capability &= ~(1 << IRQ_POSTING_CAP);
2910 #endif
2911 }
2912 
2913 /*
2914  * Suspend/Resume support
2915  * disable suspend until real resume implemented
2916  */
2917 
amd_iommu_resume(void)2918 static void amd_iommu_resume(void)
2919 {
2920 	struct amd_iommu *iommu;
2921 
2922 	for_each_iommu(iommu)
2923 		iommu_apply_resume_quirks(iommu);
2924 
2925 	/* re-load the hardware */
2926 	for_each_iommu(iommu)
2927 		early_enable_iommu(iommu);
2928 
2929 	amd_iommu_enable_interrupts();
2930 }
2931 
amd_iommu_suspend(void)2932 static int amd_iommu_suspend(void)
2933 {
2934 	/* disable IOMMUs to go out of the way for BIOS */
2935 	disable_iommus();
2936 
2937 	return 0;
2938 }
2939 
2940 static struct syscore_ops amd_iommu_syscore_ops = {
2941 	.suspend = amd_iommu_suspend,
2942 	.resume = amd_iommu_resume,
2943 };
2944 
free_iommu_resources(void)2945 static void __init free_iommu_resources(void)
2946 {
2947 	free_iommu_all();
2948 	free_pci_segments();
2949 }
2950 
2951 /* SB IOAPIC is always on this device in AMD systems */
2952 #define IOAPIC_SB_DEVID		((0x00 << 8) | PCI_DEVFN(0x14, 0))
2953 
check_ioapic_information(void)2954 static bool __init check_ioapic_information(void)
2955 {
2956 	const char *fw_bug = FW_BUG;
2957 	bool ret, has_sb_ioapic;
2958 	int idx;
2959 
2960 	has_sb_ioapic = false;
2961 	ret           = false;
2962 
2963 	/*
2964 	 * If we have map overrides on the kernel command line the
2965 	 * messages in this function might not describe firmware bugs
2966 	 * anymore - so be careful
2967 	 */
2968 	if (cmdline_maps)
2969 		fw_bug = "";
2970 
2971 	for (idx = 0; idx < nr_ioapics; idx++) {
2972 		int devid, id = mpc_ioapic_id(idx);
2973 
2974 		devid = get_ioapic_devid(id);
2975 		if (devid < 0) {
2976 			pr_err("%s: IOAPIC[%d] not in IVRS table\n",
2977 				fw_bug, id);
2978 			ret = false;
2979 		} else if (devid == IOAPIC_SB_DEVID) {
2980 			has_sb_ioapic = true;
2981 			ret           = true;
2982 		}
2983 	}
2984 
2985 	if (!has_sb_ioapic) {
2986 		/*
2987 		 * We expect the SB IOAPIC to be listed in the IVRS
2988 		 * table. The system timer is connected to the SB IOAPIC
2989 		 * and if we don't have it in the list the system will
2990 		 * panic at boot time.  This situation usually happens
2991 		 * when the BIOS is buggy and provides us the wrong
2992 		 * device id for the IOAPIC in the system.
2993 		 */
2994 		pr_err("%s: No southbridge IOAPIC found\n", fw_bug);
2995 	}
2996 
2997 	if (!ret)
2998 		pr_err("Disabling interrupt remapping\n");
2999 
3000 	return ret;
3001 }
3002 
free_dma_resources(void)3003 static void __init free_dma_resources(void)
3004 {
3005 	ida_destroy(&pdom_ids);
3006 
3007 	free_unity_maps();
3008 }
3009 
ivinfo_init(void * ivrs)3010 static void __init ivinfo_init(void *ivrs)
3011 {
3012 	amd_iommu_ivinfo = *((u32 *)(ivrs + IOMMU_IVINFO_OFFSET));
3013 }
3014 
3015 /*
3016  * This is the hardware init function for AMD IOMMU in the system.
3017  * This function is called either from amd_iommu_init or from the interrupt
3018  * remapping setup code.
3019  *
3020  * This function basically parses the ACPI table for AMD IOMMU (IVRS)
3021  * four times:
3022  *
3023  *	1 pass) Discover the most comprehensive IVHD type to use.
3024  *
3025  *	2 pass) Find the highest PCI device id the driver has to handle.
3026  *		Upon this information the size of the data structures is
3027  *		determined that needs to be allocated.
3028  *
3029  *	3 pass) Initialize the data structures just allocated with the
3030  *		information in the ACPI table about available AMD IOMMUs
3031  *		in the system. It also maps the PCI devices in the
3032  *		system to specific IOMMUs
3033  *
3034  *	4 pass) After the basic data structures are allocated and
3035  *		initialized we update them with information about memory
3036  *		remapping requirements parsed out of the ACPI table in
3037  *		this last pass.
3038  *
3039  * After everything is set up the IOMMUs are enabled and the necessary
3040  * hotplug and suspend notifiers are registered.
3041  */
early_amd_iommu_init(void)3042 static int __init early_amd_iommu_init(void)
3043 {
3044 	struct acpi_table_header *ivrs_base;
3045 	int ret;
3046 	acpi_status status;
3047 
3048 	if (!amd_iommu_detected)
3049 		return -ENODEV;
3050 
3051 	status = acpi_get_table("IVRS", 0, &ivrs_base);
3052 	if (status == AE_NOT_FOUND)
3053 		return -ENODEV;
3054 	else if (ACPI_FAILURE(status)) {
3055 		const char *err = acpi_format_exception(status);
3056 		pr_err("IVRS table error: %s\n", err);
3057 		return -EINVAL;
3058 	}
3059 
3060 	if (!boot_cpu_has(X86_FEATURE_CX16)) {
3061 		pr_err("Failed to initialize. The CMPXCHG16B feature is required.\n");
3062 		return -EINVAL;
3063 	}
3064 
3065 	/*
3066 	 * Validate checksum here so we don't need to do it when
3067 	 * we actually parse the table
3068 	 */
3069 	ret = check_ivrs_checksum(ivrs_base);
3070 	if (ret)
3071 		goto out;
3072 
3073 	ivinfo_init(ivrs_base);
3074 
3075 	amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base);
3076 	DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type);
3077 
3078 	/*
3079 	 * now the data structures are allocated and basically initialized
3080 	 * start the real acpi table scan
3081 	 */
3082 	ret = init_iommu_all(ivrs_base);
3083 	if (ret)
3084 		goto out;
3085 
3086 	/* 5 level guest page table */
3087 	if (cpu_feature_enabled(X86_FEATURE_LA57) &&
3088 	    FIELD_GET(FEATURE_GATS, amd_iommu_efr) == GUEST_PGTABLE_5_LEVEL)
3089 		amd_iommu_gpt_level = PAGE_MODE_5_LEVEL;
3090 
3091 	if (amd_iommu_pgtable == PD_MODE_V2) {
3092 		if (!amd_iommu_v2_pgtbl_supported()) {
3093 			pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n");
3094 			amd_iommu_pgtable = PD_MODE_V1;
3095 		}
3096 	}
3097 
3098 	/* Disable any previously enabled IOMMUs */
3099 	if (!is_kdump_kernel() || amd_iommu_disabled)
3100 		disable_iommus();
3101 
3102 	if (amd_iommu_irq_remap)
3103 		amd_iommu_irq_remap = check_ioapic_information();
3104 
3105 	if (amd_iommu_irq_remap) {
3106 		struct amd_iommu_pci_seg *pci_seg;
3107 		ret = -ENOMEM;
3108 		for_each_pci_segment(pci_seg) {
3109 			if (alloc_irq_lookup_table(pci_seg))
3110 				goto out;
3111 		}
3112 	}
3113 
3114 	ret = init_memory_definitions(ivrs_base);
3115 	if (ret)
3116 		goto out;
3117 
3118 	/* init the device table */
3119 	init_device_table();
3120 
3121 out:
3122 	/* Don't leak any ACPI memory */
3123 	acpi_put_table(ivrs_base);
3124 
3125 	return ret;
3126 }
3127 
amd_iommu_enable_interrupts(void)3128 static int amd_iommu_enable_interrupts(void)
3129 {
3130 	struct amd_iommu *iommu;
3131 	int ret = 0;
3132 
3133 	for_each_iommu(iommu) {
3134 		ret = iommu_init_irq(iommu);
3135 		if (ret)
3136 			goto out;
3137 	}
3138 
3139 	/*
3140 	 * Interrupt handler is ready to process interrupts. Enable
3141 	 * PPR and GA log interrupt for all IOMMUs.
3142 	 */
3143 	enable_iommus_vapic();
3144 	enable_iommus_ppr();
3145 
3146 out:
3147 	return ret;
3148 }
3149 
detect_ivrs(void)3150 static bool __init detect_ivrs(void)
3151 {
3152 	struct acpi_table_header *ivrs_base;
3153 	acpi_status status;
3154 	int i;
3155 
3156 	status = acpi_get_table("IVRS", 0, &ivrs_base);
3157 	if (status == AE_NOT_FOUND)
3158 		return false;
3159 	else if (ACPI_FAILURE(status)) {
3160 		const char *err = acpi_format_exception(status);
3161 		pr_err("IVRS table error: %s\n", err);
3162 		return false;
3163 	}
3164 
3165 	acpi_put_table(ivrs_base);
3166 
3167 	if (amd_iommu_force_enable)
3168 		goto out;
3169 
3170 	/* Don't use IOMMU if there is Stoney Ridge graphics */
3171 	for (i = 0; i < 32; i++) {
3172 		u32 pci_id;
3173 
3174 		pci_id = read_pci_config(0, i, 0, 0);
3175 		if ((pci_id & 0xffff) == 0x1002 && (pci_id >> 16) == 0x98e4) {
3176 			pr_info("Disable IOMMU on Stoney Ridge\n");
3177 			return false;
3178 		}
3179 	}
3180 
3181 out:
3182 	/* Make sure ACS will be enabled during PCI probe */
3183 	pci_request_acs();
3184 
3185 	return true;
3186 }
3187 
iommu_snp_enable(void)3188 static __init void iommu_snp_enable(void)
3189 {
3190 #ifdef CONFIG_KVM_AMD_SEV
3191 	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
3192 		return;
3193 	/*
3194 	 * The SNP support requires that IOMMU must be enabled, and is
3195 	 * configured with V1 page table (DTE[Mode] = 0 is not supported).
3196 	 */
3197 	if (no_iommu || iommu_default_passthrough()) {
3198 		pr_warn("SNP: IOMMU disabled or configured in passthrough mode, SNP cannot be supported.\n");
3199 		goto disable_snp;
3200 	}
3201 
3202 	if (amd_iommu_pgtable != PD_MODE_V1) {
3203 		pr_warn("SNP: IOMMU is configured with V2 page table mode, SNP cannot be supported.\n");
3204 		goto disable_snp;
3205 	}
3206 
3207 	amd_iommu_snp_en = check_feature(FEATURE_SNP);
3208 	if (!amd_iommu_snp_en) {
3209 		pr_warn("SNP: IOMMU SNP feature not enabled, SNP cannot be supported.\n");
3210 		goto disable_snp;
3211 	}
3212 
3213 	/*
3214 	 * Enable host SNP support once SNP support is checked on IOMMU.
3215 	 */
3216 	if (snp_rmptable_init()) {
3217 		pr_warn("SNP: RMP initialization failed, SNP cannot be supported.\n");
3218 		goto disable_snp;
3219 	}
3220 
3221 	pr_info("IOMMU SNP support enabled.\n");
3222 	return;
3223 
3224 disable_snp:
3225 	cc_platform_clear(CC_ATTR_HOST_SEV_SNP);
3226 #endif
3227 }
3228 
3229 /****************************************************************************
3230  *
3231  * AMD IOMMU Initialization State Machine
3232  *
3233  ****************************************************************************/
3234 
state_next(void)3235 static int __init state_next(void)
3236 {
3237 	int ret = 0;
3238 
3239 	switch (init_state) {
3240 	case IOMMU_START_STATE:
3241 		if (!detect_ivrs()) {
3242 			init_state	= IOMMU_NOT_FOUND;
3243 			ret		= -ENODEV;
3244 		} else {
3245 			init_state	= IOMMU_IVRS_DETECTED;
3246 		}
3247 		break;
3248 	case IOMMU_IVRS_DETECTED:
3249 		if (amd_iommu_disabled) {
3250 			init_state = IOMMU_CMDLINE_DISABLED;
3251 			ret = -EINVAL;
3252 		} else {
3253 			ret = early_amd_iommu_init();
3254 			init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
3255 		}
3256 		break;
3257 	case IOMMU_ACPI_FINISHED:
3258 		early_enable_iommus();
3259 		x86_platform.iommu_shutdown = disable_iommus;
3260 		init_state = IOMMU_ENABLED;
3261 		break;
3262 	case IOMMU_ENABLED:
3263 		register_syscore_ops(&amd_iommu_syscore_ops);
3264 		iommu_snp_enable();
3265 		ret = amd_iommu_init_pci();
3266 		init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT;
3267 		break;
3268 	case IOMMU_PCI_INIT:
3269 		ret = amd_iommu_enable_interrupts();
3270 		init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN;
3271 		break;
3272 	case IOMMU_INTERRUPTS_EN:
3273 		init_state = IOMMU_INITIALIZED;
3274 		break;
3275 	case IOMMU_INITIALIZED:
3276 		/* Nothing to do */
3277 		break;
3278 	case IOMMU_NOT_FOUND:
3279 	case IOMMU_INIT_ERROR:
3280 	case IOMMU_CMDLINE_DISABLED:
3281 		/* Error states => do nothing */
3282 		ret = -EINVAL;
3283 		break;
3284 	default:
3285 		/* Unknown state */
3286 		BUG();
3287 	}
3288 
3289 	if (ret) {
3290 		free_dma_resources();
3291 		if (!irq_remapping_enabled) {
3292 			disable_iommus();
3293 			free_iommu_resources();
3294 		} else {
3295 			struct amd_iommu *iommu;
3296 			struct amd_iommu_pci_seg *pci_seg;
3297 
3298 			for_each_pci_segment(pci_seg)
3299 				uninit_device_table_dma(pci_seg);
3300 
3301 			for_each_iommu(iommu)
3302 				amd_iommu_flush_all_caches(iommu);
3303 		}
3304 	}
3305 	return ret;
3306 }
3307 
iommu_go_to_state(enum iommu_init_state state)3308 static int __init iommu_go_to_state(enum iommu_init_state state)
3309 {
3310 	int ret = -EINVAL;
3311 
3312 	while (init_state != state) {
3313 		if (init_state == IOMMU_NOT_FOUND         ||
3314 		    init_state == IOMMU_INIT_ERROR        ||
3315 		    init_state == IOMMU_CMDLINE_DISABLED)
3316 			break;
3317 		ret = state_next();
3318 	}
3319 
3320 	/*
3321 	 * SNP platform initilazation requires IOMMUs to be fully configured.
3322 	 * If the SNP support on IOMMUs has NOT been checked, simply mark SNP
3323 	 * as unsupported. If the SNP support on IOMMUs has been checked and
3324 	 * host SNP support enabled but RMP enforcement has not been enabled
3325 	 * in IOMMUs, then the system is in a half-baked state, but can limp
3326 	 * along as all memory should be Hypervisor-Owned in the RMP. WARN,
3327 	 * but leave SNP as "supported" to avoid confusing the kernel.
3328 	 */
3329 	if (ret && cc_platform_has(CC_ATTR_HOST_SEV_SNP) &&
3330 	    !WARN_ON_ONCE(amd_iommu_snp_en))
3331 		cc_platform_clear(CC_ATTR_HOST_SEV_SNP);
3332 
3333 	return ret;
3334 }
3335 
3336 #ifdef CONFIG_IRQ_REMAP
amd_iommu_prepare(void)3337 int __init amd_iommu_prepare(void)
3338 {
3339 	int ret;
3340 
3341 	amd_iommu_irq_remap = true;
3342 
3343 	ret = iommu_go_to_state(IOMMU_ACPI_FINISHED);
3344 	if (ret) {
3345 		amd_iommu_irq_remap = false;
3346 		return ret;
3347 	}
3348 
3349 	return amd_iommu_irq_remap ? 0 : -ENODEV;
3350 }
3351 
amd_iommu_enable(void)3352 int __init amd_iommu_enable(void)
3353 {
3354 	int ret;
3355 
3356 	ret = iommu_go_to_state(IOMMU_ENABLED);
3357 	if (ret)
3358 		return ret;
3359 
3360 	irq_remapping_enabled = 1;
3361 	return amd_iommu_xt_mode;
3362 }
3363 
amd_iommu_disable(void)3364 void amd_iommu_disable(void)
3365 {
3366 	amd_iommu_suspend();
3367 }
3368 
amd_iommu_reenable(int mode)3369 int amd_iommu_reenable(int mode)
3370 {
3371 	amd_iommu_resume();
3372 
3373 	return 0;
3374 }
3375 
amd_iommu_enable_faulting(unsigned int cpu)3376 int amd_iommu_enable_faulting(unsigned int cpu)
3377 {
3378 	/* We enable MSI later when PCI is initialized */
3379 	return 0;
3380 }
3381 #endif
3382 
3383 /*
3384  * This is the core init function for AMD IOMMU hardware in the system.
3385  * This function is called from the generic x86 DMA layer initialization
3386  * code.
3387  */
amd_iommu_init(void)3388 static int __init amd_iommu_init(void)
3389 {
3390 	struct amd_iommu *iommu;
3391 	int ret;
3392 
3393 	ret = iommu_go_to_state(IOMMU_INITIALIZED);
3394 #ifdef CONFIG_GART_IOMMU
3395 	if (ret && list_empty(&amd_iommu_list)) {
3396 		/*
3397 		 * We failed to initialize the AMD IOMMU - try fallback
3398 		 * to GART if possible.
3399 		 */
3400 		gart_iommu_init();
3401 	}
3402 #endif
3403 
3404 	for_each_iommu(iommu)
3405 		amd_iommu_debugfs_setup(iommu);
3406 
3407 	return ret;
3408 }
3409 
amd_iommu_sme_check(void)3410 static bool amd_iommu_sme_check(void)
3411 {
3412 	if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) ||
3413 	    (boot_cpu_data.x86 != 0x17))
3414 		return true;
3415 
3416 	/* For Fam17h, a specific level of support is required */
3417 	if (boot_cpu_data.microcode >= 0x08001205)
3418 		return true;
3419 
3420 	if ((boot_cpu_data.microcode >= 0x08001126) &&
3421 	    (boot_cpu_data.microcode <= 0x080011ff))
3422 		return true;
3423 
3424 	pr_notice("IOMMU not currently supported when SME is active\n");
3425 
3426 	return false;
3427 }
3428 
3429 /****************************************************************************
3430  *
3431  * Early detect code. This code runs at IOMMU detection time in the DMA
3432  * layer. It just looks if there is an IVRS ACPI table to detect AMD
3433  * IOMMUs
3434  *
3435  ****************************************************************************/
amd_iommu_detect(void)3436 void __init amd_iommu_detect(void)
3437 {
3438 	int ret;
3439 
3440 	if (no_iommu || (iommu_detected && !gart_iommu_aperture))
3441 		goto disable_snp;
3442 
3443 	if (!amd_iommu_sme_check())
3444 		goto disable_snp;
3445 
3446 	ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
3447 	if (ret)
3448 		goto disable_snp;
3449 
3450 	amd_iommu_detected = true;
3451 	iommu_detected = 1;
3452 	x86_init.iommu.iommu_init = amd_iommu_init;
3453 	return;
3454 
3455 disable_snp:
3456 	if (cc_platform_has(CC_ATTR_HOST_SEV_SNP))
3457 		cc_platform_clear(CC_ATTR_HOST_SEV_SNP);
3458 }
3459 
3460 /****************************************************************************
3461  *
3462  * Parsing functions for the AMD IOMMU specific kernel command line
3463  * options.
3464  *
3465  ****************************************************************************/
3466 
parse_amd_iommu_dump(char * str)3467 static int __init parse_amd_iommu_dump(char *str)
3468 {
3469 	amd_iommu_dump = true;
3470 
3471 	return 1;
3472 }
3473 
parse_amd_iommu_intr(char * str)3474 static int __init parse_amd_iommu_intr(char *str)
3475 {
3476 	for (; *str; ++str) {
3477 		if (strncmp(str, "legacy", 6) == 0) {
3478 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
3479 			break;
3480 		}
3481 		if (strncmp(str, "vapic", 5) == 0) {
3482 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
3483 			break;
3484 		}
3485 	}
3486 	return 1;
3487 }
3488 
parse_amd_iommu_options(char * str)3489 static int __init parse_amd_iommu_options(char *str)
3490 {
3491 	if (!str)
3492 		return -EINVAL;
3493 
3494 	while (*str) {
3495 		if (strncmp(str, "fullflush", 9) == 0) {
3496 			pr_warn("amd_iommu=fullflush deprecated; use iommu.strict=1 instead\n");
3497 			iommu_set_dma_strict();
3498 		} else if (strncmp(str, "force_enable", 12) == 0) {
3499 			amd_iommu_force_enable = true;
3500 		} else if (strncmp(str, "off", 3) == 0) {
3501 			amd_iommu_disabled = true;
3502 		} else if (strncmp(str, "force_isolation", 15) == 0) {
3503 			amd_iommu_force_isolation = true;
3504 		} else if (strncmp(str, "pgtbl_v1", 8) == 0) {
3505 			amd_iommu_pgtable = PD_MODE_V1;
3506 		} else if (strncmp(str, "pgtbl_v2", 8) == 0) {
3507 			amd_iommu_pgtable = PD_MODE_V2;
3508 		} else if (strncmp(str, "irtcachedis", 11) == 0) {
3509 			amd_iommu_irtcachedis = true;
3510 		} else if (strncmp(str, "nohugepages", 11) == 0) {
3511 			pr_info("Restricting V1 page-sizes to 4KiB");
3512 			amd_iommu_pgsize_bitmap = AMD_IOMMU_PGSIZES_4K;
3513 		} else if (strncmp(str, "v2_pgsizes_only", 15) == 0) {
3514 			pr_info("Restricting V1 page-sizes to 4KiB/2MiB/1GiB");
3515 			amd_iommu_pgsize_bitmap = AMD_IOMMU_PGSIZES_V2;
3516 		} else {
3517 			pr_notice("Unknown option - '%s'\n", str);
3518 		}
3519 
3520 		str += strcspn(str, ",");
3521 		while (*str == ',')
3522 			str++;
3523 	}
3524 
3525 	return 1;
3526 }
3527 
parse_ivrs_ioapic(char * str)3528 static int __init parse_ivrs_ioapic(char *str)
3529 {
3530 	u32 seg = 0, bus, dev, fn;
3531 	int id, i;
3532 	u32 devid;
3533 
3534 	if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3535 	    sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
3536 		goto found;
3537 
3538 	if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3539 	    sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
3540 		pr_warn("ivrs_ioapic%s option format deprecated; use ivrs_ioapic=%d@%04x:%02x:%02x.%d instead\n",
3541 			str, id, seg, bus, dev, fn);
3542 		goto found;
3543 	}
3544 
3545 	pr_err("Invalid command line: ivrs_ioapic%s\n", str);
3546 	return 1;
3547 
3548 found:
3549 	if (early_ioapic_map_size == EARLY_MAP_SIZE) {
3550 		pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n",
3551 			str);
3552 		return 1;
3553 	}
3554 
3555 	devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3556 
3557 	cmdline_maps			= true;
3558 	i				= early_ioapic_map_size++;
3559 	early_ioapic_map[i].id		= id;
3560 	early_ioapic_map[i].devid	= devid;
3561 	early_ioapic_map[i].cmd_line	= true;
3562 
3563 	return 1;
3564 }
3565 
parse_ivrs_hpet(char * str)3566 static int __init parse_ivrs_hpet(char *str)
3567 {
3568 	u32 seg = 0, bus, dev, fn;
3569 	int id, i;
3570 	u32 devid;
3571 
3572 	if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3573 	    sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
3574 		goto found;
3575 
3576 	if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3577 	    sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
3578 		pr_warn("ivrs_hpet%s option format deprecated; use ivrs_hpet=%d@%04x:%02x:%02x.%d instead\n",
3579 			str, id, seg, bus, dev, fn);
3580 		goto found;
3581 	}
3582 
3583 	pr_err("Invalid command line: ivrs_hpet%s\n", str);
3584 	return 1;
3585 
3586 found:
3587 	if (early_hpet_map_size == EARLY_MAP_SIZE) {
3588 		pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n",
3589 			str);
3590 		return 1;
3591 	}
3592 
3593 	devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3594 
3595 	cmdline_maps			= true;
3596 	i				= early_hpet_map_size++;
3597 	early_hpet_map[i].id		= id;
3598 	early_hpet_map[i].devid		= devid;
3599 	early_hpet_map[i].cmd_line	= true;
3600 
3601 	return 1;
3602 }
3603 
3604 #define ACPIID_LEN (ACPIHID_UID_LEN + ACPIHID_HID_LEN)
3605 
parse_ivrs_acpihid(char * str)3606 static int __init parse_ivrs_acpihid(char *str)
3607 {
3608 	u32 seg = 0, bus, dev, fn;
3609 	char *hid, *uid, *p, *addr;
3610 	char acpiid[ACPIID_LEN] = {0};
3611 	int i;
3612 
3613 	addr = strchr(str, '@');
3614 	if (!addr) {
3615 		addr = strchr(str, '=');
3616 		if (!addr)
3617 			goto not_found;
3618 
3619 		++addr;
3620 
3621 		if (strlen(addr) > ACPIID_LEN)
3622 			goto not_found;
3623 
3624 		if (sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid) == 4 ||
3625 		    sscanf(str, "[%x:%x:%x.%x]=%s", &seg, &bus, &dev, &fn, acpiid) == 5) {
3626 			pr_warn("ivrs_acpihid%s option format deprecated; use ivrs_acpihid=%s@%04x:%02x:%02x.%d instead\n",
3627 				str, acpiid, seg, bus, dev, fn);
3628 			goto found;
3629 		}
3630 		goto not_found;
3631 	}
3632 
3633 	/* We have the '@', make it the terminator to get just the acpiid */
3634 	*addr++ = 0;
3635 
3636 	if (strlen(str) > ACPIID_LEN + 1)
3637 		goto not_found;
3638 
3639 	if (sscanf(str, "=%s", acpiid) != 1)
3640 		goto not_found;
3641 
3642 	if (sscanf(addr, "%x:%x.%x", &bus, &dev, &fn) == 3 ||
3643 	    sscanf(addr, "%x:%x:%x.%x", &seg, &bus, &dev, &fn) == 4)
3644 		goto found;
3645 
3646 not_found:
3647 	pr_err("Invalid command line: ivrs_acpihid%s\n", str);
3648 	return 1;
3649 
3650 found:
3651 	p = acpiid;
3652 	hid = strsep(&p, ":");
3653 	uid = p;
3654 
3655 	if (!hid || !(*hid) || !uid) {
3656 		pr_err("Invalid command line: hid or uid\n");
3657 		return 1;
3658 	}
3659 
3660 	/*
3661 	 * Ignore leading zeroes after ':', so e.g., AMDI0095:00
3662 	 * will match AMDI0095:0 in the second strcmp in acpi_dev_hid_uid_match
3663 	 */
3664 	while (*uid == '0' && *(uid + 1))
3665 		uid++;
3666 
3667 	i = early_acpihid_map_size++;
3668 	memcpy(early_acpihid_map[i].hid, hid, strlen(hid));
3669 	memcpy(early_acpihid_map[i].uid, uid, strlen(uid));
3670 	early_acpihid_map[i].devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3671 	early_acpihid_map[i].cmd_line	= true;
3672 
3673 	return 1;
3674 }
3675 
3676 __setup("amd_iommu_dump",	parse_amd_iommu_dump);
3677 __setup("amd_iommu=",		parse_amd_iommu_options);
3678 __setup("amd_iommu_intr=",	parse_amd_iommu_intr);
3679 __setup("ivrs_ioapic",		parse_ivrs_ioapic);
3680 __setup("ivrs_hpet",		parse_ivrs_hpet);
3681 __setup("ivrs_acpihid",		parse_ivrs_acpihid);
3682 
amd_iommu_pasid_supported(void)3683 bool amd_iommu_pasid_supported(void)
3684 {
3685 	/* CPU page table size should match IOMMU guest page table size */
3686 	if (cpu_feature_enabled(X86_FEATURE_LA57) &&
3687 	    amd_iommu_gpt_level != PAGE_MODE_5_LEVEL)
3688 		return false;
3689 
3690 	/*
3691 	 * Since DTE[Mode]=0 is prohibited on SNP-enabled system
3692 	 * (i.e. EFR[SNPSup]=1), IOMMUv2 page table cannot be used without
3693 	 * setting up IOMMUv1 page table.
3694 	 */
3695 	return amd_iommu_gt_ppr_supported() && !amd_iommu_snp_en;
3696 }
3697 
get_amd_iommu(unsigned int idx)3698 struct amd_iommu *get_amd_iommu(unsigned int idx)
3699 {
3700 	unsigned int i = 0;
3701 	struct amd_iommu *iommu;
3702 
3703 	for_each_iommu(iommu)
3704 		if (i++ == idx)
3705 			return iommu;
3706 	return NULL;
3707 }
3708 
3709 /****************************************************************************
3710  *
3711  * IOMMU EFR Performance Counter support functionality. This code allows
3712  * access to the IOMMU PC functionality.
3713  *
3714  ****************************************************************************/
3715 
amd_iommu_pc_get_max_banks(unsigned int idx)3716 u8 amd_iommu_pc_get_max_banks(unsigned int idx)
3717 {
3718 	struct amd_iommu *iommu = get_amd_iommu(idx);
3719 
3720 	if (iommu)
3721 		return iommu->max_banks;
3722 
3723 	return 0;
3724 }
3725 
amd_iommu_pc_supported(void)3726 bool amd_iommu_pc_supported(void)
3727 {
3728 	return amd_iommu_pc_present;
3729 }
3730 
amd_iommu_pc_get_max_counters(unsigned int idx)3731 u8 amd_iommu_pc_get_max_counters(unsigned int idx)
3732 {
3733 	struct amd_iommu *iommu = get_amd_iommu(idx);
3734 
3735 	if (iommu)
3736 		return iommu->max_counters;
3737 
3738 	return 0;
3739 }
3740 
iommu_pc_get_set_reg(struct amd_iommu * iommu,u8 bank,u8 cntr,u8 fxn,u64 * value,bool is_write)3741 static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr,
3742 				u8 fxn, u64 *value, bool is_write)
3743 {
3744 	u32 offset;
3745 	u32 max_offset_lim;
3746 
3747 	/* Make sure the IOMMU PC resource is available */
3748 	if (!amd_iommu_pc_present)
3749 		return -ENODEV;
3750 
3751 	/* Check for valid iommu and pc register indexing */
3752 	if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7)))
3753 		return -ENODEV;
3754 
3755 	offset = (u32)(((0x40 | bank) << 12) | (cntr << 8) | fxn);
3756 
3757 	/* Limit the offset to the hw defined mmio region aperture */
3758 	max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) |
3759 				(iommu->max_counters << 8) | 0x28);
3760 	if ((offset < MMIO_CNTR_REG_OFFSET) ||
3761 	    (offset > max_offset_lim))
3762 		return -EINVAL;
3763 
3764 	if (is_write) {
3765 		u64 val = *value & GENMASK_ULL(47, 0);
3766 
3767 		writel((u32)val, iommu->mmio_base + offset);
3768 		writel((val >> 32), iommu->mmio_base + offset + 4);
3769 	} else {
3770 		*value = readl(iommu->mmio_base + offset + 4);
3771 		*value <<= 32;
3772 		*value |= readl(iommu->mmio_base + offset);
3773 		*value &= GENMASK_ULL(47, 0);
3774 	}
3775 
3776 	return 0;
3777 }
3778 
amd_iommu_pc_get_reg(struct amd_iommu * iommu,u8 bank,u8 cntr,u8 fxn,u64 * value)3779 int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)
3780 {
3781 	if (!iommu)
3782 		return -EINVAL;
3783 
3784 	return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false);
3785 }
3786 
amd_iommu_pc_set_reg(struct amd_iommu * iommu,u8 bank,u8 cntr,u8 fxn,u64 * value)3787 int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)
3788 {
3789 	if (!iommu)
3790 		return -EINVAL;
3791 
3792 	return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true);
3793 }
3794 
3795 #ifdef CONFIG_KVM_AMD_SEV
iommu_page_make_shared(void * page)3796 static int iommu_page_make_shared(void *page)
3797 {
3798 	unsigned long paddr, pfn;
3799 
3800 	paddr = iommu_virt_to_phys(page);
3801 	/* Cbit maybe set in the paddr */
3802 	pfn = __sme_clr(paddr) >> PAGE_SHIFT;
3803 
3804 	if (!(pfn % PTRS_PER_PMD)) {
3805 		int ret, level;
3806 		bool assigned;
3807 
3808 		ret = snp_lookup_rmpentry(pfn, &assigned, &level);
3809 		if (ret) {
3810 			pr_warn("IOMMU PFN %lx RMP lookup failed, ret %d\n", pfn, ret);
3811 			return ret;
3812 		}
3813 
3814 		if (!assigned) {
3815 			pr_warn("IOMMU PFN %lx not assigned in RMP table\n", pfn);
3816 			return -EINVAL;
3817 		}
3818 
3819 		if (level > PG_LEVEL_4K) {
3820 			ret = psmash(pfn);
3821 			if (!ret)
3822 				goto done;
3823 
3824 			pr_warn("PSMASH failed for IOMMU PFN %lx huge RMP entry, ret: %d, level: %d\n",
3825 				pfn, ret, level);
3826 			return ret;
3827 		}
3828 	}
3829 
3830 done:
3831 	return rmp_make_shared(pfn, PG_LEVEL_4K);
3832 }
3833 
iommu_make_shared(void * va,size_t size)3834 static int iommu_make_shared(void *va, size_t size)
3835 {
3836 	void *page;
3837 	int ret;
3838 
3839 	if (!va)
3840 		return 0;
3841 
3842 	for (page = va; page < (va + size); page += PAGE_SIZE) {
3843 		ret = iommu_page_make_shared(page);
3844 		if (ret)
3845 			return ret;
3846 	}
3847 
3848 	return 0;
3849 }
3850 
amd_iommu_snp_disable(void)3851 int amd_iommu_snp_disable(void)
3852 {
3853 	struct amd_iommu *iommu;
3854 	int ret;
3855 
3856 	if (!amd_iommu_snp_en)
3857 		return 0;
3858 
3859 	for_each_iommu(iommu) {
3860 		ret = iommu_make_shared(iommu->evt_buf, EVT_BUFFER_SIZE);
3861 		if (ret)
3862 			return ret;
3863 
3864 		ret = iommu_make_shared(iommu->ppr_log, PPR_LOG_SIZE);
3865 		if (ret)
3866 			return ret;
3867 
3868 		ret = iommu_make_shared((void *)iommu->cmd_sem, PAGE_SIZE);
3869 		if (ret)
3870 			return ret;
3871 	}
3872 
3873 	return 0;
3874 }
3875 EXPORT_SYMBOL_GPL(amd_iommu_snp_disable);
3876 #endif
3877