xref: /linux/drivers/iommu/amd/init.c (revision 5b84c729a82a726b3a523d0cc45cd465808562af)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4  * Author: Joerg Roedel <jroedel@suse.de>
5  *         Leo Duran <leo.duran@amd.com>
6  */
7 
8 #define pr_fmt(fmt)     "AMD-Vi: " fmt
9 #define dev_fmt(fmt)    pr_fmt(fmt)
10 
11 #include <linux/pci.h>
12 #include <linux/acpi.h>
13 #include <linux/list.h>
14 #include <linux/bitmap.h>
15 #include <linux/syscore_ops.h>
16 #include <linux/interrupt.h>
17 #include <linux/msi.h>
18 #include <linux/irq.h>
19 #include <linux/amd-iommu.h>
20 #include <linux/export.h>
21 #include <linux/kmemleak.h>
22 #include <linux/cc_platform.h>
23 #include <linux/iopoll.h>
24 #include <asm/pci-direct.h>
25 #include <asm/iommu.h>
26 #include <asm/apic.h>
27 #include <asm/gart.h>
28 #include <asm/x86_init.h>
29 #include <asm/io_apic.h>
30 #include <asm/irq_remapping.h>
31 #include <asm/set_memory.h>
32 #include <asm/sev.h>
33 
34 #include <linux/crash_dump.h>
35 
36 #include "amd_iommu.h"
37 #include "../irq_remapping.h"
38 #include "../iommu-pages.h"
39 
40 /*
41  * definitions for the ACPI scanning code
42  */
43 #define IVRS_HEADER_LENGTH 48
44 
45 #define ACPI_IVHD_TYPE_MAX_SUPPORTED	0x40
46 #define ACPI_IVMD_TYPE_ALL              0x20
47 #define ACPI_IVMD_TYPE                  0x21
48 #define ACPI_IVMD_TYPE_RANGE            0x22
49 
50 #define IVHD_DEV_ALL                    0x01
51 #define IVHD_DEV_SELECT                 0x02
52 #define IVHD_DEV_SELECT_RANGE_START     0x03
53 #define IVHD_DEV_RANGE_END              0x04
54 #define IVHD_DEV_ALIAS                  0x42
55 #define IVHD_DEV_ALIAS_RANGE            0x43
56 #define IVHD_DEV_EXT_SELECT             0x46
57 #define IVHD_DEV_EXT_SELECT_RANGE       0x47
58 #define IVHD_DEV_SPECIAL		0x48
59 #define IVHD_DEV_ACPI_HID		0xf0
60 
61 #define UID_NOT_PRESENT                 0
62 #define UID_IS_INTEGER                  1
63 #define UID_IS_CHARACTER                2
64 
65 #define IVHD_SPECIAL_IOAPIC		1
66 #define IVHD_SPECIAL_HPET		2
67 
68 #define IVHD_FLAG_HT_TUN_EN_MASK        0x01
69 #define IVHD_FLAG_PASSPW_EN_MASK        0x02
70 #define IVHD_FLAG_RESPASSPW_EN_MASK     0x04
71 #define IVHD_FLAG_ISOC_EN_MASK          0x08
72 
73 #define IVMD_FLAG_EXCL_RANGE            0x08
74 #define IVMD_FLAG_IW                    0x04
75 #define IVMD_FLAG_IR                    0x02
76 #define IVMD_FLAG_UNITY_MAP             0x01
77 
78 #define ACPI_DEVFLAG_INITPASS           0x01
79 #define ACPI_DEVFLAG_EXTINT             0x02
80 #define ACPI_DEVFLAG_NMI                0x04
81 #define ACPI_DEVFLAG_SYSMGT1            0x10
82 #define ACPI_DEVFLAG_SYSMGT2            0x20
83 #define ACPI_DEVFLAG_LINT0              0x40
84 #define ACPI_DEVFLAG_LINT1              0x80
85 #define ACPI_DEVFLAG_ATSDIS             0x10000000
86 
87 #define IVRS_GET_SBDF_ID(seg, bus, dev, fn)	(((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \
88 						 | ((dev & 0x1f) << 3) | (fn & 0x7))
89 
90 /*
91  * ACPI table definitions
92  *
93  * These data structures are laid over the table to parse the important values
94  * out of it.
95  */
96 
97 /*
98  * structure describing one IOMMU in the ACPI table. Typically followed by one
99  * or more ivhd_entrys.
100  */
101 struct ivhd_header {
102 	u8 type;
103 	u8 flags;
104 	u16 length;
105 	u16 devid;
106 	u16 cap_ptr;
107 	u64 mmio_phys;
108 	u16 pci_seg;
109 	u16 info;
110 	u32 efr_attr;
111 
112 	/* Following only valid on IVHD type 11h and 40h */
113 	u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */
114 	u64 efr_reg2;
115 } __attribute__((packed));
116 
117 /*
118  * A device entry describing which devices a specific IOMMU translates and
119  * which requestor ids they use.
120  */
121 struct ivhd_entry {
122 	u8 type;
123 	u16 devid;
124 	u8 flags;
125 	struct_group(ext_hid,
126 		u32 ext;
127 		u32 hidh;
128 	);
129 	u64 cid;
130 	u8 uidf;
131 	u8 uidl;
132 	u8 uid;
133 } __attribute__((packed));
134 
135 int amd_iommu_evtlog_size = EVTLOG_SIZE_DEF;
136 int amd_iommu_pprlog_size = PPRLOG_SIZE_DEF;
137 
138 /*
139  * An AMD IOMMU memory definition structure. It defines things like exclusion
140  * ranges for devices and regions that should be unity mapped.
141  */
142 struct ivmd_header {
143 	u8 type;
144 	u8 flags;
145 	u16 length;
146 	u16 devid;
147 	u16 aux;
148 	u16 pci_seg;
149 	u8  resv[6];
150 	u64 range_start;
151 	u64 range_length;
152 } __attribute__((packed));
153 
154 bool amd_iommu_dump;
155 bool amd_iommu_irq_remap __read_mostly;
156 
157 enum protection_domain_mode amd_iommu_pgtable = PD_MODE_V1;
158 /* Virtual address size */
159 u8 amd_iommu_hpt_vasize;
160 /* Guest page table level */
161 int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL;
162 
163 int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
164 static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
165 
166 static bool amd_iommu_disabled __initdata;
167 static bool amd_iommu_force_enable __initdata;
168 static bool amd_iommu_irtcachedis;
169 static int amd_iommu_target_ivhd_type;
170 
171 /* Global EFR and EFR2 registers */
172 u64 amd_iommu_efr;
173 u64 amd_iommu_efr2;
174 
175 /* Host (v1) page table is not supported*/
176 bool amd_iommu_hatdis;
177 
178 /* SNP is enabled on the system? */
179 bool amd_iommu_snp_en;
180 EXPORT_SYMBOL(amd_iommu_snp_en);
181 
182 /* SNP page mode 0 support */
183 bool amd_iommu_snp_mode0_sup;
184 
185 LIST_HEAD(amd_iommu_pci_seg_list);	/* list of all PCI segments */
186 LIST_HEAD(amd_iommu_list);		/* list of all AMD IOMMUs in the system */
187 LIST_HEAD(amd_ivhd_dev_flags_list);	/* list of all IVHD device entry settings */
188 
189 /* Number of IOMMUs present in the system */
190 static int amd_iommus_present;
191 
192 /* IOMMUs have a non-present cache? */
193 bool amd_iommu_np_cache __read_mostly;
194 bool amd_iommu_iotlb_sup __read_mostly = true;
195 
196 static bool amd_iommu_pc_present __read_mostly;
197 bool amdr_ivrs_remap_support __read_mostly;
198 
199 bool amd_iommu_force_isolation __read_mostly;
200 
201 unsigned long amd_iommu_pgsize_bitmap __ro_after_init = AMD_IOMMU_PGSIZES;
202 
203 enum iommu_init_state {
204 	IOMMU_START_STATE,
205 	IOMMU_IVRS_DETECTED,
206 	IOMMU_ACPI_FINISHED,
207 	IOMMU_ENABLED,
208 	IOMMU_PCI_INIT,
209 	IOMMU_INTERRUPTS_EN,
210 	IOMMU_INITIALIZED,
211 	IOMMU_NOT_FOUND,
212 	IOMMU_INIT_ERROR,
213 	IOMMU_CMDLINE_DISABLED,
214 };
215 
216 /* Early ioapic and hpet maps from kernel command line */
217 #define EARLY_MAP_SIZE		4
218 static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE];
219 static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE];
220 static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE];
221 
222 static int __initdata early_ioapic_map_size;
223 static int __initdata early_hpet_map_size;
224 static int __initdata early_acpihid_map_size;
225 
226 static bool __initdata cmdline_maps;
227 
228 static enum iommu_init_state init_state = IOMMU_START_STATE;
229 
230 static int amd_iommu_enable_interrupts(void);
231 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg);
232 
233 static bool amd_iommu_pre_enabled = true;
234 
235 static u32 amd_iommu_ivinfo __initdata;
236 
237 bool translation_pre_enabled(struct amd_iommu *iommu)
238 {
239 	return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED);
240 }
241 
242 static void clear_translation_pre_enabled(struct amd_iommu *iommu)
243 {
244 	iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
245 }
246 
247 static void init_translation_status(struct amd_iommu *iommu)
248 {
249 	u64 ctrl;
250 
251 	ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
252 	if (ctrl & (1<<CONTROL_IOMMU_EN))
253 		iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
254 }
255 
256 int amd_iommu_get_num_iommus(void)
257 {
258 	return amd_iommus_present;
259 }
260 
261 bool amd_iommu_ht_range_ignore(void)
262 {
263 	return check_feature2(FEATURE_HT_RANGE_IGNORE);
264 }
265 
266 /*
267  * Iterate through all the IOMMUs to get common EFR
268  * masks among all IOMMUs and warn if found inconsistency.
269  */
270 static __init void get_global_efr(void)
271 {
272 	struct amd_iommu *iommu;
273 
274 	for_each_iommu(iommu) {
275 		u64 tmp = iommu->features;
276 		u64 tmp2 = iommu->features2;
277 
278 		if (list_is_first(&iommu->list, &amd_iommu_list)) {
279 			amd_iommu_efr = tmp;
280 			amd_iommu_efr2 = tmp2;
281 			continue;
282 		}
283 
284 		if (amd_iommu_efr == tmp &&
285 		    amd_iommu_efr2 == tmp2)
286 			continue;
287 
288 		pr_err(FW_BUG
289 		       "Found inconsistent EFR/EFR2 %#llx,%#llx (global %#llx,%#llx) on iommu%d (%04x:%02x:%02x.%01x).\n",
290 		       tmp, tmp2, amd_iommu_efr, amd_iommu_efr2,
291 		       iommu->index, iommu->pci_seg->id,
292 		       PCI_BUS_NUM(iommu->devid), PCI_SLOT(iommu->devid),
293 		       PCI_FUNC(iommu->devid));
294 
295 		amd_iommu_efr &= tmp;
296 		amd_iommu_efr2 &= tmp2;
297 	}
298 
299 	pr_info("Using global IVHD EFR:%#llx, EFR2:%#llx\n", amd_iommu_efr, amd_iommu_efr2);
300 }
301 
302 /*
303  * For IVHD type 0x11/0x40, EFR is also available via IVHD.
304  * Default to IVHD EFR since it is available sooner
305  * (i.e. before PCI init).
306  */
307 static void __init early_iommu_features_init(struct amd_iommu *iommu,
308 					     struct ivhd_header *h)
309 {
310 	if (amd_iommu_ivinfo & IOMMU_IVINFO_EFRSUP) {
311 		iommu->features = h->efr_reg;
312 		iommu->features2 = h->efr_reg2;
313 	}
314 	if (amd_iommu_ivinfo & IOMMU_IVINFO_DMA_REMAP)
315 		amdr_ivrs_remap_support = true;
316 }
317 
318 /* Access to l1 and l2 indexed register spaces */
319 
320 static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)
321 {
322 	u32 val;
323 
324 	pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
325 	pci_read_config_dword(iommu->dev, 0xfc, &val);
326 	return val;
327 }
328 
329 static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)
330 {
331 	pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));
332 	pci_write_config_dword(iommu->dev, 0xfc, val);
333 	pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
334 }
335 
336 static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)
337 {
338 	u32 val;
339 
340 	pci_write_config_dword(iommu->dev, 0xf0, address);
341 	pci_read_config_dword(iommu->dev, 0xf4, &val);
342 	return val;
343 }
344 
345 static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
346 {
347 	pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));
348 	pci_write_config_dword(iommu->dev, 0xf4, val);
349 }
350 
351 /****************************************************************************
352  *
353  * AMD IOMMU MMIO register space handling functions
354  *
355  * These functions are used to program the IOMMU device registers in
356  * MMIO space required for that driver.
357  *
358  ****************************************************************************/
359 
360 static void iommu_set_cwwb_range(struct amd_iommu *iommu)
361 {
362 	u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem);
363 	u64 entry = start & PM_ADDR_MASK;
364 
365 	if (!check_feature(FEATURE_SNP))
366 		return;
367 
368 	/* Note:
369 	 * Re-purpose Exclusion base/limit registers for Completion wait
370 	 * write-back base/limit.
371 	 */
372 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
373 		    &entry, sizeof(entry));
374 
375 	/* Note:
376 	 * Default to 4 Kbytes, which can be specified by setting base
377 	 * address equal to the limit address.
378 	 */
379 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
380 		    &entry, sizeof(entry));
381 }
382 
383 /* Programs the physical address of the device table into the IOMMU hardware */
384 static void iommu_set_device_table(struct amd_iommu *iommu)
385 {
386 	u64 entry;
387 	u32 dev_table_size = iommu->pci_seg->dev_table_size;
388 	void *dev_table = (void *)get_dev_table(iommu);
389 
390 	BUG_ON(iommu->mmio_base == NULL);
391 
392 	/*
393 	 * A kdump kernel that reuses the previous kernel's device table must
394 	 * leave the base register alone. It already points at that table, and
395 	 * with SNP enabled the register is locked. If reuse failed, program
396 	 * the freshly allocated table like a normal boot.
397 	 */
398 	if (iommu->pci_seg->old_dev_tbl_cpy)
399 		return;
400 
401 	entry = iommu_virt_to_phys(dev_table);
402 	entry |= (dev_table_size >> 12) - 1;
403 	memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
404 			&entry, sizeof(entry));
405 }
406 
407 static void iommu_feature_set(struct amd_iommu *iommu, u64 val, u64 mask, u8 shift)
408 {
409 	u64 ctrl;
410 
411 	ctrl = readq(iommu->mmio_base +  MMIO_CONTROL_OFFSET);
412 	mask <<= shift;
413 	ctrl &= ~mask;
414 	ctrl |= (val << shift) & mask;
415 	writeq(ctrl, iommu->mmio_base +  MMIO_CONTROL_OFFSET);
416 }
417 
418 /* Generic functions to enable/disable certain features of the IOMMU. */
419 void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
420 {
421 	iommu_feature_set(iommu, 1ULL, 1ULL, bit);
422 }
423 
424 static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
425 {
426 	iommu_feature_set(iommu, 0ULL, 1ULL, bit);
427 }
428 
429 /* Function to enable the hardware */
430 static void iommu_enable(struct amd_iommu *iommu)
431 {
432 	iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
433 }
434 
435 /*
436  * Program or clear the per-IOMMU PerfOpt enable bit.
437  * Gate on this IOMMU's own EFR, not the global AND-mask.
438  * Caller serializes access using iommu->lock or during init/suspend/resume.
439  */
440 int amd_iommu_perfopt_set(struct amd_iommu *iommu, bool enable)
441 {
442 	u32 old, val, readback;
443 
444 	if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT))
445 		return enable ? -ENODEV : 0;
446 
447 	old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
448 	if (old == U32_MAX)
449 		goto fail;
450 
451 	val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN;
452 	if (val != old)
453 		writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
454 	readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
455 	if (readback == U32_MAX ||
456 	    (readback & PERF_OPT_EN) != (val & PERF_OPT_EN))
457 		goto fail;
458 
459 	iommu->perfopt_enabled = enable;
460 	return 0;
461 fail:
462 	pr_err("IOMMU%d: failed to set PerfOpt to %d\n", iommu->index, enable);
463 	return -EIO;
464 }
465 
466 static void iommu_disable(struct amd_iommu *iommu)
467 {
468 	bool perfopt_enabled = iommu->perfopt_enabled;
469 
470 	if (!iommu->mmio_base)
471 		return;
472 
473 	/* Disable command buffer */
474 	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
475 
476 	/* Disable event logging and event interrupts */
477 	iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);
478 	iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
479 
480 	/* Disable IOMMU GA_LOG */
481 	iommu_feature_disable(iommu, CONTROL_GALOG_EN);
482 	iommu_feature_disable(iommu, CONTROL_GAINT_EN);
483 
484 	/* Disable IOMMU PPR logging */
485 	iommu_feature_disable(iommu, CONTROL_PPRLOG_EN);
486 	iommu_feature_disable(iommu, CONTROL_PPRINT_EN);
487 
488 	/* Disable IOMMU hardware itself */
489 	iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
490 
491 	/* Clear IRTE cache disabling bit */
492 	iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);
493 
494 	/* Clear hardware while preserving the activation state for resume. */
495 	amd_iommu_perfopt_set(iommu, false);
496 	iommu->perfopt_enabled = perfopt_enabled;
497 }
498 
499 /*
500  * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in
501  * the system has one.
502  */
503 static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end)
504 {
505 	if (!request_mem_region(address, end, "amd_iommu")) {
506 		pr_err("Can not reserve memory region %llx-%llx for mmio\n",
507 			address, end);
508 		pr_err("This is a BIOS bug. Please contact your hardware vendor\n");
509 		return NULL;
510 	}
511 
512 	return (u8 __iomem *)ioremap(address, end);
513 }
514 
515 static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)
516 {
517 	if (iommu->mmio_base)
518 		iounmap(iommu->mmio_base);
519 	release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end);
520 }
521 
522 static inline u32 get_ivhd_header_size(struct ivhd_header *h)
523 {
524 	u32 size = 0;
525 
526 	switch (h->type) {
527 	case 0x10:
528 		size = 24;
529 		break;
530 	case 0x11:
531 	case 0x40:
532 		size = 40;
533 		break;
534 	}
535 	return size;
536 }
537 
538 /****************************************************************************
539  *
540  * The functions below belong to the first pass of AMD IOMMU ACPI table
541  * parsing. In this pass we try to find out the highest device id this
542  * code has to handle. Upon this information the size of the shared data
543  * structures is determined later.
544  *
545  ****************************************************************************/
546 
547 /*
548  * This function calculates the length of a given IVHD entry
549  */
550 static inline int ivhd_entry_length(u8 *ivhd)
551 {
552 	u32 type = ((struct ivhd_entry *)ivhd)->type;
553 
554 	if (type < 0x80) {
555 		return 0x04 << (*ivhd >> 6);
556 	} else if (type == IVHD_DEV_ACPI_HID) {
557 		/* For ACPI_HID, offset 21 is uid len */
558 		return *((u8 *)ivhd + 21) + 22;
559 	}
560 	return 0;
561 }
562 
563 /*
564  * After reading the highest device id from the IOMMU PCI capability header
565  * this function looks if there is a higher device id defined in the ACPI table
566  */
567 static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
568 {
569 	u8 *p = (void *)h, *end = (void *)h;
570 	struct ivhd_entry *dev;
571 	int last_devid = -EINVAL;
572 
573 	u32 ivhd_size = get_ivhd_header_size(h);
574 
575 	if (!ivhd_size) {
576 		pr_err("Unsupported IVHD type %#x\n", h->type);
577 		return -EINVAL;
578 	}
579 
580 	p += ivhd_size;
581 	end += h->length;
582 
583 	while (p < end) {
584 		dev = (struct ivhd_entry *)p;
585 		switch (dev->type) {
586 		case IVHD_DEV_ALL:
587 			/* Use maximum BDF value for DEV_ALL */
588 			return 0xffff;
589 		case IVHD_DEV_SELECT:
590 		case IVHD_DEV_RANGE_END:
591 		case IVHD_DEV_ALIAS:
592 		case IVHD_DEV_EXT_SELECT:
593 			/* all the above subfield types refer to device ids */
594 			if (dev->devid > last_devid)
595 				last_devid = dev->devid;
596 			break;
597 		default:
598 			break;
599 		}
600 		p += ivhd_entry_length(p);
601 	}
602 
603 	WARN_ON(p != end);
604 
605 	return last_devid;
606 }
607 
608 static int __init check_ivrs_checksum(struct acpi_table_header *table)
609 {
610 	int i;
611 	u8 checksum = 0, *p = (u8 *)table;
612 
613 	for (i = 0; i < table->length; ++i)
614 		checksum += p[i];
615 	if (checksum != 0) {
616 		/* ACPI table corrupt */
617 		pr_err(FW_BUG "IVRS invalid checksum\n");
618 		return -ENODEV;
619 	}
620 
621 	return 0;
622 }
623 
624 /*
625  * Iterate over all IVHD entries in the ACPI table and find the highest device
626  * id which we need to handle. This is the first of three functions which parse
627  * the ACPI table. So we check the checksum here.
628  */
629 static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_seg)
630 {
631 	u8 *p = (u8 *)table, *end = (u8 *)table;
632 	struct ivhd_header *h;
633 	int last_devid, last_bdf = 0;
634 
635 	p += IVRS_HEADER_LENGTH;
636 
637 	end += table->length;
638 	while (p < end) {
639 		h = (struct ivhd_header *)p;
640 		if (h->pci_seg == pci_seg &&
641 		    h->type == amd_iommu_target_ivhd_type) {
642 			last_devid = find_last_devid_from_ivhd(h);
643 
644 			if (last_devid < 0)
645 				return -EINVAL;
646 			if (last_devid > last_bdf)
647 				last_bdf = last_devid;
648 		}
649 		p += h->length;
650 	}
651 	WARN_ON(p != end);
652 
653 	return last_bdf;
654 }
655 
656 /****************************************************************************
657  *
658  * The following functions belong to the code path which parses the ACPI table
659  * the second time. In this ACPI parsing iteration we allocate IOMMU specific
660  * data structures, initialize the per PCI segment device/alias/rlookup table
661  * and also basically initialize the hardware.
662  *
663  ****************************************************************************/
664 
665 /* Allocate per PCI segment device table */
666 static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg)
667 {
668 	gfp_t gfp = GFP_KERNEL;
669 
670 	/*
671 	 * Keep normal kernel device tables below 4G because a later kdump
672 	 * kernel only trusts and reuses old tables below that limit. A kdump
673 	 * kernel's new table is either discarded when reuse succeeds or
674 	 * programmed after the IOMMU is disabled, so the table does not
675 	 * need DMA32 memory (which the crashkernel may not have).
676 	 */
677 	if (!is_kdump_kernel())
678 		gfp |= GFP_DMA32;
679 
680 	pci_seg->dev_table = iommu_alloc_pages_sz(gfp, pci_seg->dev_table_size);
681 	if (!pci_seg->dev_table)
682 		return -ENOMEM;
683 
684 	return 0;
685 }
686 
687 static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg)
688 {
689 	if (is_kdump_kernel())
690 		memunmap((void *)pci_seg->dev_table);
691 	else
692 		iommu_free_pages(pci_seg->dev_table);
693 	pci_seg->dev_table = NULL;
694 }
695 
696 /* Allocate per PCI segment IOMMU rlookup table. */
697 static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg)
698 {
699 	pci_seg->rlookup_table = kvzalloc_objs(*pci_seg->rlookup_table,
700 					       pci_seg->last_bdf + 1);
701 	if (pci_seg->rlookup_table == NULL)
702 		return -ENOMEM;
703 
704 	return 0;
705 }
706 
707 static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg)
708 {
709 	kvfree(pci_seg->rlookup_table);
710 	pci_seg->rlookup_table = NULL;
711 }
712 
713 static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)
714 {
715 	pci_seg->irq_lookup_table = kvzalloc_objs(*pci_seg->irq_lookup_table,
716 						  pci_seg->last_bdf + 1);
717 	if (pci_seg->irq_lookup_table == NULL)
718 		return -ENOMEM;
719 
720 	return 0;
721 }
722 
723 static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)
724 {
725 	kvfree(pci_seg->irq_lookup_table);
726 	pci_seg->irq_lookup_table = NULL;
727 }
728 
729 static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg)
730 {
731 	int i;
732 
733 	pci_seg->alias_table = kvmalloc_objs(*pci_seg->alias_table,
734 					     pci_seg->last_bdf + 1);
735 	if (!pci_seg->alias_table)
736 		return -ENOMEM;
737 
738 	/*
739 	 * let all alias entries point to itself
740 	 */
741 	for (i = 0; i <= pci_seg->last_bdf; ++i)
742 		pci_seg->alias_table[i] = i;
743 
744 	return 0;
745 }
746 
747 static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg)
748 {
749 	kvfree(pci_seg->alias_table);
750 	pci_seg->alias_table = NULL;
751 }
752 
753 static inline void *iommu_memremap(unsigned long paddr, size_t size)
754 {
755 	phys_addr_t phys;
756 
757 	if (!paddr)
758 		return NULL;
759 
760 	/*
761 	 * Obtain true physical address in kdump kernel when SME is enabled.
762 	 * Currently, previous kernel with SME enabled and kdump kernel
763 	 * with SME support disabled is not supported.
764 	 */
765 	phys = __sme_clr(paddr);
766 
767 	if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
768 		return (__force void *)ioremap_encrypted(phys, size);
769 	else
770 		return memremap(phys, size, MEMREMAP_WB);
771 }
772 
773 /*
774  * Allocates the command buffer. This buffer is per AMD IOMMU. We can
775  * write commands to that buffer later and the IOMMU will execute them
776  * asynchronously
777  */
778 static int __init alloc_command_buffer(struct amd_iommu *iommu)
779 {
780 	iommu->cmd_buf = iommu_alloc_pages_sz(GFP_KERNEL, CMD_BUFFER_SIZE);
781 
782 	return iommu->cmd_buf ? 0 : -ENOMEM;
783 }
784 
785 /*
786  * Interrupt handler has processed all pending events and adjusted head
787  * and tail pointer. Reset overflow mask and restart logging again.
788  */
789 void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type,
790 			   u8 cntrl_intr, u8 cntrl_log,
791 			   u32 status_run_mask, u32 status_overflow_mask)
792 {
793 	u32 status;
794 
795 	status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
796 	if (status & status_run_mask)
797 		return;
798 
799 	pr_info_ratelimited("IOMMU %s log restarting\n", evt_type);
800 
801 	iommu_feature_disable(iommu, cntrl_log);
802 	iommu_feature_disable(iommu, cntrl_intr);
803 
804 	writel(status_overflow_mask, iommu->mmio_base + MMIO_STATUS_OFFSET);
805 
806 	iommu_feature_enable(iommu, cntrl_intr);
807 	iommu_feature_enable(iommu, cntrl_log);
808 }
809 
810 /*
811  * This function restarts event logging in case the IOMMU experienced
812  * an event log buffer overflow.
813  */
814 void amd_iommu_restart_event_logging(struct amd_iommu *iommu)
815 {
816 	amd_iommu_restart_log(iommu, "Event", CONTROL_EVT_INT_EN,
817 			      CONTROL_EVT_LOG_EN, MMIO_STATUS_EVT_RUN_MASK,
818 			      MMIO_STATUS_EVT_OVERFLOW_MASK);
819 }
820 
821 /*
822  * This function restarts event logging in case the IOMMU experienced
823  * GA log overflow.
824  */
825 void amd_iommu_restart_ga_log(struct amd_iommu *iommu)
826 {
827 	amd_iommu_restart_log(iommu, "GA", CONTROL_GAINT_EN,
828 			      CONTROL_GALOG_EN, MMIO_STATUS_GALOG_RUN_MASK,
829 			      MMIO_STATUS_GALOG_OVERFLOW_MASK);
830 }
831 
832 /*
833  * This function resets the command buffer if the IOMMU stopped fetching
834  * commands from it.
835  */
836 static void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
837 {
838 	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
839 
840 	writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
841 	writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
842 	iommu->cmd_buf_head = 0;
843 	iommu->cmd_buf_tail = 0;
844 
845 	iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
846 }
847 
848 /*
849  * This function writes the command buffer address to the hardware and
850  * enables it.
851  */
852 static void iommu_enable_command_buffer(struct amd_iommu *iommu)
853 {
854 	u64 entry;
855 
856 	BUG_ON(iommu->cmd_buf == NULL);
857 
858 	if (!is_kdump_kernel()) {
859 		/*
860 		 * Command buffer is re-used for kdump kernel and setting
861 		 * of MMIO register is not required.
862 		 */
863 		entry = iommu_virt_to_phys(iommu->cmd_buf);
864 		entry |= MMIO_CMD_SIZE_512;
865 		memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
866 			    &entry, sizeof(entry));
867 	}
868 
869 	amd_iommu_reset_cmd_buffer(iommu);
870 }
871 
872 /*
873  * This function disables the command buffer
874  */
875 static void iommu_disable_command_buffer(struct amd_iommu *iommu)
876 {
877 	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
878 }
879 
880 static void __init free_command_buffer(struct amd_iommu *iommu)
881 {
882 	iommu_free_pages(iommu->cmd_buf);
883 }
884 
885 void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp,
886 				  size_t size)
887 {
888 	int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
889 	void *buf;
890 
891 	size = PAGE_ALIGN(size);
892 	buf = iommu_alloc_pages_node_sz(nid, gfp, size);
893 	if (!buf)
894 		return NULL;
895 	if (check_feature(FEATURE_SNP) &&
896 	    set_memory_4k((unsigned long)buf, size / PAGE_SIZE)) {
897 		iommu_free_pages(buf);
898 		return NULL;
899 	}
900 
901 	return buf;
902 }
903 
904 /* allocates the memory where the IOMMU will log its events to */
905 static int __init alloc_event_buffer(void)
906 {
907 	struct amd_iommu *iommu;
908 
909 	for_each_iommu(iommu) {
910 		iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL,
911 						      amd_iommu_evtlog_size);
912 		if (!iommu->evt_buf)
913 			return -ENOMEM;
914 	}
915 
916 	return 0;
917 }
918 
919 static void iommu_enable_event_buffer(void)
920 {
921 	struct amd_iommu *iommu;
922 	u64 entry;
923 
924 	for_each_iommu(iommu) {
925 		BUG_ON(iommu->evt_buf == NULL);
926 
927 		if (!is_kdump_kernel()) {
928 			/*
929 			 * Event buffer is re-used for kdump kernel and setting
930 			 * of MMIO register is not required.
931 			 */
932 			entry = iommu_virt_to_phys(iommu->evt_buf);
933 			entry |= (amd_iommu_evtlog_size == EVTLOG_SIZE_DEF) ?
934 				EVTLOG_LEN_MASK_DEF : EVTLOG_LEN_MASK_MAX;
935 
936 			memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
937 				    &entry, sizeof(entry));
938 		}
939 
940 		/* set head and tail to zero manually */
941 		writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
942 		writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
943 
944 		iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
945 	}
946 }
947 
948 /*
949  * This function disables the event log buffer
950  */
951 static void iommu_disable_event_buffer(struct amd_iommu *iommu)
952 {
953 	iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
954 }
955 
956 static void __init free_event_buffer(struct amd_iommu *iommu)
957 {
958 	iommu_free_pages(iommu->evt_buf);
959 }
960 
961 static void free_ga_log(struct amd_iommu *iommu)
962 {
963 #ifdef CONFIG_IRQ_REMAP
964 	iommu_free_pages(iommu->ga_log);
965 	iommu->ga_log = NULL;
966 	iommu_free_pages(iommu->ga_log_tail);
967 	iommu->ga_log_tail = NULL;
968 #endif
969 }
970 
971 #ifdef CONFIG_IRQ_REMAP
972 static bool iommu_ga_log_enable(struct amd_iommu *iommu)
973 {
974 	u32 status, i;
975 	u64 entry;
976 
977 	if (!iommu->ga_log)
978 		return false;
979 
980 	entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
981 	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
982 		    &entry, sizeof(entry));
983 	entry = (iommu_virt_to_phys(iommu->ga_log_tail) &
984 		 (BIT_ULL(52)-1)) & ~7ULL;
985 	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,
986 		    &entry, sizeof(entry));
987 	writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
988 	writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
989 
990 
991 	iommu_feature_enable(iommu, CONTROL_GAINT_EN);
992 	iommu_feature_enable(iommu, CONTROL_GALOG_EN);
993 
994 	for (i = 0; i < MMIO_STATUS_TIMEOUT; ++i) {
995 		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
996 		if (status & (MMIO_STATUS_GALOG_RUN_MASK))
997 			break;
998 		udelay(10);
999 	}
1000 
1001 	if (WARN_ON(i >= MMIO_STATUS_TIMEOUT))
1002 		return false;
1003 
1004 	return true;
1005 }
1006 #endif /* CONFIG_IRQ_REMAP */
1007 
1008 static int alloc_ga_log(struct amd_iommu *iommu)
1009 {
1010 #ifdef CONFIG_IRQ_REMAP
1011 	int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
1012 
1013 	iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE);
1014 	if (!iommu->ga_log)
1015 		return -ENOMEM;
1016 
1017 	iommu->ga_log_tail = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, 8);
1018 	if (!iommu->ga_log_tail) {
1019 		iommu_free_pages(iommu->ga_log);
1020 		iommu->ga_log = NULL;
1021 		return -ENOMEM;
1022 	}
1023 #endif
1024 
1025 	return 0;
1026 }
1027 
1028 static int __init alloc_cwwb_sem(struct amd_iommu *iommu)
1029 {
1030 	iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL, 1);
1031 	if (!iommu->cmd_sem)
1032 		return -ENOMEM;
1033 	iommu->cmd_sem_paddr = iommu_virt_to_phys((void *)iommu->cmd_sem);
1034 	return 0;
1035 }
1036 
1037 static int __init remap_event_buffer(void)
1038 {
1039 	struct amd_iommu *iommu;
1040 	u64 paddr;
1041 
1042 	pr_info_once("Re-using event buffer from the previous kernel\n");
1043 	for_each_iommu(iommu) {
1044 		paddr = readq(iommu->mmio_base + MMIO_EVT_BUF_OFFSET) & PM_ADDR_MASK;
1045 		iommu->evt_buf = iommu_memremap(paddr, amd_iommu_evtlog_size);
1046 		if (!iommu->evt_buf)
1047 			return -ENOMEM;
1048 	}
1049 
1050 	return 0;
1051 }
1052 
1053 static int __init remap_command_buffer(struct amd_iommu *iommu)
1054 {
1055 	u64 paddr;
1056 
1057 	pr_info_once("Re-using command buffer from the previous kernel\n");
1058 	paddr = readq(iommu->mmio_base + MMIO_CMD_BUF_OFFSET) & PM_ADDR_MASK;
1059 	iommu->cmd_buf = iommu_memremap(paddr, CMD_BUFFER_SIZE);
1060 
1061 	return iommu->cmd_buf ? 0 : -ENOMEM;
1062 }
1063 
1064 static int __init remap_or_alloc_cwwb_sem(struct amd_iommu *iommu)
1065 {
1066 	u64 paddr;
1067 
1068 	if (check_feature(FEATURE_SNP)) {
1069 		/*
1070 		 * When SNP is enabled, the exclusion base register is used for the
1071 		 * completion wait buffer (CWB) address. Read and re-use it.
1072 		 */
1073 		pr_info_once("Re-using CWB buffers from the previous kernel\n");
1074 		paddr = readq(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET) & PM_ADDR_MASK;
1075 		iommu->cmd_sem = iommu_memremap(paddr, PAGE_SIZE);
1076 		if (!iommu->cmd_sem)
1077 			return -ENOMEM;
1078 		iommu->cmd_sem_paddr = paddr;
1079 	} else {
1080 		return alloc_cwwb_sem(iommu);
1081 	}
1082 
1083 	return 0;
1084 }
1085 
1086 static int __init alloc_iommu_buffers(struct amd_iommu *iommu)
1087 {
1088 	int ret;
1089 
1090 	/*
1091 	 * Reuse/Remap the previous kernel's allocated completion wait
1092 	 * command and event buffers for kdump boot.
1093 	 */
1094 	if (is_kdump_kernel()) {
1095 		ret = remap_or_alloc_cwwb_sem(iommu);
1096 		if (ret)
1097 			return ret;
1098 
1099 		ret = remap_command_buffer(iommu);
1100 		if (ret)
1101 			return ret;
1102 	} else {
1103 		ret = alloc_cwwb_sem(iommu);
1104 		if (ret)
1105 			return ret;
1106 
1107 		ret = alloc_command_buffer(iommu);
1108 		if (ret)
1109 			return ret;
1110 	}
1111 
1112 	return 0;
1113 }
1114 
1115 static void __init free_cwwb_sem(struct amd_iommu *iommu)
1116 {
1117 	if (iommu->cmd_sem)
1118 		iommu_free_pages((void *)iommu->cmd_sem);
1119 }
1120 static void __init unmap_cwwb_sem(struct amd_iommu *iommu)
1121 {
1122 	if (iommu->cmd_sem) {
1123 		if (check_feature(FEATURE_SNP))
1124 			memunmap((void *)iommu->cmd_sem);
1125 		else
1126 			iommu_free_pages((void *)iommu->cmd_sem);
1127 	}
1128 }
1129 
1130 static void __init unmap_command_buffer(struct amd_iommu *iommu)
1131 {
1132 	memunmap((void *)iommu->cmd_buf);
1133 }
1134 
1135 static void __init unmap_event_buffer(struct amd_iommu *iommu)
1136 {
1137 	memunmap(iommu->evt_buf);
1138 }
1139 
1140 static void __init free_iommu_buffers(struct amd_iommu *iommu)
1141 {
1142 	if (is_kdump_kernel()) {
1143 		unmap_cwwb_sem(iommu);
1144 		unmap_command_buffer(iommu);
1145 		unmap_event_buffer(iommu);
1146 	} else {
1147 		free_cwwb_sem(iommu);
1148 		free_command_buffer(iommu);
1149 		free_event_buffer(iommu);
1150 	}
1151 }
1152 
1153 static void iommu_enable_xt(struct amd_iommu *iommu)
1154 {
1155 #ifdef CONFIG_IRQ_REMAP
1156 	/*
1157 	 * XT mode (32-bit APIC destination ID) requires
1158 	 * GA mode (128-bit IRTE support) as a prerequisite.
1159 	 */
1160 	if (AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir) &&
1161 	    amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
1162 		iommu_feature_enable(iommu, CONTROL_XT_EN);
1163 #endif /* CONFIG_IRQ_REMAP */
1164 }
1165 
1166 static void iommu_enable_gt(struct amd_iommu *iommu)
1167 {
1168 	if (!check_feature(FEATURE_GT))
1169 		return;
1170 
1171 	iommu_feature_enable(iommu, CONTROL_GT_EN);
1172 
1173 	/*
1174 	 * This feature needs to be enabled prior to a call
1175 	 * to iommu_snp_enable(). Since this function is called
1176 	 * in early_enable_iommu(), it is safe to enable here.
1177 	 */
1178 	if (check_feature2(FEATURE_GCR3TRPMODE))
1179 		iommu_feature_enable(iommu, CONTROL_GCR3TRPMODE);
1180 }
1181 
1182 /* sets a specific bit in the device table entry. */
1183 static void set_dte_bit(struct dev_table_entry *dte, u8 bit)
1184 {
1185 	int i = (bit >> 6) & 0x03;
1186 	int _bit = bit & 0x3f;
1187 
1188 	dte->data[i] |= (1UL << _bit);
1189 }
1190 
1191 static bool __reuse_device_table(struct amd_iommu *iommu)
1192 {
1193 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
1194 	struct dev_table_entry *old_dev_tbl_entry;
1195 	u32 lo, hi, old_devtb_size, devid;
1196 	phys_addr_t old_devtb_phys;
1197 	u16 dom_id;
1198 	bool dte_v;
1199 	u64 entry;
1200 
1201 	/* Each IOMMU use separate device table with the same size */
1202 	lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET);
1203 	hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4);
1204 	entry = (((u64) hi) << 32) + lo;
1205 
1206 	old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12;
1207 	if (old_devtb_size != pci_seg->dev_table_size) {
1208 		pr_err("The device table size of IOMMU:%d is not expected!\n",
1209 			iommu->index);
1210 		return false;
1211 	}
1212 
1213 	/*
1214 	 * When SME is enabled in the first kernel, the entry includes the
1215 	 * memory encryption mask(sme_me_mask), we must remove the memory
1216 	 * encryption mask to obtain the true physical address in kdump kernel.
1217 	 */
1218 	old_devtb_phys = __sme_clr(entry) & PAGE_MASK;
1219 
1220 	if (old_devtb_phys >= 0x100000000ULL) {
1221 		pr_err("The address of old device table is above 4G, not trustworthy!\n");
1222 		return false;
1223 	}
1224 
1225 	/*
1226 	 * Re-use the previous kernel's device table for kdump.
1227 	 */
1228 	pci_seg->old_dev_tbl_cpy = iommu_memremap(old_devtb_phys, pci_seg->dev_table_size);
1229 	if (pci_seg->old_dev_tbl_cpy == NULL) {
1230 		pr_err("Failed to remap memory for reusing old device table!\n");
1231 		return false;
1232 	}
1233 
1234 	for (devid = 0; devid <= pci_seg->last_bdf; devid++) {
1235 		old_dev_tbl_entry = &pci_seg->old_dev_tbl_cpy[devid];
1236 		dte_v = FIELD_GET(DTE_FLAG_V, old_dev_tbl_entry->data[0]);
1237 		dom_id = FIELD_GET(DTE_DOMID_MASK, old_dev_tbl_entry->data[1]);
1238 
1239 		if (!dte_v || !dom_id)
1240 			continue;
1241 		/*
1242 		 * ID reservation can fail with -ENOSPC when there
1243 		 * are multiple devices present in the same domain,
1244 		 * hence check only for -ENOMEM.
1245 		 */
1246 		if (amd_iommu_pdom_id_reserve(dom_id, GFP_KERNEL) == -ENOMEM)
1247 			return false;
1248 	}
1249 
1250 	return true;
1251 }
1252 
1253 static bool reuse_device_table(void)
1254 {
1255 	struct amd_iommu *iommu;
1256 	struct amd_iommu_pci_seg *pci_seg;
1257 
1258 	if (!amd_iommu_pre_enabled)
1259 		return false;
1260 
1261 	pr_warn("Translation is already enabled - trying to reuse translation structures\n");
1262 
1263 	/*
1264 	 * All IOMMUs within PCI segment shares common device table.
1265 	 * Hence reuse device table only once per PCI segment.
1266 	 */
1267 	for_each_pci_segment(pci_seg) {
1268 		for_each_iommu(iommu) {
1269 			if (pci_seg->id != iommu->pci_seg->id)
1270 				continue;
1271 			if (!__reuse_device_table(iommu))
1272 				return false;
1273 			break;
1274 		}
1275 	}
1276 
1277 	return true;
1278 }
1279 
1280 struct dev_table_entry *amd_iommu_get_ivhd_dte_flags(u16 segid, u16 devid)
1281 {
1282 	struct ivhd_dte_flags *e;
1283 	unsigned int best_len = UINT_MAX;
1284 	struct dev_table_entry *dte = NULL;
1285 
1286 	for_each_ivhd_dte_flags(e) {
1287 		/*
1288 		 * Need to go through the whole list to find the smallest range,
1289 		 * which contains the devid.
1290 		 */
1291 		if ((e->segid == segid) &&
1292 		    (e->devid_first <= devid) && (devid <= e->devid_last)) {
1293 			unsigned int len = e->devid_last - e->devid_first;
1294 
1295 			if (len < best_len) {
1296 				dte = &(e->dte);
1297 				best_len = len;
1298 			}
1299 		}
1300 	}
1301 	return dte;
1302 }
1303 
1304 static bool search_ivhd_dte_flags(u16 segid, u16 first, u16 last)
1305 {
1306 	struct ivhd_dte_flags *e;
1307 
1308 	for_each_ivhd_dte_flags(e) {
1309 		if ((e->segid == segid) &&
1310 		    (e->devid_first == first) &&
1311 		    (e->devid_last == last))
1312 			return true;
1313 	}
1314 	return false;
1315 }
1316 
1317 /*
1318  * This function takes the device specific flags read from the ACPI
1319  * table and sets up the device table entry with that information
1320  */
1321 static void __init
1322 set_dev_entry_from_acpi_range(struct amd_iommu *iommu, u16 first, u16 last,
1323 			      u32 flags, u32 ext_flags)
1324 {
1325 	int i;
1326 	struct dev_table_entry dte = {};
1327 
1328 	/* Parse IVHD DTE setting flags and store information */
1329 	if (flags) {
1330 		struct ivhd_dte_flags *d;
1331 
1332 		if (search_ivhd_dte_flags(iommu->pci_seg->id, first, last))
1333 			return;
1334 
1335 		d = kzalloc_obj(struct ivhd_dte_flags);
1336 		if (!d)
1337 			return;
1338 
1339 		pr_debug("%s: devid range %#x:%#x\n", __func__, first, last);
1340 
1341 		if (flags & ACPI_DEVFLAG_INITPASS)
1342 			set_dte_bit(&dte, DEV_ENTRY_INIT_PASS);
1343 		if (flags & ACPI_DEVFLAG_EXTINT)
1344 			set_dte_bit(&dte, DEV_ENTRY_EINT_PASS);
1345 		if (flags & ACPI_DEVFLAG_NMI)
1346 			set_dte_bit(&dte, DEV_ENTRY_NMI_PASS);
1347 		if (flags & ACPI_DEVFLAG_SYSMGT1)
1348 			set_dte_bit(&dte, DEV_ENTRY_SYSMGT1);
1349 		if (flags & ACPI_DEVFLAG_SYSMGT2)
1350 			set_dte_bit(&dte, DEV_ENTRY_SYSMGT2);
1351 		if (flags & ACPI_DEVFLAG_LINT0)
1352 			set_dte_bit(&dte, DEV_ENTRY_LINT0_PASS);
1353 		if (flags & ACPI_DEVFLAG_LINT1)
1354 			set_dte_bit(&dte, DEV_ENTRY_LINT1_PASS);
1355 
1356 		/* Apply erratum 63, which needs info in initial_dte */
1357 		if (FIELD_GET(DTE_DATA1_SYSMGT_MASK, dte.data[1]) == 0x1)
1358 			dte.data[0] |= DTE_FLAG_IW;
1359 
1360 		memcpy(&d->dte, &dte, sizeof(dte));
1361 		d->segid = iommu->pci_seg->id;
1362 		d->devid_first = first;
1363 		d->devid_last = last;
1364 		list_add_tail(&d->list, &amd_ivhd_dev_flags_list);
1365 	}
1366 
1367 	for (i = first; i <= last; i++)  {
1368 		if (flags) {
1369 			struct dev_table_entry *dev_table = get_dev_table(iommu);
1370 
1371 			memcpy(&dev_table[i], &dte, sizeof(dte));
1372 		}
1373 		amd_iommu_set_rlookup_table(iommu, i);
1374 	}
1375 }
1376 
1377 static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
1378 					   u16 devid, u32 flags, u32 ext_flags)
1379 {
1380 	set_dev_entry_from_acpi_range(iommu, devid, devid, flags, ext_flags);
1381 }
1382 
1383 int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line)
1384 {
1385 	struct devid_map *entry;
1386 	struct list_head *list;
1387 
1388 	if (type == IVHD_SPECIAL_IOAPIC)
1389 		list = &ioapic_map;
1390 	else if (type == IVHD_SPECIAL_HPET)
1391 		list = &hpet_map;
1392 	else
1393 		return -EINVAL;
1394 
1395 	list_for_each_entry(entry, list, list) {
1396 		if (!(entry->id == id && entry->cmd_line))
1397 			continue;
1398 
1399 		pr_info("Command-line override present for %s id %d - ignoring\n",
1400 			type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id);
1401 
1402 		*devid = entry->devid;
1403 
1404 		return 0;
1405 	}
1406 
1407 	entry = kzalloc_obj(*entry);
1408 	if (!entry)
1409 		return -ENOMEM;
1410 
1411 	entry->id	= id;
1412 	entry->devid	= *devid;
1413 	entry->cmd_line	= cmd_line;
1414 
1415 	list_add_tail(&entry->list, list);
1416 
1417 	return 0;
1418 }
1419 
1420 static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u32 *devid,
1421 				      bool cmd_line)
1422 {
1423 	struct acpihid_map_entry *entry;
1424 	struct list_head *list = &acpihid_map;
1425 
1426 	list_for_each_entry(entry, list, list) {
1427 		if (strcmp(entry->hid, hid) ||
1428 		    (*uid && *entry->uid && strcmp(entry->uid, uid)) ||
1429 		    !entry->cmd_line)
1430 			continue;
1431 
1432 		pr_info("Command-line override for hid:%s uid:%s\n",
1433 			hid, uid);
1434 		*devid = entry->devid;
1435 		return 0;
1436 	}
1437 
1438 	entry = kzalloc_obj(*entry);
1439 	if (!entry)
1440 		return -ENOMEM;
1441 
1442 	memcpy(entry->uid, uid, strlen(uid));
1443 	memcpy(entry->hid, hid, strlen(hid));
1444 	entry->devid = *devid;
1445 	entry->cmd_line	= cmd_line;
1446 	entry->root_devid = (entry->devid & (~0x7));
1447 
1448 	pr_info("%s, add hid:%s, uid:%s, rdevid:%#x\n",
1449 		entry->cmd_line ? "cmd" : "ivrs",
1450 		entry->hid, entry->uid, entry->root_devid);
1451 
1452 	list_add_tail(&entry->list, list);
1453 	return 0;
1454 }
1455 
1456 static int __init add_early_maps(void)
1457 {
1458 	int i, ret;
1459 
1460 	for (i = 0; i < early_ioapic_map_size; ++i) {
1461 		ret = add_special_device(IVHD_SPECIAL_IOAPIC,
1462 					 early_ioapic_map[i].id,
1463 					 &early_ioapic_map[i].devid,
1464 					 early_ioapic_map[i].cmd_line);
1465 		if (ret)
1466 			return ret;
1467 	}
1468 
1469 	for (i = 0; i < early_hpet_map_size; ++i) {
1470 		ret = add_special_device(IVHD_SPECIAL_HPET,
1471 					 early_hpet_map[i].id,
1472 					 &early_hpet_map[i].devid,
1473 					 early_hpet_map[i].cmd_line);
1474 		if (ret)
1475 			return ret;
1476 	}
1477 
1478 	for (i = 0; i < early_acpihid_map_size; ++i) {
1479 		ret = add_acpi_hid_device(early_acpihid_map[i].hid,
1480 					  early_acpihid_map[i].uid,
1481 					  &early_acpihid_map[i].devid,
1482 					  early_acpihid_map[i].cmd_line);
1483 		if (ret)
1484 			return ret;
1485 	}
1486 
1487 	return 0;
1488 }
1489 
1490 /*
1491  * Takes a pointer to an AMD IOMMU entry in the ACPI table and
1492  * initializes the hardware and our data structures with it.
1493  */
1494 static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
1495 					struct ivhd_header *h)
1496 {
1497 	u8 *p = (u8 *)h;
1498 	u8 *end = p, flags = 0;
1499 	u16 devid = 0, devid_start = 0, devid_to = 0, seg_id;
1500 	u32 dev_i, ext_flags = 0;
1501 	bool alias = false;
1502 	struct ivhd_entry *e;
1503 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
1504 	u32 ivhd_size;
1505 	int ret;
1506 
1507 
1508 	ret = add_early_maps();
1509 	if (ret)
1510 		return ret;
1511 
1512 	amd_iommu_apply_ivrs_quirks();
1513 
1514 	/*
1515 	 * First save the recommended feature enable bits from ACPI
1516 	 */
1517 	iommu->acpi_flags = h->flags;
1518 
1519 	/*
1520 	 * Done. Now parse the device entries
1521 	 */
1522 	ivhd_size = get_ivhd_header_size(h);
1523 	if (!ivhd_size) {
1524 		pr_err("Unsupported IVHD type %#x\n", h->type);
1525 		return -EINVAL;
1526 	}
1527 
1528 	p += ivhd_size;
1529 
1530 	end += h->length;
1531 
1532 
1533 	while (p < end) {
1534 		e = (struct ivhd_entry *)p;
1535 		seg_id = pci_seg->id;
1536 
1537 		switch (e->type) {
1538 		case IVHD_DEV_ALL:
1539 
1540 			DUMP_printk("  DEV_ALL\t\t\tsetting: %#02x\n", e->flags);
1541 			set_dev_entry_from_acpi_range(iommu, 0, pci_seg->last_bdf, e->flags, 0);
1542 			break;
1543 		case IVHD_DEV_SELECT:
1544 
1545 			DUMP_printk("  DEV_SELECT\t\t\tdevid: %04x:%02x:%02x.%x flags: %#02x\n",
1546 				    seg_id, PCI_BUS_NUM(e->devid),
1547 				    PCI_SLOT(e->devid),
1548 				    PCI_FUNC(e->devid),
1549 				    e->flags);
1550 
1551 			devid = e->devid;
1552 			set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1553 			break;
1554 		case IVHD_DEV_SELECT_RANGE_START:
1555 
1556 			DUMP_printk("  DEV_SELECT_RANGE_START\tdevid: %04x:%02x:%02x.%x flags: %#02x\n",
1557 				    seg_id, PCI_BUS_NUM(e->devid),
1558 				    PCI_SLOT(e->devid),
1559 				    PCI_FUNC(e->devid),
1560 				    e->flags);
1561 
1562 			devid_start = e->devid;
1563 			flags = e->flags;
1564 			ext_flags = 0;
1565 			alias = false;
1566 			break;
1567 		case IVHD_DEV_ALIAS:
1568 
1569 			DUMP_printk("  DEV_ALIAS\t\t\tdevid: %04x:%02x:%02x.%x flags: %#02x devid_to: %02x:%02x.%x\n",
1570 				    seg_id, PCI_BUS_NUM(e->devid),
1571 				    PCI_SLOT(e->devid),
1572 				    PCI_FUNC(e->devid),
1573 				    e->flags,
1574 				    PCI_BUS_NUM(e->ext >> 8),
1575 				    PCI_SLOT(e->ext >> 8),
1576 				    PCI_FUNC(e->ext >> 8));
1577 
1578 			devid = e->devid;
1579 			devid_to = e->ext >> 8;
1580 			set_dev_entry_from_acpi(iommu, devid   , e->flags, 0);
1581 			set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);
1582 			pci_seg->alias_table[devid] = devid_to;
1583 			break;
1584 		case IVHD_DEV_ALIAS_RANGE:
1585 
1586 			DUMP_printk("  DEV_ALIAS_RANGE\t\tdevid: %04x:%02x:%02x.%x flags: %#02x devid_to: %04x:%02x:%02x.%x\n",
1587 				    seg_id, PCI_BUS_NUM(e->devid),
1588 				    PCI_SLOT(e->devid),
1589 				    PCI_FUNC(e->devid),
1590 				    e->flags,
1591 				    seg_id, PCI_BUS_NUM(e->ext >> 8),
1592 				    PCI_SLOT(e->ext >> 8),
1593 				    PCI_FUNC(e->ext >> 8));
1594 
1595 			devid_start = e->devid;
1596 			flags = e->flags;
1597 			devid_to = e->ext >> 8;
1598 			ext_flags = 0;
1599 			alias = true;
1600 			break;
1601 		case IVHD_DEV_EXT_SELECT:
1602 
1603 			DUMP_printk("  DEV_EXT_SELECT\t\tdevid: %04x:%02x:%02x.%x flags: %#02x ext: %08x\n",
1604 				    seg_id, PCI_BUS_NUM(e->devid),
1605 				    PCI_SLOT(e->devid),
1606 				    PCI_FUNC(e->devid),
1607 				    e->flags, e->ext);
1608 
1609 			devid = e->devid;
1610 			set_dev_entry_from_acpi(iommu, devid, e->flags,
1611 						e->ext);
1612 			break;
1613 		case IVHD_DEV_EXT_SELECT_RANGE:
1614 
1615 			DUMP_printk("  DEV_EXT_SELECT_RANGE\tdevid: %04x:%02x:%02x.%x flags: %#02x ext: %08x\n",
1616 				    seg_id, PCI_BUS_NUM(e->devid),
1617 				    PCI_SLOT(e->devid),
1618 				    PCI_FUNC(e->devid),
1619 				    e->flags, e->ext);
1620 
1621 			devid_start = e->devid;
1622 			flags = e->flags;
1623 			ext_flags = e->ext;
1624 			alias = false;
1625 			break;
1626 		case IVHD_DEV_RANGE_END:
1627 
1628 			DUMP_printk("  DEV_RANGE_END\t\tdevid: %04x:%02x:%02x.%x\n",
1629 				    seg_id, PCI_BUS_NUM(e->devid),
1630 				    PCI_SLOT(e->devid),
1631 				    PCI_FUNC(e->devid));
1632 
1633 			devid = e->devid;
1634 			if (alias) {
1635 				for (dev_i = devid_start; dev_i <= devid; ++dev_i)
1636 					pci_seg->alias_table[dev_i] = devid_to;
1637 				set_dev_entry_from_acpi(iommu, devid_to, flags, ext_flags);
1638 			}
1639 			set_dev_entry_from_acpi_range(iommu, devid_start, devid, flags, ext_flags);
1640 			break;
1641 		case IVHD_DEV_SPECIAL: {
1642 			u8 handle, type;
1643 			const char *var;
1644 			u32 devid;
1645 			int ret;
1646 
1647 			handle = e->ext & 0xff;
1648 			devid = PCI_SEG_DEVID_TO_SBDF(seg_id, (e->ext >> 8));
1649 			type   = (e->ext >> 24) & 0xff;
1650 
1651 			if (type == IVHD_SPECIAL_IOAPIC)
1652 				var = "IOAPIC";
1653 			else if (type == IVHD_SPECIAL_HPET)
1654 				var = "HPET";
1655 			else
1656 				var = "UNKNOWN";
1657 
1658 			DUMP_printk("  DEV_SPECIAL(%s[%d])\t\tdevid: %04x:%02x:%02x.%x, flags: %#02x\n",
1659 				    var, (int)handle,
1660 				    seg_id, PCI_BUS_NUM(devid),
1661 				    PCI_SLOT(devid),
1662 				    PCI_FUNC(devid),
1663 				    e->flags);
1664 
1665 			ret = add_special_device(type, handle, &devid, false);
1666 			if (ret)
1667 				return ret;
1668 
1669 			/*
1670 			 * add_special_device might update the devid in case a
1671 			 * command-line override is present. So call
1672 			 * set_dev_entry_from_acpi after add_special_device.
1673 			 */
1674 			set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1675 
1676 			break;
1677 		}
1678 		case IVHD_DEV_ACPI_HID: {
1679 			u32 devid;
1680 			u8 hid[ACPIHID_HID_LEN];
1681 			u8 uid[ACPIHID_UID_LEN];
1682 			int ret;
1683 
1684 			if (h->type != 0x40) {
1685 				pr_err(FW_BUG "Invalid IVHD device type %#x\n",
1686 				       e->type);
1687 				break;
1688 			}
1689 
1690 			BUILD_BUG_ON(sizeof(e->ext_hid) != ACPIHID_HID_LEN - 1);
1691 			memcpy(hid, &e->ext_hid, ACPIHID_HID_LEN - 1);
1692 			hid[ACPIHID_HID_LEN - 1] = '\0';
1693 
1694 			if (!(*hid)) {
1695 				pr_err(FW_BUG "Invalid HID.\n");
1696 				break;
1697 			}
1698 
1699 			uid[0] = '\0';
1700 			switch (e->uidf) {
1701 			case UID_NOT_PRESENT:
1702 
1703 				if (e->uidl != 0)
1704 					pr_warn(FW_BUG "Invalid UID length.\n");
1705 
1706 				break;
1707 			case UID_IS_INTEGER:
1708 
1709 				sprintf(uid, "%d", e->uid);
1710 
1711 				break;
1712 			case UID_IS_CHARACTER:
1713 
1714 				memcpy(uid, &e->uid, e->uidl);
1715 				uid[e->uidl] = '\0';
1716 
1717 				break;
1718 			default:
1719 				break;
1720 			}
1721 
1722 			devid = PCI_SEG_DEVID_TO_SBDF(seg_id, e->devid);
1723 			DUMP_printk("  DEV_ACPI_HID(%s[%s])\t\tdevid: %04x:%02x:%02x.%x, flags: %#02x\n",
1724 				    hid, uid, seg_id,
1725 				    PCI_BUS_NUM(devid),
1726 				    PCI_SLOT(devid),
1727 				    PCI_FUNC(devid),
1728 				    e->flags);
1729 
1730 			flags = e->flags;
1731 
1732 			ret = add_acpi_hid_device(hid, uid, &devid, false);
1733 			if (ret)
1734 				return ret;
1735 
1736 			/*
1737 			 * add_special_device might update the devid in case a
1738 			 * command-line override is present. So call
1739 			 * set_dev_entry_from_acpi after add_special_device.
1740 			 */
1741 			set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1742 
1743 			break;
1744 		}
1745 		default:
1746 			break;
1747 		}
1748 
1749 		p += ivhd_entry_length(p);
1750 	}
1751 
1752 	return 0;
1753 }
1754 
1755 /*
1756  * Apply workaround for Zen1 to Zen5(BRH) generation of processors.
1757  * References: CVE-2024-21963
1758  */
1759 static bool __init amd_iommu_force_max_dev_table(void)
1760 {
1761 	if (boot_cpu_data.x86 >= 0x17 && boot_cpu_data.x86 <= 0x1a) {
1762 		if (cpu_feature_enabled(X86_FEATURE_ZEN6))
1763 			return false;
1764 		else
1765 			return true;
1766 	}
1767 	return false;
1768 }
1769 
1770 /* Allocate PCI segment data structure */
1771 static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id,
1772 					  struct acpi_table_header *ivrs_base)
1773 {
1774 	struct amd_iommu_pci_seg *pci_seg;
1775 	int last_bdf;
1776 
1777 	/*
1778 	 * First parse ACPI tables to find the largest Bus/Dev/Func we need to
1779 	 * handle in this PCI segment. Upon this information the shared data
1780 	 * structures for the PCI segments in the system will be allocated.
1781 	 */
1782 	last_bdf = find_last_devid_acpi(ivrs_base, id);
1783 	if (last_bdf < 0)
1784 		return NULL;
1785 
1786 	pci_seg = kzalloc_obj(struct amd_iommu_pci_seg);
1787 	if (pci_seg == NULL)
1788 		return NULL;
1789 
1790 	pci_seg->last_bdf = last_bdf;
1791 	DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf);
1792 	if (amd_iommu_force_max_dev_table()) {
1793 		pr_info("Force device table to maximum size\n");
1794 		pci_seg->dev_table_size = SZ_2M;
1795 	} else {
1796 		pci_seg->dev_table_size =
1797 			max(roundup_pow_of_two((last_bdf + 1) *
1798 					       DEV_TABLE_ENTRY_SIZE),
1799 			    SZ_4K);
1800 	}
1801 
1802 	pci_seg->id = id;
1803 	init_llist_head(&pci_seg->dev_data_list);
1804 	INIT_LIST_HEAD(&pci_seg->unity_map);
1805 	list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list);
1806 
1807 	if (alloc_dev_table(pci_seg))
1808 		goto err_free_pci_seg;
1809 	if (alloc_alias_table(pci_seg))
1810 		goto err_free_dev_table;
1811 	if (alloc_rlookup_table(pci_seg))
1812 		goto err_free_alias_table;
1813 
1814 	return pci_seg;
1815 
1816 err_free_alias_table:
1817 	free_alias_table(pci_seg);
1818 err_free_dev_table:
1819 	free_dev_table(pci_seg);
1820 err_free_pci_seg:
1821 	list_del(&pci_seg->list);
1822 	kfree(pci_seg);
1823 	return NULL;
1824 }
1825 
1826 static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id,
1827 					struct acpi_table_header *ivrs_base)
1828 {
1829 	struct amd_iommu_pci_seg *pci_seg;
1830 
1831 	for_each_pci_segment(pci_seg) {
1832 		if (pci_seg->id == id)
1833 			return pci_seg;
1834 	}
1835 
1836 	return alloc_pci_segment(id, ivrs_base);
1837 }
1838 
1839 static void __init free_pci_segments(void)
1840 {
1841 	struct amd_iommu_pci_seg *pci_seg, *next;
1842 
1843 	for_each_pci_segment_safe(pci_seg, next) {
1844 		list_del(&pci_seg->list);
1845 		free_irq_lookup_table(pci_seg);
1846 		free_rlookup_table(pci_seg);
1847 		free_alias_table(pci_seg);
1848 		free_dev_table(pci_seg);
1849 		kfree(pci_seg);
1850 	}
1851 }
1852 
1853 static void __init free_sysfs(struct amd_iommu *iommu)
1854 {
1855 	if (iommu->iommu.dev) {
1856 		iommu_device_unregister(&iommu->iommu);
1857 		iommu_device_sysfs_remove(&iommu->iommu);
1858 	}
1859 }
1860 
1861 static void __init free_iommu_one(struct amd_iommu *iommu)
1862 {
1863 	free_sysfs(iommu);
1864 	free_iommu_buffers(iommu);
1865 	amd_iommu_free_ppr_log(iommu);
1866 	free_ga_log(iommu);
1867 	iommu_unmap_mmio_space(iommu);
1868 	amd_iommu_iopf_uninit(iommu);
1869 }
1870 
1871 static void __init free_iommu_all(void)
1872 {
1873 	struct amd_iommu *iommu, *next;
1874 
1875 	for_each_iommu_safe(iommu, next) {
1876 		list_del(&iommu->list);
1877 		free_iommu_one(iommu);
1878 		kfree(iommu);
1879 	}
1880 }
1881 
1882 /*
1883  * Family15h Model 10h-1fh erratum 746 (IOMMU Logging May Stall Translations)
1884  * Workaround:
1885  *     BIOS should disable L2B micellaneous clock gating by setting
1886  *     L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b
1887  */
1888 static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu)
1889 {
1890 	u32 value;
1891 
1892 	if ((boot_cpu_data.x86 != 0x15) ||
1893 	    (boot_cpu_data.x86_model < 0x10) ||
1894 	    (boot_cpu_data.x86_model > 0x1f))
1895 		return;
1896 
1897 	pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1898 	pci_read_config_dword(iommu->dev, 0xf4, &value);
1899 
1900 	if (value & BIT(2))
1901 		return;
1902 
1903 	/* Select NB indirect register 0x90 and enable writing */
1904 	pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8));
1905 
1906 	pci_write_config_dword(iommu->dev, 0xf4, value | 0x4);
1907 	pci_info(iommu->dev, "Applying erratum 746 workaround\n");
1908 
1909 	/* Clear the enable writing bit */
1910 	pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1911 }
1912 
1913 /*
1914  * Family15h Model 30h-3fh (IOMMU Mishandles ATS Write Permission)
1915  * Workaround:
1916  *     BIOS should enable ATS write permission check by setting
1917  *     L2_DEBUG_3[AtsIgnoreIWDis](D0F2xF4_x47[0]) = 1b
1918  */
1919 static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu)
1920 {
1921 	u32 value;
1922 
1923 	if ((boot_cpu_data.x86 != 0x15) ||
1924 	    (boot_cpu_data.x86_model < 0x30) ||
1925 	    (boot_cpu_data.x86_model > 0x3f))
1926 		return;
1927 
1928 	/* Test L2_DEBUG_3[AtsIgnoreIWDis] == 1 */
1929 	value = iommu_read_l2(iommu, 0x47);
1930 
1931 	if (value & BIT(0))
1932 		return;
1933 
1934 	/* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */
1935 	iommu_write_l2(iommu, 0x47, value | BIT(0));
1936 
1937 	pci_info(iommu->dev, "Applying ATS write check workaround\n");
1938 }
1939 
1940 /*
1941  * This function glues the initialization function for one IOMMU
1942  * together and also allocates the command buffer and programs the
1943  * hardware. It does NOT enable the IOMMU. This is done afterwards.
1944  */
1945 static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
1946 				 struct acpi_table_header *ivrs_base)
1947 {
1948 	struct amd_iommu_pci_seg *pci_seg;
1949 	int ret;
1950 
1951 	pci_seg = get_pci_segment(h->pci_seg, ivrs_base);
1952 	if (pci_seg == NULL)
1953 		return -ENOMEM;
1954 	iommu->pci_seg = pci_seg;
1955 
1956 	raw_spin_lock_init(&iommu->lock);
1957 	iommu->cmd_sem_val = 0;
1958 
1959 	/* Add IOMMU to internal data structures */
1960 	list_add_tail(&iommu->list, &amd_iommu_list);
1961 	iommu->index = amd_iommus_present++;
1962 
1963 	if (unlikely(iommu->index >= MAX_IOMMUS)) {
1964 		WARN(1, "System has more IOMMUs than supported by this driver\n");
1965 		return -ENOSYS;
1966 	}
1967 
1968 	/*
1969 	 * Copy data from ACPI table entry to the iommu struct
1970 	 */
1971 	iommu->devid   = h->devid;
1972 	iommu->cap_ptr = h->cap_ptr;
1973 	iommu->mmio_phys = h->mmio_phys;
1974 
1975 	switch (h->type) {
1976 	case 0x10:
1977 		/* Check if IVHD EFR contains proper max banks/counters */
1978 		if ((h->efr_attr != 0) &&
1979 		    ((h->efr_attr & (0xF << 13)) != 0) &&
1980 		    ((h->efr_attr & (0x3F << 17)) != 0))
1981 			iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
1982 		else
1983 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1984 
1985 		/* GAM requires GA mode. */
1986 		if ((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0)
1987 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1988 		break;
1989 	case 0x11:
1990 	case 0x40:
1991 		if (h->efr_reg & (1 << 9))
1992 			iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
1993 		else
1994 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1995 
1996 		if (h->efr_attr & BIT(IOMMU_IVHD_ATTR_HATDIS_SHIFT)) {
1997 			pr_warn_once("Host Address Translation is not supported.\n");
1998 			amd_iommu_hatdis = true;
1999 		}
2000 
2001 		/* XT and GAM require GA mode. */
2002 		if ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0) {
2003 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
2004 			break;
2005 		} else {
2006 			if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT))
2007 				amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE;
2008 		}
2009 
2010 		early_iommu_features_init(iommu, h);
2011 
2012 		break;
2013 	default:
2014 		return -EINVAL;
2015 	}
2016 
2017 	iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys,
2018 						iommu->mmio_phys_end);
2019 	if (!iommu->mmio_base)
2020 		return -ENOMEM;
2021 
2022 	/* A crash kernel may reuse device tables with PerfOpt still enabled. */
2023 	ret = amd_iommu_perfopt_set(iommu, false);
2024 	if (ret)
2025 		return ret;
2026 
2027 	return init_iommu_from_acpi(iommu, h);
2028 }
2029 
2030 static int __init init_iommu_one_late(struct amd_iommu *iommu)
2031 {
2032 	int ret;
2033 
2034 	ret = alloc_iommu_buffers(iommu);
2035 	if (ret)
2036 		return ret;
2037 
2038 	iommu->int_enabled = false;
2039 
2040 	init_translation_status(iommu);
2041 	if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
2042 		iommu_disable(iommu);
2043 		clear_translation_pre_enabled(iommu);
2044 		pr_warn("Translation was enabled for IOMMU:%d but we are not in kdump mode\n",
2045 			iommu->index);
2046 	}
2047 	if (amd_iommu_pre_enabled)
2048 		amd_iommu_pre_enabled = translation_pre_enabled(iommu);
2049 
2050 	if (amd_iommu_irq_remap) {
2051 		ret = amd_iommu_create_irq_domain(iommu);
2052 		if (ret)
2053 			return ret;
2054 	}
2055 
2056 	/*
2057 	 * Make sure IOMMU is not considered to translate itself. The IVRS
2058 	 * table tells us so, but this is a lie!
2059 	 */
2060 	iommu->pci_seg->rlookup_table[iommu->devid] = NULL;
2061 
2062 	return 0;
2063 }
2064 
2065 /**
2066  * get_highest_supported_ivhd_type - Look up the appropriate IVHD type
2067  * @ivrs: Pointer to the IVRS header
2068  *
2069  * This function search through all IVDB of the maximum supported IVHD
2070  */
2071 static u8 get_highest_supported_ivhd_type(struct acpi_table_header *ivrs)
2072 {
2073 	u8 *base = (u8 *)ivrs;
2074 	struct ivhd_header *ivhd = (struct ivhd_header *)
2075 					(base + IVRS_HEADER_LENGTH);
2076 	u8 last_type = ivhd->type;
2077 	u16 devid = ivhd->devid;
2078 
2079 	while (((u8 *)ivhd - base < ivrs->length) &&
2080 	       (ivhd->type <= ACPI_IVHD_TYPE_MAX_SUPPORTED)) {
2081 		u8 *p = (u8 *) ivhd;
2082 
2083 		if (ivhd->devid == devid)
2084 			last_type = ivhd->type;
2085 		ivhd = (struct ivhd_header *)(p + ivhd->length);
2086 	}
2087 
2088 	return last_type;
2089 }
2090 
2091 /*
2092  * Iterates over all IOMMU entries in the ACPI table, allocates the
2093  * IOMMU structure and initializes it with init_iommu_one()
2094  */
2095 static int __init init_iommu_all(struct acpi_table_header *table)
2096 {
2097 	u8 *p = (u8 *)table, *end = (u8 *)table;
2098 	struct ivhd_header *h;
2099 	struct amd_iommu *iommu;
2100 	int ret;
2101 
2102 	end += table->length;
2103 	p += IVRS_HEADER_LENGTH;
2104 
2105 	/* Phase 1: Process all IVHD blocks */
2106 	while (p < end) {
2107 		h = (struct ivhd_header *)p;
2108 		if (*p == amd_iommu_target_ivhd_type) {
2109 
2110 			DUMP_printk("device: %04x:%02x:%02x.%01x cap: %04x "
2111 				    "flags: %01x info %04x\n",
2112 				    h->pci_seg, PCI_BUS_NUM(h->devid),
2113 				    PCI_SLOT(h->devid), PCI_FUNC(h->devid),
2114 				    h->cap_ptr, h->flags, h->info);
2115 			DUMP_printk("       mmio-addr: %016llx\n",
2116 				    h->mmio_phys);
2117 
2118 			iommu = kzalloc_obj(struct amd_iommu);
2119 			if (iommu == NULL)
2120 				return -ENOMEM;
2121 
2122 			ret = init_iommu_one(iommu, h, table);
2123 			if (ret)
2124 				return ret;
2125 		}
2126 		p += h->length;
2127 
2128 	}
2129 	WARN_ON(p != end);
2130 
2131 	/* Phase 2 : Early feature support check */
2132 	get_global_efr();
2133 
2134 	/* Phase 3 : Enabling IOMMU features */
2135 	for_each_iommu(iommu) {
2136 		ret = init_iommu_one_late(iommu);
2137 		if (ret)
2138 			return ret;
2139 	}
2140 
2141 	return 0;
2142 }
2143 
2144 static void init_iommu_perf_ctr(struct amd_iommu *iommu)
2145 {
2146 	u64 val;
2147 	struct pci_dev *pdev = iommu->dev;
2148 
2149 	if (!check_feature(FEATURE_PC))
2150 		return;
2151 
2152 	amd_iommu_pc_present = true;
2153 
2154 	pci_info(pdev, "IOMMU performance counters supported\n");
2155 
2156 	val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET);
2157 	iommu->max_banks = (u8) ((val >> 12) & 0x3f);
2158 	iommu->max_counters = (u8) ((val >> 7) & 0xf);
2159 
2160 	return;
2161 }
2162 
2163 static ssize_t amd_iommu_show_cap(struct device *dev,
2164 				  struct device_attribute *attr,
2165 				  char *buf)
2166 {
2167 	struct amd_iommu *iommu = dev_to_amd_iommu(dev);
2168 	return sysfs_emit(buf, "%x\n", iommu->cap);
2169 }
2170 static DEVICE_ATTR(cap, S_IRUGO, amd_iommu_show_cap, NULL);
2171 
2172 static ssize_t amd_iommu_show_features(struct device *dev,
2173 				       struct device_attribute *attr,
2174 				       char *buf)
2175 {
2176 	return sysfs_emit(buf, "%llx:%llx\n", amd_iommu_efr, amd_iommu_efr2);
2177 }
2178 static DEVICE_ATTR(features, S_IRUGO, amd_iommu_show_features, NULL);
2179 
2180 static struct attribute *amd_iommu_attrs[] = {
2181 	&dev_attr_cap.attr,
2182 	&dev_attr_features.attr,
2183 	NULL,
2184 };
2185 
2186 static struct attribute_group amd_iommu_group = {
2187 	.name = "amd-iommu",
2188 	.attrs = amd_iommu_attrs,
2189 };
2190 
2191 static const struct attribute_group *amd_iommu_groups[] = {
2192 	&amd_iommu_group,
2193 	NULL,
2194 };
2195 
2196 /*
2197  * Note: IVHD 0x11 and 0x40 also contains exact copy
2198  * of the IOMMU Extended Feature Register [MMIO Offset 0030h].
2199  * Default to EFR in IVHD since it is available sooner (i.e. before PCI init).
2200  */
2201 static void __init late_iommu_features_init(struct amd_iommu *iommu)
2202 {
2203 	u64 features, features2;
2204 
2205 	if (!(iommu->cap & (1 << IOMMU_CAP_EFR)))
2206 		return;
2207 
2208 	/* read extended feature bits */
2209 	features = readq(iommu->mmio_base + MMIO_EXT_FEATURES);
2210 	features2 = readq(iommu->mmio_base + MMIO_EXT_FEATURES2);
2211 
2212 	if (!amd_iommu_efr) {
2213 		amd_iommu_efr = features;
2214 		amd_iommu_efr2 = features2;
2215 		return;
2216 	}
2217 
2218 	/*
2219 	 * Sanity check and warn if EFR values from
2220 	 * IVHD and MMIO conflict.
2221 	 */
2222 	if (features != amd_iommu_efr ||
2223 	    features2 != amd_iommu_efr2) {
2224 		pr_warn(FW_WARN
2225 			"EFR mismatch. Use IVHD EFR (%#llx : %#llx), EFR2 (%#llx : %#llx).\n",
2226 			features, amd_iommu_efr,
2227 			features2, amd_iommu_efr2);
2228 	}
2229 }
2230 
2231 /* Must be called after SNP support check is complete (iommu_snp_enable()) */
2232 static bool check_vapic_support(void)
2233 {
2234 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
2235 		return false;
2236 
2237 	if (!check_feature(FEATURE_GAM_VAPIC)) {
2238 		amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
2239 		return false;
2240 	}
2241 
2242 	if (amd_iommu_snp_en &&
2243 	    !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) {
2244 		pr_warn("Force to disable Virtual APIC due to SNP\n");
2245 		amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
2246 		return false;
2247 	}
2248 
2249 	return true;
2250 }
2251 
2252 static int __init iommu_init_pci(struct amd_iommu *iommu)
2253 {
2254 	int cap_ptr = iommu->cap_ptr;
2255 	int ret;
2256 
2257 	iommu->dev = pci_get_domain_bus_and_slot(iommu->pci_seg->id,
2258 						 PCI_BUS_NUM(iommu->devid),
2259 						 iommu->devid & 0xff);
2260 	if (!iommu->dev)
2261 		return -ENODEV;
2262 
2263 	/* ACPI _PRT won't have an IRQ for IOMMU */
2264 	iommu->dev->irq_managed = 1;
2265 
2266 	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
2267 			      &iommu->cap);
2268 
2269 	if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
2270 		amd_iommu_iotlb_sup = false;
2271 
2272 	late_iommu_features_init(iommu);
2273 
2274 	if (check_feature(FEATURE_GT)) {
2275 		int glxval;
2276 		u64 pasmax;
2277 
2278 		pasmax = FIELD_GET(FEATURE_PASMAX, amd_iommu_efr);
2279 		iommu->iommu.max_pasids = (1 << (pasmax + 1)) - 1;
2280 
2281 		BUG_ON(iommu->iommu.max_pasids & ~PASID_MASK);
2282 
2283 		glxval = FIELD_GET(FEATURE_GLX, amd_iommu_efr);
2284 
2285 		if (amd_iommu_max_glx_val == -1)
2286 			amd_iommu_max_glx_val = glxval;
2287 		else
2288 			amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
2289 
2290 		iommu_enable_gt(iommu);
2291 	}
2292 
2293 	if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu))
2294 		return -ENOMEM;
2295 
2296 	if (check_vapic_support()) {
2297 		ret = alloc_ga_log(iommu);
2298 		if (ret)
2299 			return ret;
2300 	}
2301 
2302 	if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) {
2303 		pr_info("Using strict mode due to virtualization\n");
2304 		iommu_set_dma_strict();
2305 		amd_iommu_np_cache = true;
2306 	}
2307 
2308 	init_iommu_perf_ctr(iommu);
2309 
2310 	if (is_rd890_iommu(iommu->dev)) {
2311 		int i, j;
2312 
2313 		iommu->root_pdev =
2314 			pci_get_domain_bus_and_slot(iommu->pci_seg->id,
2315 						    iommu->dev->bus->number,
2316 						    PCI_DEVFN(0, 0));
2317 
2318 		/*
2319 		 * Some rd890 systems may not be fully reconfigured by the
2320 		 * BIOS, so it's necessary for us to store this information so
2321 		 * it can be reprogrammed on resume
2322 		 */
2323 		pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,
2324 				&iommu->stored_addr_lo);
2325 		pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,
2326 				&iommu->stored_addr_hi);
2327 
2328 		/* Low bit locks writes to configuration space */
2329 		iommu->stored_addr_lo &= ~1;
2330 
2331 		for (i = 0; i < 6; i++)
2332 			for (j = 0; j < 0x12; j++)
2333 				iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);
2334 
2335 		for (i = 0; i < 0x83; i++)
2336 			iommu->stored_l2[i] = iommu_read_l2(iommu, i);
2337 	}
2338 
2339 	amd_iommu_erratum_746_workaround(iommu);
2340 	amd_iommu_ats_write_check_workaround(iommu);
2341 
2342 	ret = iommu_device_sysfs_add(&iommu->iommu, &iommu->dev->dev,
2343 			       amd_iommu_groups, "ivhd%d", iommu->index);
2344 	if (ret)
2345 		return ret;
2346 
2347 	/*
2348 	 * Allocate per IOMMU IOPF queue here so that in attach device path,
2349 	 * PRI capable device can be added to IOPF queue
2350 	 */
2351 	if (amd_iommu_gt_ppr_supported()) {
2352 		ret = amd_iommu_iopf_init(iommu);
2353 		if (ret)
2354 			return ret;
2355 	}
2356 
2357 	ret = iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL);
2358 	if (ret || amd_iommu_pgtable == PD_MODE_NONE) {
2359 		/*
2360 		 * Remove sysfs if DMA translation is not supported by the
2361 		 * IOMMU. Do not return an error to enable IRQ remapping
2362 		 * in state_next(), DTE[V, TV] must eventually be set to 0.
2363 		 */
2364 		iommu_device_sysfs_remove(&iommu->iommu);
2365 	}
2366 
2367 	return pci_enable_device(iommu->dev);
2368 }
2369 
2370 static void print_iommu_info(void)
2371 {
2372 	int i;
2373 	static const char * const feat_str[] = {
2374 		"PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
2375 		"IA", "GA", "HE", "PC"
2376 	};
2377 
2378 	if (amd_iommu_efr) {
2379 		pr_info("Extended features (%#llx, %#llx):", amd_iommu_efr, amd_iommu_efr2);
2380 
2381 		for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
2382 			if (check_feature(1ULL << i))
2383 				pr_cont(" %s", feat_str[i]);
2384 		}
2385 
2386 		if (check_feature(FEATURE_GAM_VAPIC))
2387 			pr_cont(" GA_vAPIC");
2388 
2389 		if (check_feature(FEATURE_SNP))
2390 			pr_cont(" SNP");
2391 
2392 		if (check_feature2(FEATURE_SEVSNPIO_SUP))
2393 			pr_cont(" SEV-TIO");
2394 
2395 		if (check_feature(FEATURE_PERF_OPT))
2396 			pr_cont(" PerfOpt");
2397 
2398 		pr_cont("\n");
2399 	}
2400 
2401 	if (irq_remapping_enabled) {
2402 		pr_info("Interrupt remapping enabled\n");
2403 		if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2404 			pr_info("X2APIC enabled\n");
2405 	}
2406 	if (amd_iommu_pgtable == PD_MODE_V2) {
2407 		pr_info("V2 page table enabled (Paging mode : %d level)\n",
2408 			amd_iommu_gpt_level);
2409 	}
2410 }
2411 
2412 static int __init amd_iommu_init_pci(void)
2413 {
2414 	struct amd_iommu *iommu;
2415 	struct amd_iommu_pci_seg *pci_seg;
2416 	int ret;
2417 
2418 	/* Init global identity domain before registering IOMMU */
2419 	amd_iommu_init_identity_domain();
2420 
2421 	for_each_iommu(iommu) {
2422 		ret = iommu_init_pci(iommu);
2423 		if (ret) {
2424 			pr_err("IOMMU%d: Failed to initialize IOMMU Hardware (error=%d)!\n",
2425 			       iommu->index, ret);
2426 			goto out;
2427 		}
2428 		/* Need to setup range after PCI init */
2429 		iommu_set_cwwb_range(iommu);
2430 	}
2431 
2432 	/*
2433 	 * Order is important here to make sure any unity map requirements are
2434 	 * fulfilled. The unity mappings are created and written to the device
2435 	 * table during the iommu_init_pci() call.
2436 	 *
2437 	 * After that we call init_device_table_dma() to make sure any
2438 	 * uninitialized DTE will block DMA, and in the end we flush the caches
2439 	 * of all IOMMUs to make sure the changes to the device table are
2440 	 * active.
2441 	 */
2442 	for_each_pci_segment(pci_seg)
2443 		init_device_table_dma(pci_seg);
2444 
2445 	for_each_iommu(iommu)
2446 		amd_iommu_flush_all_caches(iommu);
2447 
2448 	print_iommu_info();
2449 
2450 out:
2451 	return ret;
2452 }
2453 
2454 /****************************************************************************
2455  *
2456  * The following functions initialize the MSI interrupts for all IOMMUs
2457  * in the system. It's a bit challenging because there could be multiple
2458  * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per
2459  * pci_dev.
2460  *
2461  ****************************************************************************/
2462 
2463 static int iommu_setup_msi(struct amd_iommu *iommu)
2464 {
2465 	int r;
2466 
2467 	r = pci_enable_msi(iommu->dev);
2468 	if (r)
2469 		return r;
2470 
2471 	r = request_threaded_irq(iommu->dev->irq, NULL, amd_iommu_int_thread,
2472 				 IRQF_ONESHOT, "AMD-Vi", iommu);
2473 	if (r) {
2474 		pci_disable_msi(iommu->dev);
2475 		return r;
2476 	}
2477 
2478 	return 0;
2479 }
2480 
2481 union intcapxt {
2482 	u64	capxt;
2483 	struct {
2484 		u64	reserved_0		:  2,
2485 			dest_mode_logical	:  1,
2486 			reserved_1		:  5,
2487 			destid_0_23		: 24,
2488 			vector			:  8,
2489 			reserved_2		: 16,
2490 			destid_24_31		:  8;
2491 	};
2492 } __attribute__ ((packed));
2493 
2494 
2495 static struct irq_chip intcapxt_controller;
2496 
2497 static int intcapxt_irqdomain_activate(struct irq_domain *domain,
2498 				       struct irq_data *irqd, bool reserve)
2499 {
2500 	return 0;
2501 }
2502 
2503 static void intcapxt_irqdomain_deactivate(struct irq_domain *domain,
2504 					  struct irq_data *irqd)
2505 {
2506 }
2507 
2508 
2509 static int intcapxt_irqdomain_alloc(struct irq_domain *domain, unsigned int virq,
2510 				    unsigned int nr_irqs, void *arg)
2511 {
2512 	struct irq_alloc_info *info = arg;
2513 	int i, ret;
2514 
2515 	if (!info || info->type != X86_IRQ_ALLOC_TYPE_AMDVI)
2516 		return -EINVAL;
2517 
2518 	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
2519 	if (ret < 0)
2520 		return ret;
2521 
2522 	for (i = virq; i < virq + nr_irqs; i++) {
2523 		struct irq_data *irqd = irq_domain_get_irq_data(domain, i);
2524 
2525 		irqd->chip = &intcapxt_controller;
2526 		irqd->hwirq = info->hwirq;
2527 		irqd->chip_data = info->data;
2528 		__irq_set_handler(i, handle_edge_irq, 0, "edge");
2529 	}
2530 
2531 	return ret;
2532 }
2533 
2534 static void intcapxt_irqdomain_free(struct irq_domain *domain, unsigned int virq,
2535 				    unsigned int nr_irqs)
2536 {
2537 	irq_domain_free_irqs_top(domain, virq, nr_irqs);
2538 }
2539 
2540 
2541 static void intcapxt_unmask_irq(struct irq_data *irqd)
2542 {
2543 	struct amd_iommu *iommu = irqd->chip_data;
2544 	struct irq_cfg *cfg = irqd_cfg(irqd);
2545 	union intcapxt xt;
2546 
2547 	xt.capxt = 0ULL;
2548 	xt.dest_mode_logical = apic->dest_mode_logical;
2549 	xt.vector = cfg->vector;
2550 	xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0);
2551 	xt.destid_24_31 = cfg->dest_apicid >> 24;
2552 
2553 	writeq(xt.capxt, iommu->mmio_base + irqd->hwirq);
2554 }
2555 
2556 static void intcapxt_mask_irq(struct irq_data *irqd)
2557 {
2558 	struct amd_iommu *iommu = irqd->chip_data;
2559 
2560 	writeq(0, iommu->mmio_base + irqd->hwirq);
2561 }
2562 
2563 
2564 static int intcapxt_set_affinity(struct irq_data *irqd,
2565 				 const struct cpumask *mask, bool force)
2566 {
2567 	struct irq_data *parent = irqd->parent_data;
2568 	int ret;
2569 
2570 	ret = parent->chip->irq_set_affinity(parent, mask, force);
2571 	if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
2572 		return ret;
2573 	return 0;
2574 }
2575 
2576 static int intcapxt_set_wake(struct irq_data *irqd, unsigned int on)
2577 {
2578 	return on ? -EOPNOTSUPP : 0;
2579 }
2580 
2581 static struct irq_chip intcapxt_controller = {
2582 	.name			= "IOMMU-MSI",
2583 	.irq_unmask		= intcapxt_unmask_irq,
2584 	.irq_mask		= intcapxt_mask_irq,
2585 	.irq_ack		= irq_chip_ack_parent,
2586 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
2587 	.irq_set_affinity       = intcapxt_set_affinity,
2588 	.irq_set_wake		= intcapxt_set_wake,
2589 	.flags			= IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_MOVE_DEFERRED,
2590 };
2591 
2592 static const struct irq_domain_ops intcapxt_domain_ops = {
2593 	.alloc			= intcapxt_irqdomain_alloc,
2594 	.free			= intcapxt_irqdomain_free,
2595 	.activate		= intcapxt_irqdomain_activate,
2596 	.deactivate		= intcapxt_irqdomain_deactivate,
2597 };
2598 
2599 
2600 static struct irq_domain *iommu_irqdomain;
2601 
2602 static struct irq_domain *iommu_get_irqdomain(void)
2603 {
2604 	struct fwnode_handle *fn;
2605 
2606 	/* No need for locking here (yet) as the init is single-threaded */
2607 	if (iommu_irqdomain)
2608 		return iommu_irqdomain;
2609 
2610 	fn = irq_domain_alloc_named_fwnode("AMD-Vi-MSI");
2611 	if (!fn)
2612 		return NULL;
2613 
2614 	iommu_irqdomain = irq_domain_create_hierarchy(x86_vector_domain, 0, 0,
2615 						      fn, &intcapxt_domain_ops,
2616 						      NULL);
2617 	if (!iommu_irqdomain)
2618 		irq_domain_free_fwnode(fn);
2619 
2620 	return iommu_irqdomain;
2621 }
2622 
2623 static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname,
2624 				  int hwirq, irq_handler_t thread_fn)
2625 {
2626 	struct irq_domain *domain;
2627 	struct irq_alloc_info info;
2628 	int irq, ret;
2629 	int node = dev_to_node(&iommu->dev->dev);
2630 
2631 	domain = iommu_get_irqdomain();
2632 	if (!domain)
2633 		return -ENXIO;
2634 
2635 	init_irq_alloc_info(&info, NULL);
2636 	info.type = X86_IRQ_ALLOC_TYPE_AMDVI;
2637 	info.data = iommu;
2638 	info.hwirq = hwirq;
2639 
2640 	irq = irq_domain_alloc_irqs(domain, 1, node, &info);
2641 	if (irq < 0) {
2642 		irq_domain_remove(domain);
2643 		return irq;
2644 	}
2645 
2646 	ret = request_threaded_irq(irq, NULL, thread_fn, IRQF_ONESHOT, devname,
2647 				   iommu);
2648 	if (ret) {
2649 		irq_domain_free_irqs(irq, 1);
2650 		irq_domain_remove(domain);
2651 		return ret;
2652 	}
2653 
2654 	return 0;
2655 }
2656 
2657 static int iommu_setup_intcapxt(struct amd_iommu *iommu)
2658 {
2659 	int ret;
2660 
2661 	snprintf(iommu->evt_irq_name, sizeof(iommu->evt_irq_name),
2662 		 "AMD-Vi%d-Evt", iommu->index);
2663 	ret = __iommu_setup_intcapxt(iommu, iommu->evt_irq_name,
2664 				     MMIO_INTCAPXT_EVT_OFFSET,
2665 				     amd_iommu_int_thread_evtlog);
2666 	if (ret)
2667 		return ret;
2668 
2669 	snprintf(iommu->ppr_irq_name, sizeof(iommu->ppr_irq_name),
2670 		 "AMD-Vi%d-PPR", iommu->index);
2671 	ret = __iommu_setup_intcapxt(iommu, iommu->ppr_irq_name,
2672 				     MMIO_INTCAPXT_PPR_OFFSET,
2673 				     amd_iommu_int_thread_pprlog);
2674 	if (ret)
2675 		return ret;
2676 
2677 #ifdef CONFIG_IRQ_REMAP
2678 	snprintf(iommu->ga_irq_name, sizeof(iommu->ga_irq_name),
2679 		 "AMD-Vi%d-GA", iommu->index);
2680 	ret = __iommu_setup_intcapxt(iommu, iommu->ga_irq_name,
2681 				     MMIO_INTCAPXT_GALOG_OFFSET,
2682 				     amd_iommu_int_thread_galog);
2683 #endif
2684 
2685 	return ret;
2686 }
2687 
2688 static int iommu_init_irq(struct amd_iommu *iommu)
2689 {
2690 	int ret;
2691 
2692 	if (iommu->int_enabled)
2693 		goto enable_faults;
2694 
2695 	if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2696 		ret = iommu_setup_intcapxt(iommu);
2697 	else if (iommu->dev->msi_cap)
2698 		ret = iommu_setup_msi(iommu);
2699 	else
2700 		ret = -ENODEV;
2701 
2702 	if (ret)
2703 		return ret;
2704 
2705 	iommu->int_enabled = true;
2706 enable_faults:
2707 
2708 	if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2709 		iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);
2710 
2711 	iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
2712 
2713 	return 0;
2714 }
2715 
2716 /****************************************************************************
2717  *
2718  * The next functions belong to the third pass of parsing the ACPI
2719  * table. In this last pass the memory mapping requirements are
2720  * gathered (like exclusion and unity mapping ranges).
2721  *
2722  ****************************************************************************/
2723 
2724 static void __init free_unity_maps(void)
2725 {
2726 	struct unity_map_entry *entry, *next;
2727 	struct amd_iommu_pci_seg *p, *pci_seg;
2728 
2729 	for_each_pci_segment_safe(pci_seg, p) {
2730 		list_for_each_entry_safe(entry, next, &pci_seg->unity_map, list) {
2731 			list_del(&entry->list);
2732 			kfree(entry);
2733 		}
2734 	}
2735 }
2736 
2737 /* called for unity map ACPI definition */
2738 static int __init init_unity_map_range(struct ivmd_header *m,
2739 				       struct acpi_table_header *ivrs_base)
2740 {
2741 	struct unity_map_entry *e = NULL;
2742 	struct amd_iommu_pci_seg *pci_seg;
2743 	char *s;
2744 
2745 	pci_seg = get_pci_segment(m->pci_seg, ivrs_base);
2746 	if (pci_seg == NULL)
2747 		return -ENOMEM;
2748 
2749 	e = kzalloc_obj(*e);
2750 	if (e == NULL)
2751 		return -ENOMEM;
2752 
2753 	switch (m->type) {
2754 	default:
2755 		kfree(e);
2756 		return 0;
2757 	case ACPI_IVMD_TYPE:
2758 		s = "IVMD_TYPEi\t\t\t";
2759 		e->devid_start = e->devid_end = m->devid;
2760 		break;
2761 	case ACPI_IVMD_TYPE_ALL:
2762 		s = "IVMD_TYPE_ALL\t\t";
2763 		e->devid_start = 0;
2764 		e->devid_end = pci_seg->last_bdf;
2765 		break;
2766 	case ACPI_IVMD_TYPE_RANGE:
2767 		s = "IVMD_TYPE_RANGE\t\t";
2768 		e->devid_start = m->devid;
2769 		e->devid_end = m->aux;
2770 		break;
2771 	}
2772 	e->address_start = PAGE_ALIGN(m->range_start);
2773 	e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
2774 	e->prot = m->flags >> 1;
2775 
2776 	/*
2777 	 * Treat per-device exclusion ranges as r/w unity-mapped regions
2778 	 * since some buggy BIOSes might lead to the overwritten exclusion
2779 	 * range (exclusion_start and exclusion_length members). This
2780 	 * happens when there are multiple exclusion ranges (IVMD entries)
2781 	 * defined in ACPI table.
2782 	 */
2783 	if (m->flags & IVMD_FLAG_EXCL_RANGE)
2784 		e->prot = (IVMD_FLAG_IW | IVMD_FLAG_IR) >> 1;
2785 
2786 	DUMP_printk("%s devid_start: %04x:%02x:%02x.%x devid_end: "
2787 		    "%04x:%02x:%02x.%x range_start: %016llx range_end: %016llx"
2788 		    " flags: %x\n", s, m->pci_seg,
2789 		    PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start),
2790 		    PCI_FUNC(e->devid_start), m->pci_seg,
2791 		    PCI_BUS_NUM(e->devid_end),
2792 		    PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
2793 		    e->address_start, e->address_end, m->flags);
2794 
2795 	list_add_tail(&e->list, &pci_seg->unity_map);
2796 
2797 	return 0;
2798 }
2799 
2800 static bool ivrs_block_type_ivmd(u8 type)
2801 {
2802 	if (type == ACPI_IVMD_TYPE ||
2803 	    type == ACPI_IVMD_TYPE_ALL ||
2804 	    type == ACPI_IVMD_TYPE_RANGE)
2805 		return true;
2806 	return false;
2807 }
2808 
2809 /* iterates over all memory definitions we find in the ACPI table */
2810 static int __init init_memory_definitions(struct acpi_table_header *table)
2811 {
2812 	u8 *p = (u8 *)table, *end = (u8 *)table;
2813 	struct ivmd_header *m;
2814 
2815 	end += table->length;
2816 	p += IVRS_HEADER_LENGTH;
2817 
2818 	while (p < end) {
2819 		m = (struct ivmd_header *)p;
2820 		p += m->length;
2821 		if (!ivrs_block_type_ivmd(m->type))
2822 			continue;
2823 		if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE))
2824 			init_unity_map_range(m, table);
2825 	}
2826 
2827 	return 0;
2828 }
2829 
2830 /*
2831  * Init the device table to not allow DMA access for devices
2832  */
2833 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg)
2834 {
2835 	u32 devid;
2836 	struct dev_table_entry *dev_table = pci_seg->dev_table;
2837 
2838 	if (!dev_table || amd_iommu_pgtable == PD_MODE_NONE)
2839 		return;
2840 
2841 	for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
2842 		set_dte_bit(&dev_table[devid], DEV_ENTRY_VALID);
2843 		if (!amd_iommu_snp_en)
2844 			set_dte_bit(&dev_table[devid], DEV_ENTRY_TRANSLATION);
2845 	}
2846 }
2847 
2848 static void __init uninit_device_table_dma(struct amd_iommu_pci_seg *pci_seg)
2849 {
2850 	u32 devid;
2851 	struct dev_table_entry *dev_table = pci_seg->dev_table;
2852 
2853 	if (dev_table == NULL)
2854 		return;
2855 
2856 	for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
2857 		dev_table[devid].data[0] = 0ULL;
2858 		dev_table[devid].data[1] = 0ULL;
2859 	}
2860 }
2861 
2862 static void init_device_table(void)
2863 {
2864 	struct amd_iommu_pci_seg *pci_seg;
2865 	u32 devid;
2866 
2867 	if (!amd_iommu_irq_remap)
2868 		return;
2869 
2870 	for_each_pci_segment(pci_seg) {
2871 		for (devid = 0; devid <= pci_seg->last_bdf; ++devid)
2872 			set_dte_bit(&pci_seg->dev_table[devid], DEV_ENTRY_IRQ_TBL_EN);
2873 	}
2874 }
2875 
2876 static void iommu_init_flags(struct amd_iommu *iommu)
2877 {
2878 	iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?
2879 		iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :
2880 		iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);
2881 
2882 	iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?
2883 		iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :
2884 		iommu_feature_disable(iommu, CONTROL_PASSPW_EN);
2885 
2886 	iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?
2887 		iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :
2888 		iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);
2889 
2890 	iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?
2891 		iommu_feature_enable(iommu, CONTROL_ISOC_EN) :
2892 		iommu_feature_disable(iommu, CONTROL_ISOC_EN);
2893 
2894 	/*
2895 	 * make IOMMU memory accesses cache coherent
2896 	 */
2897 	iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
2898 
2899 	/* Set IOTLB invalidation timeout to 1s */
2900 	iommu_feature_set(iommu, CTRL_INV_TO_1S, CTRL_INV_TO_MASK, CONTROL_INV_TIMEOUT);
2901 
2902 	/* Enable Enhanced Peripheral Page Request Handling */
2903 	if (check_feature(FEATURE_EPHSUP))
2904 		iommu_feature_enable(iommu, CONTROL_EPH_EN);
2905 }
2906 
2907 static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
2908 {
2909 	int i, j;
2910 	u32 ioc_feature_control;
2911 	struct pci_dev *pdev = iommu->root_pdev;
2912 
2913 	/* RD890 BIOSes may not have completely reconfigured the iommu */
2914 	if (!is_rd890_iommu(iommu->dev) || !pdev)
2915 		return;
2916 
2917 	/*
2918 	 * First, we need to ensure that the iommu is enabled. This is
2919 	 * controlled by a register in the northbridge
2920 	 */
2921 
2922 	/* Select Northbridge indirect register 0x75 and enable writing */
2923 	pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));
2924 	pci_read_config_dword(pdev, 0x64, &ioc_feature_control);
2925 
2926 	/* Enable the iommu */
2927 	if (!(ioc_feature_control & 0x1))
2928 		pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);
2929 
2930 	/* Restore the iommu BAR */
2931 	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
2932 			       iommu->stored_addr_lo);
2933 	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,
2934 			       iommu->stored_addr_hi);
2935 
2936 	/* Restore the l1 indirect regs for each of the 6 l1s */
2937 	for (i = 0; i < 6; i++)
2938 		for (j = 0; j < 0x12; j++)
2939 			iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);
2940 
2941 	/* Restore the l2 indirect regs */
2942 	for (i = 0; i < 0x83; i++)
2943 		iommu_write_l2(iommu, i, iommu->stored_l2[i]);
2944 
2945 	/* Lock PCI setup registers */
2946 	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
2947 			       iommu->stored_addr_lo | 1);
2948 }
2949 
2950 static void iommu_enable_ga(struct amd_iommu *iommu)
2951 {
2952 #ifdef CONFIG_IRQ_REMAP
2953 	switch (amd_iommu_guest_ir) {
2954 	case AMD_IOMMU_GUEST_IR_VAPIC:
2955 	case AMD_IOMMU_GUEST_IR_LEGACY_GA:
2956 		iommu_feature_enable(iommu, CONTROL_GA_EN);
2957 		iommu->irte_ops = &irte_128_ops;
2958 		break;
2959 	default:
2960 		iommu->irte_ops = &irte_32_ops;
2961 		break;
2962 	}
2963 #endif
2964 }
2965 
2966 static void iommu_disable_irtcachedis(struct amd_iommu *iommu)
2967 {
2968 	iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);
2969 }
2970 
2971 static void iommu_enable_irtcachedis(struct amd_iommu *iommu)
2972 {
2973 	u64 ctrl;
2974 
2975 	if (!amd_iommu_irtcachedis)
2976 		return;
2977 
2978 	/*
2979 	 * Note:
2980 	 * The support for IRTCacheDis feature is dertermined by
2981 	 * checking if the bit is writable.
2982 	 */
2983 	iommu_feature_enable(iommu, CONTROL_IRTCACHEDIS);
2984 	ctrl = readq(iommu->mmio_base +  MMIO_CONTROL_OFFSET);
2985 	ctrl &= (1ULL << CONTROL_IRTCACHEDIS);
2986 	if (ctrl)
2987 		iommu->irtcachedis_enabled = true;
2988 	pr_info("iommu%d (%#06x) : IRT cache is %s\n",
2989 		iommu->index, iommu->devid,
2990 		iommu->irtcachedis_enabled ? "disabled" : "enabled");
2991 }
2992 
2993 static void iommu_enable_2k_int(struct amd_iommu *iommu)
2994 {
2995 	if (!FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))
2996 		return;
2997 
2998 	iommu_feature_set(iommu,
2999 			  CONTROL_NUM_INT_REMAP_MODE_2K,
3000 			  CONTROL_NUM_INT_REMAP_MODE_MASK,
3001 			  CONTROL_NUM_INT_REMAP_MODE);
3002 }
3003 
3004 static void early_enable_iommu(struct amd_iommu *iommu)
3005 {
3006 	iommu_disable(iommu);
3007 	iommu_init_flags(iommu);
3008 	iommu_set_device_table(iommu);
3009 	iommu_enable_command_buffer(iommu);
3010 	iommu_enable_gt(iommu);
3011 	iommu_enable_ga(iommu);
3012 	iommu_enable_xt(iommu);
3013 	iommu_enable_irtcachedis(iommu);
3014 	iommu_enable_2k_int(iommu);
3015 	iommu_enable(iommu);
3016 	amd_iommu_flush_all_caches(iommu);
3017 }
3018 
3019 /*
3020  * This function finally enables all IOMMUs found in the system after
3021  * they have been initialized.
3022  *
3023  * Or if in kdump kernel and IOMMUs are all pre-enabled, try to reuse
3024  * the old content of device table entries. Not this case or reuse failed,
3025  * just continue as normal kernel does.
3026  */
3027 static void early_enable_iommus(void)
3028 {
3029 	struct amd_iommu *iommu;
3030 	struct amd_iommu_pci_seg *pci_seg;
3031 
3032 	if (!reuse_device_table()) {
3033 		/*
3034 		 * If come here because of failure in reusing device table from old
3035 		 * kernel with all IOMMUs enabled, print error message and try to
3036 		 * free allocated old_dev_tbl_cpy.
3037 		 */
3038 		if (amd_iommu_pre_enabled) {
3039 			pr_err("Failed to reuse DEV table from previous kernel.\n");
3040 			/*
3041 			 * Bail out early if unable to remap/reuse DEV table from
3042 			 * previous kernel if SNP enabled as IOMMU commands will
3043 			 * time out without DEV table and cause kdump boot panic.
3044 			 */
3045 			BUG_ON(check_feature(FEATURE_SNP));
3046 		}
3047 
3048 		for_each_pci_segment(pci_seg) {
3049 			if (pci_seg->old_dev_tbl_cpy != NULL) {
3050 				memunmap((void *)pci_seg->old_dev_tbl_cpy);
3051 				pci_seg->old_dev_tbl_cpy = NULL;
3052 			}
3053 		}
3054 
3055 		for_each_iommu(iommu) {
3056 			clear_translation_pre_enabled(iommu);
3057 			early_enable_iommu(iommu);
3058 		}
3059 	} else {
3060 		pr_info("Reused DEV table from previous kernel.\n");
3061 
3062 		for_each_pci_segment(pci_seg) {
3063 			iommu_free_pages(pci_seg->dev_table);
3064 			pci_seg->dev_table = pci_seg->old_dev_tbl_cpy;
3065 		}
3066 
3067 		for_each_iommu(iommu) {
3068 			iommu_disable_command_buffer(iommu);
3069 			iommu_disable_event_buffer(iommu);
3070 			iommu_disable_irtcachedis(iommu);
3071 			iommu_enable_command_buffer(iommu);
3072 			iommu_enable_ga(iommu);
3073 			iommu_enable_xt(iommu);
3074 			iommu_enable_irtcachedis(iommu);
3075 			iommu_enable_2k_int(iommu);
3076 			iommu_set_device_table(iommu);
3077 			amd_iommu_flush_all_caches(iommu);
3078 		}
3079 	}
3080 }
3081 
3082 static void enable_iommus_ppr(void)
3083 {
3084 	struct amd_iommu *iommu;
3085 
3086 	if (!amd_iommu_gt_ppr_supported())
3087 		return;
3088 
3089 	for_each_iommu(iommu)
3090 		amd_iommu_enable_ppr_log(iommu);
3091 }
3092 
3093 static void enable_iommus_vapic(void)
3094 {
3095 #ifdef CONFIG_IRQ_REMAP
3096 	u32 status, i;
3097 	struct amd_iommu *iommu;
3098 
3099 	for_each_iommu(iommu) {
3100 		/*
3101 		 * Disable GALog if already running. It could have been enabled
3102 		 * in the previous boot before kdump.
3103 		 */
3104 		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
3105 		if (!(status & MMIO_STATUS_GALOG_RUN_MASK))
3106 			continue;
3107 
3108 		iommu_feature_disable(iommu, CONTROL_GALOG_EN);
3109 		iommu_feature_disable(iommu, CONTROL_GAINT_EN);
3110 
3111 		/*
3112 		 * Need to set and poll check the GALOGRun bit to zero before
3113 		 * we can set/ modify GA Log registers safely.
3114 		 */
3115 		for (i = 0; i < MMIO_STATUS_TIMEOUT; ++i) {
3116 			status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
3117 			if (!(status & MMIO_STATUS_GALOG_RUN_MASK))
3118 				break;
3119 			udelay(10);
3120 		}
3121 
3122 		if (WARN_ON(i >= MMIO_STATUS_TIMEOUT))
3123 			return;
3124 	}
3125 
3126 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
3127 		return;
3128 
3129 	/* Enabling GAM and SNPAVIC support */
3130 	for_each_iommu(iommu) {
3131 		if (!iommu_ga_log_enable(iommu))
3132 			return;
3133 
3134 		iommu_feature_enable(iommu, CONTROL_GAM_EN);
3135 		if (amd_iommu_snp_en)
3136 			iommu_feature_enable(iommu, CONTROL_SNPAVIC_EN);
3137 	}
3138 
3139 	amd_iommu_irq_ops.capability |= (1 << IRQ_POSTING_CAP);
3140 	pr_info("Virtual APIC enabled\n");
3141 #endif
3142 }
3143 
3144 static void disable_iommus(void)
3145 {
3146 	struct amd_iommu *iommu;
3147 
3148 	for_each_iommu(iommu)
3149 		iommu_disable(iommu);
3150 
3151 #ifdef CONFIG_IRQ_REMAP
3152 	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
3153 		amd_iommu_irq_ops.capability &= ~(1 << IRQ_POSTING_CAP);
3154 #endif
3155 }
3156 
3157 /*
3158  * Suspend/Resume support
3159  * disable suspend until real resume implemented
3160  */
3161 
3162 static void amd_iommu_resume(void *data)
3163 {
3164 	struct amd_iommu *iommu;
3165 
3166 	for_each_iommu(iommu)
3167 		iommu_apply_resume_quirks(iommu);
3168 
3169 	/* re-load the hardware */
3170 	for_each_iommu(iommu)
3171 		early_enable_iommu(iommu);
3172 
3173 	/* Restore the activation state saved at suspend. */
3174 	for_each_iommu(iommu) {
3175 		if (iommu->perfopt_enabled)
3176 			amd_iommu_perfopt_set(iommu, true);
3177 	}
3178 
3179 	iommu_enable_event_buffer();
3180 	amd_iommu_enable_interrupts();
3181 }
3182 
3183 static int amd_iommu_suspend(void *data)
3184 {
3185 	/* disable IOMMUs to go out of the way for BIOS */
3186 	disable_iommus();
3187 
3188 	return 0;
3189 }
3190 
3191 static const struct syscore_ops amd_iommu_syscore_ops = {
3192 	.suspend = amd_iommu_suspend,
3193 	.resume = amd_iommu_resume,
3194 };
3195 
3196 static struct syscore amd_iommu_syscore = {
3197 	.ops = &amd_iommu_syscore_ops,
3198 };
3199 
3200 static void __init free_iommu_resources(void)
3201 {
3202 	free_iommu_all();
3203 	free_pci_segments();
3204 }
3205 
3206 static bool __init check_sb_ioapic(int devid)
3207 {
3208 	u8 bus = PCI_BUS_NUM(devid);
3209 	u8 devfn = devid & 0xff;
3210 	u16 val;
3211 
3212 	val = read_pci_config_16(bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
3213 				 PCI_CLASS_DEVICE);
3214 
3215 	/*
3216 	 * The SB IOAPIC is integrated into the FCH (Southbridge), which is
3217 	 * exposed as an SMBus or ISA bridge in PCI config space.
3218 	 */
3219 	return val == PCI_CLASS_SERIAL_SMBUS || val == PCI_CLASS_BRIDGE_ISA;
3220 }
3221 
3222 /*
3223  * The Southbridge IOAPIC is assigned a GSI Base of 0 (handling interrupts
3224  * 0 through 23).
3225  */
3226 static int __init get_sb_ioapic_id(void)
3227 {
3228 	int idx = mp_find_ioapic(0);
3229 
3230 	if (idx < 0)
3231 		return -ENODEV;
3232 
3233 	return mpc_ioapic_id(idx);
3234 }
3235 
3236 static bool __init check_ioapic_information(void)
3237 {
3238 	const char *fw_bug = FW_BUG;
3239 	bool ret, has_sb_ioapic;
3240 	int idx, sb_apicid;
3241 
3242 	has_sb_ioapic = false;
3243 	ret           = true;
3244 
3245 	/*
3246 	 * If we have map overrides on the kernel command line the
3247 	 * messages in this function might not describe firmware bugs
3248 	 * anymore - so be careful
3249 	 */
3250 	if (cmdline_maps)
3251 		fw_bug = "";
3252 
3253 	sb_apicid = get_sb_ioapic_id();
3254 	if (sb_apicid < 0) {
3255 		/*
3256 		 * Lack of SB IOAPIC registration is not a firmware bug,
3257 		 * e.g. kernel booted with noapic or noacpi.
3258 		 */
3259 		fw_bug = "";
3260 		goto out;
3261 	}
3262 
3263 	for (idx = 0; idx < nr_ioapics; idx++) {
3264 		int devid, id = mpc_ioapic_id(idx);
3265 
3266 		devid = get_ioapic_devid(id);
3267 		if (devid < 0) {
3268 			pr_err("%s: IOAPIC[%d] not in IVRS table\n",
3269 				fw_bug, id);
3270 			ret = false;
3271 		} else if (id == sb_apicid && check_sb_ioapic(devid)) {
3272 			has_sb_ioapic = true;
3273 		}
3274 	}
3275 out:
3276 	if (!has_sb_ioapic) {
3277 		/*
3278 		 * We expect the SB IOAPIC to be listed in the IVRS
3279 		 * table. The system timer is connected to the SB IOAPIC
3280 		 * and if we don't have it in the list the system will
3281 		 * panic at boot time.  This situation usually happens
3282 		 * when the BIOS is buggy and provides us the wrong
3283 		 * device id for the IOAPIC in the system.
3284 		 */
3285 		pr_err("%s: No southbridge IOAPIC found\n", fw_bug);
3286 		ret = false;
3287 	}
3288 
3289 	if (!ret)
3290 		pr_err("Disabling interrupt remapping\n");
3291 
3292 	return ret;
3293 }
3294 
3295 static void __init free_dma_resources(void)
3296 {
3297 	amd_iommu_pdom_id_destroy();
3298 	free_unity_maps();
3299 }
3300 
3301 static void __init ivinfo_init(void *ivrs)
3302 {
3303 	amd_iommu_ivinfo = *((u32 *)(ivrs + IOMMU_IVINFO_OFFSET));
3304 }
3305 
3306 /*
3307  * This is the hardware init function for AMD IOMMU in the system.
3308  * This function is called either from amd_iommu_init or from the interrupt
3309  * remapping setup code.
3310  *
3311  * This function basically parses the ACPI table for AMD IOMMU (IVRS)
3312  * four times:
3313  *
3314  *	1 pass) Discover the most comprehensive IVHD type to use.
3315  *
3316  *	2 pass) Find the highest PCI device id the driver has to handle.
3317  *		Upon this information the size of the data structures is
3318  *		determined that needs to be allocated.
3319  *
3320  *	3 pass) Initialize the data structures just allocated with the
3321  *		information in the ACPI table about available AMD IOMMUs
3322  *		in the system. It also maps the PCI devices in the
3323  *		system to specific IOMMUs
3324  *
3325  *	4 pass) After the basic data structures are allocated and
3326  *		initialized we update them with information about memory
3327  *		remapping requirements parsed out of the ACPI table in
3328  *		this last pass.
3329  *
3330  * After everything is set up the IOMMUs are enabled and the necessary
3331  * hotplug and suspend notifiers are registered.
3332  */
3333 static int __init early_amd_iommu_init(void)
3334 {
3335 	struct acpi_table_header *ivrs_base;
3336 	int ret;
3337 	acpi_status status;
3338 	u8 efr_hats, max_vasize;
3339 
3340 	status = acpi_get_table("IVRS", 0, &ivrs_base);
3341 	if (status == AE_NOT_FOUND)
3342 		return -ENODEV;
3343 	else if (ACPI_FAILURE(status)) {
3344 		const char *err = acpi_format_exception(status);
3345 		pr_err("IVRS table error: %s\n", err);
3346 		return -EINVAL;
3347 	}
3348 
3349 	if (!boot_cpu_has(X86_FEATURE_CX16)) {
3350 		pr_err("Failed to initialize. The CMPXCHG16B feature is required.\n");
3351 		ret = -EINVAL;
3352 		goto out;
3353 	}
3354 
3355 	/*
3356 	 * Validate checksum here so we don't need to do it when
3357 	 * we actually parse the table
3358 	 */
3359 	ret = check_ivrs_checksum(ivrs_base);
3360 	if (ret)
3361 		goto out;
3362 
3363 	ivinfo_init(ivrs_base);
3364 
3365 	max_vasize = FIELD_GET(IOMMU_IVINFO_VASIZE, amd_iommu_ivinfo);
3366 	if (!max_vasize)
3367 		max_vasize = 64;
3368 
3369 	amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base);
3370 	DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type);
3371 
3372 	/*
3373 	 * now the data structures are allocated and basically initialized
3374 	 * start the real acpi table scan
3375 	 */
3376 	ret = init_iommu_all(ivrs_base);
3377 	if (ret)
3378 		goto out;
3379 
3380 	/* 5 level guest page table */
3381 	if (cpu_feature_enabled(X86_FEATURE_LA57) &&
3382 	    FIELD_GET(FEATURE_GATS, amd_iommu_efr) == GUEST_PGTABLE_5_LEVEL)
3383 		amd_iommu_gpt_level = PAGE_MODE_5_LEVEL;
3384 
3385 	efr_hats = FIELD_GET(FEATURE_HATS, amd_iommu_efr);
3386 	if (efr_hats != 0x3) {
3387 		/*
3388 		 * efr[HATS] bits specify the maximum host translation level
3389 		 * supported, with LEVEL 4 being initial max level.
3390 		 */
3391 		amd_iommu_hpt_vasize = min_t(unsigned int, max_vasize,
3392 					 (efr_hats + PAGE_MODE_4_LEVEL - 1) * 9 + 21);
3393 	} else {
3394 		pr_warn_once(FW_BUG "Disable host address translation due to invalid translation level (%#x).\n",
3395 			     efr_hats);
3396 		amd_iommu_hatdis = true;
3397 	}
3398 
3399 	if (amd_iommu_pgtable == PD_MODE_V2) {
3400 		if (!amd_iommu_v2_pgtbl_supported()) {
3401 			pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n");
3402 			amd_iommu_pgtable = PD_MODE_V1;
3403 		}
3404 	}
3405 
3406 	if (amd_iommu_hatdis) {
3407 		/*
3408 		 * Host (v1) page table is not available. Attempt to use
3409 		 * Guest (v2) page table.
3410 		 */
3411 		if (amd_iommu_v2_pgtbl_supported())
3412 			amd_iommu_pgtable = PD_MODE_V2;
3413 		else
3414 			amd_iommu_pgtable = PD_MODE_NONE;
3415 	}
3416 
3417 	/* Disable any previously enabled IOMMUs */
3418 	if (!is_kdump_kernel())
3419 		disable_iommus();
3420 
3421 	if (amd_iommu_irq_remap)
3422 		amd_iommu_irq_remap = check_ioapic_information();
3423 
3424 	if (amd_iommu_irq_remap) {
3425 		struct amd_iommu_pci_seg *pci_seg;
3426 		ret = -ENOMEM;
3427 		for_each_pci_segment(pci_seg) {
3428 			if (alloc_irq_lookup_table(pci_seg))
3429 				goto out;
3430 		}
3431 	}
3432 
3433 	ret = init_memory_definitions(ivrs_base);
3434 	if (ret)
3435 		goto out;
3436 
3437 	/* init the device table */
3438 	init_device_table();
3439 
3440 out:
3441 	/* Don't leak any ACPI memory */
3442 	acpi_put_table(ivrs_base);
3443 
3444 	return ret;
3445 }
3446 
3447 static int amd_iommu_enable_interrupts(void)
3448 {
3449 	struct amd_iommu *iommu;
3450 	int ret = 0;
3451 
3452 	for_each_iommu(iommu) {
3453 		ret = iommu_init_irq(iommu);
3454 		if (ret)
3455 			goto out;
3456 	}
3457 
3458 	/*
3459 	 * Interrupt handler is ready to process interrupts. Enable
3460 	 * PPR and GA log interrupt for all IOMMUs.
3461 	 */
3462 	enable_iommus_vapic();
3463 	enable_iommus_ppr();
3464 
3465 out:
3466 	return ret;
3467 }
3468 
3469 static bool __init detect_ivrs(void)
3470 {
3471 	struct acpi_table_header *ivrs_base;
3472 	acpi_status status;
3473 	int i;
3474 
3475 	status = acpi_get_table("IVRS", 0, &ivrs_base);
3476 	if (status == AE_NOT_FOUND)
3477 		return false;
3478 	else if (ACPI_FAILURE(status)) {
3479 		const char *err = acpi_format_exception(status);
3480 		pr_err("IVRS table error: %s\n", err);
3481 		return false;
3482 	}
3483 
3484 	acpi_put_table(ivrs_base);
3485 
3486 	if (amd_iommu_force_enable)
3487 		goto out;
3488 
3489 	/* Don't use IOMMU if there is Stoney Ridge graphics */
3490 	for (i = 0; i < 32; i++) {
3491 		u32 pci_id;
3492 
3493 		pci_id = read_pci_config(0, i, 0, 0);
3494 		if ((pci_id & 0xffff) == 0x1002 && (pci_id >> 16) == 0x98e4) {
3495 			pr_info("Disable IOMMU on Stoney Ridge\n");
3496 			return false;
3497 		}
3498 	}
3499 
3500 out:
3501 	/* Make sure ACS will be enabled during PCI probe */
3502 	pci_request_acs();
3503 
3504 	return true;
3505 }
3506 
3507 static __init void iommu_snp_enable(void)
3508 {
3509 #ifdef CONFIG_KVM_AMD_SEV
3510 	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
3511 		return;
3512 
3513 	amd_iommu_snp_mode0_sup = check_feature2(FEATURE_SNP_PAGE_MODE0_SUP);
3514 	/*
3515 	 * If SNP page mode 0 is not enabled, then SNP support requires that IOMMU
3516 	 * must be configured with V1 page table (DTE[Mode] != 0).
3517 	 */
3518 	if (!amd_iommu_snp_mode0_sup) {
3519 		if (iommu_default_passthrough()) {
3520 			pr_warn("SNP: IOMMU configured in passthrough mode, SNP cannot be supported.\n");
3521 			goto disable_snp;
3522 		}
3523 
3524 		if (amd_iommu_pgtable != PD_MODE_V1) {
3525 			pr_warn("SNP: IOMMU is configured with V2 page table mode, SNP cannot be supported.\n");
3526 			goto disable_snp;
3527 		}
3528 	}
3529 
3530 	amd_iommu_snp_en = check_feature(FEATURE_SNP);
3531 	if (!amd_iommu_snp_en) {
3532 		pr_warn("SNP: IOMMU SNP feature not enabled, SNP cannot be supported.\n");
3533 		goto disable_snp;
3534 	}
3535 
3536 	/*
3537 	 * Enable host SNP support once SNP support is checked on IOMMU.
3538 	 */
3539 	if (snp_rmptable_init()) {
3540 		pr_warn("SNP: RMP initialization failed, SNP cannot be supported.\n");
3541 		goto disable_snp;
3542 	}
3543 
3544 	pr_info("IOMMU SNP support enabled.\n");
3545 	return;
3546 
3547 disable_snp:
3548 	amd_iommu_snp_en = false;
3549 	cc_platform_clear(CC_ATTR_HOST_SEV_SNP);
3550 #endif
3551 }
3552 
3553 static void amd_iommu_apply_erratum_snp(void)
3554 {
3555 #ifdef CONFIG_KVM_AMD_SEV
3556 	if (!amd_iommu_snp_en)
3557 		return;
3558 
3559 	/* Errata fix for Family 0x19 */
3560 	if (boot_cpu_data.x86 != 0x19)
3561 		return;
3562 
3563 	/* Set event log buffer size to max */
3564 	amd_iommu_evtlog_size = EVTLOG_SIZE_MAX;
3565 	pr_info("Applying erratum: Increase Event log size to 0x%x\n",
3566 		amd_iommu_evtlog_size);
3567 
3568 	/*
3569 	 * Set PPR log buffer size to max.
3570 	 * (Family 0x19, model < 0x10 doesn't support PPR when SNP is enabled).
3571 	 */
3572 	if (boot_cpu_data.x86_model >= 0x10) {
3573 		amd_iommu_pprlog_size = PPRLOG_SIZE_MAX;
3574 		pr_info("Applying erratum: Increase PPR log size to 0x%x\n",
3575 			amd_iommu_pprlog_size);
3576 	}
3577 #endif
3578 }
3579 
3580 static bool amd_iommu_sme_check(void);
3581 
3582 /****************************************************************************
3583  *
3584  * AMD IOMMU Initialization State Machine
3585  *
3586  ****************************************************************************/
3587 
3588 static int __init state_next(void)
3589 {
3590 	int ret = 0;
3591 
3592 	switch (init_state) {
3593 	case IOMMU_START_STATE:
3594 		if (no_iommu || amd_iommu_disabled) {
3595 			init_state	= IOMMU_CMDLINE_DISABLED;
3596 			ret		= -EINVAL;
3597 		} else if (!amd_iommu_sme_check()) {
3598 			init_state	= IOMMU_INIT_ERROR;
3599 			ret		= -EINVAL;
3600 		} else if (!detect_ivrs()) {
3601 			init_state	= IOMMU_NOT_FOUND;
3602 			ret		= -ENODEV;
3603 		} else {
3604 			init_state	= IOMMU_IVRS_DETECTED;
3605 		}
3606 		break;
3607 	case IOMMU_IVRS_DETECTED:
3608 		ret = early_amd_iommu_init();
3609 		init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
3610 		break;
3611 	case IOMMU_ACPI_FINISHED:
3612 		early_enable_iommus();
3613 		x86_platform.iommu_shutdown = disable_iommus;
3614 		init_state = IOMMU_ENABLED;
3615 		break;
3616 	case IOMMU_ENABLED:
3617 		register_syscore(&amd_iommu_syscore);
3618 		iommu_snp_enable();
3619 
3620 		amd_iommu_apply_erratum_snp();
3621 
3622 		/* Allocate/enable event log buffer */
3623 		if (is_kdump_kernel())
3624 			ret = remap_event_buffer();
3625 		else
3626 			ret = alloc_event_buffer();
3627 
3628 		if (ret) {
3629 			init_state = IOMMU_INIT_ERROR;
3630 			break;
3631 		}
3632 		iommu_enable_event_buffer();
3633 
3634 		ret = amd_iommu_init_pci();
3635 		init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT;
3636 		break;
3637 	case IOMMU_PCI_INIT:
3638 		ret = amd_iommu_enable_interrupts();
3639 		init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN;
3640 		break;
3641 	case IOMMU_INTERRUPTS_EN:
3642 		init_state = IOMMU_INITIALIZED;
3643 		break;
3644 	case IOMMU_INITIALIZED:
3645 		/* Nothing to do */
3646 		break;
3647 	case IOMMU_NOT_FOUND:
3648 	case IOMMU_INIT_ERROR:
3649 	case IOMMU_CMDLINE_DISABLED:
3650 		/* Error states => do nothing */
3651 		ret = -EINVAL;
3652 		break;
3653 	default:
3654 		/* Unknown state */
3655 		BUG();
3656 	}
3657 
3658 	if (ret) {
3659 		free_dma_resources();
3660 		if (!irq_remapping_enabled) {
3661 			disable_iommus();
3662 			free_iommu_resources();
3663 		} else {
3664 			struct amd_iommu *iommu;
3665 			struct amd_iommu_pci_seg *pci_seg;
3666 
3667 			for_each_pci_segment(pci_seg)
3668 				uninit_device_table_dma(pci_seg);
3669 
3670 			for_each_iommu(iommu)
3671 				amd_iommu_flush_all_caches(iommu);
3672 		}
3673 	}
3674 	return ret;
3675 }
3676 
3677 static int __init iommu_go_to_state(enum iommu_init_state state)
3678 {
3679 	int ret = -EINVAL;
3680 
3681 	/*
3682 	 * Some essential housekeeping work is done by amd_iommu_detect().
3683 	 * Skipping calling it implies that the platform (e.g., Xen hypervisor)
3684 	 * has taken over the hardware. Progressing the state machine in this
3685 	 * case is worthless and fragile.
3686 	 *
3687 	 * There are several paths requesting later states, so disallow implicit
3688 	 * START_STATE => IVRS_DETECTED transition to prevent these paths from
3689 	 * accidentally progressing the state machine.
3690 	 */
3691 	if (init_state == IOMMU_START_STATE && state != IOMMU_IVRS_DETECTED)
3692 		goto out;
3693 
3694 	while (init_state != state) {
3695 		if (init_state == IOMMU_NOT_FOUND         ||
3696 		    init_state == IOMMU_INIT_ERROR        ||
3697 		    init_state == IOMMU_CMDLINE_DISABLED)
3698 			break;
3699 		ret = state_next();
3700 	}
3701 
3702 out:
3703 	/*
3704 	 * SNP platform initilazation requires IOMMUs to be fully configured.
3705 	 * If the SNP support on IOMMUs has NOT been checked, simply mark SNP
3706 	 * as unsupported. If the SNP support on IOMMUs has been checked and
3707 	 * host SNP support enabled but RMP enforcement has not been enabled
3708 	 * in IOMMUs, then the system is in a half-baked state, but can limp
3709 	 * along as all memory should be Hypervisor-Owned in the RMP. WARN,
3710 	 * but leave SNP as "supported" to avoid confusing the kernel.
3711 	 */
3712 	if (ret && cc_platform_has(CC_ATTR_HOST_SEV_SNP) &&
3713 	    !WARN_ON_ONCE(amd_iommu_snp_en))
3714 		cc_platform_clear(CC_ATTR_HOST_SEV_SNP);
3715 
3716 	return ret;
3717 }
3718 
3719 #ifdef CONFIG_IRQ_REMAP
3720 int __init amd_iommu_prepare(void)
3721 {
3722 	int ret;
3723 
3724 	amd_iommu_irq_remap = true;
3725 
3726 	ret = iommu_go_to_state(IOMMU_ACPI_FINISHED);
3727 	if (ret) {
3728 		amd_iommu_irq_remap = false;
3729 		return ret;
3730 	}
3731 
3732 	return amd_iommu_irq_remap ? 0 : -ENODEV;
3733 }
3734 
3735 int __init amd_iommu_enable(void)
3736 {
3737 	int ret;
3738 
3739 	ret = iommu_go_to_state(IOMMU_ENABLED);
3740 	if (ret)
3741 		return ret;
3742 
3743 	irq_remapping_enabled = 1;
3744 	return amd_iommu_xt_mode;
3745 }
3746 
3747 void amd_iommu_disable(void)
3748 {
3749 	amd_iommu_suspend(NULL);
3750 }
3751 
3752 int amd_iommu_reenable(int mode)
3753 {
3754 	amd_iommu_resume(NULL);
3755 
3756 	return 0;
3757 }
3758 
3759 int amd_iommu_enable_faulting(unsigned int cpu)
3760 {
3761 	/* We enable MSI later when PCI is initialized */
3762 	return 0;
3763 }
3764 #endif
3765 
3766 /*
3767  * This is the core init function for AMD IOMMU hardware in the system.
3768  * This function is called from the generic x86 DMA layer initialization
3769  * code.
3770  */
3771 static int __init amd_iommu_init(void)
3772 {
3773 	int ret;
3774 
3775 	ret = iommu_go_to_state(IOMMU_INITIALIZED);
3776 #ifdef CONFIG_GART_IOMMU
3777 	if (ret && list_empty(&amd_iommu_list)) {
3778 		/*
3779 		 * We failed to initialize the AMD IOMMU - try fallback
3780 		 * to GART if possible.
3781 		 */
3782 		gart_iommu_init();
3783 	}
3784 #endif
3785 
3786 	if (!ret)
3787 		amd_iommu_debugfs_setup();
3788 
3789 	return ret;
3790 }
3791 
3792 static bool amd_iommu_sme_check(void)
3793 {
3794 	if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) ||
3795 	    (boot_cpu_data.x86 != 0x17))
3796 		return true;
3797 
3798 	/* For Fam17h, a specific level of support is required */
3799 	if (boot_cpu_data.microcode >= 0x08001205)
3800 		return true;
3801 
3802 	if ((boot_cpu_data.microcode >= 0x08001126) &&
3803 	    (boot_cpu_data.microcode <= 0x080011ff))
3804 		return true;
3805 
3806 	pr_notice("IOMMU not currently supported when SME is active\n");
3807 
3808 	return false;
3809 }
3810 
3811 /****************************************************************************
3812  *
3813  * Early detect code. This code runs at IOMMU detection time in the DMA
3814  * layer. It just looks if there is an IVRS ACPI table to detect AMD
3815  * IOMMUs
3816  *
3817  ****************************************************************************/
3818 void __init amd_iommu_detect(void)
3819 {
3820 	int ret;
3821 
3822 	ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
3823 	if (ret)
3824 		goto disable_snp;
3825 
3826 	iommu_detected = 1;
3827 	x86_init.iommu.iommu_init = amd_iommu_init;
3828 	return;
3829 
3830 disable_snp:
3831 	if (cc_platform_has(CC_ATTR_HOST_SEV_SNP))
3832 		cc_platform_clear(CC_ATTR_HOST_SEV_SNP);
3833 }
3834 
3835 /****************************************************************************
3836  *
3837  * Parsing functions for the AMD IOMMU specific kernel command line
3838  * options.
3839  *
3840  ****************************************************************************/
3841 
3842 static int __init parse_amd_iommu_dump(char *str)
3843 {
3844 	amd_iommu_dump = true;
3845 
3846 	return 1;
3847 }
3848 
3849 static int __init parse_amd_iommu_intr(char *str)
3850 {
3851 	for (; *str; ++str) {
3852 		if (strncmp(str, "legacy", 6) == 0) {
3853 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
3854 			break;
3855 		}
3856 		if (strncmp(str, "vapic", 5) == 0) {
3857 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
3858 			break;
3859 		}
3860 	}
3861 	return 1;
3862 }
3863 
3864 static int __init parse_amd_iommu_options(char *str)
3865 {
3866 	if (!str)
3867 		return -EINVAL;
3868 
3869 	while (*str) {
3870 		if (strncmp(str, "fullflush", 9) == 0) {
3871 			pr_warn("amd_iommu=fullflush deprecated; use iommu.strict=1 instead\n");
3872 			iommu_set_dma_strict();
3873 		} else if (strncmp(str, "force_enable", 12) == 0) {
3874 			amd_iommu_force_enable = true;
3875 		} else if (strncmp(str, "off", 3) == 0) {
3876 			amd_iommu_disabled = true;
3877 		} else if (strncmp(str, "force_isolation", 15) == 0) {
3878 			amd_iommu_force_isolation = true;
3879 		} else if (strncmp(str, "pgtbl_v1", 8) == 0) {
3880 			amd_iommu_pgtable = PD_MODE_V1;
3881 		} else if (strncmp(str, "pgtbl_v2", 8) == 0) {
3882 			amd_iommu_pgtable = PD_MODE_V2;
3883 		} else if (strncmp(str, "irtcachedis", 11) == 0) {
3884 			amd_iommu_irtcachedis = true;
3885 		} else if (strncmp(str, "nohugepages", 11) == 0) {
3886 			pr_info("Restricting V1 page-sizes to 4KiB");
3887 			amd_iommu_pgsize_bitmap = AMD_IOMMU_PGSIZES_4K;
3888 		} else if (strncmp(str, "v2_pgsizes_only", 15) == 0) {
3889 			pr_info("Restricting V1 page-sizes to 4KiB/2MiB/1GiB");
3890 			amd_iommu_pgsize_bitmap = AMD_IOMMU_PGSIZES_V2;
3891 		} else {
3892 			pr_notice("Unknown option - '%s'\n", str);
3893 		}
3894 
3895 		str += strcspn(str, ",");
3896 		while (*str == ',')
3897 			str++;
3898 	}
3899 
3900 	return 1;
3901 }
3902 
3903 static int __init parse_ivrs_ioapic(char *str)
3904 {
3905 	u32 seg = 0, bus, dev, fn;
3906 	int id, i;
3907 	u32 devid;
3908 
3909 	if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3910 	    sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
3911 		goto found;
3912 
3913 	if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3914 	    sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
3915 		pr_warn("ivrs_ioapic%s option format deprecated; use ivrs_ioapic=%d@%04x:%02x:%02x.%d instead\n",
3916 			str, id, seg, bus, dev, fn);
3917 		goto found;
3918 	}
3919 
3920 	pr_err("Invalid command line: ivrs_ioapic%s\n", str);
3921 	return 1;
3922 
3923 found:
3924 	if (early_ioapic_map_size == EARLY_MAP_SIZE) {
3925 		pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n",
3926 			str);
3927 		return 1;
3928 	}
3929 
3930 	devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3931 
3932 	cmdline_maps			= true;
3933 	i				= early_ioapic_map_size++;
3934 	early_ioapic_map[i].id		= id;
3935 	early_ioapic_map[i].devid	= devid;
3936 	early_ioapic_map[i].cmd_line	= true;
3937 
3938 	return 1;
3939 }
3940 
3941 static int __init parse_ivrs_hpet(char *str)
3942 {
3943 	u32 seg = 0, bus, dev, fn;
3944 	int id, i;
3945 	u32 devid;
3946 
3947 	if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3948 	    sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
3949 		goto found;
3950 
3951 	if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3952 	    sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
3953 		pr_warn("ivrs_hpet%s option format deprecated; use ivrs_hpet=%d@%04x:%02x:%02x.%d instead\n",
3954 			str, id, seg, bus, dev, fn);
3955 		goto found;
3956 	}
3957 
3958 	pr_err("Invalid command line: ivrs_hpet%s\n", str);
3959 	return 1;
3960 
3961 found:
3962 	if (early_hpet_map_size == EARLY_MAP_SIZE) {
3963 		pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n",
3964 			str);
3965 		return 1;
3966 	}
3967 
3968 	devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3969 
3970 	cmdline_maps			= true;
3971 	i				= early_hpet_map_size++;
3972 	early_hpet_map[i].id		= id;
3973 	early_hpet_map[i].devid		= devid;
3974 	early_hpet_map[i].cmd_line	= true;
3975 
3976 	return 1;
3977 }
3978 
3979 #define ACPIID_LEN (ACPIHID_UID_LEN + ACPIHID_HID_LEN)
3980 
3981 static int __init parse_ivrs_acpihid(char *str)
3982 {
3983 	u32 seg = 0, bus, dev, fn;
3984 	char *hid, *uid, *p, *addr;
3985 	char acpiid[ACPIID_LEN + 1] = { }; /* size with NULL terminator */
3986 	int i;
3987 
3988 	addr = strchr(str, '@');
3989 	if (!addr) {
3990 		addr = strchr(str, '=');
3991 		if (!addr)
3992 			goto not_found;
3993 
3994 		++addr;
3995 
3996 		if (strlen(addr) > ACPIID_LEN)
3997 			goto not_found;
3998 
3999 		if (sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid) == 4 ||
4000 		    sscanf(str, "[%x:%x:%x.%x]=%s", &seg, &bus, &dev, &fn, acpiid) == 5) {
4001 			pr_warn("ivrs_acpihid%s option format deprecated; use ivrs_acpihid=%s@%04x:%02x:%02x.%d instead\n",
4002 				str, acpiid, seg, bus, dev, fn);
4003 			goto found;
4004 		}
4005 		goto not_found;
4006 	}
4007 
4008 	/* We have the '@', make it the terminator to get just the acpiid */
4009 	*addr++ = 0;
4010 
4011 	if (strlen(str) > ACPIID_LEN)
4012 		goto not_found;
4013 
4014 	if (sscanf(str, "=%s", acpiid) != 1)
4015 		goto not_found;
4016 
4017 	if (sscanf(addr, "%x:%x.%x", &bus, &dev, &fn) == 3 ||
4018 	    sscanf(addr, "%x:%x:%x.%x", &seg, &bus, &dev, &fn) == 4)
4019 		goto found;
4020 
4021 not_found:
4022 	pr_err("Invalid command line: ivrs_acpihid%s\n", str);
4023 	return 1;
4024 
4025 found:
4026 	if (early_acpihid_map_size == EARLY_MAP_SIZE) {
4027 		pr_err("Early ACPI HID map overflow - ignoring ivrs_acpihid%s\n",
4028 		       str);
4029 		return 1;
4030 	}
4031 
4032 	p = acpiid;
4033 	hid = strsep(&p, ":");
4034 	uid = p;
4035 
4036 	if (!hid || !(*hid) || !uid) {
4037 		pr_err("Invalid command line: hid or uid\n");
4038 		return 1;
4039 	}
4040 
4041 	/*
4042 	 * Ignore leading zeroes after ':', so e.g., AMDI0095:00
4043 	 * will match AMDI0095:0 in the second strcmp in acpi_dev_hid_uid_match
4044 	 */
4045 	while (*uid == '0' && *(uid + 1))
4046 		uid++;
4047 
4048 	if (strlen(hid) >= ACPIHID_HID_LEN) {
4049 		pr_err("Invalid command line: hid is too long\n");
4050 		return 1;
4051 	} else if (strlen(uid) >= ACPIHID_UID_LEN) {
4052 		pr_err("Invalid command line: uid is too long\n");
4053 		return 1;
4054 	}
4055 
4056 	i = early_acpihid_map_size++;
4057 	memcpy(early_acpihid_map[i].hid, hid, strlen(hid));
4058 	memcpy(early_acpihid_map[i].uid, uid, strlen(uid));
4059 	early_acpihid_map[i].devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
4060 	early_acpihid_map[i].cmd_line	= true;
4061 
4062 	return 1;
4063 }
4064 
4065 __setup("amd_iommu_dump",	parse_amd_iommu_dump);
4066 __setup("amd_iommu=",		parse_amd_iommu_options);
4067 __setup("amd_iommu_intr=",	parse_amd_iommu_intr);
4068 __setup("ivrs_ioapic",		parse_ivrs_ioapic);
4069 __setup("ivrs_hpet",		parse_ivrs_hpet);
4070 __setup("ivrs_acpihid",		parse_ivrs_acpihid);
4071 
4072 bool amd_iommu_pasid_supported(void)
4073 {
4074 	/* CPU page table size should match IOMMU guest page table size */
4075 	if (cpu_feature_enabled(X86_FEATURE_LA57) &&
4076 	    amd_iommu_gpt_level != PAGE_MODE_5_LEVEL)
4077 		return false;
4078 
4079 	if (!amd_iommu_gt_ppr_supported())
4080 		return false;
4081 
4082 	/*
4083 	 * If SNP page mode 0 is not supported, then DTE[Mode]=0 is prohibited
4084 	 * on SNP-enabled system (i.e. EFR[SNPSup]=1). IOMMUv2 page table
4085 	 * cannot be used without setting up IOMMUv1 page table.
4086 	 */
4087 	return (!amd_iommu_snp_en) || (amd_iommu_snp_en && amd_iommu_snp_mode0_sup);
4088 }
4089 
4090 struct amd_iommu *get_amd_iommu(unsigned int idx)
4091 {
4092 	unsigned int i = 0;
4093 	struct amd_iommu *iommu;
4094 
4095 	for_each_iommu(iommu)
4096 		if (i++ == idx)
4097 			return iommu;
4098 	return NULL;
4099 }
4100 
4101 /****************************************************************************
4102  *
4103  * IOMMU EFR Performance Counter support functionality. This code allows
4104  * access to the IOMMU PC functionality.
4105  *
4106  ****************************************************************************/
4107 
4108 u8 amd_iommu_pc_get_max_banks(unsigned int idx)
4109 {
4110 	struct amd_iommu *iommu = get_amd_iommu(idx);
4111 
4112 	if (iommu)
4113 		return iommu->max_banks;
4114 
4115 	return 0;
4116 }
4117 
4118 bool amd_iommu_pc_supported(void)
4119 {
4120 	return amd_iommu_pc_present;
4121 }
4122 
4123 u8 amd_iommu_pc_get_max_counters(unsigned int idx)
4124 {
4125 	struct amd_iommu *iommu = get_amd_iommu(idx);
4126 
4127 	if (iommu)
4128 		return iommu->max_counters;
4129 
4130 	return 0;
4131 }
4132 
4133 static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr,
4134 				u8 fxn, u64 *value, bool is_write)
4135 {
4136 	u32 offset;
4137 	u32 max_offset_lim;
4138 
4139 	/* Make sure the IOMMU PC resource is available */
4140 	if (!amd_iommu_pc_present)
4141 		return -ENODEV;
4142 
4143 	/* Check for valid iommu and pc register indexing */
4144 	if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7)))
4145 		return -ENODEV;
4146 
4147 	offset = (u32)(((0x40 | bank) << 12) | (cntr << 8) | fxn);
4148 
4149 	/* Limit the offset to the hw defined mmio region aperture */
4150 	max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) |
4151 				(iommu->max_counters << 8) | 0x28);
4152 	if ((offset < MMIO_CNTR_REG_OFFSET) ||
4153 	    (offset > max_offset_lim))
4154 		return -EINVAL;
4155 
4156 	if (is_write) {
4157 		u64 val = *value & GENMASK_ULL(47, 0);
4158 
4159 		writel((u32)val, iommu->mmio_base + offset);
4160 		writel((val >> 32), iommu->mmio_base + offset + 4);
4161 	} else {
4162 		*value = readl(iommu->mmio_base + offset + 4);
4163 		*value <<= 32;
4164 		*value |= readl(iommu->mmio_base + offset);
4165 		*value &= GENMASK_ULL(47, 0);
4166 	}
4167 
4168 	return 0;
4169 }
4170 
4171 int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)
4172 {
4173 	if (!iommu)
4174 		return -EINVAL;
4175 
4176 	return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false);
4177 }
4178 
4179 int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)
4180 {
4181 	if (!iommu)
4182 		return -EINVAL;
4183 
4184 	return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true);
4185 }
4186 
4187 #ifdef CONFIG_KVM_AMD_SEV
4188 static int iommu_page_make_shared(void *page)
4189 {
4190 	unsigned long paddr, pfn;
4191 
4192 	paddr = iommu_virt_to_phys(page);
4193 	/* Cbit maybe set in the paddr */
4194 	pfn = __sme_clr(paddr) >> PAGE_SHIFT;
4195 
4196 	if (!(pfn % PTRS_PER_PMD)) {
4197 		int ret, level;
4198 		bool assigned;
4199 
4200 		ret = snp_lookup_rmpentry(pfn, &assigned, &level);
4201 		if (ret) {
4202 			pr_warn("IOMMU PFN %lx RMP lookup failed, ret %d\n", pfn, ret);
4203 			return ret;
4204 		}
4205 
4206 		if (!assigned) {
4207 			pr_warn("IOMMU PFN %lx not assigned in RMP table\n", pfn);
4208 			return -EINVAL;
4209 		}
4210 
4211 		if (level > PG_LEVEL_4K) {
4212 			ret = psmash(pfn);
4213 			if (!ret)
4214 				goto done;
4215 
4216 			pr_warn("PSMASH failed for IOMMU PFN %lx huge RMP entry, ret: %d, level: %d\n",
4217 				pfn, ret, level);
4218 			return ret;
4219 		}
4220 	}
4221 
4222 done:
4223 	return rmp_make_shared(pfn, PG_LEVEL_4K);
4224 }
4225 
4226 static int iommu_make_shared(void *va, size_t size)
4227 {
4228 	void *page;
4229 	int ret;
4230 
4231 	if (!va)
4232 		return 0;
4233 
4234 	for (page = va; page < (va + size); page += PAGE_SIZE) {
4235 		ret = iommu_page_make_shared(page);
4236 		if (ret)
4237 			return ret;
4238 	}
4239 
4240 	return 0;
4241 }
4242 
4243 int amd_iommu_snp_disable(void)
4244 {
4245 	struct amd_iommu *iommu;
4246 	int ret;
4247 
4248 	if (!amd_iommu_snp_en)
4249 		return 0;
4250 
4251 	for_each_iommu(iommu) {
4252 		ret = iommu_make_shared(iommu->evt_buf, amd_iommu_evtlog_size);
4253 		if (ret)
4254 			return ret;
4255 
4256 		ret = iommu_make_shared(iommu->ppr_log, amd_iommu_pprlog_size);
4257 		if (ret)
4258 			return ret;
4259 
4260 		ret = iommu_make_shared((void *)iommu->cmd_sem, PAGE_SIZE);
4261 		if (ret)
4262 			return ret;
4263 	}
4264 
4265 	return 0;
4266 }
4267 EXPORT_SYMBOL_GPL(amd_iommu_snp_disable);
4268 
4269 bool amd_iommu_sev_tio_supported(void)
4270 {
4271 	return check_feature2(FEATURE_SEVSNPIO_SUP);
4272 }
4273 EXPORT_SYMBOL_GPL(amd_iommu_sev_tio_supported);
4274 #endif
4275