xref: /linux/drivers/iommu/amd/iommu.c (revision d457a0e329b0bfd3a1450e0b1a18cd2b47a25a08)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4  * Author: Joerg Roedel <jroedel@suse.de>
5  *         Leo Duran <leo.duran@amd.com>
6  */
7 
8 #define pr_fmt(fmt)     "AMD-Vi: " fmt
9 #define dev_fmt(fmt)    pr_fmt(fmt)
10 
11 #include <linux/ratelimit.h>
12 #include <linux/pci.h>
13 #include <linux/acpi.h>
14 #include <linux/pci-ats.h>
15 #include <linux/bitmap.h>
16 #include <linux/slab.h>
17 #include <linux/debugfs.h>
18 #include <linux/scatterlist.h>
19 #include <linux/dma-map-ops.h>
20 #include <linux/dma-direct.h>
21 #include <linux/iommu-helper.h>
22 #include <linux/delay.h>
23 #include <linux/amd-iommu.h>
24 #include <linux/notifier.h>
25 #include <linux/export.h>
26 #include <linux/irq.h>
27 #include <linux/msi.h>
28 #include <linux/irqdomain.h>
29 #include <linux/percpu.h>
30 #include <linux/io-pgtable.h>
31 #include <linux/cc_platform.h>
32 #include <asm/irq_remapping.h>
33 #include <asm/io_apic.h>
34 #include <asm/apic.h>
35 #include <asm/hw_irq.h>
36 #include <asm/proto.h>
37 #include <asm/iommu.h>
38 #include <asm/gart.h>
39 #include <asm/dma.h>
40 
41 #include "amd_iommu.h"
42 #include "../dma-iommu.h"
43 #include "../irq_remapping.h"
44 
45 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
46 
47 #define LOOP_TIMEOUT	100000
48 
49 /* IO virtual address start page frame number */
50 #define IOVA_START_PFN		(1)
51 #define IOVA_PFN(addr)		((addr) >> PAGE_SHIFT)
52 
53 /* Reserved IOVA ranges */
54 #define MSI_RANGE_START		(0xfee00000)
55 #define MSI_RANGE_END		(0xfeefffff)
56 #define HT_RANGE_START		(0xfd00000000ULL)
57 #define HT_RANGE_END		(0xffffffffffULL)
58 
59 #define DEFAULT_PGTABLE_LEVEL	PAGE_MODE_3_LEVEL
60 
61 static DEFINE_SPINLOCK(pd_bitmap_lock);
62 
63 LIST_HEAD(ioapic_map);
64 LIST_HEAD(hpet_map);
65 LIST_HEAD(acpihid_map);
66 
67 const struct iommu_ops amd_iommu_ops;
68 
69 static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
70 int amd_iommu_max_glx_val = -1;
71 
72 /*
73  * general struct to manage commands send to an IOMMU
74  */
75 struct iommu_cmd {
76 	u32 data[4];
77 };
78 
79 struct kmem_cache *amd_iommu_irq_cache;
80 
81 static void detach_device(struct device *dev);
82 static int domain_enable_v2(struct protection_domain *domain, int pasids);
83 
84 /****************************************************************************
85  *
86  * Helper functions
87  *
88  ****************************************************************************/
89 
90 static inline int get_acpihid_device_id(struct device *dev,
91 					struct acpihid_map_entry **entry)
92 {
93 	struct acpi_device *adev = ACPI_COMPANION(dev);
94 	struct acpihid_map_entry *p;
95 
96 	if (!adev)
97 		return -ENODEV;
98 
99 	list_for_each_entry(p, &acpihid_map, list) {
100 		if (acpi_dev_hid_uid_match(adev, p->hid,
101 					   p->uid[0] ? p->uid : NULL)) {
102 			if (entry)
103 				*entry = p;
104 			return p->devid;
105 		}
106 	}
107 	return -EINVAL;
108 }
109 
110 static inline int get_device_sbdf_id(struct device *dev)
111 {
112 	int sbdf;
113 
114 	if (dev_is_pci(dev))
115 		sbdf = get_pci_sbdf_id(to_pci_dev(dev));
116 	else
117 		sbdf = get_acpihid_device_id(dev, NULL);
118 
119 	return sbdf;
120 }
121 
122 struct dev_table_entry *get_dev_table(struct amd_iommu *iommu)
123 {
124 	struct dev_table_entry *dev_table;
125 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
126 
127 	BUG_ON(pci_seg == NULL);
128 	dev_table = pci_seg->dev_table;
129 	BUG_ON(dev_table == NULL);
130 
131 	return dev_table;
132 }
133 
134 static inline u16 get_device_segment(struct device *dev)
135 {
136 	u16 seg;
137 
138 	if (dev_is_pci(dev)) {
139 		struct pci_dev *pdev = to_pci_dev(dev);
140 
141 		seg = pci_domain_nr(pdev->bus);
142 	} else {
143 		u32 devid = get_acpihid_device_id(dev, NULL);
144 
145 		seg = PCI_SBDF_TO_SEGID(devid);
146 	}
147 
148 	return seg;
149 }
150 
151 /* Writes the specific IOMMU for a device into the PCI segment rlookup table */
152 void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid)
153 {
154 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
155 
156 	pci_seg->rlookup_table[devid] = iommu;
157 }
158 
159 static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid)
160 {
161 	struct amd_iommu_pci_seg *pci_seg;
162 
163 	for_each_pci_segment(pci_seg) {
164 		if (pci_seg->id == seg)
165 			return pci_seg->rlookup_table[devid];
166 	}
167 	return NULL;
168 }
169 
170 static struct amd_iommu *rlookup_amd_iommu(struct device *dev)
171 {
172 	u16 seg = get_device_segment(dev);
173 	int devid = get_device_sbdf_id(dev);
174 
175 	if (devid < 0)
176 		return NULL;
177 	return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid));
178 }
179 
180 static struct protection_domain *to_pdomain(struct iommu_domain *dom)
181 {
182 	return container_of(dom, struct protection_domain, domain);
183 }
184 
185 static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid)
186 {
187 	struct iommu_dev_data *dev_data;
188 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
189 
190 	dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
191 	if (!dev_data)
192 		return NULL;
193 
194 	spin_lock_init(&dev_data->lock);
195 	dev_data->devid = devid;
196 	ratelimit_default_init(&dev_data->rs);
197 
198 	llist_add(&dev_data->dev_data_list, &pci_seg->dev_data_list);
199 	return dev_data;
200 }
201 
202 static struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid)
203 {
204 	struct iommu_dev_data *dev_data;
205 	struct llist_node *node;
206 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
207 
208 	if (llist_empty(&pci_seg->dev_data_list))
209 		return NULL;
210 
211 	node = pci_seg->dev_data_list.first;
212 	llist_for_each_entry(dev_data, node, dev_data_list) {
213 		if (dev_data->devid == devid)
214 			return dev_data;
215 	}
216 
217 	return NULL;
218 }
219 
220 static int clone_alias(struct pci_dev *pdev, u16 alias, void *data)
221 {
222 	struct amd_iommu *iommu;
223 	struct dev_table_entry *dev_table;
224 	u16 devid = pci_dev_id(pdev);
225 
226 	if (devid == alias)
227 		return 0;
228 
229 	iommu = rlookup_amd_iommu(&pdev->dev);
230 	if (!iommu)
231 		return 0;
232 
233 	amd_iommu_set_rlookup_table(iommu, alias);
234 	dev_table = get_dev_table(iommu);
235 	memcpy(dev_table[alias].data,
236 	       dev_table[devid].data,
237 	       sizeof(dev_table[alias].data));
238 
239 	return 0;
240 }
241 
242 static void clone_aliases(struct amd_iommu *iommu, struct device *dev)
243 {
244 	struct pci_dev *pdev;
245 
246 	if (!dev_is_pci(dev))
247 		return;
248 	pdev = to_pci_dev(dev);
249 
250 	/*
251 	 * The IVRS alias stored in the alias table may not be
252 	 * part of the PCI DMA aliases if it's bus differs
253 	 * from the original device.
254 	 */
255 	clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL);
256 
257 	pci_for_each_dma_alias(pdev, clone_alias, NULL);
258 }
259 
260 static void setup_aliases(struct amd_iommu *iommu, struct device *dev)
261 {
262 	struct pci_dev *pdev = to_pci_dev(dev);
263 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
264 	u16 ivrs_alias;
265 
266 	/* For ACPI HID devices, there are no aliases */
267 	if (!dev_is_pci(dev))
268 		return;
269 
270 	/*
271 	 * Add the IVRS alias to the pci aliases if it is on the same
272 	 * bus. The IVRS table may know about a quirk that we don't.
273 	 */
274 	ivrs_alias = pci_seg->alias_table[pci_dev_id(pdev)];
275 	if (ivrs_alias != pci_dev_id(pdev) &&
276 	    PCI_BUS_NUM(ivrs_alias) == pdev->bus->number)
277 		pci_add_dma_alias(pdev, ivrs_alias & 0xff, 1);
278 
279 	clone_aliases(iommu, dev);
280 }
281 
282 static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid)
283 {
284 	struct iommu_dev_data *dev_data;
285 
286 	dev_data = search_dev_data(iommu, devid);
287 
288 	if (dev_data == NULL) {
289 		dev_data = alloc_dev_data(iommu, devid);
290 		if (!dev_data)
291 			return NULL;
292 
293 		if (translation_pre_enabled(iommu))
294 			dev_data->defer_attach = true;
295 	}
296 
297 	return dev_data;
298 }
299 
300 /*
301 * Find or create an IOMMU group for a acpihid device.
302 */
303 static struct iommu_group *acpihid_device_group(struct device *dev)
304 {
305 	struct acpihid_map_entry *p, *entry = NULL;
306 	int devid;
307 
308 	devid = get_acpihid_device_id(dev, &entry);
309 	if (devid < 0)
310 		return ERR_PTR(devid);
311 
312 	list_for_each_entry(p, &acpihid_map, list) {
313 		if ((devid == p->devid) && p->group)
314 			entry->group = p->group;
315 	}
316 
317 	if (!entry->group)
318 		entry->group = generic_device_group(dev);
319 	else
320 		iommu_group_ref_get(entry->group);
321 
322 	return entry->group;
323 }
324 
325 static bool pci_iommuv2_capable(struct pci_dev *pdev)
326 {
327 	static const int caps[] = {
328 		PCI_EXT_CAP_ID_PRI,
329 		PCI_EXT_CAP_ID_PASID,
330 	};
331 	int i, pos;
332 
333 	if (!pci_ats_supported(pdev))
334 		return false;
335 
336 	for (i = 0; i < 2; ++i) {
337 		pos = pci_find_ext_capability(pdev, caps[i]);
338 		if (pos == 0)
339 			return false;
340 	}
341 
342 	return true;
343 }
344 
345 /*
346  * This function checks if the driver got a valid device from the caller to
347  * avoid dereferencing invalid pointers.
348  */
349 static bool check_device(struct device *dev)
350 {
351 	struct amd_iommu_pci_seg *pci_seg;
352 	struct amd_iommu *iommu;
353 	int devid, sbdf;
354 
355 	if (!dev)
356 		return false;
357 
358 	sbdf = get_device_sbdf_id(dev);
359 	if (sbdf < 0)
360 		return false;
361 	devid = PCI_SBDF_TO_DEVID(sbdf);
362 
363 	iommu = rlookup_amd_iommu(dev);
364 	if (!iommu)
365 		return false;
366 
367 	/* Out of our scope? */
368 	pci_seg = iommu->pci_seg;
369 	if (devid > pci_seg->last_bdf)
370 		return false;
371 
372 	return true;
373 }
374 
375 static int iommu_init_device(struct amd_iommu *iommu, struct device *dev)
376 {
377 	struct iommu_dev_data *dev_data;
378 	int devid, sbdf;
379 
380 	if (dev_iommu_priv_get(dev))
381 		return 0;
382 
383 	sbdf = get_device_sbdf_id(dev);
384 	if (sbdf < 0)
385 		return sbdf;
386 
387 	devid = PCI_SBDF_TO_DEVID(sbdf);
388 	dev_data = find_dev_data(iommu, devid);
389 	if (!dev_data)
390 		return -ENOMEM;
391 
392 	dev_data->dev = dev;
393 	setup_aliases(iommu, dev);
394 
395 	/*
396 	 * By default we use passthrough mode for IOMMUv2 capable device.
397 	 * But if amd_iommu=force_isolation is set (e.g. to debug DMA to
398 	 * invalid address), we ignore the capability for the device so
399 	 * it'll be forced to go into translation mode.
400 	 */
401 	if ((iommu_default_passthrough() || !amd_iommu_force_isolation) &&
402 	    dev_is_pci(dev) && pci_iommuv2_capable(to_pci_dev(dev))) {
403 		dev_data->iommu_v2 = iommu->is_iommu_v2;
404 	}
405 
406 	dev_iommu_priv_set(dev, dev_data);
407 
408 	return 0;
409 }
410 
411 static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev)
412 {
413 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
414 	struct dev_table_entry *dev_table = get_dev_table(iommu);
415 	int devid, sbdf;
416 
417 	sbdf = get_device_sbdf_id(dev);
418 	if (sbdf < 0)
419 		return;
420 
421 	devid = PCI_SBDF_TO_DEVID(sbdf);
422 	pci_seg->rlookup_table[devid] = NULL;
423 	memset(&dev_table[devid], 0, sizeof(struct dev_table_entry));
424 
425 	setup_aliases(iommu, dev);
426 }
427 
428 static void amd_iommu_uninit_device(struct device *dev)
429 {
430 	struct iommu_dev_data *dev_data;
431 
432 	dev_data = dev_iommu_priv_get(dev);
433 	if (!dev_data)
434 		return;
435 
436 	if (dev_data->domain)
437 		detach_device(dev);
438 
439 	dev_iommu_priv_set(dev, NULL);
440 
441 	/*
442 	 * We keep dev_data around for unplugged devices and reuse it when the
443 	 * device is re-plugged - not doing so would introduce a ton of races.
444 	 */
445 }
446 
447 /****************************************************************************
448  *
449  * Interrupt handling functions
450  *
451  ****************************************************************************/
452 
453 static void dump_dte_entry(struct amd_iommu *iommu, u16 devid)
454 {
455 	int i;
456 	struct dev_table_entry *dev_table = get_dev_table(iommu);
457 
458 	for (i = 0; i < 4; ++i)
459 		pr_err("DTE[%d]: %016llx\n", i, dev_table[devid].data[i]);
460 }
461 
462 static void dump_command(unsigned long phys_addr)
463 {
464 	struct iommu_cmd *cmd = iommu_phys_to_virt(phys_addr);
465 	int i;
466 
467 	for (i = 0; i < 4; ++i)
468 		pr_err("CMD[%d]: %08x\n", i, cmd->data[i]);
469 }
470 
471 static void amd_iommu_report_rmp_hw_error(struct amd_iommu *iommu, volatile u32 *event)
472 {
473 	struct iommu_dev_data *dev_data = NULL;
474 	int devid, vmg_tag, flags;
475 	struct pci_dev *pdev;
476 	u64 spa;
477 
478 	devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
479 	vmg_tag = (event[1]) & 0xFFFF;
480 	flags   = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
481 	spa     = ((u64)event[3] << 32) | (event[2] & 0xFFFFFFF8);
482 
483 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),
484 					   devid & 0xff);
485 	if (pdev)
486 		dev_data = dev_iommu_priv_get(&pdev->dev);
487 
488 	if (dev_data) {
489 		if (__ratelimit(&dev_data->rs)) {
490 			pci_err(pdev, "Event logged [RMP_HW_ERROR vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n",
491 				vmg_tag, spa, flags);
492 		}
493 	} else {
494 		pr_err_ratelimited("Event logged [RMP_HW_ERROR device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n",
495 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
496 			vmg_tag, spa, flags);
497 	}
498 
499 	if (pdev)
500 		pci_dev_put(pdev);
501 }
502 
503 static void amd_iommu_report_rmp_fault(struct amd_iommu *iommu, volatile u32 *event)
504 {
505 	struct iommu_dev_data *dev_data = NULL;
506 	int devid, flags_rmp, vmg_tag, flags;
507 	struct pci_dev *pdev;
508 	u64 gpa;
509 
510 	devid     = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
511 	flags_rmp = (event[0] >> EVENT_FLAGS_SHIFT) & 0xFF;
512 	vmg_tag   = (event[1]) & 0xFFFF;
513 	flags     = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
514 	gpa       = ((u64)event[3] << 32) | event[2];
515 
516 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),
517 					   devid & 0xff);
518 	if (pdev)
519 		dev_data = dev_iommu_priv_get(&pdev->dev);
520 
521 	if (dev_data) {
522 		if (__ratelimit(&dev_data->rs)) {
523 			pci_err(pdev, "Event logged [RMP_PAGE_FAULT vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n",
524 				vmg_tag, gpa, flags_rmp, flags);
525 		}
526 	} else {
527 		pr_err_ratelimited("Event logged [RMP_PAGE_FAULT device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n",
528 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
529 			vmg_tag, gpa, flags_rmp, flags);
530 	}
531 
532 	if (pdev)
533 		pci_dev_put(pdev);
534 }
535 
536 #define IS_IOMMU_MEM_TRANSACTION(flags)		\
537 	(((flags) & EVENT_FLAG_I) == 0)
538 
539 #define IS_WRITE_REQUEST(flags)			\
540 	((flags) & EVENT_FLAG_RW)
541 
542 static void amd_iommu_report_page_fault(struct amd_iommu *iommu,
543 					u16 devid, u16 domain_id,
544 					u64 address, int flags)
545 {
546 	struct iommu_dev_data *dev_data = NULL;
547 	struct pci_dev *pdev;
548 
549 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),
550 					   devid & 0xff);
551 	if (pdev)
552 		dev_data = dev_iommu_priv_get(&pdev->dev);
553 
554 	if (dev_data) {
555 		/*
556 		 * If this is a DMA fault (for which the I(nterrupt)
557 		 * bit will be unset), allow report_iommu_fault() to
558 		 * prevent logging it.
559 		 */
560 		if (IS_IOMMU_MEM_TRANSACTION(flags)) {
561 			/* Device not attached to domain properly */
562 			if (dev_data->domain == NULL) {
563 				pr_err_ratelimited("Event logged [Device not attached to domain properly]\n");
564 				pr_err_ratelimited("  device=%04x:%02x:%02x.%x domain=0x%04x\n",
565 						   iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid),
566 						   PCI_FUNC(devid), domain_id);
567 				goto out;
568 			}
569 
570 			if (!report_iommu_fault(&dev_data->domain->domain,
571 						&pdev->dev, address,
572 						IS_WRITE_REQUEST(flags) ?
573 							IOMMU_FAULT_WRITE :
574 							IOMMU_FAULT_READ))
575 				goto out;
576 		}
577 
578 		if (__ratelimit(&dev_data->rs)) {
579 			pci_err(pdev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n",
580 				domain_id, address, flags);
581 		}
582 	} else {
583 		pr_err_ratelimited("Event logged [IO_PAGE_FAULT device=%04x:%02x:%02x.%x domain=0x%04x address=0x%llx flags=0x%04x]\n",
584 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
585 			domain_id, address, flags);
586 	}
587 
588 out:
589 	if (pdev)
590 		pci_dev_put(pdev);
591 }
592 
593 static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
594 {
595 	struct device *dev = iommu->iommu.dev;
596 	int type, devid, flags, tag;
597 	volatile u32 *event = __evt;
598 	int count = 0;
599 	u64 address;
600 	u32 pasid;
601 
602 retry:
603 	type    = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
604 	devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
605 	pasid   = (event[0] & EVENT_DOMID_MASK_HI) |
606 		  (event[1] & EVENT_DOMID_MASK_LO);
607 	flags   = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
608 	address = (u64)(((u64)event[3]) << 32) | event[2];
609 
610 	if (type == 0) {
611 		/* Did we hit the erratum? */
612 		if (++count == LOOP_TIMEOUT) {
613 			pr_err("No event written to event log\n");
614 			return;
615 		}
616 		udelay(1);
617 		goto retry;
618 	}
619 
620 	if (type == EVENT_TYPE_IO_FAULT) {
621 		amd_iommu_report_page_fault(iommu, devid, pasid, address, flags);
622 		return;
623 	}
624 
625 	switch (type) {
626 	case EVENT_TYPE_ILL_DEV:
627 		dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n",
628 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
629 			pasid, address, flags);
630 		dump_dte_entry(iommu, devid);
631 		break;
632 	case EVENT_TYPE_DEV_TAB_ERR:
633 		dev_err(dev, "Event logged [DEV_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x "
634 			"address=0x%llx flags=0x%04x]\n",
635 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
636 			address, flags);
637 		break;
638 	case EVENT_TYPE_PAGE_TAB_ERR:
639 		dev_err(dev, "Event logged [PAGE_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x pasid=0x%04x address=0x%llx flags=0x%04x]\n",
640 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
641 			pasid, address, flags);
642 		break;
643 	case EVENT_TYPE_ILL_CMD:
644 		dev_err(dev, "Event logged [ILLEGAL_COMMAND_ERROR address=0x%llx]\n", address);
645 		dump_command(address);
646 		break;
647 	case EVENT_TYPE_CMD_HARD_ERR:
648 		dev_err(dev, "Event logged [COMMAND_HARDWARE_ERROR address=0x%llx flags=0x%04x]\n",
649 			address, flags);
650 		break;
651 	case EVENT_TYPE_IOTLB_INV_TO:
652 		dev_err(dev, "Event logged [IOTLB_INV_TIMEOUT device=%04x:%02x:%02x.%x address=0x%llx]\n",
653 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
654 			address);
655 		break;
656 	case EVENT_TYPE_INV_DEV_REQ:
657 		dev_err(dev, "Event logged [INVALID_DEVICE_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n",
658 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
659 			pasid, address, flags);
660 		break;
661 	case EVENT_TYPE_RMP_FAULT:
662 		amd_iommu_report_rmp_fault(iommu, event);
663 		break;
664 	case EVENT_TYPE_RMP_HW_ERR:
665 		amd_iommu_report_rmp_hw_error(iommu, event);
666 		break;
667 	case EVENT_TYPE_INV_PPR_REQ:
668 		pasid = PPR_PASID(*((u64 *)__evt));
669 		tag = event[1] & 0x03FF;
670 		dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x tag=0x%03x]\n",
671 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
672 			pasid, address, flags, tag);
673 		break;
674 	default:
675 		dev_err(dev, "Event logged [UNKNOWN event[0]=0x%08x event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n",
676 			event[0], event[1], event[2], event[3]);
677 	}
678 
679 	/*
680 	 * To detect the hardware errata 732 we need to clear the
681 	 * entry back to zero. This issue does not exist on SNP
682 	 * enabled system. Also this buffer is not writeable on
683 	 * SNP enabled system.
684 	 */
685 	if (!amd_iommu_snp_en)
686 		memset(__evt, 0, 4 * sizeof(u32));
687 }
688 
689 static void iommu_poll_events(struct amd_iommu *iommu)
690 {
691 	u32 head, tail;
692 
693 	head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
694 	tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
695 
696 	while (head != tail) {
697 		iommu_print_event(iommu, iommu->evt_buf + head);
698 		head = (head + EVENT_ENTRY_SIZE) % EVT_BUFFER_SIZE;
699 	}
700 
701 	writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
702 }
703 
704 static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw)
705 {
706 	struct amd_iommu_fault fault;
707 
708 	if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
709 		pr_err_ratelimited("Unknown PPR request received\n");
710 		return;
711 	}
712 
713 	fault.address   = raw[1];
714 	fault.pasid     = PPR_PASID(raw[0]);
715 	fault.sbdf      = PCI_SEG_DEVID_TO_SBDF(iommu->pci_seg->id, PPR_DEVID(raw[0]));
716 	fault.tag       = PPR_TAG(raw[0]);
717 	fault.flags     = PPR_FLAGS(raw[0]);
718 
719 	atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
720 }
721 
722 static void iommu_poll_ppr_log(struct amd_iommu *iommu)
723 {
724 	u32 head, tail;
725 
726 	if (iommu->ppr_log == NULL)
727 		return;
728 
729 	head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
730 	tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
731 
732 	while (head != tail) {
733 		volatile u64 *raw;
734 		u64 entry[2];
735 		int i;
736 
737 		raw = (u64 *)(iommu->ppr_log + head);
738 
739 		/*
740 		 * Hardware bug: Interrupt may arrive before the entry is
741 		 * written to memory. If this happens we need to wait for the
742 		 * entry to arrive.
743 		 */
744 		for (i = 0; i < LOOP_TIMEOUT; ++i) {
745 			if (PPR_REQ_TYPE(raw[0]) != 0)
746 				break;
747 			udelay(1);
748 		}
749 
750 		/* Avoid memcpy function-call overhead */
751 		entry[0] = raw[0];
752 		entry[1] = raw[1];
753 
754 		/*
755 		 * To detect the hardware errata 733 we need to clear the
756 		 * entry back to zero. This issue does not exist on SNP
757 		 * enabled system. Also this buffer is not writeable on
758 		 * SNP enabled system.
759 		 */
760 		if (!amd_iommu_snp_en)
761 			raw[0] = raw[1] = 0UL;
762 
763 		/* Update head pointer of hardware ring-buffer */
764 		head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
765 		writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
766 
767 		/* Handle PPR entry */
768 		iommu_handle_ppr_entry(iommu, entry);
769 
770 		/* Refresh ring-buffer information */
771 		head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
772 		tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
773 	}
774 }
775 
776 #ifdef CONFIG_IRQ_REMAP
777 static int (*iommu_ga_log_notifier)(u32);
778 
779 int amd_iommu_register_ga_log_notifier(int (*notifier)(u32))
780 {
781 	iommu_ga_log_notifier = notifier;
782 
783 	return 0;
784 }
785 EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier);
786 
787 static void iommu_poll_ga_log(struct amd_iommu *iommu)
788 {
789 	u32 head, tail;
790 
791 	if (iommu->ga_log == NULL)
792 		return;
793 
794 	head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
795 	tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
796 
797 	while (head != tail) {
798 		volatile u64 *raw;
799 		u64 log_entry;
800 
801 		raw = (u64 *)(iommu->ga_log + head);
802 
803 		/* Avoid memcpy function-call overhead */
804 		log_entry = *raw;
805 
806 		/* Update head pointer of hardware ring-buffer */
807 		head = (head + GA_ENTRY_SIZE) % GA_LOG_SIZE;
808 		writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
809 
810 		/* Handle GA entry */
811 		switch (GA_REQ_TYPE(log_entry)) {
812 		case GA_GUEST_NR:
813 			if (!iommu_ga_log_notifier)
814 				break;
815 
816 			pr_debug("%s: devid=%#x, ga_tag=%#x\n",
817 				 __func__, GA_DEVID(log_entry),
818 				 GA_TAG(log_entry));
819 
820 			if (iommu_ga_log_notifier(GA_TAG(log_entry)) != 0)
821 				pr_err("GA log notifier failed.\n");
822 			break;
823 		default:
824 			break;
825 		}
826 	}
827 }
828 
829 static void
830 amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu)
831 {
832 	if (!irq_remapping_enabled || !dev_is_pci(dev) ||
833 	    !pci_dev_has_default_msi_parent_domain(to_pci_dev(dev)))
834 		return;
835 
836 	dev_set_msi_domain(dev, iommu->ir_domain);
837 }
838 
839 #else /* CONFIG_IRQ_REMAP */
840 static inline void
841 amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) { }
842 #endif /* !CONFIG_IRQ_REMAP */
843 
844 #define AMD_IOMMU_INT_MASK	\
845 	(MMIO_STATUS_EVT_OVERFLOW_INT_MASK | \
846 	 MMIO_STATUS_EVT_INT_MASK | \
847 	 MMIO_STATUS_PPR_INT_MASK | \
848 	 MMIO_STATUS_GALOG_OVERFLOW_MASK | \
849 	 MMIO_STATUS_GALOG_INT_MASK)
850 
851 irqreturn_t amd_iommu_int_thread(int irq, void *data)
852 {
853 	struct amd_iommu *iommu = (struct amd_iommu *) data;
854 	u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
855 
856 	while (status & AMD_IOMMU_INT_MASK) {
857 		/* Enable interrupt sources again */
858 		writel(AMD_IOMMU_INT_MASK,
859 			iommu->mmio_base + MMIO_STATUS_OFFSET);
860 
861 		if (status & MMIO_STATUS_EVT_INT_MASK) {
862 			pr_devel("Processing IOMMU Event Log\n");
863 			iommu_poll_events(iommu);
864 		}
865 
866 		if (status & MMIO_STATUS_PPR_INT_MASK) {
867 			pr_devel("Processing IOMMU PPR Log\n");
868 			iommu_poll_ppr_log(iommu);
869 		}
870 
871 #ifdef CONFIG_IRQ_REMAP
872 		if (status & (MMIO_STATUS_GALOG_INT_MASK |
873 			      MMIO_STATUS_GALOG_OVERFLOW_MASK)) {
874 			pr_devel("Processing IOMMU GA Log\n");
875 			iommu_poll_ga_log(iommu);
876 		}
877 
878 		if (status & MMIO_STATUS_GALOG_OVERFLOW_MASK) {
879 			pr_info_ratelimited("IOMMU GA Log overflow\n");
880 			amd_iommu_restart_ga_log(iommu);
881 		}
882 #endif
883 
884 		if (status & MMIO_STATUS_EVT_OVERFLOW_INT_MASK) {
885 			pr_info_ratelimited("IOMMU event log overflow\n");
886 			amd_iommu_restart_event_logging(iommu);
887 		}
888 
889 		/*
890 		 * Hardware bug: ERBT1312
891 		 * When re-enabling interrupt (by writing 1
892 		 * to clear the bit), the hardware might also try to set
893 		 * the interrupt bit in the event status register.
894 		 * In this scenario, the bit will be set, and disable
895 		 * subsequent interrupts.
896 		 *
897 		 * Workaround: The IOMMU driver should read back the
898 		 * status register and check if the interrupt bits are cleared.
899 		 * If not, driver will need to go through the interrupt handler
900 		 * again and re-clear the bits
901 		 */
902 		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
903 	}
904 	return IRQ_HANDLED;
905 }
906 
907 irqreturn_t amd_iommu_int_handler(int irq, void *data)
908 {
909 	return IRQ_WAKE_THREAD;
910 }
911 
912 /****************************************************************************
913  *
914  * IOMMU command queuing functions
915  *
916  ****************************************************************************/
917 
918 static int wait_on_sem(struct amd_iommu *iommu, u64 data)
919 {
920 	int i = 0;
921 
922 	while (*iommu->cmd_sem != data && i < LOOP_TIMEOUT) {
923 		udelay(1);
924 		i += 1;
925 	}
926 
927 	if (i == LOOP_TIMEOUT) {
928 		pr_alert("Completion-Wait loop timed out\n");
929 		return -EIO;
930 	}
931 
932 	return 0;
933 }
934 
935 static void copy_cmd_to_buffer(struct amd_iommu *iommu,
936 			       struct iommu_cmd *cmd)
937 {
938 	u8 *target;
939 	u32 tail;
940 
941 	/* Copy command to buffer */
942 	tail = iommu->cmd_buf_tail;
943 	target = iommu->cmd_buf + tail;
944 	memcpy(target, cmd, sizeof(*cmd));
945 
946 	tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
947 	iommu->cmd_buf_tail = tail;
948 
949 	/* Tell the IOMMU about it */
950 	writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
951 }
952 
953 static void build_completion_wait(struct iommu_cmd *cmd,
954 				  struct amd_iommu *iommu,
955 				  u64 data)
956 {
957 	u64 paddr = iommu_virt_to_phys((void *)iommu->cmd_sem);
958 
959 	memset(cmd, 0, sizeof(*cmd));
960 	cmd->data[0] = lower_32_bits(paddr) | CMD_COMPL_WAIT_STORE_MASK;
961 	cmd->data[1] = upper_32_bits(paddr);
962 	cmd->data[2] = lower_32_bits(data);
963 	cmd->data[3] = upper_32_bits(data);
964 	CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
965 }
966 
967 static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
968 {
969 	memset(cmd, 0, sizeof(*cmd));
970 	cmd->data[0] = devid;
971 	CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
972 }
973 
974 /*
975  * Builds an invalidation address which is suitable for one page or multiple
976  * pages. Sets the size bit (S) as needed is more than one page is flushed.
977  */
978 static inline u64 build_inv_address(u64 address, size_t size)
979 {
980 	u64 pages, end, msb_diff;
981 
982 	pages = iommu_num_pages(address, size, PAGE_SIZE);
983 
984 	if (pages == 1)
985 		return address & PAGE_MASK;
986 
987 	end = address + size - 1;
988 
989 	/*
990 	 * msb_diff would hold the index of the most significant bit that
991 	 * flipped between the start and end.
992 	 */
993 	msb_diff = fls64(end ^ address) - 1;
994 
995 	/*
996 	 * Bits 63:52 are sign extended. If for some reason bit 51 is different
997 	 * between the start and the end, invalidate everything.
998 	 */
999 	if (unlikely(msb_diff > 51)) {
1000 		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
1001 	} else {
1002 		/*
1003 		 * The msb-bit must be clear on the address. Just set all the
1004 		 * lower bits.
1005 		 */
1006 		address |= (1ull << msb_diff) - 1;
1007 	}
1008 
1009 	/* Clear bits 11:0 */
1010 	address &= PAGE_MASK;
1011 
1012 	/* Set the size bit - we flush more than one 4kb page */
1013 	return address | CMD_INV_IOMMU_PAGES_SIZE_MASK;
1014 }
1015 
1016 static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
1017 				  size_t size, u16 domid, int pde)
1018 {
1019 	u64 inv_address = build_inv_address(address, size);
1020 
1021 	memset(cmd, 0, sizeof(*cmd));
1022 	cmd->data[1] |= domid;
1023 	cmd->data[2]  = lower_32_bits(inv_address);
1024 	cmd->data[3]  = upper_32_bits(inv_address);
1025 	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
1026 	if (pde) /* PDE bit - we want to flush everything, not only the PTEs */
1027 		cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
1028 }
1029 
1030 static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
1031 				  u64 address, size_t size)
1032 {
1033 	u64 inv_address = build_inv_address(address, size);
1034 
1035 	memset(cmd, 0, sizeof(*cmd));
1036 	cmd->data[0]  = devid;
1037 	cmd->data[0] |= (qdep & 0xff) << 24;
1038 	cmd->data[1]  = devid;
1039 	cmd->data[2]  = lower_32_bits(inv_address);
1040 	cmd->data[3]  = upper_32_bits(inv_address);
1041 	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
1042 }
1043 
1044 static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, u32 pasid,
1045 				  u64 address, bool size)
1046 {
1047 	memset(cmd, 0, sizeof(*cmd));
1048 
1049 	address &= ~(0xfffULL);
1050 
1051 	cmd->data[0]  = pasid;
1052 	cmd->data[1]  = domid;
1053 	cmd->data[2]  = lower_32_bits(address);
1054 	cmd->data[3]  = upper_32_bits(address);
1055 	cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
1056 	cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
1057 	if (size)
1058 		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
1059 	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
1060 }
1061 
1062 static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, u32 pasid,
1063 				  int qdep, u64 address, bool size)
1064 {
1065 	memset(cmd, 0, sizeof(*cmd));
1066 
1067 	address &= ~(0xfffULL);
1068 
1069 	cmd->data[0]  = devid;
1070 	cmd->data[0] |= ((pasid >> 8) & 0xff) << 16;
1071 	cmd->data[0] |= (qdep  & 0xff) << 24;
1072 	cmd->data[1]  = devid;
1073 	cmd->data[1] |= (pasid & 0xff) << 16;
1074 	cmd->data[2]  = lower_32_bits(address);
1075 	cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
1076 	cmd->data[3]  = upper_32_bits(address);
1077 	if (size)
1078 		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
1079 	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
1080 }
1081 
1082 static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, u32 pasid,
1083 			       int status, int tag, bool gn)
1084 {
1085 	memset(cmd, 0, sizeof(*cmd));
1086 
1087 	cmd->data[0]  = devid;
1088 	if (gn) {
1089 		cmd->data[1]  = pasid;
1090 		cmd->data[2]  = CMD_INV_IOMMU_PAGES_GN_MASK;
1091 	}
1092 	cmd->data[3]  = tag & 0x1ff;
1093 	cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;
1094 
1095 	CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
1096 }
1097 
1098 static void build_inv_all(struct iommu_cmd *cmd)
1099 {
1100 	memset(cmd, 0, sizeof(*cmd));
1101 	CMD_SET_TYPE(cmd, CMD_INV_ALL);
1102 }
1103 
1104 static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)
1105 {
1106 	memset(cmd, 0, sizeof(*cmd));
1107 	cmd->data[0] = devid;
1108 	CMD_SET_TYPE(cmd, CMD_INV_IRT);
1109 }
1110 
1111 /*
1112  * Writes the command to the IOMMUs command buffer and informs the
1113  * hardware about the new command.
1114  */
1115 static int __iommu_queue_command_sync(struct amd_iommu *iommu,
1116 				      struct iommu_cmd *cmd,
1117 				      bool sync)
1118 {
1119 	unsigned int count = 0;
1120 	u32 left, next_tail;
1121 
1122 	next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
1123 again:
1124 	left      = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE;
1125 
1126 	if (left <= 0x20) {
1127 		/* Skip udelay() the first time around */
1128 		if (count++) {
1129 			if (count == LOOP_TIMEOUT) {
1130 				pr_err("Command buffer timeout\n");
1131 				return -EIO;
1132 			}
1133 
1134 			udelay(1);
1135 		}
1136 
1137 		/* Update head and recheck remaining space */
1138 		iommu->cmd_buf_head = readl(iommu->mmio_base +
1139 					    MMIO_CMD_HEAD_OFFSET);
1140 
1141 		goto again;
1142 	}
1143 
1144 	copy_cmd_to_buffer(iommu, cmd);
1145 
1146 	/* Do we need to make sure all commands are processed? */
1147 	iommu->need_sync = sync;
1148 
1149 	return 0;
1150 }
1151 
1152 static int iommu_queue_command_sync(struct amd_iommu *iommu,
1153 				    struct iommu_cmd *cmd,
1154 				    bool sync)
1155 {
1156 	unsigned long flags;
1157 	int ret;
1158 
1159 	raw_spin_lock_irqsave(&iommu->lock, flags);
1160 	ret = __iommu_queue_command_sync(iommu, cmd, sync);
1161 	raw_spin_unlock_irqrestore(&iommu->lock, flags);
1162 
1163 	return ret;
1164 }
1165 
1166 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
1167 {
1168 	return iommu_queue_command_sync(iommu, cmd, true);
1169 }
1170 
1171 /*
1172  * This function queues a completion wait command into the command
1173  * buffer of an IOMMU
1174  */
1175 static int iommu_completion_wait(struct amd_iommu *iommu)
1176 {
1177 	struct iommu_cmd cmd;
1178 	unsigned long flags;
1179 	int ret;
1180 	u64 data;
1181 
1182 	if (!iommu->need_sync)
1183 		return 0;
1184 
1185 	raw_spin_lock_irqsave(&iommu->lock, flags);
1186 
1187 	data = ++iommu->cmd_sem_val;
1188 	build_completion_wait(&cmd, iommu, data);
1189 
1190 	ret = __iommu_queue_command_sync(iommu, &cmd, false);
1191 	if (ret)
1192 		goto out_unlock;
1193 
1194 	ret = wait_on_sem(iommu, data);
1195 
1196 out_unlock:
1197 	raw_spin_unlock_irqrestore(&iommu->lock, flags);
1198 
1199 	return ret;
1200 }
1201 
1202 static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
1203 {
1204 	struct iommu_cmd cmd;
1205 
1206 	build_inv_dte(&cmd, devid);
1207 
1208 	return iommu_queue_command(iommu, &cmd);
1209 }
1210 
1211 static void amd_iommu_flush_dte_all(struct amd_iommu *iommu)
1212 {
1213 	u32 devid;
1214 	u16 last_bdf = iommu->pci_seg->last_bdf;
1215 
1216 	for (devid = 0; devid <= last_bdf; ++devid)
1217 		iommu_flush_dte(iommu, devid);
1218 
1219 	iommu_completion_wait(iommu);
1220 }
1221 
1222 /*
1223  * This function uses heavy locking and may disable irqs for some time. But
1224  * this is no issue because it is only called during resume.
1225  */
1226 static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu)
1227 {
1228 	u32 dom_id;
1229 	u16 last_bdf = iommu->pci_seg->last_bdf;
1230 
1231 	for (dom_id = 0; dom_id <= last_bdf; ++dom_id) {
1232 		struct iommu_cmd cmd;
1233 		build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
1234 				      dom_id, 1);
1235 		iommu_queue_command(iommu, &cmd);
1236 	}
1237 
1238 	iommu_completion_wait(iommu);
1239 }
1240 
1241 static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id)
1242 {
1243 	struct iommu_cmd cmd;
1244 
1245 	build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
1246 			      dom_id, 1);
1247 	iommu_queue_command(iommu, &cmd);
1248 
1249 	iommu_completion_wait(iommu);
1250 }
1251 
1252 static void amd_iommu_flush_all(struct amd_iommu *iommu)
1253 {
1254 	struct iommu_cmd cmd;
1255 
1256 	build_inv_all(&cmd);
1257 
1258 	iommu_queue_command(iommu, &cmd);
1259 	iommu_completion_wait(iommu);
1260 }
1261 
1262 static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)
1263 {
1264 	struct iommu_cmd cmd;
1265 
1266 	build_inv_irt(&cmd, devid);
1267 
1268 	iommu_queue_command(iommu, &cmd);
1269 }
1270 
1271 static void amd_iommu_flush_irt_all(struct amd_iommu *iommu)
1272 {
1273 	u32 devid;
1274 	u16 last_bdf = iommu->pci_seg->last_bdf;
1275 
1276 	for (devid = 0; devid <= last_bdf; devid++)
1277 		iommu_flush_irt(iommu, devid);
1278 
1279 	iommu_completion_wait(iommu);
1280 }
1281 
1282 void iommu_flush_all_caches(struct amd_iommu *iommu)
1283 {
1284 	if (iommu_feature(iommu, FEATURE_IA)) {
1285 		amd_iommu_flush_all(iommu);
1286 	} else {
1287 		amd_iommu_flush_dte_all(iommu);
1288 		amd_iommu_flush_irt_all(iommu);
1289 		amd_iommu_flush_tlb_all(iommu);
1290 	}
1291 }
1292 
1293 /*
1294  * Command send function for flushing on-device TLB
1295  */
1296 static int device_flush_iotlb(struct iommu_dev_data *dev_data,
1297 			      u64 address, size_t size)
1298 {
1299 	struct amd_iommu *iommu;
1300 	struct iommu_cmd cmd;
1301 	int qdep;
1302 
1303 	qdep     = dev_data->ats.qdep;
1304 	iommu    = rlookup_amd_iommu(dev_data->dev);
1305 	if (!iommu)
1306 		return -EINVAL;
1307 
1308 	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
1309 
1310 	return iommu_queue_command(iommu, &cmd);
1311 }
1312 
1313 static int device_flush_dte_alias(struct pci_dev *pdev, u16 alias, void *data)
1314 {
1315 	struct amd_iommu *iommu = data;
1316 
1317 	return iommu_flush_dte(iommu, alias);
1318 }
1319 
1320 /*
1321  * Command send function for invalidating a device table entry
1322  */
1323 static int device_flush_dte(struct iommu_dev_data *dev_data)
1324 {
1325 	struct amd_iommu *iommu;
1326 	struct pci_dev *pdev = NULL;
1327 	struct amd_iommu_pci_seg *pci_seg;
1328 	u16 alias;
1329 	int ret;
1330 
1331 	iommu = rlookup_amd_iommu(dev_data->dev);
1332 	if (!iommu)
1333 		return -EINVAL;
1334 
1335 	if (dev_is_pci(dev_data->dev))
1336 		pdev = to_pci_dev(dev_data->dev);
1337 
1338 	if (pdev)
1339 		ret = pci_for_each_dma_alias(pdev,
1340 					     device_flush_dte_alias, iommu);
1341 	else
1342 		ret = iommu_flush_dte(iommu, dev_data->devid);
1343 	if (ret)
1344 		return ret;
1345 
1346 	pci_seg = iommu->pci_seg;
1347 	alias = pci_seg->alias_table[dev_data->devid];
1348 	if (alias != dev_data->devid) {
1349 		ret = iommu_flush_dte(iommu, alias);
1350 		if (ret)
1351 			return ret;
1352 	}
1353 
1354 	if (dev_data->ats.enabled)
1355 		ret = device_flush_iotlb(dev_data, 0, ~0UL);
1356 
1357 	return ret;
1358 }
1359 
1360 /*
1361  * TLB invalidation function which is called from the mapping functions.
1362  * It invalidates a single PTE if the range to flush is within a single
1363  * page. Otherwise it flushes the whole TLB of the IOMMU.
1364  */
1365 static void __domain_flush_pages(struct protection_domain *domain,
1366 				 u64 address, size_t size, int pde)
1367 {
1368 	struct iommu_dev_data *dev_data;
1369 	struct iommu_cmd cmd;
1370 	int ret = 0, i;
1371 
1372 	build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
1373 
1374 	for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {
1375 		if (!domain->dev_iommu[i])
1376 			continue;
1377 
1378 		/*
1379 		 * Devices of this domain are behind this IOMMU
1380 		 * We need a TLB flush
1381 		 */
1382 		ret |= iommu_queue_command(amd_iommus[i], &cmd);
1383 	}
1384 
1385 	list_for_each_entry(dev_data, &domain->dev_list, list) {
1386 
1387 		if (!dev_data->ats.enabled)
1388 			continue;
1389 
1390 		ret |= device_flush_iotlb(dev_data, address, size);
1391 	}
1392 
1393 	WARN_ON(ret);
1394 }
1395 
1396 static void domain_flush_pages(struct protection_domain *domain,
1397 			       u64 address, size_t size, int pde)
1398 {
1399 	if (likely(!amd_iommu_np_cache)) {
1400 		__domain_flush_pages(domain, address, size, pde);
1401 		return;
1402 	}
1403 
1404 	/*
1405 	 * When NpCache is on, we infer that we run in a VM and use a vIOMMU.
1406 	 * In such setups it is best to avoid flushes of ranges which are not
1407 	 * naturally aligned, since it would lead to flushes of unmodified
1408 	 * PTEs. Such flushes would require the hypervisor to do more work than
1409 	 * necessary. Therefore, perform repeated flushes of aligned ranges
1410 	 * until you cover the range. Each iteration flushes the smaller
1411 	 * between the natural alignment of the address that we flush and the
1412 	 * greatest naturally aligned region that fits in the range.
1413 	 */
1414 	while (size != 0) {
1415 		int addr_alignment = __ffs(address);
1416 		int size_alignment = __fls(size);
1417 		int min_alignment;
1418 		size_t flush_size;
1419 
1420 		/*
1421 		 * size is always non-zero, but address might be zero, causing
1422 		 * addr_alignment to be negative. As the casting of the
1423 		 * argument in __ffs(address) to long might trim the high bits
1424 		 * of the address on x86-32, cast to long when doing the check.
1425 		 */
1426 		if (likely((unsigned long)address != 0))
1427 			min_alignment = min(addr_alignment, size_alignment);
1428 		else
1429 			min_alignment = size_alignment;
1430 
1431 		flush_size = 1ul << min_alignment;
1432 
1433 		__domain_flush_pages(domain, address, flush_size, pde);
1434 		address += flush_size;
1435 		size -= flush_size;
1436 	}
1437 }
1438 
1439 /* Flush the whole IO/TLB for a given protection domain - including PDE */
1440 void amd_iommu_domain_flush_tlb_pde(struct protection_domain *domain)
1441 {
1442 	domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
1443 }
1444 
1445 void amd_iommu_domain_flush_complete(struct protection_domain *domain)
1446 {
1447 	int i;
1448 
1449 	for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {
1450 		if (domain && !domain->dev_iommu[i])
1451 			continue;
1452 
1453 		/*
1454 		 * Devices of this domain are behind this IOMMU
1455 		 * We need to wait for completion of all commands.
1456 		 */
1457 		iommu_completion_wait(amd_iommus[i]);
1458 	}
1459 }
1460 
1461 /* Flush the not present cache if it exists */
1462 static void domain_flush_np_cache(struct protection_domain *domain,
1463 		dma_addr_t iova, size_t size)
1464 {
1465 	if (unlikely(amd_iommu_np_cache)) {
1466 		unsigned long flags;
1467 
1468 		spin_lock_irqsave(&domain->lock, flags);
1469 		domain_flush_pages(domain, iova, size, 1);
1470 		amd_iommu_domain_flush_complete(domain);
1471 		spin_unlock_irqrestore(&domain->lock, flags);
1472 	}
1473 }
1474 
1475 
1476 /*
1477  * This function flushes the DTEs for all devices in domain
1478  */
1479 static void domain_flush_devices(struct protection_domain *domain)
1480 {
1481 	struct iommu_dev_data *dev_data;
1482 
1483 	list_for_each_entry(dev_data, &domain->dev_list, list)
1484 		device_flush_dte(dev_data);
1485 }
1486 
1487 /****************************************************************************
1488  *
1489  * The next functions belong to the domain allocation. A domain is
1490  * allocated for every IOMMU as the default domain. If device isolation
1491  * is enabled, every device get its own domain. The most important thing
1492  * about domains is the page table mapping the DMA address space they
1493  * contain.
1494  *
1495  ****************************************************************************/
1496 
1497 static u16 domain_id_alloc(void)
1498 {
1499 	int id;
1500 
1501 	spin_lock(&pd_bitmap_lock);
1502 	id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
1503 	BUG_ON(id == 0);
1504 	if (id > 0 && id < MAX_DOMAIN_ID)
1505 		__set_bit(id, amd_iommu_pd_alloc_bitmap);
1506 	else
1507 		id = 0;
1508 	spin_unlock(&pd_bitmap_lock);
1509 
1510 	return id;
1511 }
1512 
1513 static void domain_id_free(int id)
1514 {
1515 	spin_lock(&pd_bitmap_lock);
1516 	if (id > 0 && id < MAX_DOMAIN_ID)
1517 		__clear_bit(id, amd_iommu_pd_alloc_bitmap);
1518 	spin_unlock(&pd_bitmap_lock);
1519 }
1520 
1521 static void free_gcr3_tbl_level1(u64 *tbl)
1522 {
1523 	u64 *ptr;
1524 	int i;
1525 
1526 	for (i = 0; i < 512; ++i) {
1527 		if (!(tbl[i] & GCR3_VALID))
1528 			continue;
1529 
1530 		ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK);
1531 
1532 		free_page((unsigned long)ptr);
1533 	}
1534 }
1535 
1536 static void free_gcr3_tbl_level2(u64 *tbl)
1537 {
1538 	u64 *ptr;
1539 	int i;
1540 
1541 	for (i = 0; i < 512; ++i) {
1542 		if (!(tbl[i] & GCR3_VALID))
1543 			continue;
1544 
1545 		ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK);
1546 
1547 		free_gcr3_tbl_level1(ptr);
1548 	}
1549 }
1550 
1551 static void free_gcr3_table(struct protection_domain *domain)
1552 {
1553 	if (domain->glx == 2)
1554 		free_gcr3_tbl_level2(domain->gcr3_tbl);
1555 	else if (domain->glx == 1)
1556 		free_gcr3_tbl_level1(domain->gcr3_tbl);
1557 	else
1558 		BUG_ON(domain->glx != 0);
1559 
1560 	free_page((unsigned long)domain->gcr3_tbl);
1561 }
1562 
1563 static void set_dte_entry(struct amd_iommu *iommu, u16 devid,
1564 			  struct protection_domain *domain, bool ats, bool ppr)
1565 {
1566 	u64 pte_root = 0;
1567 	u64 flags = 0;
1568 	u32 old_domid;
1569 	struct dev_table_entry *dev_table = get_dev_table(iommu);
1570 
1571 	if (domain->iop.mode != PAGE_MODE_NONE)
1572 		pte_root = iommu_virt_to_phys(domain->iop.root);
1573 
1574 	pte_root |= (domain->iop.mode & DEV_ENTRY_MODE_MASK)
1575 		    << DEV_ENTRY_MODE_SHIFT;
1576 
1577 	pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V;
1578 
1579 	/*
1580 	 * When SNP is enabled, Only set TV bit when IOMMU
1581 	 * page translation is in use.
1582 	 */
1583 	if (!amd_iommu_snp_en || (domain->id != 0))
1584 		pte_root |= DTE_FLAG_TV;
1585 
1586 	flags = dev_table[devid].data[1];
1587 
1588 	if (ats)
1589 		flags |= DTE_FLAG_IOTLB;
1590 
1591 	if (ppr) {
1592 		if (iommu_feature(iommu, FEATURE_EPHSUP))
1593 			pte_root |= 1ULL << DEV_ENTRY_PPR;
1594 	}
1595 
1596 	if (domain->flags & PD_IOMMUV2_MASK) {
1597 		u64 gcr3 = iommu_virt_to_phys(domain->gcr3_tbl);
1598 		u64 glx  = domain->glx;
1599 		u64 tmp;
1600 
1601 		pte_root |= DTE_FLAG_GV;
1602 		pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
1603 
1604 		/* First mask out possible old values for GCR3 table */
1605 		tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
1606 		flags    &= ~tmp;
1607 
1608 		tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
1609 		flags    &= ~tmp;
1610 
1611 		/* Encode GCR3 table into DTE */
1612 		tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
1613 		pte_root |= tmp;
1614 
1615 		tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
1616 		flags    |= tmp;
1617 
1618 		tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
1619 		flags    |= tmp;
1620 
1621 		if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) {
1622 			dev_table[devid].data[2] |=
1623 				((u64)GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT);
1624 		}
1625 
1626 		if (domain->flags & PD_GIOV_MASK)
1627 			pte_root |= DTE_FLAG_GIOV;
1628 	}
1629 
1630 	flags &= ~DEV_DOMID_MASK;
1631 	flags |= domain->id;
1632 
1633 	old_domid = dev_table[devid].data[1] & DEV_DOMID_MASK;
1634 	dev_table[devid].data[1]  = flags;
1635 	dev_table[devid].data[0]  = pte_root;
1636 
1637 	/*
1638 	 * A kdump kernel might be replacing a domain ID that was copied from
1639 	 * the previous kernel--if so, it needs to flush the translation cache
1640 	 * entries for the old domain ID that is being overwritten
1641 	 */
1642 	if (old_domid) {
1643 		amd_iommu_flush_tlb_domid(iommu, old_domid);
1644 	}
1645 }
1646 
1647 static void clear_dte_entry(struct amd_iommu *iommu, u16 devid)
1648 {
1649 	struct dev_table_entry *dev_table = get_dev_table(iommu);
1650 
1651 	/* remove entry from the device table seen by the hardware */
1652 	dev_table[devid].data[0]  = DTE_FLAG_V;
1653 
1654 	if (!amd_iommu_snp_en)
1655 		dev_table[devid].data[0] |= DTE_FLAG_TV;
1656 
1657 	dev_table[devid].data[1] &= DTE_FLAG_MASK;
1658 
1659 	amd_iommu_apply_erratum_63(iommu, devid);
1660 }
1661 
1662 static void do_attach(struct iommu_dev_data *dev_data,
1663 		      struct protection_domain *domain)
1664 {
1665 	struct amd_iommu *iommu;
1666 	bool ats;
1667 
1668 	iommu = rlookup_amd_iommu(dev_data->dev);
1669 	if (!iommu)
1670 		return;
1671 	ats   = dev_data->ats.enabled;
1672 
1673 	/* Update data structures */
1674 	dev_data->domain = domain;
1675 	list_add(&dev_data->list, &domain->dev_list);
1676 
1677 	/* Update NUMA Node ID */
1678 	if (domain->nid == NUMA_NO_NODE)
1679 		domain->nid = dev_to_node(dev_data->dev);
1680 
1681 	/* Do reference counting */
1682 	domain->dev_iommu[iommu->index] += 1;
1683 	domain->dev_cnt                 += 1;
1684 
1685 	/* Update device table */
1686 	set_dte_entry(iommu, dev_data->devid, domain,
1687 		      ats, dev_data->iommu_v2);
1688 	clone_aliases(iommu, dev_data->dev);
1689 
1690 	device_flush_dte(dev_data);
1691 }
1692 
1693 static void do_detach(struct iommu_dev_data *dev_data)
1694 {
1695 	struct protection_domain *domain = dev_data->domain;
1696 	struct amd_iommu *iommu;
1697 
1698 	iommu = rlookup_amd_iommu(dev_data->dev);
1699 	if (!iommu)
1700 		return;
1701 
1702 	/* Update data structures */
1703 	dev_data->domain = NULL;
1704 	list_del(&dev_data->list);
1705 	clear_dte_entry(iommu, dev_data->devid);
1706 	clone_aliases(iommu, dev_data->dev);
1707 
1708 	/* Flush the DTE entry */
1709 	device_flush_dte(dev_data);
1710 
1711 	/* Flush IOTLB */
1712 	amd_iommu_domain_flush_tlb_pde(domain);
1713 
1714 	/* Wait for the flushes to finish */
1715 	amd_iommu_domain_flush_complete(domain);
1716 
1717 	/* decrease reference counters - needs to happen after the flushes */
1718 	domain->dev_iommu[iommu->index] -= 1;
1719 	domain->dev_cnt                 -= 1;
1720 }
1721 
1722 static void pdev_iommuv2_disable(struct pci_dev *pdev)
1723 {
1724 	pci_disable_ats(pdev);
1725 	pci_disable_pri(pdev);
1726 	pci_disable_pasid(pdev);
1727 }
1728 
1729 static int pdev_pri_ats_enable(struct pci_dev *pdev)
1730 {
1731 	int ret;
1732 
1733 	/* Only allow access to user-accessible pages */
1734 	ret = pci_enable_pasid(pdev, 0);
1735 	if (ret)
1736 		return ret;
1737 
1738 	/* First reset the PRI state of the device */
1739 	ret = pci_reset_pri(pdev);
1740 	if (ret)
1741 		goto out_err_pasid;
1742 
1743 	/* Enable PRI */
1744 	/* FIXME: Hardcode number of outstanding requests for now */
1745 	ret = pci_enable_pri(pdev, 32);
1746 	if (ret)
1747 		goto out_err_pasid;
1748 
1749 	ret = pci_enable_ats(pdev, PAGE_SHIFT);
1750 	if (ret)
1751 		goto out_err_pri;
1752 
1753 	return 0;
1754 
1755 out_err_pri:
1756 	pci_disable_pri(pdev);
1757 
1758 out_err_pasid:
1759 	pci_disable_pasid(pdev);
1760 
1761 	return ret;
1762 }
1763 
1764 /*
1765  * If a device is not yet associated with a domain, this function makes the
1766  * device visible in the domain
1767  */
1768 static int attach_device(struct device *dev,
1769 			 struct protection_domain *domain)
1770 {
1771 	struct iommu_dev_data *dev_data;
1772 	struct pci_dev *pdev;
1773 	unsigned long flags;
1774 	int ret;
1775 
1776 	spin_lock_irqsave(&domain->lock, flags);
1777 
1778 	dev_data = dev_iommu_priv_get(dev);
1779 
1780 	spin_lock(&dev_data->lock);
1781 
1782 	ret = -EBUSY;
1783 	if (dev_data->domain != NULL)
1784 		goto out;
1785 
1786 	if (!dev_is_pci(dev))
1787 		goto skip_ats_check;
1788 
1789 	pdev = to_pci_dev(dev);
1790 	if (domain->flags & PD_IOMMUV2_MASK) {
1791 		struct iommu_domain *def_domain = iommu_get_dma_domain(dev);
1792 
1793 		ret = -EINVAL;
1794 
1795 		/*
1796 		 * In case of using AMD_IOMMU_V1 page table mode and the device
1797 		 * is enabling for PPR/ATS support (using v2 table),
1798 		 * we need to make sure that the domain type is identity map.
1799 		 */
1800 		if ((amd_iommu_pgtable == AMD_IOMMU_V1) &&
1801 		    def_domain->type != IOMMU_DOMAIN_IDENTITY) {
1802 			goto out;
1803 		}
1804 
1805 		if (dev_data->iommu_v2) {
1806 			if (pdev_pri_ats_enable(pdev) != 0)
1807 				goto out;
1808 
1809 			dev_data->ats.enabled = true;
1810 			dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
1811 			dev_data->pri_tlp     = pci_prg_resp_pasid_required(pdev);
1812 		}
1813 	} else if (amd_iommu_iotlb_sup &&
1814 		   pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
1815 		dev_data->ats.enabled = true;
1816 		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
1817 	}
1818 
1819 skip_ats_check:
1820 	ret = 0;
1821 
1822 	do_attach(dev_data, domain);
1823 
1824 	/*
1825 	 * We might boot into a crash-kernel here. The crashed kernel
1826 	 * left the caches in the IOMMU dirty. So we have to flush
1827 	 * here to evict all dirty stuff.
1828 	 */
1829 	amd_iommu_domain_flush_tlb_pde(domain);
1830 
1831 	amd_iommu_domain_flush_complete(domain);
1832 
1833 out:
1834 	spin_unlock(&dev_data->lock);
1835 
1836 	spin_unlock_irqrestore(&domain->lock, flags);
1837 
1838 	return ret;
1839 }
1840 
1841 /*
1842  * Removes a device from a protection domain (with devtable_lock held)
1843  */
1844 static void detach_device(struct device *dev)
1845 {
1846 	struct protection_domain *domain;
1847 	struct iommu_dev_data *dev_data;
1848 	unsigned long flags;
1849 
1850 	dev_data = dev_iommu_priv_get(dev);
1851 	domain   = dev_data->domain;
1852 
1853 	spin_lock_irqsave(&domain->lock, flags);
1854 
1855 	spin_lock(&dev_data->lock);
1856 
1857 	/*
1858 	 * First check if the device is still attached. It might already
1859 	 * be detached from its domain because the generic
1860 	 * iommu_detach_group code detached it and we try again here in
1861 	 * our alias handling.
1862 	 */
1863 	if (WARN_ON(!dev_data->domain))
1864 		goto out;
1865 
1866 	do_detach(dev_data);
1867 
1868 	if (!dev_is_pci(dev))
1869 		goto out;
1870 
1871 	if (domain->flags & PD_IOMMUV2_MASK && dev_data->iommu_v2)
1872 		pdev_iommuv2_disable(to_pci_dev(dev));
1873 	else if (dev_data->ats.enabled)
1874 		pci_disable_ats(to_pci_dev(dev));
1875 
1876 	dev_data->ats.enabled = false;
1877 
1878 out:
1879 	spin_unlock(&dev_data->lock);
1880 
1881 	spin_unlock_irqrestore(&domain->lock, flags);
1882 }
1883 
1884 static struct iommu_device *amd_iommu_probe_device(struct device *dev)
1885 {
1886 	struct iommu_device *iommu_dev;
1887 	struct amd_iommu *iommu;
1888 	int ret;
1889 
1890 	if (!check_device(dev))
1891 		return ERR_PTR(-ENODEV);
1892 
1893 	iommu = rlookup_amd_iommu(dev);
1894 	if (!iommu)
1895 		return ERR_PTR(-ENODEV);
1896 
1897 	/* Not registered yet? */
1898 	if (!iommu->iommu.ops)
1899 		return ERR_PTR(-ENODEV);
1900 
1901 	if (dev_iommu_priv_get(dev))
1902 		return &iommu->iommu;
1903 
1904 	ret = iommu_init_device(iommu, dev);
1905 	if (ret) {
1906 		if (ret != -ENOTSUPP)
1907 			dev_err(dev, "Failed to initialize - trying to proceed anyway\n");
1908 		iommu_dev = ERR_PTR(ret);
1909 		iommu_ignore_device(iommu, dev);
1910 	} else {
1911 		amd_iommu_set_pci_msi_domain(dev, iommu);
1912 		iommu_dev = &iommu->iommu;
1913 	}
1914 
1915 	iommu_completion_wait(iommu);
1916 
1917 	return iommu_dev;
1918 }
1919 
1920 static void amd_iommu_probe_finalize(struct device *dev)
1921 {
1922 	/* Domains are initialized for this device - have a look what we ended up with */
1923 	set_dma_ops(dev, NULL);
1924 	iommu_setup_dma_ops(dev, 0, U64_MAX);
1925 }
1926 
1927 static void amd_iommu_release_device(struct device *dev)
1928 {
1929 	struct amd_iommu *iommu;
1930 
1931 	if (!check_device(dev))
1932 		return;
1933 
1934 	iommu = rlookup_amd_iommu(dev);
1935 	if (!iommu)
1936 		return;
1937 
1938 	amd_iommu_uninit_device(dev);
1939 	iommu_completion_wait(iommu);
1940 }
1941 
1942 static struct iommu_group *amd_iommu_device_group(struct device *dev)
1943 {
1944 	if (dev_is_pci(dev))
1945 		return pci_device_group(dev);
1946 
1947 	return acpihid_device_group(dev);
1948 }
1949 
1950 /*****************************************************************************
1951  *
1952  * The next functions belong to the dma_ops mapping/unmapping code.
1953  *
1954  *****************************************************************************/
1955 
1956 static void update_device_table(struct protection_domain *domain)
1957 {
1958 	struct iommu_dev_data *dev_data;
1959 
1960 	list_for_each_entry(dev_data, &domain->dev_list, list) {
1961 		struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev);
1962 
1963 		if (!iommu)
1964 			continue;
1965 		set_dte_entry(iommu, dev_data->devid, domain,
1966 			      dev_data->ats.enabled, dev_data->iommu_v2);
1967 		clone_aliases(iommu, dev_data->dev);
1968 	}
1969 }
1970 
1971 void amd_iommu_update_and_flush_device_table(struct protection_domain *domain)
1972 {
1973 	update_device_table(domain);
1974 	domain_flush_devices(domain);
1975 }
1976 
1977 void amd_iommu_domain_update(struct protection_domain *domain)
1978 {
1979 	/* Update device table */
1980 	amd_iommu_update_and_flush_device_table(domain);
1981 
1982 	/* Flush domain TLB(s) and wait for completion */
1983 	amd_iommu_domain_flush_tlb_pde(domain);
1984 	amd_iommu_domain_flush_complete(domain);
1985 }
1986 
1987 /*****************************************************************************
1988  *
1989  * The following functions belong to the exported interface of AMD IOMMU
1990  *
1991  * This interface allows access to lower level functions of the IOMMU
1992  * like protection domain handling and assignement of devices to domains
1993  * which is not possible with the dma_ops interface.
1994  *
1995  *****************************************************************************/
1996 
1997 static void cleanup_domain(struct protection_domain *domain)
1998 {
1999 	struct iommu_dev_data *entry;
2000 	unsigned long flags;
2001 
2002 	spin_lock_irqsave(&domain->lock, flags);
2003 
2004 	while (!list_empty(&domain->dev_list)) {
2005 		entry = list_first_entry(&domain->dev_list,
2006 					 struct iommu_dev_data, list);
2007 		BUG_ON(!entry->domain);
2008 		do_detach(entry);
2009 	}
2010 
2011 	spin_unlock_irqrestore(&domain->lock, flags);
2012 }
2013 
2014 static void protection_domain_free(struct protection_domain *domain)
2015 {
2016 	if (!domain)
2017 		return;
2018 
2019 	if (domain->iop.pgtbl_cfg.tlb)
2020 		free_io_pgtable_ops(&domain->iop.iop.ops);
2021 
2022 	if (domain->id)
2023 		domain_id_free(domain->id);
2024 
2025 	kfree(domain);
2026 }
2027 
2028 static int protection_domain_init_v1(struct protection_domain *domain, int mode)
2029 {
2030 	u64 *pt_root = NULL;
2031 
2032 	BUG_ON(mode < PAGE_MODE_NONE || mode > PAGE_MODE_6_LEVEL);
2033 
2034 	spin_lock_init(&domain->lock);
2035 	domain->id = domain_id_alloc();
2036 	if (!domain->id)
2037 		return -ENOMEM;
2038 	INIT_LIST_HEAD(&domain->dev_list);
2039 
2040 	if (mode != PAGE_MODE_NONE) {
2041 		pt_root = (void *)get_zeroed_page(GFP_KERNEL);
2042 		if (!pt_root) {
2043 			domain_id_free(domain->id);
2044 			return -ENOMEM;
2045 		}
2046 	}
2047 
2048 	amd_iommu_domain_set_pgtable(domain, pt_root, mode);
2049 
2050 	return 0;
2051 }
2052 
2053 static int protection_domain_init_v2(struct protection_domain *domain)
2054 {
2055 	spin_lock_init(&domain->lock);
2056 	domain->id = domain_id_alloc();
2057 	if (!domain->id)
2058 		return -ENOMEM;
2059 	INIT_LIST_HEAD(&domain->dev_list);
2060 
2061 	domain->flags |= PD_GIOV_MASK;
2062 
2063 	domain->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2;
2064 
2065 	if (domain_enable_v2(domain, 1)) {
2066 		domain_id_free(domain->id);
2067 		return -ENOMEM;
2068 	}
2069 
2070 	return 0;
2071 }
2072 
2073 static struct protection_domain *protection_domain_alloc(unsigned int type)
2074 {
2075 	struct io_pgtable_ops *pgtbl_ops;
2076 	struct protection_domain *domain;
2077 	int pgtable;
2078 	int mode = DEFAULT_PGTABLE_LEVEL;
2079 	int ret;
2080 
2081 	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
2082 	if (!domain)
2083 		return NULL;
2084 
2085 	/*
2086 	 * Force IOMMU v1 page table when iommu=pt and
2087 	 * when allocating domain for pass-through devices.
2088 	 */
2089 	if (type == IOMMU_DOMAIN_IDENTITY) {
2090 		pgtable = AMD_IOMMU_V1;
2091 		mode = PAGE_MODE_NONE;
2092 	} else if (type == IOMMU_DOMAIN_UNMANAGED) {
2093 		pgtable = AMD_IOMMU_V1;
2094 	} else if (type == IOMMU_DOMAIN_DMA || type == IOMMU_DOMAIN_DMA_FQ) {
2095 		pgtable = amd_iommu_pgtable;
2096 	} else {
2097 		return NULL;
2098 	}
2099 
2100 	switch (pgtable) {
2101 	case AMD_IOMMU_V1:
2102 		ret = protection_domain_init_v1(domain, mode);
2103 		break;
2104 	case AMD_IOMMU_V2:
2105 		ret = protection_domain_init_v2(domain);
2106 		break;
2107 	default:
2108 		ret = -EINVAL;
2109 	}
2110 
2111 	if (ret)
2112 		goto out_err;
2113 
2114 	/* No need to allocate io pgtable ops in passthrough mode */
2115 	if (type == IOMMU_DOMAIN_IDENTITY)
2116 		return domain;
2117 
2118 	domain->nid = NUMA_NO_NODE;
2119 
2120 	pgtbl_ops = alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl_cfg, domain);
2121 	if (!pgtbl_ops) {
2122 		domain_id_free(domain->id);
2123 		goto out_err;
2124 	}
2125 
2126 	return domain;
2127 out_err:
2128 	kfree(domain);
2129 	return NULL;
2130 }
2131 
2132 static inline u64 dma_max_address(void)
2133 {
2134 	if (amd_iommu_pgtable == AMD_IOMMU_V1)
2135 		return ~0ULL;
2136 
2137 	/* V2 with 4/5 level page table */
2138 	return ((1ULL << PM_LEVEL_SHIFT(amd_iommu_gpt_level)) - 1);
2139 }
2140 
2141 static struct iommu_domain *amd_iommu_domain_alloc(unsigned type)
2142 {
2143 	struct protection_domain *domain;
2144 
2145 	/*
2146 	 * Since DTE[Mode]=0 is prohibited on SNP-enabled system,
2147 	 * default to use IOMMU_DOMAIN_DMA[_FQ].
2148 	 */
2149 	if (amd_iommu_snp_en && (type == IOMMU_DOMAIN_IDENTITY))
2150 		return NULL;
2151 
2152 	domain = protection_domain_alloc(type);
2153 	if (!domain)
2154 		return NULL;
2155 
2156 	domain->domain.geometry.aperture_start = 0;
2157 	domain->domain.geometry.aperture_end   = dma_max_address();
2158 	domain->domain.geometry.force_aperture = true;
2159 
2160 	return &domain->domain;
2161 }
2162 
2163 static void amd_iommu_domain_free(struct iommu_domain *dom)
2164 {
2165 	struct protection_domain *domain;
2166 
2167 	domain = to_pdomain(dom);
2168 
2169 	if (domain->dev_cnt > 0)
2170 		cleanup_domain(domain);
2171 
2172 	BUG_ON(domain->dev_cnt != 0);
2173 
2174 	if (!dom)
2175 		return;
2176 
2177 	if (domain->flags & PD_IOMMUV2_MASK)
2178 		free_gcr3_table(domain);
2179 
2180 	protection_domain_free(domain);
2181 }
2182 
2183 static int amd_iommu_attach_device(struct iommu_domain *dom,
2184 				   struct device *dev)
2185 {
2186 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2187 	struct protection_domain *domain = to_pdomain(dom);
2188 	struct amd_iommu *iommu = rlookup_amd_iommu(dev);
2189 	int ret;
2190 
2191 	/*
2192 	 * Skip attach device to domain if new domain is same as
2193 	 * devices current domain
2194 	 */
2195 	if (dev_data->domain == domain)
2196 		return 0;
2197 
2198 	dev_data->defer_attach = false;
2199 
2200 	if (dev_data->domain)
2201 		detach_device(dev);
2202 
2203 	ret = attach_device(dev, domain);
2204 
2205 #ifdef CONFIG_IRQ_REMAP
2206 	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) {
2207 		if (dom->type == IOMMU_DOMAIN_UNMANAGED)
2208 			dev_data->use_vapic = 1;
2209 		else
2210 			dev_data->use_vapic = 0;
2211 	}
2212 #endif
2213 
2214 	iommu_completion_wait(iommu);
2215 
2216 	return ret;
2217 }
2218 
2219 static void amd_iommu_iotlb_sync_map(struct iommu_domain *dom,
2220 				     unsigned long iova, size_t size)
2221 {
2222 	struct protection_domain *domain = to_pdomain(dom);
2223 	struct io_pgtable_ops *ops = &domain->iop.iop.ops;
2224 
2225 	if (ops->map_pages)
2226 		domain_flush_np_cache(domain, iova, size);
2227 }
2228 
2229 static int amd_iommu_map_pages(struct iommu_domain *dom, unsigned long iova,
2230 			       phys_addr_t paddr, size_t pgsize, size_t pgcount,
2231 			       int iommu_prot, gfp_t gfp, size_t *mapped)
2232 {
2233 	struct protection_domain *domain = to_pdomain(dom);
2234 	struct io_pgtable_ops *ops = &domain->iop.iop.ops;
2235 	int prot = 0;
2236 	int ret = -EINVAL;
2237 
2238 	if ((amd_iommu_pgtable == AMD_IOMMU_V1) &&
2239 	    (domain->iop.mode == PAGE_MODE_NONE))
2240 		return -EINVAL;
2241 
2242 	if (iommu_prot & IOMMU_READ)
2243 		prot |= IOMMU_PROT_IR;
2244 	if (iommu_prot & IOMMU_WRITE)
2245 		prot |= IOMMU_PROT_IW;
2246 
2247 	if (ops->map_pages) {
2248 		ret = ops->map_pages(ops, iova, paddr, pgsize,
2249 				     pgcount, prot, gfp, mapped);
2250 	}
2251 
2252 	return ret;
2253 }
2254 
2255 static void amd_iommu_iotlb_gather_add_page(struct iommu_domain *domain,
2256 					    struct iommu_iotlb_gather *gather,
2257 					    unsigned long iova, size_t size)
2258 {
2259 	/*
2260 	 * AMD's IOMMU can flush as many pages as necessary in a single flush.
2261 	 * Unless we run in a virtual machine, which can be inferred according
2262 	 * to whether "non-present cache" is on, it is probably best to prefer
2263 	 * (potentially) too extensive TLB flushing (i.e., more misses) over
2264 	 * mutliple TLB flushes (i.e., more flushes). For virtual machines the
2265 	 * hypervisor needs to synchronize the host IOMMU PTEs with those of
2266 	 * the guest, and the trade-off is different: unnecessary TLB flushes
2267 	 * should be avoided.
2268 	 */
2269 	if (amd_iommu_np_cache &&
2270 	    iommu_iotlb_gather_is_disjoint(gather, iova, size))
2271 		iommu_iotlb_sync(domain, gather);
2272 
2273 	iommu_iotlb_gather_add_range(gather, iova, size);
2274 }
2275 
2276 static size_t amd_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova,
2277 				    size_t pgsize, size_t pgcount,
2278 				    struct iommu_iotlb_gather *gather)
2279 {
2280 	struct protection_domain *domain = to_pdomain(dom);
2281 	struct io_pgtable_ops *ops = &domain->iop.iop.ops;
2282 	size_t r;
2283 
2284 	if ((amd_iommu_pgtable == AMD_IOMMU_V1) &&
2285 	    (domain->iop.mode == PAGE_MODE_NONE))
2286 		return 0;
2287 
2288 	r = (ops->unmap_pages) ? ops->unmap_pages(ops, iova, pgsize, pgcount, NULL) : 0;
2289 
2290 	if (r)
2291 		amd_iommu_iotlb_gather_add_page(dom, gather, iova, r);
2292 
2293 	return r;
2294 }
2295 
2296 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
2297 					  dma_addr_t iova)
2298 {
2299 	struct protection_domain *domain = to_pdomain(dom);
2300 	struct io_pgtable_ops *ops = &domain->iop.iop.ops;
2301 
2302 	return ops->iova_to_phys(ops, iova);
2303 }
2304 
2305 static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap)
2306 {
2307 	switch (cap) {
2308 	case IOMMU_CAP_CACHE_COHERENCY:
2309 		return true;
2310 	case IOMMU_CAP_NOEXEC:
2311 		return false;
2312 	case IOMMU_CAP_PRE_BOOT_PROTECTION:
2313 		return amdr_ivrs_remap_support;
2314 	case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
2315 		return true;
2316 	default:
2317 		break;
2318 	}
2319 
2320 	return false;
2321 }
2322 
2323 static void amd_iommu_get_resv_regions(struct device *dev,
2324 				       struct list_head *head)
2325 {
2326 	struct iommu_resv_region *region;
2327 	struct unity_map_entry *entry;
2328 	struct amd_iommu *iommu;
2329 	struct amd_iommu_pci_seg *pci_seg;
2330 	int devid, sbdf;
2331 
2332 	sbdf = get_device_sbdf_id(dev);
2333 	if (sbdf < 0)
2334 		return;
2335 
2336 	devid = PCI_SBDF_TO_DEVID(sbdf);
2337 	iommu = rlookup_amd_iommu(dev);
2338 	if (!iommu)
2339 		return;
2340 	pci_seg = iommu->pci_seg;
2341 
2342 	list_for_each_entry(entry, &pci_seg->unity_map, list) {
2343 		int type, prot = 0;
2344 		size_t length;
2345 
2346 		if (devid < entry->devid_start || devid > entry->devid_end)
2347 			continue;
2348 
2349 		type   = IOMMU_RESV_DIRECT;
2350 		length = entry->address_end - entry->address_start;
2351 		if (entry->prot & IOMMU_PROT_IR)
2352 			prot |= IOMMU_READ;
2353 		if (entry->prot & IOMMU_PROT_IW)
2354 			prot |= IOMMU_WRITE;
2355 		if (entry->prot & IOMMU_UNITY_MAP_FLAG_EXCL_RANGE)
2356 			/* Exclusion range */
2357 			type = IOMMU_RESV_RESERVED;
2358 
2359 		region = iommu_alloc_resv_region(entry->address_start,
2360 						 length, prot, type,
2361 						 GFP_KERNEL);
2362 		if (!region) {
2363 			dev_err(dev, "Out of memory allocating dm-regions\n");
2364 			return;
2365 		}
2366 		list_add_tail(&region->list, head);
2367 	}
2368 
2369 	region = iommu_alloc_resv_region(MSI_RANGE_START,
2370 					 MSI_RANGE_END - MSI_RANGE_START + 1,
2371 					 0, IOMMU_RESV_MSI, GFP_KERNEL);
2372 	if (!region)
2373 		return;
2374 	list_add_tail(&region->list, head);
2375 
2376 	region = iommu_alloc_resv_region(HT_RANGE_START,
2377 					 HT_RANGE_END - HT_RANGE_START + 1,
2378 					 0, IOMMU_RESV_RESERVED, GFP_KERNEL);
2379 	if (!region)
2380 		return;
2381 	list_add_tail(&region->list, head);
2382 }
2383 
2384 bool amd_iommu_is_attach_deferred(struct device *dev)
2385 {
2386 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2387 
2388 	return dev_data->defer_attach;
2389 }
2390 EXPORT_SYMBOL_GPL(amd_iommu_is_attach_deferred);
2391 
2392 static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
2393 {
2394 	struct protection_domain *dom = to_pdomain(domain);
2395 	unsigned long flags;
2396 
2397 	spin_lock_irqsave(&dom->lock, flags);
2398 	amd_iommu_domain_flush_tlb_pde(dom);
2399 	amd_iommu_domain_flush_complete(dom);
2400 	spin_unlock_irqrestore(&dom->lock, flags);
2401 }
2402 
2403 static void amd_iommu_iotlb_sync(struct iommu_domain *domain,
2404 				 struct iommu_iotlb_gather *gather)
2405 {
2406 	struct protection_domain *dom = to_pdomain(domain);
2407 	unsigned long flags;
2408 
2409 	spin_lock_irqsave(&dom->lock, flags);
2410 	domain_flush_pages(dom, gather->start, gather->end - gather->start + 1, 1);
2411 	amd_iommu_domain_flush_complete(dom);
2412 	spin_unlock_irqrestore(&dom->lock, flags);
2413 }
2414 
2415 static int amd_iommu_def_domain_type(struct device *dev)
2416 {
2417 	struct iommu_dev_data *dev_data;
2418 
2419 	dev_data = dev_iommu_priv_get(dev);
2420 	if (!dev_data)
2421 		return 0;
2422 
2423 	/*
2424 	 * Do not identity map IOMMUv2 capable devices when:
2425 	 *  - memory encryption is active, because some of those devices
2426 	 *    (AMD GPUs) don't have the encryption bit in their DMA-mask
2427 	 *    and require remapping.
2428 	 *  - SNP is enabled, because it prohibits DTE[Mode]=0.
2429 	 */
2430 	if (dev_data->iommu_v2 &&
2431 	    !cc_platform_has(CC_ATTR_MEM_ENCRYPT) &&
2432 	    !amd_iommu_snp_en) {
2433 		return IOMMU_DOMAIN_IDENTITY;
2434 	}
2435 
2436 	return 0;
2437 }
2438 
2439 static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain)
2440 {
2441 	/* IOMMU_PTE_FC is always set */
2442 	return true;
2443 }
2444 
2445 const struct iommu_ops amd_iommu_ops = {
2446 	.capable = amd_iommu_capable,
2447 	.domain_alloc = amd_iommu_domain_alloc,
2448 	.probe_device = amd_iommu_probe_device,
2449 	.release_device = amd_iommu_release_device,
2450 	.probe_finalize = amd_iommu_probe_finalize,
2451 	.device_group = amd_iommu_device_group,
2452 	.get_resv_regions = amd_iommu_get_resv_regions,
2453 	.is_attach_deferred = amd_iommu_is_attach_deferred,
2454 	.pgsize_bitmap	= AMD_IOMMU_PGSIZES,
2455 	.def_domain_type = amd_iommu_def_domain_type,
2456 	.default_domain_ops = &(const struct iommu_domain_ops) {
2457 		.attach_dev	= amd_iommu_attach_device,
2458 		.map_pages	= amd_iommu_map_pages,
2459 		.unmap_pages	= amd_iommu_unmap_pages,
2460 		.iotlb_sync_map	= amd_iommu_iotlb_sync_map,
2461 		.iova_to_phys	= amd_iommu_iova_to_phys,
2462 		.flush_iotlb_all = amd_iommu_flush_iotlb_all,
2463 		.iotlb_sync	= amd_iommu_iotlb_sync,
2464 		.free		= amd_iommu_domain_free,
2465 		.enforce_cache_coherency = amd_iommu_enforce_cache_coherency,
2466 	}
2467 };
2468 
2469 /*****************************************************************************
2470  *
2471  * The next functions do a basic initialization of IOMMU for pass through
2472  * mode
2473  *
2474  * In passthrough mode the IOMMU is initialized and enabled but not used for
2475  * DMA-API translation.
2476  *
2477  *****************************************************************************/
2478 
2479 /* IOMMUv2 specific functions */
2480 int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
2481 {
2482 	return atomic_notifier_chain_register(&ppr_notifier, nb);
2483 }
2484 EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);
2485 
2486 int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
2487 {
2488 	return atomic_notifier_chain_unregister(&ppr_notifier, nb);
2489 }
2490 EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
2491 
2492 void amd_iommu_domain_direct_map(struct iommu_domain *dom)
2493 {
2494 	struct protection_domain *domain = to_pdomain(dom);
2495 	unsigned long flags;
2496 
2497 	spin_lock_irqsave(&domain->lock, flags);
2498 
2499 	if (domain->iop.pgtbl_cfg.tlb)
2500 		free_io_pgtable_ops(&domain->iop.iop.ops);
2501 
2502 	spin_unlock_irqrestore(&domain->lock, flags);
2503 }
2504 EXPORT_SYMBOL(amd_iommu_domain_direct_map);
2505 
2506 /* Note: This function expects iommu_domain->lock to be held prior calling the function. */
2507 static int domain_enable_v2(struct protection_domain *domain, int pasids)
2508 {
2509 	int levels;
2510 
2511 	/* Number of GCR3 table levels required */
2512 	for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
2513 		levels += 1;
2514 
2515 	if (levels > amd_iommu_max_glx_val)
2516 		return -EINVAL;
2517 
2518 	domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
2519 	if (domain->gcr3_tbl == NULL)
2520 		return -ENOMEM;
2521 
2522 	domain->glx      = levels;
2523 	domain->flags   |= PD_IOMMUV2_MASK;
2524 
2525 	amd_iommu_domain_update(domain);
2526 
2527 	return 0;
2528 }
2529 
2530 int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
2531 {
2532 	struct protection_domain *pdom = to_pdomain(dom);
2533 	unsigned long flags;
2534 	int ret;
2535 
2536 	spin_lock_irqsave(&pdom->lock, flags);
2537 
2538 	/*
2539 	 * Save us all sanity checks whether devices already in the
2540 	 * domain support IOMMUv2. Just force that the domain has no
2541 	 * devices attached when it is switched into IOMMUv2 mode.
2542 	 */
2543 	ret = -EBUSY;
2544 	if (pdom->dev_cnt > 0 || pdom->flags & PD_IOMMUV2_MASK)
2545 		goto out;
2546 
2547 	if (!pdom->gcr3_tbl)
2548 		ret = domain_enable_v2(pdom, pasids);
2549 
2550 out:
2551 	spin_unlock_irqrestore(&pdom->lock, flags);
2552 	return ret;
2553 }
2554 EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
2555 
2556 static int __flush_pasid(struct protection_domain *domain, u32 pasid,
2557 			 u64 address, bool size)
2558 {
2559 	struct iommu_dev_data *dev_data;
2560 	struct iommu_cmd cmd;
2561 	int i, ret;
2562 
2563 	if (!(domain->flags & PD_IOMMUV2_MASK))
2564 		return -EINVAL;
2565 
2566 	build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);
2567 
2568 	/*
2569 	 * IOMMU TLB needs to be flushed before Device TLB to
2570 	 * prevent device TLB refill from IOMMU TLB
2571 	 */
2572 	for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {
2573 		if (domain->dev_iommu[i] == 0)
2574 			continue;
2575 
2576 		ret = iommu_queue_command(amd_iommus[i], &cmd);
2577 		if (ret != 0)
2578 			goto out;
2579 	}
2580 
2581 	/* Wait until IOMMU TLB flushes are complete */
2582 	amd_iommu_domain_flush_complete(domain);
2583 
2584 	/* Now flush device TLBs */
2585 	list_for_each_entry(dev_data, &domain->dev_list, list) {
2586 		struct amd_iommu *iommu;
2587 		int qdep;
2588 
2589 		/*
2590 		   There might be non-IOMMUv2 capable devices in an IOMMUv2
2591 		 * domain.
2592 		 */
2593 		if (!dev_data->ats.enabled)
2594 			continue;
2595 
2596 		qdep  = dev_data->ats.qdep;
2597 		iommu = rlookup_amd_iommu(dev_data->dev);
2598 		if (!iommu)
2599 			continue;
2600 		build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,
2601 				      qdep, address, size);
2602 
2603 		ret = iommu_queue_command(iommu, &cmd);
2604 		if (ret != 0)
2605 			goto out;
2606 	}
2607 
2608 	/* Wait until all device TLBs are flushed */
2609 	amd_iommu_domain_flush_complete(domain);
2610 
2611 	ret = 0;
2612 
2613 out:
2614 
2615 	return ret;
2616 }
2617 
2618 static int __amd_iommu_flush_page(struct protection_domain *domain, u32 pasid,
2619 				  u64 address)
2620 {
2621 	return __flush_pasid(domain, pasid, address, false);
2622 }
2623 
2624 int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid,
2625 			 u64 address)
2626 {
2627 	struct protection_domain *domain = to_pdomain(dom);
2628 	unsigned long flags;
2629 	int ret;
2630 
2631 	spin_lock_irqsave(&domain->lock, flags);
2632 	ret = __amd_iommu_flush_page(domain, pasid, address);
2633 	spin_unlock_irqrestore(&domain->lock, flags);
2634 
2635 	return ret;
2636 }
2637 EXPORT_SYMBOL(amd_iommu_flush_page);
2638 
2639 static int __amd_iommu_flush_tlb(struct protection_domain *domain, u32 pasid)
2640 {
2641 	return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
2642 			     true);
2643 }
2644 
2645 int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid)
2646 {
2647 	struct protection_domain *domain = to_pdomain(dom);
2648 	unsigned long flags;
2649 	int ret;
2650 
2651 	spin_lock_irqsave(&domain->lock, flags);
2652 	ret = __amd_iommu_flush_tlb(domain, pasid);
2653 	spin_unlock_irqrestore(&domain->lock, flags);
2654 
2655 	return ret;
2656 }
2657 EXPORT_SYMBOL(amd_iommu_flush_tlb);
2658 
2659 static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc)
2660 {
2661 	int index;
2662 	u64 *pte;
2663 
2664 	while (true) {
2665 
2666 		index = (pasid >> (9 * level)) & 0x1ff;
2667 		pte   = &root[index];
2668 
2669 		if (level == 0)
2670 			break;
2671 
2672 		if (!(*pte & GCR3_VALID)) {
2673 			if (!alloc)
2674 				return NULL;
2675 
2676 			root = (void *)get_zeroed_page(GFP_ATOMIC);
2677 			if (root == NULL)
2678 				return NULL;
2679 
2680 			*pte = iommu_virt_to_phys(root) | GCR3_VALID;
2681 		}
2682 
2683 		root = iommu_phys_to_virt(*pte & PAGE_MASK);
2684 
2685 		level -= 1;
2686 	}
2687 
2688 	return pte;
2689 }
2690 
2691 static int __set_gcr3(struct protection_domain *domain, u32 pasid,
2692 		      unsigned long cr3)
2693 {
2694 	u64 *pte;
2695 
2696 	if (domain->iop.mode != PAGE_MODE_NONE)
2697 		return -EINVAL;
2698 
2699 	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
2700 	if (pte == NULL)
2701 		return -ENOMEM;
2702 
2703 	*pte = (cr3 & PAGE_MASK) | GCR3_VALID;
2704 
2705 	return __amd_iommu_flush_tlb(domain, pasid);
2706 }
2707 
2708 static int __clear_gcr3(struct protection_domain *domain, u32 pasid)
2709 {
2710 	u64 *pte;
2711 
2712 	if (domain->iop.mode != PAGE_MODE_NONE)
2713 		return -EINVAL;
2714 
2715 	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
2716 	if (pte == NULL)
2717 		return 0;
2718 
2719 	*pte = 0;
2720 
2721 	return __amd_iommu_flush_tlb(domain, pasid);
2722 }
2723 
2724 int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid,
2725 			      unsigned long cr3)
2726 {
2727 	struct protection_domain *domain = to_pdomain(dom);
2728 	unsigned long flags;
2729 	int ret;
2730 
2731 	spin_lock_irqsave(&domain->lock, flags);
2732 	ret = __set_gcr3(domain, pasid, cr3);
2733 	spin_unlock_irqrestore(&domain->lock, flags);
2734 
2735 	return ret;
2736 }
2737 EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);
2738 
2739 int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid)
2740 {
2741 	struct protection_domain *domain = to_pdomain(dom);
2742 	unsigned long flags;
2743 	int ret;
2744 
2745 	spin_lock_irqsave(&domain->lock, flags);
2746 	ret = __clear_gcr3(domain, pasid);
2747 	spin_unlock_irqrestore(&domain->lock, flags);
2748 
2749 	return ret;
2750 }
2751 EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
2752 
2753 int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid,
2754 			   int status, int tag)
2755 {
2756 	struct iommu_dev_data *dev_data;
2757 	struct amd_iommu *iommu;
2758 	struct iommu_cmd cmd;
2759 
2760 	dev_data = dev_iommu_priv_get(&pdev->dev);
2761 	iommu    = rlookup_amd_iommu(&pdev->dev);
2762 	if (!iommu)
2763 		return -ENODEV;
2764 
2765 	build_complete_ppr(&cmd, dev_data->devid, pasid, status,
2766 			   tag, dev_data->pri_tlp);
2767 
2768 	return iommu_queue_command(iommu, &cmd);
2769 }
2770 EXPORT_SYMBOL(amd_iommu_complete_ppr);
2771 
2772 int amd_iommu_device_info(struct pci_dev *pdev,
2773                           struct amd_iommu_device_info *info)
2774 {
2775 	int max_pasids;
2776 	int pos;
2777 
2778 	if (pdev == NULL || info == NULL)
2779 		return -EINVAL;
2780 
2781 	if (!amd_iommu_v2_supported())
2782 		return -EINVAL;
2783 
2784 	memset(info, 0, sizeof(*info));
2785 
2786 	if (pci_ats_supported(pdev))
2787 		info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
2788 
2789 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2790 	if (pos)
2791 		info->flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;
2792 
2793 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
2794 	if (pos) {
2795 		int features;
2796 
2797 		max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));
2798 		max_pasids = min(max_pasids, (1 << 20));
2799 
2800 		info->flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
2801 		info->max_pasids = min(pci_max_pasids(pdev), max_pasids);
2802 
2803 		features = pci_pasid_features(pdev);
2804 		if (features & PCI_PASID_CAP_EXEC)
2805 			info->flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
2806 		if (features & PCI_PASID_CAP_PRIV)
2807 			info->flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
2808 	}
2809 
2810 	return 0;
2811 }
2812 EXPORT_SYMBOL(amd_iommu_device_info);
2813 
2814 #ifdef CONFIG_IRQ_REMAP
2815 
2816 /*****************************************************************************
2817  *
2818  * Interrupt Remapping Implementation
2819  *
2820  *****************************************************************************/
2821 
2822 static struct irq_chip amd_ir_chip;
2823 static DEFINE_SPINLOCK(iommu_table_lock);
2824 
2825 static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid,
2826 			      struct irq_remap_table *table)
2827 {
2828 	u64 dte;
2829 	struct dev_table_entry *dev_table = get_dev_table(iommu);
2830 
2831 	dte	= dev_table[devid].data[2];
2832 	dte	&= ~DTE_IRQ_PHYS_ADDR_MASK;
2833 	dte	|= iommu_virt_to_phys(table->table);
2834 	dte	|= DTE_IRQ_REMAP_INTCTL;
2835 	dte	|= DTE_INTTABLEN;
2836 	dte	|= DTE_IRQ_REMAP_ENABLE;
2837 
2838 	dev_table[devid].data[2] = dte;
2839 }
2840 
2841 static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid)
2842 {
2843 	struct irq_remap_table *table;
2844 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
2845 
2846 	if (WARN_ONCE(!pci_seg->rlookup_table[devid],
2847 		      "%s: no iommu for devid %x:%x\n",
2848 		      __func__, pci_seg->id, devid))
2849 		return NULL;
2850 
2851 	table = pci_seg->irq_lookup_table[devid];
2852 	if (WARN_ONCE(!table, "%s: no table for devid %x:%x\n",
2853 		      __func__, pci_seg->id, devid))
2854 		return NULL;
2855 
2856 	return table;
2857 }
2858 
2859 static struct irq_remap_table *__alloc_irq_table(void)
2860 {
2861 	struct irq_remap_table *table;
2862 
2863 	table = kzalloc(sizeof(*table), GFP_KERNEL);
2864 	if (!table)
2865 		return NULL;
2866 
2867 	table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_KERNEL);
2868 	if (!table->table) {
2869 		kfree(table);
2870 		return NULL;
2871 	}
2872 	raw_spin_lock_init(&table->lock);
2873 
2874 	if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
2875 		memset(table->table, 0,
2876 		       MAX_IRQS_PER_TABLE * sizeof(u32));
2877 	else
2878 		memset(table->table, 0,
2879 		       (MAX_IRQS_PER_TABLE * (sizeof(u64) * 2)));
2880 	return table;
2881 }
2882 
2883 static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid,
2884 				  struct irq_remap_table *table)
2885 {
2886 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
2887 
2888 	pci_seg->irq_lookup_table[devid] = table;
2889 	set_dte_irq_entry(iommu, devid, table);
2890 	iommu_flush_dte(iommu, devid);
2891 }
2892 
2893 static int set_remap_table_entry_alias(struct pci_dev *pdev, u16 alias,
2894 				       void *data)
2895 {
2896 	struct irq_remap_table *table = data;
2897 	struct amd_iommu_pci_seg *pci_seg;
2898 	struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);
2899 
2900 	if (!iommu)
2901 		return -EINVAL;
2902 
2903 	pci_seg = iommu->pci_seg;
2904 	pci_seg->irq_lookup_table[alias] = table;
2905 	set_dte_irq_entry(iommu, alias, table);
2906 	iommu_flush_dte(pci_seg->rlookup_table[alias], alias);
2907 
2908 	return 0;
2909 }
2910 
2911 static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu,
2912 					       u16 devid, struct pci_dev *pdev)
2913 {
2914 	struct irq_remap_table *table = NULL;
2915 	struct irq_remap_table *new_table = NULL;
2916 	struct amd_iommu_pci_seg *pci_seg;
2917 	unsigned long flags;
2918 	u16 alias;
2919 
2920 	spin_lock_irqsave(&iommu_table_lock, flags);
2921 
2922 	pci_seg = iommu->pci_seg;
2923 	table = pci_seg->irq_lookup_table[devid];
2924 	if (table)
2925 		goto out_unlock;
2926 
2927 	alias = pci_seg->alias_table[devid];
2928 	table = pci_seg->irq_lookup_table[alias];
2929 	if (table) {
2930 		set_remap_table_entry(iommu, devid, table);
2931 		goto out_wait;
2932 	}
2933 	spin_unlock_irqrestore(&iommu_table_lock, flags);
2934 
2935 	/* Nothing there yet, allocate new irq remapping table */
2936 	new_table = __alloc_irq_table();
2937 	if (!new_table)
2938 		return NULL;
2939 
2940 	spin_lock_irqsave(&iommu_table_lock, flags);
2941 
2942 	table = pci_seg->irq_lookup_table[devid];
2943 	if (table)
2944 		goto out_unlock;
2945 
2946 	table = pci_seg->irq_lookup_table[alias];
2947 	if (table) {
2948 		set_remap_table_entry(iommu, devid, table);
2949 		goto out_wait;
2950 	}
2951 
2952 	table = new_table;
2953 	new_table = NULL;
2954 
2955 	if (pdev)
2956 		pci_for_each_dma_alias(pdev, set_remap_table_entry_alias,
2957 				       table);
2958 	else
2959 		set_remap_table_entry(iommu, devid, table);
2960 
2961 	if (devid != alias)
2962 		set_remap_table_entry(iommu, alias, table);
2963 
2964 out_wait:
2965 	iommu_completion_wait(iommu);
2966 
2967 out_unlock:
2968 	spin_unlock_irqrestore(&iommu_table_lock, flags);
2969 
2970 	if (new_table) {
2971 		kmem_cache_free(amd_iommu_irq_cache, new_table->table);
2972 		kfree(new_table);
2973 	}
2974 	return table;
2975 }
2976 
2977 static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count,
2978 			   bool align, struct pci_dev *pdev)
2979 {
2980 	struct irq_remap_table *table;
2981 	int index, c, alignment = 1;
2982 	unsigned long flags;
2983 
2984 	table = alloc_irq_table(iommu, devid, pdev);
2985 	if (!table)
2986 		return -ENODEV;
2987 
2988 	if (align)
2989 		alignment = roundup_pow_of_two(count);
2990 
2991 	raw_spin_lock_irqsave(&table->lock, flags);
2992 
2993 	/* Scan table for free entries */
2994 	for (index = ALIGN(table->min_index, alignment), c = 0;
2995 	     index < MAX_IRQS_PER_TABLE;) {
2996 		if (!iommu->irte_ops->is_allocated(table, index)) {
2997 			c += 1;
2998 		} else {
2999 			c     = 0;
3000 			index = ALIGN(index + 1, alignment);
3001 			continue;
3002 		}
3003 
3004 		if (c == count)	{
3005 			for (; c != 0; --c)
3006 				iommu->irte_ops->set_allocated(table, index - c + 1);
3007 
3008 			index -= count - 1;
3009 			goto out;
3010 		}
3011 
3012 		index++;
3013 	}
3014 
3015 	index = -ENOSPC;
3016 
3017 out:
3018 	raw_spin_unlock_irqrestore(&table->lock, flags);
3019 
3020 	return index;
3021 }
3022 
3023 static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
3024 			  struct irte_ga *irte, struct amd_ir_data *data)
3025 {
3026 	bool ret;
3027 	struct irq_remap_table *table;
3028 	unsigned long flags;
3029 	struct irte_ga *entry;
3030 
3031 	table = get_irq_table(iommu, devid);
3032 	if (!table)
3033 		return -ENOMEM;
3034 
3035 	raw_spin_lock_irqsave(&table->lock, flags);
3036 
3037 	entry = (struct irte_ga *)table->table;
3038 	entry = &entry[index];
3039 
3040 	ret = cmpxchg_double(&entry->lo.val, &entry->hi.val,
3041 			     entry->lo.val, entry->hi.val,
3042 			     irte->lo.val, irte->hi.val);
3043 	/*
3044 	 * We use cmpxchg16 to atomically update the 128-bit IRTE,
3045 	 * and it cannot be updated by the hardware or other processors
3046 	 * behind us, so the return value of cmpxchg16 should be the
3047 	 * same as the old value.
3048 	 */
3049 	WARN_ON(!ret);
3050 
3051 	if (data)
3052 		data->ref = entry;
3053 
3054 	raw_spin_unlock_irqrestore(&table->lock, flags);
3055 
3056 	iommu_flush_irt(iommu, devid);
3057 	iommu_completion_wait(iommu);
3058 
3059 	return 0;
3060 }
3061 
3062 static int modify_irte(struct amd_iommu *iommu,
3063 		       u16 devid, int index, union irte *irte)
3064 {
3065 	struct irq_remap_table *table;
3066 	unsigned long flags;
3067 
3068 	table = get_irq_table(iommu, devid);
3069 	if (!table)
3070 		return -ENOMEM;
3071 
3072 	raw_spin_lock_irqsave(&table->lock, flags);
3073 	table->table[index] = irte->val;
3074 	raw_spin_unlock_irqrestore(&table->lock, flags);
3075 
3076 	iommu_flush_irt(iommu, devid);
3077 	iommu_completion_wait(iommu);
3078 
3079 	return 0;
3080 }
3081 
3082 static void free_irte(struct amd_iommu *iommu, u16 devid, int index)
3083 {
3084 	struct irq_remap_table *table;
3085 	unsigned long flags;
3086 
3087 	table = get_irq_table(iommu, devid);
3088 	if (!table)
3089 		return;
3090 
3091 	raw_spin_lock_irqsave(&table->lock, flags);
3092 	iommu->irte_ops->clear_allocated(table, index);
3093 	raw_spin_unlock_irqrestore(&table->lock, flags);
3094 
3095 	iommu_flush_irt(iommu, devid);
3096 	iommu_completion_wait(iommu);
3097 }
3098 
3099 static void irte_prepare(void *entry,
3100 			 u32 delivery_mode, bool dest_mode,
3101 			 u8 vector, u32 dest_apicid, int devid)
3102 {
3103 	union irte *irte = (union irte *) entry;
3104 
3105 	irte->val                = 0;
3106 	irte->fields.vector      = vector;
3107 	irte->fields.int_type    = delivery_mode;
3108 	irte->fields.destination = dest_apicid;
3109 	irte->fields.dm          = dest_mode;
3110 	irte->fields.valid       = 1;
3111 }
3112 
3113 static void irte_ga_prepare(void *entry,
3114 			    u32 delivery_mode, bool dest_mode,
3115 			    u8 vector, u32 dest_apicid, int devid)
3116 {
3117 	struct irte_ga *irte = (struct irte_ga *) entry;
3118 
3119 	irte->lo.val                      = 0;
3120 	irte->hi.val                      = 0;
3121 	irte->lo.fields_remap.int_type    = delivery_mode;
3122 	irte->lo.fields_remap.dm          = dest_mode;
3123 	irte->hi.fields.vector            = vector;
3124 	irte->lo.fields_remap.destination = APICID_TO_IRTE_DEST_LO(dest_apicid);
3125 	irte->hi.fields.destination       = APICID_TO_IRTE_DEST_HI(dest_apicid);
3126 	irte->lo.fields_remap.valid       = 1;
3127 }
3128 
3129 static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
3130 {
3131 	union irte *irte = (union irte *) entry;
3132 
3133 	irte->fields.valid = 1;
3134 	modify_irte(iommu, devid, index, irte);
3135 }
3136 
3137 static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
3138 {
3139 	struct irte_ga *irte = (struct irte_ga *) entry;
3140 
3141 	irte->lo.fields_remap.valid = 1;
3142 	modify_irte_ga(iommu, devid, index, irte, NULL);
3143 }
3144 
3145 static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
3146 {
3147 	union irte *irte = (union irte *) entry;
3148 
3149 	irte->fields.valid = 0;
3150 	modify_irte(iommu, devid, index, irte);
3151 }
3152 
3153 static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
3154 {
3155 	struct irte_ga *irte = (struct irte_ga *) entry;
3156 
3157 	irte->lo.fields_remap.valid = 0;
3158 	modify_irte_ga(iommu, devid, index, irte, NULL);
3159 }
3160 
3161 static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,
3162 			      u8 vector, u32 dest_apicid)
3163 {
3164 	union irte *irte = (union irte *) entry;
3165 
3166 	irte->fields.vector = vector;
3167 	irte->fields.destination = dest_apicid;
3168 	modify_irte(iommu, devid, index, irte);
3169 }
3170 
3171 static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,
3172 				 u8 vector, u32 dest_apicid)
3173 {
3174 	struct irte_ga *irte = (struct irte_ga *) entry;
3175 
3176 	if (!irte->lo.fields_remap.guest_mode) {
3177 		irte->hi.fields.vector = vector;
3178 		irte->lo.fields_remap.destination =
3179 					APICID_TO_IRTE_DEST_LO(dest_apicid);
3180 		irte->hi.fields.destination =
3181 					APICID_TO_IRTE_DEST_HI(dest_apicid);
3182 		modify_irte_ga(iommu, devid, index, irte, NULL);
3183 	}
3184 }
3185 
3186 #define IRTE_ALLOCATED (~1U)
3187 static void irte_set_allocated(struct irq_remap_table *table, int index)
3188 {
3189 	table->table[index] = IRTE_ALLOCATED;
3190 }
3191 
3192 static void irte_ga_set_allocated(struct irq_remap_table *table, int index)
3193 {
3194 	struct irte_ga *ptr = (struct irte_ga *)table->table;
3195 	struct irte_ga *irte = &ptr[index];
3196 
3197 	memset(&irte->lo.val, 0, sizeof(u64));
3198 	memset(&irte->hi.val, 0, sizeof(u64));
3199 	irte->hi.fields.vector = 0xff;
3200 }
3201 
3202 static bool irte_is_allocated(struct irq_remap_table *table, int index)
3203 {
3204 	union irte *ptr = (union irte *)table->table;
3205 	union irte *irte = &ptr[index];
3206 
3207 	return irte->val != 0;
3208 }
3209 
3210 static bool irte_ga_is_allocated(struct irq_remap_table *table, int index)
3211 {
3212 	struct irte_ga *ptr = (struct irte_ga *)table->table;
3213 	struct irte_ga *irte = &ptr[index];
3214 
3215 	return irte->hi.fields.vector != 0;
3216 }
3217 
3218 static void irte_clear_allocated(struct irq_remap_table *table, int index)
3219 {
3220 	table->table[index] = 0;
3221 }
3222 
3223 static void irte_ga_clear_allocated(struct irq_remap_table *table, int index)
3224 {
3225 	struct irte_ga *ptr = (struct irte_ga *)table->table;
3226 	struct irte_ga *irte = &ptr[index];
3227 
3228 	memset(&irte->lo.val, 0, sizeof(u64));
3229 	memset(&irte->hi.val, 0, sizeof(u64));
3230 }
3231 
3232 static int get_devid(struct irq_alloc_info *info)
3233 {
3234 	switch (info->type) {
3235 	case X86_IRQ_ALLOC_TYPE_IOAPIC:
3236 		return get_ioapic_devid(info->devid);
3237 	case X86_IRQ_ALLOC_TYPE_HPET:
3238 		return get_hpet_devid(info->devid);
3239 	case X86_IRQ_ALLOC_TYPE_PCI_MSI:
3240 	case X86_IRQ_ALLOC_TYPE_PCI_MSIX:
3241 		return get_device_sbdf_id(msi_desc_to_dev(info->desc));
3242 	default:
3243 		WARN_ON_ONCE(1);
3244 		return -1;
3245 	}
3246 }
3247 
3248 struct irq_remap_ops amd_iommu_irq_ops = {
3249 	.prepare		= amd_iommu_prepare,
3250 	.enable			= amd_iommu_enable,
3251 	.disable		= amd_iommu_disable,
3252 	.reenable		= amd_iommu_reenable,
3253 	.enable_faulting	= amd_iommu_enable_faulting,
3254 };
3255 
3256 static void fill_msi_msg(struct msi_msg *msg, u32 index)
3257 {
3258 	msg->data = index;
3259 	msg->address_lo = 0;
3260 	msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;
3261 	msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;
3262 }
3263 
3264 static void irq_remapping_prepare_irte(struct amd_ir_data *data,
3265 				       struct irq_cfg *irq_cfg,
3266 				       struct irq_alloc_info *info,
3267 				       int devid, int index, int sub_handle)
3268 {
3269 	struct irq_2_irte *irte_info = &data->irq_2_irte;
3270 	struct amd_iommu *iommu = data->iommu;
3271 
3272 	if (!iommu)
3273 		return;
3274 
3275 	data->irq_2_irte.devid = devid;
3276 	data->irq_2_irte.index = index + sub_handle;
3277 	iommu->irte_ops->prepare(data->entry, apic->delivery_mode,
3278 				 apic->dest_mode_logical, irq_cfg->vector,
3279 				 irq_cfg->dest_apicid, devid);
3280 
3281 	switch (info->type) {
3282 	case X86_IRQ_ALLOC_TYPE_IOAPIC:
3283 	case X86_IRQ_ALLOC_TYPE_HPET:
3284 	case X86_IRQ_ALLOC_TYPE_PCI_MSI:
3285 	case X86_IRQ_ALLOC_TYPE_PCI_MSIX:
3286 		fill_msi_msg(&data->msi_entry, irte_info->index);
3287 		break;
3288 
3289 	default:
3290 		BUG_ON(1);
3291 		break;
3292 	}
3293 }
3294 
3295 struct amd_irte_ops irte_32_ops = {
3296 	.prepare = irte_prepare,
3297 	.activate = irte_activate,
3298 	.deactivate = irte_deactivate,
3299 	.set_affinity = irte_set_affinity,
3300 	.set_allocated = irte_set_allocated,
3301 	.is_allocated = irte_is_allocated,
3302 	.clear_allocated = irte_clear_allocated,
3303 };
3304 
3305 struct amd_irte_ops irte_128_ops = {
3306 	.prepare = irte_ga_prepare,
3307 	.activate = irte_ga_activate,
3308 	.deactivate = irte_ga_deactivate,
3309 	.set_affinity = irte_ga_set_affinity,
3310 	.set_allocated = irte_ga_set_allocated,
3311 	.is_allocated = irte_ga_is_allocated,
3312 	.clear_allocated = irte_ga_clear_allocated,
3313 };
3314 
3315 static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
3316 			       unsigned int nr_irqs, void *arg)
3317 {
3318 	struct irq_alloc_info *info = arg;
3319 	struct irq_data *irq_data;
3320 	struct amd_ir_data *data = NULL;
3321 	struct amd_iommu *iommu;
3322 	struct irq_cfg *cfg;
3323 	int i, ret, devid, seg, sbdf;
3324 	int index;
3325 
3326 	if (!info)
3327 		return -EINVAL;
3328 	if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_PCI_MSI)
3329 		return -EINVAL;
3330 
3331 	sbdf = get_devid(info);
3332 	if (sbdf < 0)
3333 		return -EINVAL;
3334 
3335 	seg = PCI_SBDF_TO_SEGID(sbdf);
3336 	devid = PCI_SBDF_TO_DEVID(sbdf);
3337 	iommu = __rlookup_amd_iommu(seg, devid);
3338 	if (!iommu)
3339 		return -EINVAL;
3340 
3341 	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
3342 	if (ret < 0)
3343 		return ret;
3344 
3345 	if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) {
3346 		struct irq_remap_table *table;
3347 
3348 		table = alloc_irq_table(iommu, devid, NULL);
3349 		if (table) {
3350 			if (!table->min_index) {
3351 				/*
3352 				 * Keep the first 32 indexes free for IOAPIC
3353 				 * interrupts.
3354 				 */
3355 				table->min_index = 32;
3356 				for (i = 0; i < 32; ++i)
3357 					iommu->irte_ops->set_allocated(table, i);
3358 			}
3359 			WARN_ON(table->min_index != 32);
3360 			index = info->ioapic.pin;
3361 		} else {
3362 			index = -ENOMEM;
3363 		}
3364 	} else if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI ||
3365 		   info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX) {
3366 		bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI);
3367 
3368 		index = alloc_irq_index(iommu, devid, nr_irqs, align,
3369 					msi_desc_to_pci_dev(info->desc));
3370 	} else {
3371 		index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL);
3372 	}
3373 
3374 	if (index < 0) {
3375 		pr_warn("Failed to allocate IRTE\n");
3376 		ret = index;
3377 		goto out_free_parent;
3378 	}
3379 
3380 	for (i = 0; i < nr_irqs; i++) {
3381 		irq_data = irq_domain_get_irq_data(domain, virq + i);
3382 		cfg = irq_data ? irqd_cfg(irq_data) : NULL;
3383 		if (!cfg) {
3384 			ret = -EINVAL;
3385 			goto out_free_data;
3386 		}
3387 
3388 		ret = -ENOMEM;
3389 		data = kzalloc(sizeof(*data), GFP_KERNEL);
3390 		if (!data)
3391 			goto out_free_data;
3392 
3393 		if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
3394 			data->entry = kzalloc(sizeof(union irte), GFP_KERNEL);
3395 		else
3396 			data->entry = kzalloc(sizeof(struct irte_ga),
3397 						     GFP_KERNEL);
3398 		if (!data->entry) {
3399 			kfree(data);
3400 			goto out_free_data;
3401 		}
3402 
3403 		data->iommu = iommu;
3404 		irq_data->hwirq = (devid << 16) + i;
3405 		irq_data->chip_data = data;
3406 		irq_data->chip = &amd_ir_chip;
3407 		irq_remapping_prepare_irte(data, cfg, info, devid, index, i);
3408 		irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT);
3409 	}
3410 
3411 	return 0;
3412 
3413 out_free_data:
3414 	for (i--; i >= 0; i--) {
3415 		irq_data = irq_domain_get_irq_data(domain, virq + i);
3416 		if (irq_data)
3417 			kfree(irq_data->chip_data);
3418 	}
3419 	for (i = 0; i < nr_irqs; i++)
3420 		free_irte(iommu, devid, index + i);
3421 out_free_parent:
3422 	irq_domain_free_irqs_common(domain, virq, nr_irqs);
3423 	return ret;
3424 }
3425 
3426 static void irq_remapping_free(struct irq_domain *domain, unsigned int virq,
3427 			       unsigned int nr_irqs)
3428 {
3429 	struct irq_2_irte *irte_info;
3430 	struct irq_data *irq_data;
3431 	struct amd_ir_data *data;
3432 	int i;
3433 
3434 	for (i = 0; i < nr_irqs; i++) {
3435 		irq_data = irq_domain_get_irq_data(domain, virq  + i);
3436 		if (irq_data && irq_data->chip_data) {
3437 			data = irq_data->chip_data;
3438 			irte_info = &data->irq_2_irte;
3439 			free_irte(data->iommu, irte_info->devid, irte_info->index);
3440 			kfree(data->entry);
3441 			kfree(data);
3442 		}
3443 	}
3444 	irq_domain_free_irqs_common(domain, virq, nr_irqs);
3445 }
3446 
3447 static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,
3448 			       struct amd_ir_data *ir_data,
3449 			       struct irq_2_irte *irte_info,
3450 			       struct irq_cfg *cfg);
3451 
3452 static int irq_remapping_activate(struct irq_domain *domain,
3453 				  struct irq_data *irq_data, bool reserve)
3454 {
3455 	struct amd_ir_data *data = irq_data->chip_data;
3456 	struct irq_2_irte *irte_info = &data->irq_2_irte;
3457 	struct amd_iommu *iommu = data->iommu;
3458 	struct irq_cfg *cfg = irqd_cfg(irq_data);
3459 
3460 	if (!iommu)
3461 		return 0;
3462 
3463 	iommu->irte_ops->activate(iommu, data->entry, irte_info->devid,
3464 				  irte_info->index);
3465 	amd_ir_update_irte(irq_data, iommu, data, irte_info, cfg);
3466 	return 0;
3467 }
3468 
3469 static void irq_remapping_deactivate(struct irq_domain *domain,
3470 				     struct irq_data *irq_data)
3471 {
3472 	struct amd_ir_data *data = irq_data->chip_data;
3473 	struct irq_2_irte *irte_info = &data->irq_2_irte;
3474 	struct amd_iommu *iommu = data->iommu;
3475 
3476 	if (iommu)
3477 		iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid,
3478 					    irte_info->index);
3479 }
3480 
3481 static int irq_remapping_select(struct irq_domain *d, struct irq_fwspec *fwspec,
3482 				enum irq_domain_bus_token bus_token)
3483 {
3484 	struct amd_iommu *iommu;
3485 	int devid = -1;
3486 
3487 	if (!amd_iommu_irq_remap)
3488 		return 0;
3489 
3490 	if (x86_fwspec_is_ioapic(fwspec))
3491 		devid = get_ioapic_devid(fwspec->param[0]);
3492 	else if (x86_fwspec_is_hpet(fwspec))
3493 		devid = get_hpet_devid(fwspec->param[0]);
3494 
3495 	if (devid < 0)
3496 		return 0;
3497 	iommu = __rlookup_amd_iommu((devid >> 16), (devid & 0xffff));
3498 
3499 	return iommu && iommu->ir_domain == d;
3500 }
3501 
3502 static const struct irq_domain_ops amd_ir_domain_ops = {
3503 	.select = irq_remapping_select,
3504 	.alloc = irq_remapping_alloc,
3505 	.free = irq_remapping_free,
3506 	.activate = irq_remapping_activate,
3507 	.deactivate = irq_remapping_deactivate,
3508 };
3509 
3510 int amd_iommu_activate_guest_mode(void *data)
3511 {
3512 	struct amd_ir_data *ir_data = (struct amd_ir_data *)data;
3513 	struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
3514 	u64 valid;
3515 
3516 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) || !entry)
3517 		return 0;
3518 
3519 	valid = entry->lo.fields_vapic.valid;
3520 
3521 	entry->lo.val = 0;
3522 	entry->hi.val = 0;
3523 
3524 	entry->lo.fields_vapic.valid       = valid;
3525 	entry->lo.fields_vapic.guest_mode  = 1;
3526 	entry->lo.fields_vapic.ga_log_intr = 1;
3527 	entry->hi.fields.ga_root_ptr       = ir_data->ga_root_ptr;
3528 	entry->hi.fields.vector            = ir_data->ga_vector;
3529 	entry->lo.fields_vapic.ga_tag      = ir_data->ga_tag;
3530 
3531 	return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,
3532 			      ir_data->irq_2_irte.index, entry, ir_data);
3533 }
3534 EXPORT_SYMBOL(amd_iommu_activate_guest_mode);
3535 
3536 int amd_iommu_deactivate_guest_mode(void *data)
3537 {
3538 	struct amd_ir_data *ir_data = (struct amd_ir_data *)data;
3539 	struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
3540 	struct irq_cfg *cfg = ir_data->cfg;
3541 	u64 valid;
3542 
3543 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) ||
3544 	    !entry || !entry->lo.fields_vapic.guest_mode)
3545 		return 0;
3546 
3547 	valid = entry->lo.fields_remap.valid;
3548 
3549 	entry->lo.val = 0;
3550 	entry->hi.val = 0;
3551 
3552 	entry->lo.fields_remap.valid       = valid;
3553 	entry->lo.fields_remap.dm          = apic->dest_mode_logical;
3554 	entry->lo.fields_remap.int_type    = apic->delivery_mode;
3555 	entry->hi.fields.vector            = cfg->vector;
3556 	entry->lo.fields_remap.destination =
3557 				APICID_TO_IRTE_DEST_LO(cfg->dest_apicid);
3558 	entry->hi.fields.destination =
3559 				APICID_TO_IRTE_DEST_HI(cfg->dest_apicid);
3560 
3561 	return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,
3562 			      ir_data->irq_2_irte.index, entry, ir_data);
3563 }
3564 EXPORT_SYMBOL(amd_iommu_deactivate_guest_mode);
3565 
3566 static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
3567 {
3568 	int ret;
3569 	struct amd_iommu_pi_data *pi_data = vcpu_info;
3570 	struct vcpu_data *vcpu_pi_info = pi_data->vcpu_data;
3571 	struct amd_ir_data *ir_data = data->chip_data;
3572 	struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
3573 	struct iommu_dev_data *dev_data;
3574 
3575 	if (ir_data->iommu == NULL)
3576 		return -EINVAL;
3577 
3578 	dev_data = search_dev_data(ir_data->iommu, irte_info->devid);
3579 
3580 	/* Note:
3581 	 * This device has never been set up for guest mode.
3582 	 * we should not modify the IRTE
3583 	 */
3584 	if (!dev_data || !dev_data->use_vapic)
3585 		return 0;
3586 
3587 	ir_data->cfg = irqd_cfg(data);
3588 	pi_data->ir_data = ir_data;
3589 
3590 	/* Note:
3591 	 * SVM tries to set up for VAPIC mode, but we are in
3592 	 * legacy mode. So, we force legacy mode instead.
3593 	 */
3594 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) {
3595 		pr_debug("%s: Fall back to using intr legacy remap\n",
3596 			 __func__);
3597 		pi_data->is_guest_mode = false;
3598 	}
3599 
3600 	pi_data->prev_ga_tag = ir_data->cached_ga_tag;
3601 	if (pi_data->is_guest_mode) {
3602 		ir_data->ga_root_ptr = (pi_data->base >> 12);
3603 		ir_data->ga_vector = vcpu_pi_info->vector;
3604 		ir_data->ga_tag = pi_data->ga_tag;
3605 		ret = amd_iommu_activate_guest_mode(ir_data);
3606 		if (!ret)
3607 			ir_data->cached_ga_tag = pi_data->ga_tag;
3608 	} else {
3609 		ret = amd_iommu_deactivate_guest_mode(ir_data);
3610 
3611 		/*
3612 		 * This communicates the ga_tag back to the caller
3613 		 * so that it can do all the necessary clean up.
3614 		 */
3615 		if (!ret)
3616 			ir_data->cached_ga_tag = 0;
3617 	}
3618 
3619 	return ret;
3620 }
3621 
3622 
3623 static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,
3624 			       struct amd_ir_data *ir_data,
3625 			       struct irq_2_irte *irte_info,
3626 			       struct irq_cfg *cfg)
3627 {
3628 
3629 	/*
3630 	 * Atomically updates the IRTE with the new destination, vector
3631 	 * and flushes the interrupt entry cache.
3632 	 */
3633 	iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid,
3634 				      irte_info->index, cfg->vector,
3635 				      cfg->dest_apicid);
3636 }
3637 
3638 static int amd_ir_set_affinity(struct irq_data *data,
3639 			       const struct cpumask *mask, bool force)
3640 {
3641 	struct amd_ir_data *ir_data = data->chip_data;
3642 	struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
3643 	struct irq_cfg *cfg = irqd_cfg(data);
3644 	struct irq_data *parent = data->parent_data;
3645 	struct amd_iommu *iommu = ir_data->iommu;
3646 	int ret;
3647 
3648 	if (!iommu)
3649 		return -ENODEV;
3650 
3651 	ret = parent->chip->irq_set_affinity(parent, mask, force);
3652 	if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
3653 		return ret;
3654 
3655 	amd_ir_update_irte(data, iommu, ir_data, irte_info, cfg);
3656 	/*
3657 	 * After this point, all the interrupts will start arriving
3658 	 * at the new destination. So, time to cleanup the previous
3659 	 * vector allocation.
3660 	 */
3661 	send_cleanup_vector(cfg);
3662 
3663 	return IRQ_SET_MASK_OK_DONE;
3664 }
3665 
3666 static void ir_compose_msi_msg(struct irq_data *irq_data, struct msi_msg *msg)
3667 {
3668 	struct amd_ir_data *ir_data = irq_data->chip_data;
3669 
3670 	*msg = ir_data->msi_entry;
3671 }
3672 
3673 static struct irq_chip amd_ir_chip = {
3674 	.name			= "AMD-IR",
3675 	.irq_ack		= apic_ack_irq,
3676 	.irq_set_affinity	= amd_ir_set_affinity,
3677 	.irq_set_vcpu_affinity	= amd_ir_set_vcpu_affinity,
3678 	.irq_compose_msi_msg	= ir_compose_msi_msg,
3679 };
3680 
3681 static const struct msi_parent_ops amdvi_msi_parent_ops = {
3682 	.supported_flags	= X86_VECTOR_MSI_FLAGS_SUPPORTED |
3683 				  MSI_FLAG_MULTI_PCI_MSI |
3684 				  MSI_FLAG_PCI_IMS,
3685 	.prefix			= "IR-",
3686 	.init_dev_msi_info	= msi_parent_init_dev_msi_info,
3687 };
3688 
3689 static const struct msi_parent_ops virt_amdvi_msi_parent_ops = {
3690 	.supported_flags	= X86_VECTOR_MSI_FLAGS_SUPPORTED |
3691 				  MSI_FLAG_MULTI_PCI_MSI,
3692 	.prefix			= "vIR-",
3693 	.init_dev_msi_info	= msi_parent_init_dev_msi_info,
3694 };
3695 
3696 int amd_iommu_create_irq_domain(struct amd_iommu *iommu)
3697 {
3698 	struct fwnode_handle *fn;
3699 
3700 	fn = irq_domain_alloc_named_id_fwnode("AMD-IR", iommu->index);
3701 	if (!fn)
3702 		return -ENOMEM;
3703 	iommu->ir_domain = irq_domain_create_hierarchy(arch_get_ir_parent_domain(), 0, 0,
3704 						       fn, &amd_ir_domain_ops, iommu);
3705 	if (!iommu->ir_domain) {
3706 		irq_domain_free_fwnode(fn);
3707 		return -ENOMEM;
3708 	}
3709 
3710 	irq_domain_update_bus_token(iommu->ir_domain,  DOMAIN_BUS_AMDVI);
3711 	iommu->ir_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT |
3712 				   IRQ_DOMAIN_FLAG_ISOLATED_MSI;
3713 
3714 	if (amd_iommu_np_cache)
3715 		iommu->ir_domain->msi_parent_ops = &virt_amdvi_msi_parent_ops;
3716 	else
3717 		iommu->ir_domain->msi_parent_ops = &amdvi_msi_parent_ops;
3718 
3719 	return 0;
3720 }
3721 
3722 int amd_iommu_update_ga(int cpu, bool is_run, void *data)
3723 {
3724 	unsigned long flags;
3725 	struct amd_iommu *iommu;
3726 	struct irq_remap_table *table;
3727 	struct amd_ir_data *ir_data = (struct amd_ir_data *)data;
3728 	int devid = ir_data->irq_2_irte.devid;
3729 	struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
3730 	struct irte_ga *ref = (struct irte_ga *) ir_data->ref;
3731 
3732 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) ||
3733 	    !ref || !entry || !entry->lo.fields_vapic.guest_mode)
3734 		return 0;
3735 
3736 	iommu = ir_data->iommu;
3737 	if (!iommu)
3738 		return -ENODEV;
3739 
3740 	table = get_irq_table(iommu, devid);
3741 	if (!table)
3742 		return -ENODEV;
3743 
3744 	raw_spin_lock_irqsave(&table->lock, flags);
3745 
3746 	if (ref->lo.fields_vapic.guest_mode) {
3747 		if (cpu >= 0) {
3748 			ref->lo.fields_vapic.destination =
3749 						APICID_TO_IRTE_DEST_LO(cpu);
3750 			ref->hi.fields.destination =
3751 						APICID_TO_IRTE_DEST_HI(cpu);
3752 		}
3753 		ref->lo.fields_vapic.is_run = is_run;
3754 		barrier();
3755 	}
3756 
3757 	raw_spin_unlock_irqrestore(&table->lock, flags);
3758 
3759 	iommu_flush_irt(iommu, devid);
3760 	iommu_completion_wait(iommu);
3761 	return 0;
3762 }
3763 EXPORT_SYMBOL(amd_iommu_update_ga);
3764 #endif
3765