xref: /linux/drivers/iommu/amd/iommu.c (revision b50ecc5aca4d18f1f0c4942f5c797bc85edef144)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4  * Author: Joerg Roedel <jroedel@suse.de>
5  *         Leo Duran <leo.duran@amd.com>
6  */
7 
8 #define pr_fmt(fmt)     "AMD-Vi: " fmt
9 #define dev_fmt(fmt)    pr_fmt(fmt)
10 
11 #include <linux/ratelimit.h>
12 #include <linux/pci.h>
13 #include <linux/acpi.h>
14 #include <linux/pci-ats.h>
15 #include <linux/bitmap.h>
16 #include <linux/slab.h>
17 #include <linux/debugfs.h>
18 #include <linux/scatterlist.h>
19 #include <linux/dma-map-ops.h>
20 #include <linux/dma-direct.h>
21 #include <linux/idr.h>
22 #include <linux/iommu-helper.h>
23 #include <linux/delay.h>
24 #include <linux/amd-iommu.h>
25 #include <linux/notifier.h>
26 #include <linux/export.h>
27 #include <linux/irq.h>
28 #include <linux/msi.h>
29 #include <linux/irqdomain.h>
30 #include <linux/percpu.h>
31 #include <linux/io-pgtable.h>
32 #include <linux/cc_platform.h>
33 #include <asm/irq_remapping.h>
34 #include <asm/io_apic.h>
35 #include <asm/apic.h>
36 #include <asm/hw_irq.h>
37 #include <asm/proto.h>
38 #include <asm/iommu.h>
39 #include <asm/gart.h>
40 #include <asm/dma.h>
41 #include <uapi/linux/iommufd.h>
42 
43 #include "amd_iommu.h"
44 #include "../dma-iommu.h"
45 #include "../irq_remapping.h"
46 #include "../iommu-pages.h"
47 
48 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
49 
50 /* Reserved IOVA ranges */
51 #define MSI_RANGE_START		(0xfee00000)
52 #define MSI_RANGE_END		(0xfeefffff)
53 #define HT_RANGE_START		(0xfd00000000ULL)
54 #define HT_RANGE_END		(0xffffffffffULL)
55 
56 LIST_HEAD(ioapic_map);
57 LIST_HEAD(hpet_map);
58 LIST_HEAD(acpihid_map);
59 
60 const struct iommu_ops amd_iommu_ops;
61 static const struct iommu_dirty_ops amd_dirty_ops;
62 
63 int amd_iommu_max_glx_val = -1;
64 
65 /*
66  * general struct to manage commands send to an IOMMU
67  */
68 struct iommu_cmd {
69 	u32 data[4];
70 };
71 
72 /*
73  * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap
74  * to know which ones are already in use.
75  */
76 DEFINE_IDA(pdom_ids);
77 
78 struct kmem_cache *amd_iommu_irq_cache;
79 
80 static int amd_iommu_attach_device(struct iommu_domain *dom,
81 				   struct device *dev);
82 
83 static void set_dte_entry(struct amd_iommu *iommu,
84 			  struct iommu_dev_data *dev_data);
85 
86 /****************************************************************************
87  *
88  * Helper functions
89  *
90  ****************************************************************************/
91 
92 static inline bool pdom_is_v2_pgtbl_mode(struct protection_domain *pdom)
93 {
94 	return (pdom && (pdom->pd_mode == PD_MODE_V2));
95 }
96 
97 static inline bool pdom_is_in_pt_mode(struct protection_domain *pdom)
98 {
99 	return (pdom->domain.type == IOMMU_DOMAIN_IDENTITY);
100 }
101 
102 /*
103  * We cannot support PASID w/ existing v1 page table in the same domain
104  * since it will be nested. However, existing domain w/ v2 page table
105  * or passthrough mode can be used for PASID.
106  */
107 static inline bool pdom_is_sva_capable(struct protection_domain *pdom)
108 {
109 	return pdom_is_v2_pgtbl_mode(pdom) || pdom_is_in_pt_mode(pdom);
110 }
111 
112 static inline int get_acpihid_device_id(struct device *dev,
113 					struct acpihid_map_entry **entry)
114 {
115 	struct acpi_device *adev = ACPI_COMPANION(dev);
116 	struct acpihid_map_entry *p;
117 
118 	if (!adev)
119 		return -ENODEV;
120 
121 	list_for_each_entry(p, &acpihid_map, list) {
122 		if (acpi_dev_hid_uid_match(adev, p->hid,
123 					   p->uid[0] ? p->uid : NULL)) {
124 			if (entry)
125 				*entry = p;
126 			return p->devid;
127 		}
128 	}
129 	return -EINVAL;
130 }
131 
132 static inline int get_device_sbdf_id(struct device *dev)
133 {
134 	int sbdf;
135 
136 	if (dev_is_pci(dev))
137 		sbdf = get_pci_sbdf_id(to_pci_dev(dev));
138 	else
139 		sbdf = get_acpihid_device_id(dev, NULL);
140 
141 	return sbdf;
142 }
143 
144 struct dev_table_entry *get_dev_table(struct amd_iommu *iommu)
145 {
146 	struct dev_table_entry *dev_table;
147 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
148 
149 	BUG_ON(pci_seg == NULL);
150 	dev_table = pci_seg->dev_table;
151 	BUG_ON(dev_table == NULL);
152 
153 	return dev_table;
154 }
155 
156 static inline u16 get_device_segment(struct device *dev)
157 {
158 	u16 seg;
159 
160 	if (dev_is_pci(dev)) {
161 		struct pci_dev *pdev = to_pci_dev(dev);
162 
163 		seg = pci_domain_nr(pdev->bus);
164 	} else {
165 		u32 devid = get_acpihid_device_id(dev, NULL);
166 
167 		seg = PCI_SBDF_TO_SEGID(devid);
168 	}
169 
170 	return seg;
171 }
172 
173 /* Writes the specific IOMMU for a device into the PCI segment rlookup table */
174 void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid)
175 {
176 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
177 
178 	pci_seg->rlookup_table[devid] = iommu;
179 }
180 
181 static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid)
182 {
183 	struct amd_iommu_pci_seg *pci_seg;
184 
185 	for_each_pci_segment(pci_seg) {
186 		if (pci_seg->id == seg)
187 			return pci_seg->rlookup_table[devid];
188 	}
189 	return NULL;
190 }
191 
192 static struct amd_iommu *rlookup_amd_iommu(struct device *dev)
193 {
194 	u16 seg = get_device_segment(dev);
195 	int devid = get_device_sbdf_id(dev);
196 
197 	if (devid < 0)
198 		return NULL;
199 	return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid));
200 }
201 
202 static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid)
203 {
204 	struct iommu_dev_data *dev_data;
205 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
206 
207 	dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
208 	if (!dev_data)
209 		return NULL;
210 
211 	mutex_init(&dev_data->mutex);
212 	dev_data->devid = devid;
213 	ratelimit_default_init(&dev_data->rs);
214 
215 	llist_add(&dev_data->dev_data_list, &pci_seg->dev_data_list);
216 	return dev_data;
217 }
218 
219 static struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid)
220 {
221 	struct iommu_dev_data *dev_data;
222 	struct llist_node *node;
223 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
224 
225 	if (llist_empty(&pci_seg->dev_data_list))
226 		return NULL;
227 
228 	node = pci_seg->dev_data_list.first;
229 	llist_for_each_entry(dev_data, node, dev_data_list) {
230 		if (dev_data->devid == devid)
231 			return dev_data;
232 	}
233 
234 	return NULL;
235 }
236 
237 static int clone_alias(struct pci_dev *pdev, u16 alias, void *data)
238 {
239 	struct amd_iommu *iommu;
240 	struct dev_table_entry *dev_table;
241 	u16 devid = pci_dev_id(pdev);
242 
243 	if (devid == alias)
244 		return 0;
245 
246 	iommu = rlookup_amd_iommu(&pdev->dev);
247 	if (!iommu)
248 		return 0;
249 
250 	amd_iommu_set_rlookup_table(iommu, alias);
251 	dev_table = get_dev_table(iommu);
252 	memcpy(dev_table[alias].data,
253 	       dev_table[devid].data,
254 	       sizeof(dev_table[alias].data));
255 
256 	return 0;
257 }
258 
259 static void clone_aliases(struct amd_iommu *iommu, struct device *dev)
260 {
261 	struct pci_dev *pdev;
262 
263 	if (!dev_is_pci(dev))
264 		return;
265 	pdev = to_pci_dev(dev);
266 
267 	/*
268 	 * The IVRS alias stored in the alias table may not be
269 	 * part of the PCI DMA aliases if it's bus differs
270 	 * from the original device.
271 	 */
272 	clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL);
273 
274 	pci_for_each_dma_alias(pdev, clone_alias, NULL);
275 }
276 
277 static void setup_aliases(struct amd_iommu *iommu, struct device *dev)
278 {
279 	struct pci_dev *pdev = to_pci_dev(dev);
280 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
281 	u16 ivrs_alias;
282 
283 	/* For ACPI HID devices, there are no aliases */
284 	if (!dev_is_pci(dev))
285 		return;
286 
287 	/*
288 	 * Add the IVRS alias to the pci aliases if it is on the same
289 	 * bus. The IVRS table may know about a quirk that we don't.
290 	 */
291 	ivrs_alias = pci_seg->alias_table[pci_dev_id(pdev)];
292 	if (ivrs_alias != pci_dev_id(pdev) &&
293 	    PCI_BUS_NUM(ivrs_alias) == pdev->bus->number)
294 		pci_add_dma_alias(pdev, ivrs_alias & 0xff, 1);
295 
296 	clone_aliases(iommu, dev);
297 }
298 
299 static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid)
300 {
301 	struct iommu_dev_data *dev_data;
302 
303 	dev_data = search_dev_data(iommu, devid);
304 
305 	if (dev_data == NULL) {
306 		dev_data = alloc_dev_data(iommu, devid);
307 		if (!dev_data)
308 			return NULL;
309 
310 		if (translation_pre_enabled(iommu))
311 			dev_data->defer_attach = true;
312 	}
313 
314 	return dev_data;
315 }
316 
317 /*
318 * Find or create an IOMMU group for a acpihid device.
319 */
320 static struct iommu_group *acpihid_device_group(struct device *dev)
321 {
322 	struct acpihid_map_entry *p, *entry = NULL;
323 	int devid;
324 
325 	devid = get_acpihid_device_id(dev, &entry);
326 	if (devid < 0)
327 		return ERR_PTR(devid);
328 
329 	list_for_each_entry(p, &acpihid_map, list) {
330 		if ((devid == p->devid) && p->group)
331 			entry->group = p->group;
332 	}
333 
334 	if (!entry->group)
335 		entry->group = generic_device_group(dev);
336 	else
337 		iommu_group_ref_get(entry->group);
338 
339 	return entry->group;
340 }
341 
342 static inline bool pdev_pasid_supported(struct iommu_dev_data *dev_data)
343 {
344 	return (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP);
345 }
346 
347 static u32 pdev_get_caps(struct pci_dev *pdev)
348 {
349 	int features;
350 	u32 flags = 0;
351 
352 	if (pci_ats_supported(pdev))
353 		flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
354 
355 	if (pci_pri_supported(pdev))
356 		flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;
357 
358 	features = pci_pasid_features(pdev);
359 	if (features >= 0) {
360 		flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
361 
362 		if (features & PCI_PASID_CAP_EXEC)
363 			flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
364 
365 		if (features & PCI_PASID_CAP_PRIV)
366 			flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
367 	}
368 
369 	return flags;
370 }
371 
372 static inline int pdev_enable_cap_ats(struct pci_dev *pdev)
373 {
374 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
375 	int ret = -EINVAL;
376 
377 	if (dev_data->ats_enabled)
378 		return 0;
379 
380 	if (amd_iommu_iotlb_sup &&
381 	    (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP)) {
382 		ret = pci_enable_ats(pdev, PAGE_SHIFT);
383 		if (!ret) {
384 			dev_data->ats_enabled = 1;
385 			dev_data->ats_qdep    = pci_ats_queue_depth(pdev);
386 		}
387 	}
388 
389 	return ret;
390 }
391 
392 static inline void pdev_disable_cap_ats(struct pci_dev *pdev)
393 {
394 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
395 
396 	if (dev_data->ats_enabled) {
397 		pci_disable_ats(pdev);
398 		dev_data->ats_enabled = 0;
399 	}
400 }
401 
402 static inline int pdev_enable_cap_pri(struct pci_dev *pdev)
403 {
404 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
405 	int ret = -EINVAL;
406 
407 	if (dev_data->pri_enabled)
408 		return 0;
409 
410 	if (!dev_data->ats_enabled)
411 		return 0;
412 
413 	if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) {
414 		/*
415 		 * First reset the PRI state of the device.
416 		 * FIXME: Hardcode number of outstanding requests for now
417 		 */
418 		if (!pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32)) {
419 			dev_data->pri_enabled = 1;
420 			dev_data->pri_tlp     = pci_prg_resp_pasid_required(pdev);
421 
422 			ret = 0;
423 		}
424 	}
425 
426 	return ret;
427 }
428 
429 static inline void pdev_disable_cap_pri(struct pci_dev *pdev)
430 {
431 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
432 
433 	if (dev_data->pri_enabled) {
434 		pci_disable_pri(pdev);
435 		dev_data->pri_enabled = 0;
436 	}
437 }
438 
439 static inline int pdev_enable_cap_pasid(struct pci_dev *pdev)
440 {
441 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
442 	int ret = -EINVAL;
443 
444 	if (dev_data->pasid_enabled)
445 		return 0;
446 
447 	if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) {
448 		/* Only allow access to user-accessible pages */
449 		ret = pci_enable_pasid(pdev, 0);
450 		if (!ret)
451 			dev_data->pasid_enabled = 1;
452 	}
453 
454 	return ret;
455 }
456 
457 static inline void pdev_disable_cap_pasid(struct pci_dev *pdev)
458 {
459 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
460 
461 	if (dev_data->pasid_enabled) {
462 		pci_disable_pasid(pdev);
463 		dev_data->pasid_enabled = 0;
464 	}
465 }
466 
467 static void pdev_enable_caps(struct pci_dev *pdev)
468 {
469 	pdev_enable_cap_ats(pdev);
470 	pdev_enable_cap_pasid(pdev);
471 	pdev_enable_cap_pri(pdev);
472 }
473 
474 static void pdev_disable_caps(struct pci_dev *pdev)
475 {
476 	pdev_disable_cap_ats(pdev);
477 	pdev_disable_cap_pasid(pdev);
478 	pdev_disable_cap_pri(pdev);
479 }
480 
481 /*
482  * This function checks if the driver got a valid device from the caller to
483  * avoid dereferencing invalid pointers.
484  */
485 static bool check_device(struct device *dev)
486 {
487 	struct amd_iommu_pci_seg *pci_seg;
488 	struct amd_iommu *iommu;
489 	int devid, sbdf;
490 
491 	if (!dev)
492 		return false;
493 
494 	sbdf = get_device_sbdf_id(dev);
495 	if (sbdf < 0)
496 		return false;
497 	devid = PCI_SBDF_TO_DEVID(sbdf);
498 
499 	iommu = rlookup_amd_iommu(dev);
500 	if (!iommu)
501 		return false;
502 
503 	/* Out of our scope? */
504 	pci_seg = iommu->pci_seg;
505 	if (devid > pci_seg->last_bdf)
506 		return false;
507 
508 	return true;
509 }
510 
511 static int iommu_init_device(struct amd_iommu *iommu, struct device *dev)
512 {
513 	struct iommu_dev_data *dev_data;
514 	int devid, sbdf;
515 
516 	if (dev_iommu_priv_get(dev))
517 		return 0;
518 
519 	sbdf = get_device_sbdf_id(dev);
520 	if (sbdf < 0)
521 		return sbdf;
522 
523 	devid = PCI_SBDF_TO_DEVID(sbdf);
524 	dev_data = find_dev_data(iommu, devid);
525 	if (!dev_data)
526 		return -ENOMEM;
527 
528 	dev_data->dev = dev;
529 	setup_aliases(iommu, dev);
530 
531 	/*
532 	 * By default we use passthrough mode for IOMMUv2 capable device.
533 	 * But if amd_iommu=force_isolation is set (e.g. to debug DMA to
534 	 * invalid address), we ignore the capability for the device so
535 	 * it'll be forced to go into translation mode.
536 	 */
537 	if ((iommu_default_passthrough() || !amd_iommu_force_isolation) &&
538 	    dev_is_pci(dev) && amd_iommu_gt_ppr_supported()) {
539 		dev_data->flags = pdev_get_caps(to_pci_dev(dev));
540 	}
541 
542 	dev_iommu_priv_set(dev, dev_data);
543 
544 	return 0;
545 }
546 
547 static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev)
548 {
549 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
550 	struct dev_table_entry *dev_table = get_dev_table(iommu);
551 	int devid, sbdf;
552 
553 	sbdf = get_device_sbdf_id(dev);
554 	if (sbdf < 0)
555 		return;
556 
557 	devid = PCI_SBDF_TO_DEVID(sbdf);
558 	pci_seg->rlookup_table[devid] = NULL;
559 	memset(&dev_table[devid], 0, sizeof(struct dev_table_entry));
560 
561 	setup_aliases(iommu, dev);
562 }
563 
564 
565 /****************************************************************************
566  *
567  * Interrupt handling functions
568  *
569  ****************************************************************************/
570 
571 static void dump_dte_entry(struct amd_iommu *iommu, u16 devid)
572 {
573 	int i;
574 	struct dev_table_entry *dev_table = get_dev_table(iommu);
575 
576 	for (i = 0; i < 4; ++i)
577 		pr_err("DTE[%d]: %016llx\n", i, dev_table[devid].data[i]);
578 }
579 
580 static void dump_command(unsigned long phys_addr)
581 {
582 	struct iommu_cmd *cmd = iommu_phys_to_virt(phys_addr);
583 	int i;
584 
585 	for (i = 0; i < 4; ++i)
586 		pr_err("CMD[%d]: %08x\n", i, cmd->data[i]);
587 }
588 
589 static void amd_iommu_report_rmp_hw_error(struct amd_iommu *iommu, volatile u32 *event)
590 {
591 	struct iommu_dev_data *dev_data = NULL;
592 	int devid, vmg_tag, flags;
593 	struct pci_dev *pdev;
594 	u64 spa;
595 
596 	devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
597 	vmg_tag = (event[1]) & 0xFFFF;
598 	flags   = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
599 	spa     = ((u64)event[3] << 32) | (event[2] & 0xFFFFFFF8);
600 
601 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),
602 					   devid & 0xff);
603 	if (pdev)
604 		dev_data = dev_iommu_priv_get(&pdev->dev);
605 
606 	if (dev_data) {
607 		if (__ratelimit(&dev_data->rs)) {
608 			pci_err(pdev, "Event logged [RMP_HW_ERROR vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n",
609 				vmg_tag, spa, flags);
610 		}
611 	} else {
612 		pr_err_ratelimited("Event logged [RMP_HW_ERROR device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n",
613 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
614 			vmg_tag, spa, flags);
615 	}
616 
617 	if (pdev)
618 		pci_dev_put(pdev);
619 }
620 
621 static void amd_iommu_report_rmp_fault(struct amd_iommu *iommu, volatile u32 *event)
622 {
623 	struct iommu_dev_data *dev_data = NULL;
624 	int devid, flags_rmp, vmg_tag, flags;
625 	struct pci_dev *pdev;
626 	u64 gpa;
627 
628 	devid     = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
629 	flags_rmp = (event[0] >> EVENT_FLAGS_SHIFT) & 0xFF;
630 	vmg_tag   = (event[1]) & 0xFFFF;
631 	flags     = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
632 	gpa       = ((u64)event[3] << 32) | event[2];
633 
634 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),
635 					   devid & 0xff);
636 	if (pdev)
637 		dev_data = dev_iommu_priv_get(&pdev->dev);
638 
639 	if (dev_data) {
640 		if (__ratelimit(&dev_data->rs)) {
641 			pci_err(pdev, "Event logged [RMP_PAGE_FAULT vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n",
642 				vmg_tag, gpa, flags_rmp, flags);
643 		}
644 	} else {
645 		pr_err_ratelimited("Event logged [RMP_PAGE_FAULT device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n",
646 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
647 			vmg_tag, gpa, flags_rmp, flags);
648 	}
649 
650 	if (pdev)
651 		pci_dev_put(pdev);
652 }
653 
654 #define IS_IOMMU_MEM_TRANSACTION(flags)		\
655 	(((flags) & EVENT_FLAG_I) == 0)
656 
657 #define IS_WRITE_REQUEST(flags)			\
658 	((flags) & EVENT_FLAG_RW)
659 
660 static void amd_iommu_report_page_fault(struct amd_iommu *iommu,
661 					u16 devid, u16 domain_id,
662 					u64 address, int flags)
663 {
664 	struct iommu_dev_data *dev_data = NULL;
665 	struct pci_dev *pdev;
666 
667 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),
668 					   devid & 0xff);
669 	if (pdev)
670 		dev_data = dev_iommu_priv_get(&pdev->dev);
671 
672 	if (dev_data) {
673 		/*
674 		 * If this is a DMA fault (for which the I(nterrupt)
675 		 * bit will be unset), allow report_iommu_fault() to
676 		 * prevent logging it.
677 		 */
678 		if (IS_IOMMU_MEM_TRANSACTION(flags)) {
679 			/* Device not attached to domain properly */
680 			if (dev_data->domain == NULL) {
681 				pr_err_ratelimited("Event logged [Device not attached to domain properly]\n");
682 				pr_err_ratelimited("  device=%04x:%02x:%02x.%x domain=0x%04x\n",
683 						   iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid),
684 						   PCI_FUNC(devid), domain_id);
685 				goto out;
686 			}
687 
688 			if (!report_iommu_fault(&dev_data->domain->domain,
689 						&pdev->dev, address,
690 						IS_WRITE_REQUEST(flags) ?
691 							IOMMU_FAULT_WRITE :
692 							IOMMU_FAULT_READ))
693 				goto out;
694 		}
695 
696 		if (__ratelimit(&dev_data->rs)) {
697 			pci_err(pdev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n",
698 				domain_id, address, flags);
699 		}
700 	} else {
701 		pr_err_ratelimited("Event logged [IO_PAGE_FAULT device=%04x:%02x:%02x.%x domain=0x%04x address=0x%llx flags=0x%04x]\n",
702 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
703 			domain_id, address, flags);
704 	}
705 
706 out:
707 	if (pdev)
708 		pci_dev_put(pdev);
709 }
710 
711 static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
712 {
713 	struct device *dev = iommu->iommu.dev;
714 	int type, devid, flags, tag;
715 	volatile u32 *event = __evt;
716 	int count = 0;
717 	u64 address;
718 	u32 pasid;
719 
720 retry:
721 	type    = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
722 	devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
723 	pasid   = (event[0] & EVENT_DOMID_MASK_HI) |
724 		  (event[1] & EVENT_DOMID_MASK_LO);
725 	flags   = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
726 	address = (u64)(((u64)event[3]) << 32) | event[2];
727 
728 	if (type == 0) {
729 		/* Did we hit the erratum? */
730 		if (++count == LOOP_TIMEOUT) {
731 			pr_err("No event written to event log\n");
732 			return;
733 		}
734 		udelay(1);
735 		goto retry;
736 	}
737 
738 	if (type == EVENT_TYPE_IO_FAULT) {
739 		amd_iommu_report_page_fault(iommu, devid, pasid, address, flags);
740 		return;
741 	}
742 
743 	switch (type) {
744 	case EVENT_TYPE_ILL_DEV:
745 		dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n",
746 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
747 			pasid, address, flags);
748 		dump_dte_entry(iommu, devid);
749 		break;
750 	case EVENT_TYPE_DEV_TAB_ERR:
751 		dev_err(dev, "Event logged [DEV_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x "
752 			"address=0x%llx flags=0x%04x]\n",
753 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
754 			address, flags);
755 		break;
756 	case EVENT_TYPE_PAGE_TAB_ERR:
757 		dev_err(dev, "Event logged [PAGE_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x pasid=0x%04x address=0x%llx flags=0x%04x]\n",
758 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
759 			pasid, address, flags);
760 		break;
761 	case EVENT_TYPE_ILL_CMD:
762 		dev_err(dev, "Event logged [ILLEGAL_COMMAND_ERROR address=0x%llx]\n", address);
763 		dump_command(address);
764 		break;
765 	case EVENT_TYPE_CMD_HARD_ERR:
766 		dev_err(dev, "Event logged [COMMAND_HARDWARE_ERROR address=0x%llx flags=0x%04x]\n",
767 			address, flags);
768 		break;
769 	case EVENT_TYPE_IOTLB_INV_TO:
770 		dev_err(dev, "Event logged [IOTLB_INV_TIMEOUT device=%04x:%02x:%02x.%x address=0x%llx]\n",
771 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
772 			address);
773 		break;
774 	case EVENT_TYPE_INV_DEV_REQ:
775 		dev_err(dev, "Event logged [INVALID_DEVICE_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n",
776 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
777 			pasid, address, flags);
778 		break;
779 	case EVENT_TYPE_RMP_FAULT:
780 		amd_iommu_report_rmp_fault(iommu, event);
781 		break;
782 	case EVENT_TYPE_RMP_HW_ERR:
783 		amd_iommu_report_rmp_hw_error(iommu, event);
784 		break;
785 	case EVENT_TYPE_INV_PPR_REQ:
786 		pasid = PPR_PASID(*((u64 *)__evt));
787 		tag = event[1] & 0x03FF;
788 		dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x tag=0x%03x]\n",
789 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
790 			pasid, address, flags, tag);
791 		break;
792 	default:
793 		dev_err(dev, "Event logged [UNKNOWN event[0]=0x%08x event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n",
794 			event[0], event[1], event[2], event[3]);
795 	}
796 
797 	/*
798 	 * To detect the hardware errata 732 we need to clear the
799 	 * entry back to zero. This issue does not exist on SNP
800 	 * enabled system. Also this buffer is not writeable on
801 	 * SNP enabled system.
802 	 */
803 	if (!amd_iommu_snp_en)
804 		memset(__evt, 0, 4 * sizeof(u32));
805 }
806 
807 static void iommu_poll_events(struct amd_iommu *iommu)
808 {
809 	u32 head, tail;
810 
811 	head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
812 	tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
813 
814 	while (head != tail) {
815 		iommu_print_event(iommu, iommu->evt_buf + head);
816 
817 		/* Update head pointer of hardware ring-buffer */
818 		head = (head + EVENT_ENTRY_SIZE) % EVT_BUFFER_SIZE;
819 		writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
820 	}
821 
822 }
823 
824 #ifdef CONFIG_IRQ_REMAP
825 static int (*iommu_ga_log_notifier)(u32);
826 
827 int amd_iommu_register_ga_log_notifier(int (*notifier)(u32))
828 {
829 	iommu_ga_log_notifier = notifier;
830 
831 	return 0;
832 }
833 EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier);
834 
835 static void iommu_poll_ga_log(struct amd_iommu *iommu)
836 {
837 	u32 head, tail;
838 
839 	if (iommu->ga_log == NULL)
840 		return;
841 
842 	head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
843 	tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
844 
845 	while (head != tail) {
846 		volatile u64 *raw;
847 		u64 log_entry;
848 
849 		raw = (u64 *)(iommu->ga_log + head);
850 
851 		/* Avoid memcpy function-call overhead */
852 		log_entry = *raw;
853 
854 		/* Update head pointer of hardware ring-buffer */
855 		head = (head + GA_ENTRY_SIZE) % GA_LOG_SIZE;
856 		writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
857 
858 		/* Handle GA entry */
859 		switch (GA_REQ_TYPE(log_entry)) {
860 		case GA_GUEST_NR:
861 			if (!iommu_ga_log_notifier)
862 				break;
863 
864 			pr_debug("%s: devid=%#x, ga_tag=%#x\n",
865 				 __func__, GA_DEVID(log_entry),
866 				 GA_TAG(log_entry));
867 
868 			if (iommu_ga_log_notifier(GA_TAG(log_entry)) != 0)
869 				pr_err("GA log notifier failed.\n");
870 			break;
871 		default:
872 			break;
873 		}
874 	}
875 }
876 
877 static void
878 amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu)
879 {
880 	if (!irq_remapping_enabled || !dev_is_pci(dev) ||
881 	    !pci_dev_has_default_msi_parent_domain(to_pci_dev(dev)))
882 		return;
883 
884 	dev_set_msi_domain(dev, iommu->ir_domain);
885 }
886 
887 #else /* CONFIG_IRQ_REMAP */
888 static inline void
889 amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) { }
890 #endif /* !CONFIG_IRQ_REMAP */
891 
892 static void amd_iommu_handle_irq(void *data, const char *evt_type,
893 				 u32 int_mask, u32 overflow_mask,
894 				 void (*int_handler)(struct amd_iommu *),
895 				 void (*overflow_handler)(struct amd_iommu *))
896 {
897 	struct amd_iommu *iommu = (struct amd_iommu *) data;
898 	u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
899 	u32 mask = int_mask | overflow_mask;
900 
901 	while (status & mask) {
902 		/* Enable interrupt sources again */
903 		writel(mask, iommu->mmio_base + MMIO_STATUS_OFFSET);
904 
905 		if (int_handler) {
906 			pr_devel("Processing IOMMU (ivhd%d) %s Log\n",
907 				 iommu->index, evt_type);
908 			int_handler(iommu);
909 		}
910 
911 		if ((status & overflow_mask) && overflow_handler)
912 			overflow_handler(iommu);
913 
914 		/*
915 		 * Hardware bug: ERBT1312
916 		 * When re-enabling interrupt (by writing 1
917 		 * to clear the bit), the hardware might also try to set
918 		 * the interrupt bit in the event status register.
919 		 * In this scenario, the bit will be set, and disable
920 		 * subsequent interrupts.
921 		 *
922 		 * Workaround: The IOMMU driver should read back the
923 		 * status register and check if the interrupt bits are cleared.
924 		 * If not, driver will need to go through the interrupt handler
925 		 * again and re-clear the bits
926 		 */
927 		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
928 	}
929 }
930 
931 irqreturn_t amd_iommu_int_thread_evtlog(int irq, void *data)
932 {
933 	amd_iommu_handle_irq(data, "Evt", MMIO_STATUS_EVT_INT_MASK,
934 			     MMIO_STATUS_EVT_OVERFLOW_MASK,
935 			     iommu_poll_events, amd_iommu_restart_event_logging);
936 
937 	return IRQ_HANDLED;
938 }
939 
940 irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data)
941 {
942 	amd_iommu_handle_irq(data, "PPR", MMIO_STATUS_PPR_INT_MASK,
943 			     MMIO_STATUS_PPR_OVERFLOW_MASK,
944 			     amd_iommu_poll_ppr_log, amd_iommu_restart_ppr_log);
945 
946 	return IRQ_HANDLED;
947 }
948 
949 irqreturn_t amd_iommu_int_thread_galog(int irq, void *data)
950 {
951 #ifdef CONFIG_IRQ_REMAP
952 	amd_iommu_handle_irq(data, "GA", MMIO_STATUS_GALOG_INT_MASK,
953 			     MMIO_STATUS_GALOG_OVERFLOW_MASK,
954 			     iommu_poll_ga_log, amd_iommu_restart_ga_log);
955 #endif
956 
957 	return IRQ_HANDLED;
958 }
959 
960 irqreturn_t amd_iommu_int_thread(int irq, void *data)
961 {
962 	amd_iommu_int_thread_evtlog(irq, data);
963 	amd_iommu_int_thread_pprlog(irq, data);
964 	amd_iommu_int_thread_galog(irq, data);
965 
966 	return IRQ_HANDLED;
967 }
968 
969 irqreturn_t amd_iommu_int_handler(int irq, void *data)
970 {
971 	return IRQ_WAKE_THREAD;
972 }
973 
974 /****************************************************************************
975  *
976  * IOMMU command queuing functions
977  *
978  ****************************************************************************/
979 
980 static int wait_on_sem(struct amd_iommu *iommu, u64 data)
981 {
982 	int i = 0;
983 
984 	while (*iommu->cmd_sem != data && i < LOOP_TIMEOUT) {
985 		udelay(1);
986 		i += 1;
987 	}
988 
989 	if (i == LOOP_TIMEOUT) {
990 		pr_alert("Completion-Wait loop timed out\n");
991 		return -EIO;
992 	}
993 
994 	return 0;
995 }
996 
997 static void copy_cmd_to_buffer(struct amd_iommu *iommu,
998 			       struct iommu_cmd *cmd)
999 {
1000 	u8 *target;
1001 	u32 tail;
1002 
1003 	/* Copy command to buffer */
1004 	tail = iommu->cmd_buf_tail;
1005 	target = iommu->cmd_buf + tail;
1006 	memcpy(target, cmd, sizeof(*cmd));
1007 
1008 	tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
1009 	iommu->cmd_buf_tail = tail;
1010 
1011 	/* Tell the IOMMU about it */
1012 	writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
1013 }
1014 
1015 static void build_completion_wait(struct iommu_cmd *cmd,
1016 				  struct amd_iommu *iommu,
1017 				  u64 data)
1018 {
1019 	u64 paddr = iommu_virt_to_phys((void *)iommu->cmd_sem);
1020 
1021 	memset(cmd, 0, sizeof(*cmd));
1022 	cmd->data[0] = lower_32_bits(paddr) | CMD_COMPL_WAIT_STORE_MASK;
1023 	cmd->data[1] = upper_32_bits(paddr);
1024 	cmd->data[2] = lower_32_bits(data);
1025 	cmd->data[3] = upper_32_bits(data);
1026 	CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
1027 }
1028 
1029 static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
1030 {
1031 	memset(cmd, 0, sizeof(*cmd));
1032 	cmd->data[0] = devid;
1033 	CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
1034 }
1035 
1036 /*
1037  * Builds an invalidation address which is suitable for one page or multiple
1038  * pages. Sets the size bit (S) as needed is more than one page is flushed.
1039  */
1040 static inline u64 build_inv_address(u64 address, size_t size)
1041 {
1042 	u64 pages, end, msb_diff;
1043 
1044 	pages = iommu_num_pages(address, size, PAGE_SIZE);
1045 
1046 	if (pages == 1)
1047 		return address & PAGE_MASK;
1048 
1049 	end = address + size - 1;
1050 
1051 	/*
1052 	 * msb_diff would hold the index of the most significant bit that
1053 	 * flipped between the start and end.
1054 	 */
1055 	msb_diff = fls64(end ^ address) - 1;
1056 
1057 	/*
1058 	 * Bits 63:52 are sign extended. If for some reason bit 51 is different
1059 	 * between the start and the end, invalidate everything.
1060 	 */
1061 	if (unlikely(msb_diff > 51)) {
1062 		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
1063 	} else {
1064 		/*
1065 		 * The msb-bit must be clear on the address. Just set all the
1066 		 * lower bits.
1067 		 */
1068 		address |= (1ull << msb_diff) - 1;
1069 	}
1070 
1071 	/* Clear bits 11:0 */
1072 	address &= PAGE_MASK;
1073 
1074 	/* Set the size bit - we flush more than one 4kb page */
1075 	return address | CMD_INV_IOMMU_PAGES_SIZE_MASK;
1076 }
1077 
1078 static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
1079 				  size_t size, u16 domid,
1080 				  ioasid_t pasid, bool gn)
1081 {
1082 	u64 inv_address = build_inv_address(address, size);
1083 
1084 	memset(cmd, 0, sizeof(*cmd));
1085 
1086 	cmd->data[1] |= domid;
1087 	cmd->data[2]  = lower_32_bits(inv_address);
1088 	cmd->data[3]  = upper_32_bits(inv_address);
1089 	/* PDE bit - we want to flush everything, not only the PTEs */
1090 	cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
1091 	if (gn) {
1092 		cmd->data[0] |= pasid;
1093 		cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
1094 	}
1095 	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
1096 }
1097 
1098 static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
1099 				  u64 address, size_t size,
1100 				  ioasid_t pasid, bool gn)
1101 {
1102 	u64 inv_address = build_inv_address(address, size);
1103 
1104 	memset(cmd, 0, sizeof(*cmd));
1105 
1106 	cmd->data[0]  = devid;
1107 	cmd->data[0] |= (qdep & 0xff) << 24;
1108 	cmd->data[1]  = devid;
1109 	cmd->data[2]  = lower_32_bits(inv_address);
1110 	cmd->data[3]  = upper_32_bits(inv_address);
1111 	if (gn) {
1112 		cmd->data[0] |= ((pasid >> 8) & 0xff) << 16;
1113 		cmd->data[1] |= (pasid & 0xff) << 16;
1114 		cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
1115 	}
1116 
1117 	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
1118 }
1119 
1120 static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, u32 pasid,
1121 			       int status, int tag, u8 gn)
1122 {
1123 	memset(cmd, 0, sizeof(*cmd));
1124 
1125 	cmd->data[0]  = devid;
1126 	if (gn) {
1127 		cmd->data[1]  = pasid;
1128 		cmd->data[2]  = CMD_INV_IOMMU_PAGES_GN_MASK;
1129 	}
1130 	cmd->data[3]  = tag & 0x1ff;
1131 	cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;
1132 
1133 	CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
1134 }
1135 
1136 static void build_inv_all(struct iommu_cmd *cmd)
1137 {
1138 	memset(cmd, 0, sizeof(*cmd));
1139 	CMD_SET_TYPE(cmd, CMD_INV_ALL);
1140 }
1141 
1142 static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)
1143 {
1144 	memset(cmd, 0, sizeof(*cmd));
1145 	cmd->data[0] = devid;
1146 	CMD_SET_TYPE(cmd, CMD_INV_IRT);
1147 }
1148 
1149 /*
1150  * Writes the command to the IOMMUs command buffer and informs the
1151  * hardware about the new command.
1152  */
1153 static int __iommu_queue_command_sync(struct amd_iommu *iommu,
1154 				      struct iommu_cmd *cmd,
1155 				      bool sync)
1156 {
1157 	unsigned int count = 0;
1158 	u32 left, next_tail;
1159 
1160 	next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
1161 again:
1162 	left      = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE;
1163 
1164 	if (left <= 0x20) {
1165 		/* Skip udelay() the first time around */
1166 		if (count++) {
1167 			if (count == LOOP_TIMEOUT) {
1168 				pr_err("Command buffer timeout\n");
1169 				return -EIO;
1170 			}
1171 
1172 			udelay(1);
1173 		}
1174 
1175 		/* Update head and recheck remaining space */
1176 		iommu->cmd_buf_head = readl(iommu->mmio_base +
1177 					    MMIO_CMD_HEAD_OFFSET);
1178 
1179 		goto again;
1180 	}
1181 
1182 	copy_cmd_to_buffer(iommu, cmd);
1183 
1184 	/* Do we need to make sure all commands are processed? */
1185 	iommu->need_sync = sync;
1186 
1187 	return 0;
1188 }
1189 
1190 static int iommu_queue_command_sync(struct amd_iommu *iommu,
1191 				    struct iommu_cmd *cmd,
1192 				    bool sync)
1193 {
1194 	unsigned long flags;
1195 	int ret;
1196 
1197 	raw_spin_lock_irqsave(&iommu->lock, flags);
1198 	ret = __iommu_queue_command_sync(iommu, cmd, sync);
1199 	raw_spin_unlock_irqrestore(&iommu->lock, flags);
1200 
1201 	return ret;
1202 }
1203 
1204 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
1205 {
1206 	return iommu_queue_command_sync(iommu, cmd, true);
1207 }
1208 
1209 /*
1210  * This function queues a completion wait command into the command
1211  * buffer of an IOMMU
1212  */
1213 static int iommu_completion_wait(struct amd_iommu *iommu)
1214 {
1215 	struct iommu_cmd cmd;
1216 	unsigned long flags;
1217 	int ret;
1218 	u64 data;
1219 
1220 	if (!iommu->need_sync)
1221 		return 0;
1222 
1223 	data = atomic64_inc_return(&iommu->cmd_sem_val);
1224 	build_completion_wait(&cmd, iommu, data);
1225 
1226 	raw_spin_lock_irqsave(&iommu->lock, flags);
1227 
1228 	ret = __iommu_queue_command_sync(iommu, &cmd, false);
1229 	if (ret)
1230 		goto out_unlock;
1231 
1232 	ret = wait_on_sem(iommu, data);
1233 
1234 out_unlock:
1235 	raw_spin_unlock_irqrestore(&iommu->lock, flags);
1236 
1237 	return ret;
1238 }
1239 
1240 static void domain_flush_complete(struct protection_domain *domain)
1241 {
1242 	struct pdom_iommu_info *pdom_iommu_info;
1243 	unsigned long i;
1244 
1245 	lockdep_assert_held(&domain->lock);
1246 
1247 	/*
1248 	 * Devices of this domain are behind this IOMMU
1249 	 * We need to wait for completion of all commands.
1250 	 */
1251 	 xa_for_each(&domain->iommu_array, i, pdom_iommu_info)
1252 		iommu_completion_wait(pdom_iommu_info->iommu);
1253 }
1254 
1255 static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
1256 {
1257 	struct iommu_cmd cmd;
1258 
1259 	build_inv_dte(&cmd, devid);
1260 
1261 	return iommu_queue_command(iommu, &cmd);
1262 }
1263 
1264 static void amd_iommu_flush_dte_all(struct amd_iommu *iommu)
1265 {
1266 	u32 devid;
1267 	u16 last_bdf = iommu->pci_seg->last_bdf;
1268 
1269 	for (devid = 0; devid <= last_bdf; ++devid)
1270 		iommu_flush_dte(iommu, devid);
1271 
1272 	iommu_completion_wait(iommu);
1273 }
1274 
1275 /*
1276  * This function uses heavy locking and may disable irqs for some time. But
1277  * this is no issue because it is only called during resume.
1278  */
1279 static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu)
1280 {
1281 	u32 dom_id;
1282 	u16 last_bdf = iommu->pci_seg->last_bdf;
1283 
1284 	for (dom_id = 0; dom_id <= last_bdf; ++dom_id) {
1285 		struct iommu_cmd cmd;
1286 		build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
1287 				      dom_id, IOMMU_NO_PASID, false);
1288 		iommu_queue_command(iommu, &cmd);
1289 	}
1290 
1291 	iommu_completion_wait(iommu);
1292 }
1293 
1294 static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id)
1295 {
1296 	struct iommu_cmd cmd;
1297 
1298 	build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
1299 			      dom_id, IOMMU_NO_PASID, false);
1300 	iommu_queue_command(iommu, &cmd);
1301 
1302 	iommu_completion_wait(iommu);
1303 }
1304 
1305 static void amd_iommu_flush_all(struct amd_iommu *iommu)
1306 {
1307 	struct iommu_cmd cmd;
1308 
1309 	build_inv_all(&cmd);
1310 
1311 	iommu_queue_command(iommu, &cmd);
1312 	iommu_completion_wait(iommu);
1313 }
1314 
1315 static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)
1316 {
1317 	struct iommu_cmd cmd;
1318 
1319 	build_inv_irt(&cmd, devid);
1320 
1321 	iommu_queue_command(iommu, &cmd);
1322 }
1323 
1324 static void amd_iommu_flush_irt_all(struct amd_iommu *iommu)
1325 {
1326 	u32 devid;
1327 	u16 last_bdf = iommu->pci_seg->last_bdf;
1328 
1329 	if (iommu->irtcachedis_enabled)
1330 		return;
1331 
1332 	for (devid = 0; devid <= last_bdf; devid++)
1333 		iommu_flush_irt(iommu, devid);
1334 
1335 	iommu_completion_wait(iommu);
1336 }
1337 
1338 void amd_iommu_flush_all_caches(struct amd_iommu *iommu)
1339 {
1340 	if (check_feature(FEATURE_IA)) {
1341 		amd_iommu_flush_all(iommu);
1342 	} else {
1343 		amd_iommu_flush_dte_all(iommu);
1344 		amd_iommu_flush_irt_all(iommu);
1345 		amd_iommu_flush_tlb_all(iommu);
1346 	}
1347 }
1348 
1349 /*
1350  * Command send function for flushing on-device TLB
1351  */
1352 static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address,
1353 			      size_t size, ioasid_t pasid, bool gn)
1354 {
1355 	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
1356 	struct iommu_cmd cmd;
1357 	int qdep = dev_data->ats_qdep;
1358 
1359 	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address,
1360 			      size, pasid, gn);
1361 
1362 	return iommu_queue_command(iommu, &cmd);
1363 }
1364 
1365 static int device_flush_dte_alias(struct pci_dev *pdev, u16 alias, void *data)
1366 {
1367 	struct amd_iommu *iommu = data;
1368 
1369 	return iommu_flush_dte(iommu, alias);
1370 }
1371 
1372 /*
1373  * Command send function for invalidating a device table entry
1374  */
1375 static int device_flush_dte(struct iommu_dev_data *dev_data)
1376 {
1377 	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
1378 	struct pci_dev *pdev = NULL;
1379 	struct amd_iommu_pci_seg *pci_seg;
1380 	u16 alias;
1381 	int ret;
1382 
1383 	if (dev_is_pci(dev_data->dev))
1384 		pdev = to_pci_dev(dev_data->dev);
1385 
1386 	if (pdev)
1387 		ret = pci_for_each_dma_alias(pdev,
1388 					     device_flush_dte_alias, iommu);
1389 	else
1390 		ret = iommu_flush_dte(iommu, dev_data->devid);
1391 	if (ret)
1392 		return ret;
1393 
1394 	pci_seg = iommu->pci_seg;
1395 	alias = pci_seg->alias_table[dev_data->devid];
1396 	if (alias != dev_data->devid) {
1397 		ret = iommu_flush_dte(iommu, alias);
1398 		if (ret)
1399 			return ret;
1400 	}
1401 
1402 	if (dev_data->ats_enabled) {
1403 		/* Invalidate the entire contents of an IOTLB */
1404 		ret = device_flush_iotlb(dev_data, 0, ~0UL,
1405 					 IOMMU_NO_PASID, false);
1406 	}
1407 
1408 	return ret;
1409 }
1410 
1411 static int domain_flush_pages_v2(struct protection_domain *pdom,
1412 				 u64 address, size_t size)
1413 {
1414 	struct iommu_dev_data *dev_data;
1415 	struct iommu_cmd cmd;
1416 	int ret = 0;
1417 
1418 	list_for_each_entry(dev_data, &pdom->dev_list, list) {
1419 		struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);
1420 		u16 domid = dev_data->gcr3_info.domid;
1421 
1422 		build_inv_iommu_pages(&cmd, address, size,
1423 				      domid, IOMMU_NO_PASID, true);
1424 
1425 		ret |= iommu_queue_command(iommu, &cmd);
1426 	}
1427 
1428 	return ret;
1429 }
1430 
1431 static int domain_flush_pages_v1(struct protection_domain *pdom,
1432 				 u64 address, size_t size)
1433 {
1434 	struct pdom_iommu_info *pdom_iommu_info;
1435 	struct iommu_cmd cmd;
1436 	int ret = 0;
1437 	unsigned long i;
1438 
1439 	lockdep_assert_held(&pdom->lock);
1440 
1441 	build_inv_iommu_pages(&cmd, address, size,
1442 			      pdom->id, IOMMU_NO_PASID, false);
1443 
1444 	xa_for_each(&pdom->iommu_array, i, pdom_iommu_info) {
1445 		/*
1446 		 * Devices of this domain are behind this IOMMU
1447 		 * We need a TLB flush
1448 		 */
1449 		ret |= iommu_queue_command(pdom_iommu_info->iommu, &cmd);
1450 	}
1451 
1452 	return ret;
1453 }
1454 
1455 /*
1456  * TLB invalidation function which is called from the mapping functions.
1457  * It flushes range of PTEs of the domain.
1458  */
1459 static void __domain_flush_pages(struct protection_domain *domain,
1460 				 u64 address, size_t size)
1461 {
1462 	struct iommu_dev_data *dev_data;
1463 	int ret = 0;
1464 	ioasid_t pasid = IOMMU_NO_PASID;
1465 	bool gn = false;
1466 
1467 	if (pdom_is_v2_pgtbl_mode(domain)) {
1468 		gn = true;
1469 		ret = domain_flush_pages_v2(domain, address, size);
1470 	} else {
1471 		ret = domain_flush_pages_v1(domain, address, size);
1472 	}
1473 
1474 	list_for_each_entry(dev_data, &domain->dev_list, list) {
1475 
1476 		if (!dev_data->ats_enabled)
1477 			continue;
1478 
1479 		ret |= device_flush_iotlb(dev_data, address, size, pasid, gn);
1480 	}
1481 
1482 	WARN_ON(ret);
1483 }
1484 
1485 void amd_iommu_domain_flush_pages(struct protection_domain *domain,
1486 				  u64 address, size_t size)
1487 {
1488 	lockdep_assert_held(&domain->lock);
1489 
1490 	if (likely(!amd_iommu_np_cache)) {
1491 		__domain_flush_pages(domain, address, size);
1492 
1493 		/* Wait until IOMMU TLB and all device IOTLB flushes are complete */
1494 		domain_flush_complete(domain);
1495 
1496 		return;
1497 	}
1498 
1499 	/*
1500 	 * When NpCache is on, we infer that we run in a VM and use a vIOMMU.
1501 	 * In such setups it is best to avoid flushes of ranges which are not
1502 	 * naturally aligned, since it would lead to flushes of unmodified
1503 	 * PTEs. Such flushes would require the hypervisor to do more work than
1504 	 * necessary. Therefore, perform repeated flushes of aligned ranges
1505 	 * until you cover the range. Each iteration flushes the smaller
1506 	 * between the natural alignment of the address that we flush and the
1507 	 * greatest naturally aligned region that fits in the range.
1508 	 */
1509 	while (size != 0) {
1510 		int addr_alignment = __ffs(address);
1511 		int size_alignment = __fls(size);
1512 		int min_alignment;
1513 		size_t flush_size;
1514 
1515 		/*
1516 		 * size is always non-zero, but address might be zero, causing
1517 		 * addr_alignment to be negative. As the casting of the
1518 		 * argument in __ffs(address) to long might trim the high bits
1519 		 * of the address on x86-32, cast to long when doing the check.
1520 		 */
1521 		if (likely((unsigned long)address != 0))
1522 			min_alignment = min(addr_alignment, size_alignment);
1523 		else
1524 			min_alignment = size_alignment;
1525 
1526 		flush_size = 1ul << min_alignment;
1527 
1528 		__domain_flush_pages(domain, address, flush_size);
1529 		address += flush_size;
1530 		size -= flush_size;
1531 	}
1532 
1533 	/* Wait until IOMMU TLB and all device IOTLB flushes are complete */
1534 	domain_flush_complete(domain);
1535 }
1536 
1537 /* Flush the whole IO/TLB for a given protection domain - including PDE */
1538 static void amd_iommu_domain_flush_all(struct protection_domain *domain)
1539 {
1540 	amd_iommu_domain_flush_pages(domain, 0,
1541 				     CMD_INV_IOMMU_ALL_PAGES_ADDRESS);
1542 }
1543 
1544 void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data,
1545 				     ioasid_t pasid, u64 address, size_t size)
1546 {
1547 	struct iommu_cmd cmd;
1548 	struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);
1549 
1550 	build_inv_iommu_pages(&cmd, address, size,
1551 			      dev_data->gcr3_info.domid, pasid, true);
1552 	iommu_queue_command(iommu, &cmd);
1553 
1554 	if (dev_data->ats_enabled)
1555 		device_flush_iotlb(dev_data, address, size, pasid, true);
1556 
1557 	iommu_completion_wait(iommu);
1558 }
1559 
1560 static void dev_flush_pasid_all(struct iommu_dev_data *dev_data,
1561 				ioasid_t pasid)
1562 {
1563 	amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0,
1564 					CMD_INV_IOMMU_ALL_PAGES_ADDRESS);
1565 }
1566 
1567 /* Flush the not present cache if it exists */
1568 static void domain_flush_np_cache(struct protection_domain *domain,
1569 		dma_addr_t iova, size_t size)
1570 {
1571 	if (unlikely(amd_iommu_np_cache)) {
1572 		unsigned long flags;
1573 
1574 		spin_lock_irqsave(&domain->lock, flags);
1575 		amd_iommu_domain_flush_pages(domain, iova, size);
1576 		spin_unlock_irqrestore(&domain->lock, flags);
1577 	}
1578 }
1579 
1580 
1581 /*
1582  * This function flushes the DTEs for all devices in domain
1583  */
1584 void amd_iommu_update_and_flush_device_table(struct protection_domain *domain)
1585 {
1586 	struct iommu_dev_data *dev_data;
1587 
1588 	list_for_each_entry(dev_data, &domain->dev_list, list) {
1589 		struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev);
1590 
1591 		set_dte_entry(iommu, dev_data);
1592 		clone_aliases(iommu, dev_data->dev);
1593 	}
1594 
1595 	list_for_each_entry(dev_data, &domain->dev_list, list)
1596 		device_flush_dte(dev_data);
1597 
1598 	domain_flush_complete(domain);
1599 }
1600 
1601 void amd_iommu_domain_update(struct protection_domain *domain)
1602 {
1603 	/* Update device table */
1604 	amd_iommu_update_and_flush_device_table(domain);
1605 
1606 	/* Flush domain TLB(s) and wait for completion */
1607 	amd_iommu_domain_flush_all(domain);
1608 }
1609 
1610 int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag)
1611 {
1612 	struct iommu_dev_data *dev_data;
1613 	struct amd_iommu *iommu;
1614 	struct iommu_cmd cmd;
1615 
1616 	dev_data = dev_iommu_priv_get(dev);
1617 	iommu    = get_amd_iommu_from_dev(dev);
1618 
1619 	build_complete_ppr(&cmd, dev_data->devid, pasid, status,
1620 			   tag, dev_data->pri_tlp);
1621 
1622 	return iommu_queue_command(iommu, &cmd);
1623 }
1624 
1625 /****************************************************************************
1626  *
1627  * The next functions belong to the domain allocation. A domain is
1628  * allocated for every IOMMU as the default domain. If device isolation
1629  * is enabled, every device get its own domain. The most important thing
1630  * about domains is the page table mapping the DMA address space they
1631  * contain.
1632  *
1633  ****************************************************************************/
1634 
1635 static int pdom_id_alloc(void)
1636 {
1637 	return ida_alloc_range(&pdom_ids, 1, MAX_DOMAIN_ID - 1, GFP_ATOMIC);
1638 }
1639 
1640 static void pdom_id_free(int id)
1641 {
1642 	ida_free(&pdom_ids, id);
1643 }
1644 
1645 static void free_gcr3_tbl_level1(u64 *tbl)
1646 {
1647 	u64 *ptr;
1648 	int i;
1649 
1650 	for (i = 0; i < 512; ++i) {
1651 		if (!(tbl[i] & GCR3_VALID))
1652 			continue;
1653 
1654 		ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK);
1655 
1656 		iommu_free_page(ptr);
1657 	}
1658 }
1659 
1660 static void free_gcr3_tbl_level2(u64 *tbl)
1661 {
1662 	u64 *ptr;
1663 	int i;
1664 
1665 	for (i = 0; i < 512; ++i) {
1666 		if (!(tbl[i] & GCR3_VALID))
1667 			continue;
1668 
1669 		ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK);
1670 
1671 		free_gcr3_tbl_level1(ptr);
1672 	}
1673 }
1674 
1675 static void free_gcr3_table(struct gcr3_tbl_info *gcr3_info)
1676 {
1677 	if (gcr3_info->glx == 2)
1678 		free_gcr3_tbl_level2(gcr3_info->gcr3_tbl);
1679 	else if (gcr3_info->glx == 1)
1680 		free_gcr3_tbl_level1(gcr3_info->gcr3_tbl);
1681 	else
1682 		WARN_ON_ONCE(gcr3_info->glx != 0);
1683 
1684 	gcr3_info->glx = 0;
1685 
1686 	/* Free per device domain ID */
1687 	pdom_id_free(gcr3_info->domid);
1688 
1689 	iommu_free_page(gcr3_info->gcr3_tbl);
1690 	gcr3_info->gcr3_tbl = NULL;
1691 }
1692 
1693 /*
1694  * Number of GCR3 table levels required. Level must be 4-Kbyte
1695  * page and can contain up to 512 entries.
1696  */
1697 static int get_gcr3_levels(int pasids)
1698 {
1699 	int levels;
1700 
1701 	if (pasids == -1)
1702 		return amd_iommu_max_glx_val;
1703 
1704 	levels = get_count_order(pasids);
1705 
1706 	return levels ? (DIV_ROUND_UP(levels, 9) - 1) : levels;
1707 }
1708 
1709 static int setup_gcr3_table(struct gcr3_tbl_info *gcr3_info,
1710 			    struct amd_iommu *iommu, int pasids)
1711 {
1712 	int levels = get_gcr3_levels(pasids);
1713 	int nid = iommu ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
1714 	int domid;
1715 
1716 	if (levels > amd_iommu_max_glx_val)
1717 		return -EINVAL;
1718 
1719 	if (gcr3_info->gcr3_tbl)
1720 		return -EBUSY;
1721 
1722 	/* Allocate per device domain ID */
1723 	domid = pdom_id_alloc();
1724 	if (domid <= 0)
1725 		return -ENOSPC;
1726 	gcr3_info->domid = domid;
1727 
1728 	gcr3_info->gcr3_tbl = iommu_alloc_page_node(nid, GFP_ATOMIC);
1729 	if (gcr3_info->gcr3_tbl == NULL) {
1730 		pdom_id_free(domid);
1731 		return -ENOMEM;
1732 	}
1733 
1734 	gcr3_info->glx = levels;
1735 
1736 	return 0;
1737 }
1738 
1739 static u64 *__get_gcr3_pte(struct gcr3_tbl_info *gcr3_info,
1740 			   ioasid_t pasid, bool alloc)
1741 {
1742 	int index;
1743 	u64 *pte;
1744 	u64 *root = gcr3_info->gcr3_tbl;
1745 	int level = gcr3_info->glx;
1746 
1747 	while (true) {
1748 
1749 		index = (pasid >> (9 * level)) & 0x1ff;
1750 		pte   = &root[index];
1751 
1752 		if (level == 0)
1753 			break;
1754 
1755 		if (!(*pte & GCR3_VALID)) {
1756 			if (!alloc)
1757 				return NULL;
1758 
1759 			root = (void *)get_zeroed_page(GFP_ATOMIC);
1760 			if (root == NULL)
1761 				return NULL;
1762 
1763 			*pte = iommu_virt_to_phys(root) | GCR3_VALID;
1764 		}
1765 
1766 		root = iommu_phys_to_virt(*pte & PAGE_MASK);
1767 
1768 		level -= 1;
1769 	}
1770 
1771 	return pte;
1772 }
1773 
1774 static int update_gcr3(struct iommu_dev_data *dev_data,
1775 		       ioasid_t pasid, unsigned long gcr3, bool set)
1776 {
1777 	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
1778 	u64 *pte;
1779 
1780 	pte = __get_gcr3_pte(gcr3_info, pasid, true);
1781 	if (pte == NULL)
1782 		return -ENOMEM;
1783 
1784 	if (set)
1785 		*pte = (gcr3 & PAGE_MASK) | GCR3_VALID;
1786 	else
1787 		*pte = 0;
1788 
1789 	dev_flush_pasid_all(dev_data, pasid);
1790 	return 0;
1791 }
1792 
1793 int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid,
1794 		       unsigned long gcr3)
1795 {
1796 	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
1797 	int ret;
1798 
1799 	iommu_group_mutex_assert(dev_data->dev);
1800 
1801 	ret = update_gcr3(dev_data, pasid, gcr3, true);
1802 	if (ret)
1803 		return ret;
1804 
1805 	gcr3_info->pasid_cnt++;
1806 	return ret;
1807 }
1808 
1809 int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid)
1810 {
1811 	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
1812 	int ret;
1813 
1814 	iommu_group_mutex_assert(dev_data->dev);
1815 
1816 	ret = update_gcr3(dev_data, pasid, 0, false);
1817 	if (ret)
1818 		return ret;
1819 
1820 	gcr3_info->pasid_cnt--;
1821 	return ret;
1822 }
1823 
1824 static void set_dte_entry(struct amd_iommu *iommu,
1825 			  struct iommu_dev_data *dev_data)
1826 {
1827 	u64 pte_root = 0;
1828 	u64 flags = 0;
1829 	u32 old_domid;
1830 	u16 devid = dev_data->devid;
1831 	u16 domid;
1832 	struct protection_domain *domain = dev_data->domain;
1833 	struct dev_table_entry *dev_table = get_dev_table(iommu);
1834 	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
1835 
1836 	if (gcr3_info && gcr3_info->gcr3_tbl)
1837 		domid = dev_data->gcr3_info.domid;
1838 	else
1839 		domid = domain->id;
1840 
1841 	if (domain->iop.mode != PAGE_MODE_NONE)
1842 		pte_root = iommu_virt_to_phys(domain->iop.root);
1843 
1844 	pte_root |= (domain->iop.mode & DEV_ENTRY_MODE_MASK)
1845 		    << DEV_ENTRY_MODE_SHIFT;
1846 
1847 	pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V;
1848 
1849 	/*
1850 	 * When SNP is enabled, Only set TV bit when IOMMU
1851 	 * page translation is in use.
1852 	 */
1853 	if (!amd_iommu_snp_en || (domid != 0))
1854 		pte_root |= DTE_FLAG_TV;
1855 
1856 	flags = dev_table[devid].data[1];
1857 
1858 	if (dev_data->ats_enabled)
1859 		flags |= DTE_FLAG_IOTLB;
1860 
1861 	if (dev_data->ppr)
1862 		pte_root |= 1ULL << DEV_ENTRY_PPR;
1863 
1864 	if (domain->dirty_tracking)
1865 		pte_root |= DTE_FLAG_HAD;
1866 
1867 	if (gcr3_info && gcr3_info->gcr3_tbl) {
1868 		u64 gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl);
1869 		u64 glx  = gcr3_info->glx;
1870 		u64 tmp;
1871 
1872 		pte_root |= DTE_FLAG_GV;
1873 		pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
1874 
1875 		/* First mask out possible old values for GCR3 table */
1876 		tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
1877 		flags    &= ~tmp;
1878 
1879 		tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
1880 		flags    &= ~tmp;
1881 
1882 		/* Encode GCR3 table into DTE */
1883 		tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
1884 		pte_root |= tmp;
1885 
1886 		tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
1887 		flags    |= tmp;
1888 
1889 		tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
1890 		flags    |= tmp;
1891 
1892 		if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) {
1893 			dev_table[devid].data[2] |=
1894 				((u64)GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT);
1895 		}
1896 
1897 		/* GIOV is supported with V2 page table mode only */
1898 		if (pdom_is_v2_pgtbl_mode(domain))
1899 			pte_root |= DTE_FLAG_GIOV;
1900 	}
1901 
1902 	flags &= ~DEV_DOMID_MASK;
1903 	flags |= domid;
1904 
1905 	old_domid = dev_table[devid].data[1] & DEV_DOMID_MASK;
1906 	dev_table[devid].data[1]  = flags;
1907 	dev_table[devid].data[0]  = pte_root;
1908 
1909 	/*
1910 	 * A kdump kernel might be replacing a domain ID that was copied from
1911 	 * the previous kernel--if so, it needs to flush the translation cache
1912 	 * entries for the old domain ID that is being overwritten
1913 	 */
1914 	if (old_domid) {
1915 		amd_iommu_flush_tlb_domid(iommu, old_domid);
1916 	}
1917 }
1918 
1919 static void clear_dte_entry(struct amd_iommu *iommu, u16 devid)
1920 {
1921 	struct dev_table_entry *dev_table = get_dev_table(iommu);
1922 
1923 	/* remove entry from the device table seen by the hardware */
1924 	dev_table[devid].data[0]  = DTE_FLAG_V;
1925 
1926 	if (!amd_iommu_snp_en)
1927 		dev_table[devid].data[0] |= DTE_FLAG_TV;
1928 
1929 	dev_table[devid].data[1] &= DTE_FLAG_MASK;
1930 
1931 	amd_iommu_apply_erratum_63(iommu, devid);
1932 }
1933 
1934 /* Update and flush DTE for the given device */
1935 static void dev_update_dte(struct iommu_dev_data *dev_data, bool set)
1936 {
1937 	struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);
1938 
1939 	if (set)
1940 		set_dte_entry(iommu, dev_data);
1941 	else
1942 		clear_dte_entry(iommu, dev_data->devid);
1943 
1944 	clone_aliases(iommu, dev_data->dev);
1945 	device_flush_dte(dev_data);
1946 	iommu_completion_wait(iommu);
1947 }
1948 
1949 /*
1950  * If domain is SVA capable then initialize GCR3 table. Also if domain is
1951  * in v2 page table mode then update GCR3[0].
1952  */
1953 static int init_gcr3_table(struct iommu_dev_data *dev_data,
1954 			   struct protection_domain *pdom)
1955 {
1956 	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
1957 	int max_pasids = dev_data->max_pasids;
1958 	int ret = 0;
1959 
1960 	 /*
1961 	  * If domain is in pt mode then setup GCR3 table only if device
1962 	  * is PASID capable
1963 	  */
1964 	if (pdom_is_in_pt_mode(pdom) && !pdev_pasid_supported(dev_data))
1965 		return ret;
1966 
1967 	/*
1968 	 * By default, setup GCR3 table to support MAX PASIDs
1969 	 * supported by the device/IOMMU.
1970 	 */
1971 	ret = setup_gcr3_table(&dev_data->gcr3_info, iommu,
1972 			       max_pasids > 0 ?  max_pasids : 1);
1973 	if (ret)
1974 		return ret;
1975 
1976 	/* Setup GCR3[0] only if domain is setup with v2 page table mode */
1977 	if (!pdom_is_v2_pgtbl_mode(pdom))
1978 		return ret;
1979 
1980 	ret = update_gcr3(dev_data, 0, iommu_virt_to_phys(pdom->iop.pgd), true);
1981 	if (ret)
1982 		free_gcr3_table(&dev_data->gcr3_info);
1983 
1984 	return ret;
1985 }
1986 
1987 static void destroy_gcr3_table(struct iommu_dev_data *dev_data,
1988 			       struct protection_domain *pdom)
1989 {
1990 	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
1991 
1992 	if (pdom_is_v2_pgtbl_mode(pdom))
1993 		update_gcr3(dev_data, 0, 0, false);
1994 
1995 	if (gcr3_info->gcr3_tbl == NULL)
1996 		return;
1997 
1998 	free_gcr3_table(gcr3_info);
1999 }
2000 
2001 static int pdom_attach_iommu(struct amd_iommu *iommu,
2002 			     struct protection_domain *pdom)
2003 {
2004 	struct pdom_iommu_info *pdom_iommu_info, *curr;
2005 	struct io_pgtable_cfg *cfg = &pdom->iop.pgtbl.cfg;
2006 	unsigned long flags;
2007 	int ret = 0;
2008 
2009 	spin_lock_irqsave(&pdom->lock, flags);
2010 
2011 	pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index);
2012 	if (pdom_iommu_info) {
2013 		pdom_iommu_info->refcnt++;
2014 		goto out_unlock;
2015 	}
2016 
2017 	pdom_iommu_info = kzalloc(sizeof(*pdom_iommu_info), GFP_ATOMIC);
2018 	if (!pdom_iommu_info) {
2019 		ret = -ENOMEM;
2020 		goto out_unlock;
2021 	}
2022 
2023 	pdom_iommu_info->iommu = iommu;
2024 	pdom_iommu_info->refcnt = 1;
2025 
2026 	curr = xa_cmpxchg(&pdom->iommu_array, iommu->index,
2027 			  NULL, pdom_iommu_info, GFP_ATOMIC);
2028 	if (curr) {
2029 		kfree(pdom_iommu_info);
2030 		ret = -ENOSPC;
2031 		goto out_unlock;
2032 	}
2033 
2034 	/* Update NUMA Node ID */
2035 	if (cfg->amd.nid == NUMA_NO_NODE)
2036 		cfg->amd.nid = dev_to_node(&iommu->dev->dev);
2037 
2038 out_unlock:
2039 	spin_unlock_irqrestore(&pdom->lock, flags);
2040 	return ret;
2041 }
2042 
2043 static void pdom_detach_iommu(struct amd_iommu *iommu,
2044 			      struct protection_domain *pdom)
2045 {
2046 	struct pdom_iommu_info *pdom_iommu_info;
2047 	unsigned long flags;
2048 
2049 	spin_lock_irqsave(&pdom->lock, flags);
2050 
2051 	pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index);
2052 	if (!pdom_iommu_info) {
2053 		spin_unlock_irqrestore(&pdom->lock, flags);
2054 		return;
2055 	}
2056 
2057 	pdom_iommu_info->refcnt--;
2058 	if (pdom_iommu_info->refcnt == 0) {
2059 		xa_erase(&pdom->iommu_array, iommu->index);
2060 		kfree(pdom_iommu_info);
2061 	}
2062 
2063 	spin_unlock_irqrestore(&pdom->lock, flags);
2064 }
2065 
2066 /*
2067  * If a device is not yet associated with a domain, this function makes the
2068  * device visible in the domain
2069  */
2070 static int attach_device(struct device *dev,
2071 			 struct protection_domain *domain)
2072 {
2073 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2074 	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
2075 	struct pci_dev *pdev;
2076 	int ret = 0;
2077 
2078 	mutex_lock(&dev_data->mutex);
2079 
2080 	if (dev_data->domain != NULL) {
2081 		ret = -EBUSY;
2082 		goto out;
2083 	}
2084 
2085 	/* Do reference counting */
2086 	ret = pdom_attach_iommu(iommu, domain);
2087 	if (ret)
2088 		goto out;
2089 
2090 	/* Setup GCR3 table */
2091 	if (pdom_is_sva_capable(domain)) {
2092 		ret = init_gcr3_table(dev_data, domain);
2093 		if (ret) {
2094 			pdom_detach_iommu(iommu, domain);
2095 			goto out;
2096 		}
2097 	}
2098 
2099 	pdev = dev_is_pci(dev_data->dev) ? to_pci_dev(dev_data->dev) : NULL;
2100 	if (pdev && pdom_is_sva_capable(domain)) {
2101 		pdev_enable_caps(pdev);
2102 
2103 		/*
2104 		 * Device can continue to function even if IOPF
2105 		 * enablement failed. Hence in error path just
2106 		 * disable device PRI support.
2107 		 */
2108 		if (amd_iommu_iopf_add_device(iommu, dev_data))
2109 			pdev_disable_cap_pri(pdev);
2110 	} else if (pdev) {
2111 		pdev_enable_cap_ats(pdev);
2112 	}
2113 
2114 	/* Update data structures */
2115 	dev_data->domain = domain;
2116 	list_add(&dev_data->list, &domain->dev_list);
2117 
2118 	/* Update device table */
2119 	dev_update_dte(dev_data, true);
2120 
2121 out:
2122 	mutex_unlock(&dev_data->mutex);
2123 
2124 	return ret;
2125 }
2126 
2127 /*
2128  * Removes a device from a protection domain (with devtable_lock held)
2129  */
2130 static void detach_device(struct device *dev)
2131 {
2132 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2133 	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
2134 	struct protection_domain *domain = dev_data->domain;
2135 	unsigned long flags;
2136 
2137 	mutex_lock(&dev_data->mutex);
2138 
2139 	/*
2140 	 * First check if the device is still attached. It might already
2141 	 * be detached from its domain because the generic
2142 	 * iommu_detach_group code detached it and we try again here in
2143 	 * our alias handling.
2144 	 */
2145 	if (WARN_ON(!dev_data->domain))
2146 		goto out;
2147 
2148 	/* Remove IOPF handler */
2149 	if (dev_data->ppr) {
2150 		iopf_queue_flush_dev(dev);
2151 		amd_iommu_iopf_remove_device(iommu, dev_data);
2152 	}
2153 
2154 	if (dev_is_pci(dev))
2155 		pdev_disable_caps(to_pci_dev(dev));
2156 
2157 	/* Clear DTE and flush the entry */
2158 	dev_update_dte(dev_data, false);
2159 
2160 	/* Flush IOTLB and wait for the flushes to finish */
2161 	spin_lock_irqsave(&domain->lock, flags);
2162 	amd_iommu_domain_flush_all(domain);
2163 	spin_unlock_irqrestore(&domain->lock, flags);
2164 
2165 	/* Clear GCR3 table */
2166 	if (pdom_is_sva_capable(domain))
2167 		destroy_gcr3_table(dev_data, domain);
2168 
2169 	/* Update data structures */
2170 	dev_data->domain = NULL;
2171 	list_del(&dev_data->list);
2172 
2173 	/* decrease reference counters - needs to happen after the flushes */
2174 	pdom_detach_iommu(iommu, domain);
2175 
2176 out:
2177 	mutex_unlock(&dev_data->mutex);
2178 }
2179 
2180 static struct iommu_device *amd_iommu_probe_device(struct device *dev)
2181 {
2182 	struct iommu_device *iommu_dev;
2183 	struct amd_iommu *iommu;
2184 	struct iommu_dev_data *dev_data;
2185 	int ret;
2186 
2187 	if (!check_device(dev))
2188 		return ERR_PTR(-ENODEV);
2189 
2190 	iommu = rlookup_amd_iommu(dev);
2191 	if (!iommu)
2192 		return ERR_PTR(-ENODEV);
2193 
2194 	/* Not registered yet? */
2195 	if (!iommu->iommu.ops)
2196 		return ERR_PTR(-ENODEV);
2197 
2198 	if (dev_iommu_priv_get(dev))
2199 		return &iommu->iommu;
2200 
2201 	ret = iommu_init_device(iommu, dev);
2202 	if (ret) {
2203 		dev_err(dev, "Failed to initialize - trying to proceed anyway\n");
2204 		iommu_dev = ERR_PTR(ret);
2205 		iommu_ignore_device(iommu, dev);
2206 		goto out_err;
2207 	}
2208 
2209 	amd_iommu_set_pci_msi_domain(dev, iommu);
2210 	iommu_dev = &iommu->iommu;
2211 
2212 	/*
2213 	 * If IOMMU and device supports PASID then it will contain max
2214 	 * supported PASIDs, else it will be zero.
2215 	 */
2216 	dev_data = dev_iommu_priv_get(dev);
2217 	if (amd_iommu_pasid_supported() && dev_is_pci(dev) &&
2218 	    pdev_pasid_supported(dev_data)) {
2219 		dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids,
2220 					     pci_max_pasids(to_pci_dev(dev)));
2221 	}
2222 
2223 out_err:
2224 	iommu_completion_wait(iommu);
2225 
2226 	if (dev_is_pci(dev))
2227 		pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT);
2228 
2229 	return iommu_dev;
2230 }
2231 
2232 static void amd_iommu_release_device(struct device *dev)
2233 {
2234 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2235 
2236 	WARN_ON(dev_data->domain);
2237 
2238 	/*
2239 	 * We keep dev_data around for unplugged devices and reuse it when the
2240 	 * device is re-plugged - not doing so would introduce a ton of races.
2241 	 */
2242 }
2243 
2244 static struct iommu_group *amd_iommu_device_group(struct device *dev)
2245 {
2246 	if (dev_is_pci(dev))
2247 		return pci_device_group(dev);
2248 
2249 	return acpihid_device_group(dev);
2250 }
2251 
2252 /*****************************************************************************
2253  *
2254  * The following functions belong to the exported interface of AMD IOMMU
2255  *
2256  * This interface allows access to lower level functions of the IOMMU
2257  * like protection domain handling and assignement of devices to domains
2258  * which is not possible with the dma_ops interface.
2259  *
2260  *****************************************************************************/
2261 
2262 void protection_domain_free(struct protection_domain *domain)
2263 {
2264 	WARN_ON(!list_empty(&domain->dev_list));
2265 	if (domain->domain.type & __IOMMU_DOMAIN_PAGING)
2266 		free_io_pgtable_ops(&domain->iop.pgtbl.ops);
2267 	pdom_id_free(domain->id);
2268 	kfree(domain);
2269 }
2270 
2271 static void protection_domain_init(struct protection_domain *domain, int nid)
2272 {
2273 	spin_lock_init(&domain->lock);
2274 	INIT_LIST_HEAD(&domain->dev_list);
2275 	INIT_LIST_HEAD(&domain->dev_data_list);
2276 	xa_init(&domain->iommu_array);
2277 	domain->iop.pgtbl.cfg.amd.nid = nid;
2278 }
2279 
2280 struct protection_domain *protection_domain_alloc(unsigned int type, int nid)
2281 {
2282 	struct protection_domain *domain;
2283 	int domid;
2284 
2285 	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
2286 	if (!domain)
2287 		return NULL;
2288 
2289 	domid = pdom_id_alloc();
2290 	if (domid <= 0) {
2291 		kfree(domain);
2292 		return NULL;
2293 	}
2294 	domain->id = domid;
2295 
2296 	protection_domain_init(domain, nid);
2297 
2298 	return domain;
2299 }
2300 
2301 static int pdom_setup_pgtable(struct protection_domain *domain,
2302 			      unsigned int type, int pgtable)
2303 {
2304 	struct io_pgtable_ops *pgtbl_ops;
2305 
2306 	/* No need to allocate io pgtable ops in passthrough mode */
2307 	if (!(type & __IOMMU_DOMAIN_PAGING))
2308 		return 0;
2309 
2310 	switch (pgtable) {
2311 	case AMD_IOMMU_V1:
2312 		domain->pd_mode = PD_MODE_V1;
2313 		break;
2314 	case AMD_IOMMU_V2:
2315 		domain->pd_mode = PD_MODE_V2;
2316 		break;
2317 	default:
2318 		return -EINVAL;
2319 	}
2320 
2321 	pgtbl_ops =
2322 		alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl.cfg, domain);
2323 	if (!pgtbl_ops)
2324 		return -ENOMEM;
2325 
2326 	return 0;
2327 }
2328 
2329 static inline u64 dma_max_address(int pgtable)
2330 {
2331 	if (pgtable == AMD_IOMMU_V1)
2332 		return ~0ULL;
2333 
2334 	/* V2 with 4/5 level page table */
2335 	return ((1ULL << PM_LEVEL_SHIFT(amd_iommu_gpt_level)) - 1);
2336 }
2337 
2338 static bool amd_iommu_hd_support(struct amd_iommu *iommu)
2339 {
2340 	return iommu && (iommu->features & FEATURE_HDSUP);
2341 }
2342 
2343 static struct iommu_domain *do_iommu_domain_alloc(unsigned int type,
2344 						  struct device *dev,
2345 						  u32 flags, int pgtable)
2346 {
2347 	bool dirty_tracking = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
2348 	struct protection_domain *domain;
2349 	struct amd_iommu *iommu = NULL;
2350 	int ret;
2351 
2352 	if (dev)
2353 		iommu = get_amd_iommu_from_dev(dev);
2354 
2355 	/*
2356 	 * Since DTE[Mode]=0 is prohibited on SNP-enabled system,
2357 	 * default to use IOMMU_DOMAIN_DMA[_FQ].
2358 	 */
2359 	if (amd_iommu_snp_en && (type == IOMMU_DOMAIN_IDENTITY))
2360 		return ERR_PTR(-EINVAL);
2361 
2362 	domain = protection_domain_alloc(type,
2363 					 dev ? dev_to_node(dev) : NUMA_NO_NODE);
2364 	if (!domain)
2365 		return ERR_PTR(-ENOMEM);
2366 
2367 	ret = pdom_setup_pgtable(domain, type, pgtable);
2368 	if (ret) {
2369 		pdom_id_free(domain->id);
2370 		kfree(domain);
2371 		return ERR_PTR(ret);
2372 	}
2373 
2374 	domain->domain.geometry.aperture_start = 0;
2375 	domain->domain.geometry.aperture_end   = dma_max_address(pgtable);
2376 	domain->domain.geometry.force_aperture = true;
2377 	domain->domain.pgsize_bitmap = domain->iop.pgtbl.cfg.pgsize_bitmap;
2378 
2379 	if (iommu) {
2380 		domain->domain.type = type;
2381 		domain->domain.ops = iommu->iommu.ops->default_domain_ops;
2382 
2383 		if (dirty_tracking)
2384 			domain->domain.dirty_ops = &amd_dirty_ops;
2385 	}
2386 
2387 	return &domain->domain;
2388 }
2389 
2390 static struct iommu_domain *amd_iommu_domain_alloc(unsigned int type)
2391 {
2392 	struct iommu_domain *domain;
2393 	int pgtable = amd_iommu_pgtable;
2394 
2395 	/*
2396 	 * Force IOMMU v1 page table when allocating
2397 	 * domain for pass-through devices.
2398 	 */
2399 	if (type == IOMMU_DOMAIN_UNMANAGED)
2400 		pgtable = AMD_IOMMU_V1;
2401 
2402 	domain = do_iommu_domain_alloc(type, NULL, 0, pgtable);
2403 	if (IS_ERR(domain))
2404 		return NULL;
2405 
2406 	return domain;
2407 }
2408 
2409 static struct iommu_domain *
2410 amd_iommu_domain_alloc_user(struct device *dev, u32 flags,
2411 			    struct iommu_domain *parent,
2412 			    const struct iommu_user_data *user_data)
2413 
2414 {
2415 	unsigned int type = IOMMU_DOMAIN_UNMANAGED;
2416 	struct amd_iommu *iommu = NULL;
2417 	const u32 supported_flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
2418 						IOMMU_HWPT_ALLOC_PASID;
2419 
2420 	if (dev)
2421 		iommu = get_amd_iommu_from_dev(dev);
2422 
2423 	if ((flags & ~supported_flags) || parent || user_data)
2424 		return ERR_PTR(-EOPNOTSUPP);
2425 
2426 	/* Allocate domain with v2 page table if IOMMU supports PASID. */
2427 	if (flags & IOMMU_HWPT_ALLOC_PASID) {
2428 		if (!amd_iommu_pasid_supported())
2429 			return ERR_PTR(-EOPNOTSUPP);
2430 
2431 		return do_iommu_domain_alloc(type, dev, flags, AMD_IOMMU_V2);
2432 	}
2433 
2434 	/* Allocate domain with v1 page table for dirty tracking */
2435 	if (flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) {
2436 		if (iommu && amd_iommu_hd_support(iommu)) {
2437 			return do_iommu_domain_alloc(type, dev,
2438 						     flags, AMD_IOMMU_V1);
2439 		}
2440 
2441 		return ERR_PTR(-EOPNOTSUPP);
2442 	}
2443 
2444 	/* If nothing specific is required use the kernel commandline default */
2445 	return do_iommu_domain_alloc(type, dev, 0, amd_iommu_pgtable);
2446 }
2447 
2448 void amd_iommu_domain_free(struct iommu_domain *dom)
2449 {
2450 	struct protection_domain *domain = to_pdomain(dom);
2451 
2452 	protection_domain_free(domain);
2453 }
2454 
2455 static int blocked_domain_attach_device(struct iommu_domain *domain,
2456 					struct device *dev)
2457 {
2458 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2459 
2460 	if (dev_data->domain)
2461 		detach_device(dev);
2462 
2463 	/* Clear DTE and flush the entry */
2464 	mutex_lock(&dev_data->mutex);
2465 	dev_update_dte(dev_data, false);
2466 	mutex_unlock(&dev_data->mutex);
2467 
2468 	return 0;
2469 }
2470 
2471 static struct iommu_domain blocked_domain = {
2472 	.type = IOMMU_DOMAIN_BLOCKED,
2473 	.ops = &(const struct iommu_domain_ops) {
2474 		.attach_dev     = blocked_domain_attach_device,
2475 	}
2476 };
2477 
2478 static struct protection_domain identity_domain;
2479 
2480 static const struct iommu_domain_ops identity_domain_ops = {
2481 	.attach_dev = amd_iommu_attach_device,
2482 };
2483 
2484 void amd_iommu_init_identity_domain(void)
2485 {
2486 	struct iommu_domain *domain = &identity_domain.domain;
2487 
2488 	domain->type = IOMMU_DOMAIN_IDENTITY;
2489 	domain->ops = &identity_domain_ops;
2490 	domain->owner = &amd_iommu_ops;
2491 
2492 	identity_domain.id = pdom_id_alloc();
2493 
2494 	protection_domain_init(&identity_domain, NUMA_NO_NODE);
2495 }
2496 
2497 /* Same as blocked domain except it supports only ops->attach_dev() */
2498 static struct iommu_domain release_domain = {
2499 	.type = IOMMU_DOMAIN_BLOCKED,
2500 	.ops = &(const struct iommu_domain_ops) {
2501 		.attach_dev     = blocked_domain_attach_device,
2502 	}
2503 };
2504 
2505 static int amd_iommu_attach_device(struct iommu_domain *dom,
2506 				   struct device *dev)
2507 {
2508 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2509 	struct protection_domain *domain = to_pdomain(dom);
2510 	struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);
2511 	int ret;
2512 
2513 	/*
2514 	 * Skip attach device to domain if new domain is same as
2515 	 * devices current domain
2516 	 */
2517 	if (dev_data->domain == domain)
2518 		return 0;
2519 
2520 	dev_data->defer_attach = false;
2521 
2522 	/*
2523 	 * Restrict to devices with compatible IOMMU hardware support
2524 	 * when enforcement of dirty tracking is enabled.
2525 	 */
2526 	if (dom->dirty_ops && !amd_iommu_hd_support(iommu))
2527 		return -EINVAL;
2528 
2529 	if (dev_data->domain)
2530 		detach_device(dev);
2531 
2532 	ret = attach_device(dev, domain);
2533 
2534 #ifdef CONFIG_IRQ_REMAP
2535 	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) {
2536 		if (dom->type == IOMMU_DOMAIN_UNMANAGED)
2537 			dev_data->use_vapic = 1;
2538 		else
2539 			dev_data->use_vapic = 0;
2540 	}
2541 #endif
2542 
2543 	return ret;
2544 }
2545 
2546 static int amd_iommu_iotlb_sync_map(struct iommu_domain *dom,
2547 				    unsigned long iova, size_t size)
2548 {
2549 	struct protection_domain *domain = to_pdomain(dom);
2550 	struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops;
2551 
2552 	if (ops->map_pages)
2553 		domain_flush_np_cache(domain, iova, size);
2554 	return 0;
2555 }
2556 
2557 static int amd_iommu_map_pages(struct iommu_domain *dom, unsigned long iova,
2558 			       phys_addr_t paddr, size_t pgsize, size_t pgcount,
2559 			       int iommu_prot, gfp_t gfp, size_t *mapped)
2560 {
2561 	struct protection_domain *domain = to_pdomain(dom);
2562 	struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops;
2563 	int prot = 0;
2564 	int ret = -EINVAL;
2565 
2566 	if ((domain->pd_mode == PD_MODE_V1) &&
2567 	    (domain->iop.mode == PAGE_MODE_NONE))
2568 		return -EINVAL;
2569 
2570 	if (iommu_prot & IOMMU_READ)
2571 		prot |= IOMMU_PROT_IR;
2572 	if (iommu_prot & IOMMU_WRITE)
2573 		prot |= IOMMU_PROT_IW;
2574 
2575 	if (ops->map_pages) {
2576 		ret = ops->map_pages(ops, iova, paddr, pgsize,
2577 				     pgcount, prot, gfp, mapped);
2578 	}
2579 
2580 	return ret;
2581 }
2582 
2583 static void amd_iommu_iotlb_gather_add_page(struct iommu_domain *domain,
2584 					    struct iommu_iotlb_gather *gather,
2585 					    unsigned long iova, size_t size)
2586 {
2587 	/*
2588 	 * AMD's IOMMU can flush as many pages as necessary in a single flush.
2589 	 * Unless we run in a virtual machine, which can be inferred according
2590 	 * to whether "non-present cache" is on, it is probably best to prefer
2591 	 * (potentially) too extensive TLB flushing (i.e., more misses) over
2592 	 * mutliple TLB flushes (i.e., more flushes). For virtual machines the
2593 	 * hypervisor needs to synchronize the host IOMMU PTEs with those of
2594 	 * the guest, and the trade-off is different: unnecessary TLB flushes
2595 	 * should be avoided.
2596 	 */
2597 	if (amd_iommu_np_cache &&
2598 	    iommu_iotlb_gather_is_disjoint(gather, iova, size))
2599 		iommu_iotlb_sync(domain, gather);
2600 
2601 	iommu_iotlb_gather_add_range(gather, iova, size);
2602 }
2603 
2604 static size_t amd_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova,
2605 				    size_t pgsize, size_t pgcount,
2606 				    struct iommu_iotlb_gather *gather)
2607 {
2608 	struct protection_domain *domain = to_pdomain(dom);
2609 	struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops;
2610 	size_t r;
2611 
2612 	if ((domain->pd_mode == PD_MODE_V1) &&
2613 	    (domain->iop.mode == PAGE_MODE_NONE))
2614 		return 0;
2615 
2616 	r = (ops->unmap_pages) ? ops->unmap_pages(ops, iova, pgsize, pgcount, NULL) : 0;
2617 
2618 	if (r)
2619 		amd_iommu_iotlb_gather_add_page(dom, gather, iova, r);
2620 
2621 	return r;
2622 }
2623 
2624 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
2625 					  dma_addr_t iova)
2626 {
2627 	struct protection_domain *domain = to_pdomain(dom);
2628 	struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops;
2629 
2630 	return ops->iova_to_phys(ops, iova);
2631 }
2632 
2633 static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap)
2634 {
2635 	switch (cap) {
2636 	case IOMMU_CAP_CACHE_COHERENCY:
2637 		return true;
2638 	case IOMMU_CAP_NOEXEC:
2639 		return false;
2640 	case IOMMU_CAP_PRE_BOOT_PROTECTION:
2641 		return amdr_ivrs_remap_support;
2642 	case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
2643 		return true;
2644 	case IOMMU_CAP_DEFERRED_FLUSH:
2645 		return true;
2646 	case IOMMU_CAP_DIRTY_TRACKING: {
2647 		struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);
2648 
2649 		return amd_iommu_hd_support(iommu);
2650 	}
2651 	default:
2652 		break;
2653 	}
2654 
2655 	return false;
2656 }
2657 
2658 static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain,
2659 					bool enable)
2660 {
2661 	struct protection_domain *pdomain = to_pdomain(domain);
2662 	struct dev_table_entry *dev_table;
2663 	struct iommu_dev_data *dev_data;
2664 	bool domain_flush = false;
2665 	struct amd_iommu *iommu;
2666 	unsigned long flags;
2667 	u64 pte_root;
2668 
2669 	spin_lock_irqsave(&pdomain->lock, flags);
2670 	if (!(pdomain->dirty_tracking ^ enable)) {
2671 		spin_unlock_irqrestore(&pdomain->lock, flags);
2672 		return 0;
2673 	}
2674 
2675 	list_for_each_entry(dev_data, &pdomain->dev_list, list) {
2676 		iommu = get_amd_iommu_from_dev_data(dev_data);
2677 
2678 		dev_table = get_dev_table(iommu);
2679 		pte_root = dev_table[dev_data->devid].data[0];
2680 
2681 		pte_root = (enable ? pte_root | DTE_FLAG_HAD :
2682 				     pte_root & ~DTE_FLAG_HAD);
2683 
2684 		/* Flush device DTE */
2685 		dev_table[dev_data->devid].data[0] = pte_root;
2686 		device_flush_dte(dev_data);
2687 		domain_flush = true;
2688 	}
2689 
2690 	/* Flush IOTLB to mark IOPTE dirty on the next translation(s) */
2691 	if (domain_flush)
2692 		amd_iommu_domain_flush_all(pdomain);
2693 
2694 	pdomain->dirty_tracking = enable;
2695 	spin_unlock_irqrestore(&pdomain->lock, flags);
2696 
2697 	return 0;
2698 }
2699 
2700 static int amd_iommu_read_and_clear_dirty(struct iommu_domain *domain,
2701 					  unsigned long iova, size_t size,
2702 					  unsigned long flags,
2703 					  struct iommu_dirty_bitmap *dirty)
2704 {
2705 	struct protection_domain *pdomain = to_pdomain(domain);
2706 	struct io_pgtable_ops *ops = &pdomain->iop.pgtbl.ops;
2707 	unsigned long lflags;
2708 
2709 	if (!ops || !ops->read_and_clear_dirty)
2710 		return -EOPNOTSUPP;
2711 
2712 	spin_lock_irqsave(&pdomain->lock, lflags);
2713 	if (!pdomain->dirty_tracking && dirty->bitmap) {
2714 		spin_unlock_irqrestore(&pdomain->lock, lflags);
2715 		return -EINVAL;
2716 	}
2717 	spin_unlock_irqrestore(&pdomain->lock, lflags);
2718 
2719 	return ops->read_and_clear_dirty(ops, iova, size, flags, dirty);
2720 }
2721 
2722 static void amd_iommu_get_resv_regions(struct device *dev,
2723 				       struct list_head *head)
2724 {
2725 	struct iommu_resv_region *region;
2726 	struct unity_map_entry *entry;
2727 	struct amd_iommu *iommu;
2728 	struct amd_iommu_pci_seg *pci_seg;
2729 	int devid, sbdf;
2730 
2731 	sbdf = get_device_sbdf_id(dev);
2732 	if (sbdf < 0)
2733 		return;
2734 
2735 	devid = PCI_SBDF_TO_DEVID(sbdf);
2736 	iommu = get_amd_iommu_from_dev(dev);
2737 	pci_seg = iommu->pci_seg;
2738 
2739 	list_for_each_entry(entry, &pci_seg->unity_map, list) {
2740 		int type, prot = 0;
2741 		size_t length;
2742 
2743 		if (devid < entry->devid_start || devid > entry->devid_end)
2744 			continue;
2745 
2746 		type   = IOMMU_RESV_DIRECT;
2747 		length = entry->address_end - entry->address_start;
2748 		if (entry->prot & IOMMU_PROT_IR)
2749 			prot |= IOMMU_READ;
2750 		if (entry->prot & IOMMU_PROT_IW)
2751 			prot |= IOMMU_WRITE;
2752 		if (entry->prot & IOMMU_UNITY_MAP_FLAG_EXCL_RANGE)
2753 			/* Exclusion range */
2754 			type = IOMMU_RESV_RESERVED;
2755 
2756 		region = iommu_alloc_resv_region(entry->address_start,
2757 						 length, prot, type,
2758 						 GFP_KERNEL);
2759 		if (!region) {
2760 			dev_err(dev, "Out of memory allocating dm-regions\n");
2761 			return;
2762 		}
2763 		list_add_tail(&region->list, head);
2764 	}
2765 
2766 	region = iommu_alloc_resv_region(MSI_RANGE_START,
2767 					 MSI_RANGE_END - MSI_RANGE_START + 1,
2768 					 0, IOMMU_RESV_MSI, GFP_KERNEL);
2769 	if (!region)
2770 		return;
2771 	list_add_tail(&region->list, head);
2772 
2773 	region = iommu_alloc_resv_region(HT_RANGE_START,
2774 					 HT_RANGE_END - HT_RANGE_START + 1,
2775 					 0, IOMMU_RESV_RESERVED, GFP_KERNEL);
2776 	if (!region)
2777 		return;
2778 	list_add_tail(&region->list, head);
2779 }
2780 
2781 static bool amd_iommu_is_attach_deferred(struct device *dev)
2782 {
2783 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2784 
2785 	return dev_data->defer_attach;
2786 }
2787 
2788 static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
2789 {
2790 	struct protection_domain *dom = to_pdomain(domain);
2791 	unsigned long flags;
2792 
2793 	spin_lock_irqsave(&dom->lock, flags);
2794 	amd_iommu_domain_flush_all(dom);
2795 	spin_unlock_irqrestore(&dom->lock, flags);
2796 }
2797 
2798 static void amd_iommu_iotlb_sync(struct iommu_domain *domain,
2799 				 struct iommu_iotlb_gather *gather)
2800 {
2801 	struct protection_domain *dom = to_pdomain(domain);
2802 	unsigned long flags;
2803 
2804 	spin_lock_irqsave(&dom->lock, flags);
2805 	amd_iommu_domain_flush_pages(dom, gather->start,
2806 				     gather->end - gather->start + 1);
2807 	spin_unlock_irqrestore(&dom->lock, flags);
2808 }
2809 
2810 static int amd_iommu_def_domain_type(struct device *dev)
2811 {
2812 	struct iommu_dev_data *dev_data;
2813 
2814 	dev_data = dev_iommu_priv_get(dev);
2815 	if (!dev_data)
2816 		return 0;
2817 
2818 	/* Always use DMA domain for untrusted device */
2819 	if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted)
2820 		return IOMMU_DOMAIN_DMA;
2821 
2822 	/*
2823 	 * Do not identity map IOMMUv2 capable devices when:
2824 	 *  - memory encryption is active, because some of those devices
2825 	 *    (AMD GPUs) don't have the encryption bit in their DMA-mask
2826 	 *    and require remapping.
2827 	 *  - SNP is enabled, because it prohibits DTE[Mode]=0.
2828 	 */
2829 	if (pdev_pasid_supported(dev_data) &&
2830 	    !cc_platform_has(CC_ATTR_MEM_ENCRYPT) &&
2831 	    !amd_iommu_snp_en) {
2832 		return IOMMU_DOMAIN_IDENTITY;
2833 	}
2834 
2835 	return 0;
2836 }
2837 
2838 static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain)
2839 {
2840 	/* IOMMU_PTE_FC is always set */
2841 	return true;
2842 }
2843 
2844 static const struct iommu_dirty_ops amd_dirty_ops = {
2845 	.set_dirty_tracking = amd_iommu_set_dirty_tracking,
2846 	.read_and_clear_dirty = amd_iommu_read_and_clear_dirty,
2847 };
2848 
2849 static int amd_iommu_dev_enable_feature(struct device *dev,
2850 					enum iommu_dev_features feat)
2851 {
2852 	int ret = 0;
2853 
2854 	switch (feat) {
2855 	case IOMMU_DEV_FEAT_IOPF:
2856 	case IOMMU_DEV_FEAT_SVA:
2857 		break;
2858 	default:
2859 		ret = -EINVAL;
2860 		break;
2861 	}
2862 	return ret;
2863 }
2864 
2865 static int amd_iommu_dev_disable_feature(struct device *dev,
2866 					 enum iommu_dev_features feat)
2867 {
2868 	int ret = 0;
2869 
2870 	switch (feat) {
2871 	case IOMMU_DEV_FEAT_IOPF:
2872 	case IOMMU_DEV_FEAT_SVA:
2873 		break;
2874 	default:
2875 		ret = -EINVAL;
2876 		break;
2877 	}
2878 	return ret;
2879 }
2880 
2881 const struct iommu_ops amd_iommu_ops = {
2882 	.capable = amd_iommu_capable,
2883 	.blocked_domain = &blocked_domain,
2884 	.release_domain = &release_domain,
2885 	.identity_domain = &identity_domain.domain,
2886 	.domain_alloc = amd_iommu_domain_alloc,
2887 	.domain_alloc_user = amd_iommu_domain_alloc_user,
2888 	.domain_alloc_sva = amd_iommu_domain_alloc_sva,
2889 	.probe_device = amd_iommu_probe_device,
2890 	.release_device = amd_iommu_release_device,
2891 	.device_group = amd_iommu_device_group,
2892 	.get_resv_regions = amd_iommu_get_resv_regions,
2893 	.is_attach_deferred = amd_iommu_is_attach_deferred,
2894 	.def_domain_type = amd_iommu_def_domain_type,
2895 	.dev_enable_feat = amd_iommu_dev_enable_feature,
2896 	.dev_disable_feat = amd_iommu_dev_disable_feature,
2897 	.remove_dev_pasid = amd_iommu_remove_dev_pasid,
2898 	.page_response = amd_iommu_page_response,
2899 	.default_domain_ops = &(const struct iommu_domain_ops) {
2900 		.attach_dev	= amd_iommu_attach_device,
2901 		.map_pages	= amd_iommu_map_pages,
2902 		.unmap_pages	= amd_iommu_unmap_pages,
2903 		.iotlb_sync_map	= amd_iommu_iotlb_sync_map,
2904 		.iova_to_phys	= amd_iommu_iova_to_phys,
2905 		.flush_iotlb_all = amd_iommu_flush_iotlb_all,
2906 		.iotlb_sync	= amd_iommu_iotlb_sync,
2907 		.free		= amd_iommu_domain_free,
2908 		.enforce_cache_coherency = amd_iommu_enforce_cache_coherency,
2909 	}
2910 };
2911 
2912 #ifdef CONFIG_IRQ_REMAP
2913 
2914 /*****************************************************************************
2915  *
2916  * Interrupt Remapping Implementation
2917  *
2918  *****************************************************************************/
2919 
2920 static struct irq_chip amd_ir_chip;
2921 static DEFINE_SPINLOCK(iommu_table_lock);
2922 
2923 static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)
2924 {
2925 	int ret;
2926 	u64 data;
2927 	unsigned long flags;
2928 	struct iommu_cmd cmd, cmd2;
2929 
2930 	if (iommu->irtcachedis_enabled)
2931 		return;
2932 
2933 	build_inv_irt(&cmd, devid);
2934 	data = atomic64_inc_return(&iommu->cmd_sem_val);
2935 	build_completion_wait(&cmd2, iommu, data);
2936 
2937 	raw_spin_lock_irqsave(&iommu->lock, flags);
2938 	ret = __iommu_queue_command_sync(iommu, &cmd, true);
2939 	if (ret)
2940 		goto out;
2941 	ret = __iommu_queue_command_sync(iommu, &cmd2, false);
2942 	if (ret)
2943 		goto out;
2944 	wait_on_sem(iommu, data);
2945 out:
2946 	raw_spin_unlock_irqrestore(&iommu->lock, flags);
2947 }
2948 
2949 static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid,
2950 			      struct irq_remap_table *table)
2951 {
2952 	u64 dte;
2953 	struct dev_table_entry *dev_table = get_dev_table(iommu);
2954 
2955 	dte	= dev_table[devid].data[2];
2956 	dte	&= ~DTE_IRQ_PHYS_ADDR_MASK;
2957 	dte	|= iommu_virt_to_phys(table->table);
2958 	dte	|= DTE_IRQ_REMAP_INTCTL;
2959 	dte	|= DTE_INTTABLEN;
2960 	dte	|= DTE_IRQ_REMAP_ENABLE;
2961 
2962 	dev_table[devid].data[2] = dte;
2963 }
2964 
2965 static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid)
2966 {
2967 	struct irq_remap_table *table;
2968 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
2969 
2970 	if (WARN_ONCE(!pci_seg->rlookup_table[devid],
2971 		      "%s: no iommu for devid %x:%x\n",
2972 		      __func__, pci_seg->id, devid))
2973 		return NULL;
2974 
2975 	table = pci_seg->irq_lookup_table[devid];
2976 	if (WARN_ONCE(!table, "%s: no table for devid %x:%x\n",
2977 		      __func__, pci_seg->id, devid))
2978 		return NULL;
2979 
2980 	return table;
2981 }
2982 
2983 static struct irq_remap_table *__alloc_irq_table(void)
2984 {
2985 	struct irq_remap_table *table;
2986 
2987 	table = kzalloc(sizeof(*table), GFP_KERNEL);
2988 	if (!table)
2989 		return NULL;
2990 
2991 	table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_KERNEL);
2992 	if (!table->table) {
2993 		kfree(table);
2994 		return NULL;
2995 	}
2996 	raw_spin_lock_init(&table->lock);
2997 
2998 	if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
2999 		memset(table->table, 0,
3000 		       MAX_IRQS_PER_TABLE * sizeof(u32));
3001 	else
3002 		memset(table->table, 0,
3003 		       (MAX_IRQS_PER_TABLE * (sizeof(u64) * 2)));
3004 	return table;
3005 }
3006 
3007 static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid,
3008 				  struct irq_remap_table *table)
3009 {
3010 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
3011 
3012 	pci_seg->irq_lookup_table[devid] = table;
3013 	set_dte_irq_entry(iommu, devid, table);
3014 	iommu_flush_dte(iommu, devid);
3015 }
3016 
3017 static int set_remap_table_entry_alias(struct pci_dev *pdev, u16 alias,
3018 				       void *data)
3019 {
3020 	struct irq_remap_table *table = data;
3021 	struct amd_iommu_pci_seg *pci_seg;
3022 	struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);
3023 
3024 	if (!iommu)
3025 		return -EINVAL;
3026 
3027 	pci_seg = iommu->pci_seg;
3028 	pci_seg->irq_lookup_table[alias] = table;
3029 	set_dte_irq_entry(iommu, alias, table);
3030 	iommu_flush_dte(pci_seg->rlookup_table[alias], alias);
3031 
3032 	return 0;
3033 }
3034 
3035 static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu,
3036 					       u16 devid, struct pci_dev *pdev)
3037 {
3038 	struct irq_remap_table *table = NULL;
3039 	struct irq_remap_table *new_table = NULL;
3040 	struct amd_iommu_pci_seg *pci_seg;
3041 	unsigned long flags;
3042 	u16 alias;
3043 
3044 	spin_lock_irqsave(&iommu_table_lock, flags);
3045 
3046 	pci_seg = iommu->pci_seg;
3047 	table = pci_seg->irq_lookup_table[devid];
3048 	if (table)
3049 		goto out_unlock;
3050 
3051 	alias = pci_seg->alias_table[devid];
3052 	table = pci_seg->irq_lookup_table[alias];
3053 	if (table) {
3054 		set_remap_table_entry(iommu, devid, table);
3055 		goto out_wait;
3056 	}
3057 	spin_unlock_irqrestore(&iommu_table_lock, flags);
3058 
3059 	/* Nothing there yet, allocate new irq remapping table */
3060 	new_table = __alloc_irq_table();
3061 	if (!new_table)
3062 		return NULL;
3063 
3064 	spin_lock_irqsave(&iommu_table_lock, flags);
3065 
3066 	table = pci_seg->irq_lookup_table[devid];
3067 	if (table)
3068 		goto out_unlock;
3069 
3070 	table = pci_seg->irq_lookup_table[alias];
3071 	if (table) {
3072 		set_remap_table_entry(iommu, devid, table);
3073 		goto out_wait;
3074 	}
3075 
3076 	table = new_table;
3077 	new_table = NULL;
3078 
3079 	if (pdev)
3080 		pci_for_each_dma_alias(pdev, set_remap_table_entry_alias,
3081 				       table);
3082 	else
3083 		set_remap_table_entry(iommu, devid, table);
3084 
3085 	if (devid != alias)
3086 		set_remap_table_entry(iommu, alias, table);
3087 
3088 out_wait:
3089 	iommu_completion_wait(iommu);
3090 
3091 out_unlock:
3092 	spin_unlock_irqrestore(&iommu_table_lock, flags);
3093 
3094 	if (new_table) {
3095 		kmem_cache_free(amd_iommu_irq_cache, new_table->table);
3096 		kfree(new_table);
3097 	}
3098 	return table;
3099 }
3100 
3101 static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count,
3102 			   bool align, struct pci_dev *pdev)
3103 {
3104 	struct irq_remap_table *table;
3105 	int index, c, alignment = 1;
3106 	unsigned long flags;
3107 
3108 	table = alloc_irq_table(iommu, devid, pdev);
3109 	if (!table)
3110 		return -ENODEV;
3111 
3112 	if (align)
3113 		alignment = roundup_pow_of_two(count);
3114 
3115 	raw_spin_lock_irqsave(&table->lock, flags);
3116 
3117 	/* Scan table for free entries */
3118 	for (index = ALIGN(table->min_index, alignment), c = 0;
3119 	     index < MAX_IRQS_PER_TABLE;) {
3120 		if (!iommu->irte_ops->is_allocated(table, index)) {
3121 			c += 1;
3122 		} else {
3123 			c     = 0;
3124 			index = ALIGN(index + 1, alignment);
3125 			continue;
3126 		}
3127 
3128 		if (c == count)	{
3129 			for (; c != 0; --c)
3130 				iommu->irte_ops->set_allocated(table, index - c + 1);
3131 
3132 			index -= count - 1;
3133 			goto out;
3134 		}
3135 
3136 		index++;
3137 	}
3138 
3139 	index = -ENOSPC;
3140 
3141 out:
3142 	raw_spin_unlock_irqrestore(&table->lock, flags);
3143 
3144 	return index;
3145 }
3146 
3147 static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
3148 			    struct irte_ga *irte)
3149 {
3150 	struct irq_remap_table *table;
3151 	struct irte_ga *entry;
3152 	unsigned long flags;
3153 	u128 old;
3154 
3155 	table = get_irq_table(iommu, devid);
3156 	if (!table)
3157 		return -ENOMEM;
3158 
3159 	raw_spin_lock_irqsave(&table->lock, flags);
3160 
3161 	entry = (struct irte_ga *)table->table;
3162 	entry = &entry[index];
3163 
3164 	/*
3165 	 * We use cmpxchg16 to atomically update the 128-bit IRTE,
3166 	 * and it cannot be updated by the hardware or other processors
3167 	 * behind us, so the return value of cmpxchg16 should be the
3168 	 * same as the old value.
3169 	 */
3170 	old = entry->irte;
3171 	WARN_ON(!try_cmpxchg128(&entry->irte, &old, irte->irte));
3172 
3173 	raw_spin_unlock_irqrestore(&table->lock, flags);
3174 
3175 	return 0;
3176 }
3177 
3178 static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
3179 			  struct irte_ga *irte)
3180 {
3181 	bool ret;
3182 
3183 	ret = __modify_irte_ga(iommu, devid, index, irte);
3184 	if (ret)
3185 		return ret;
3186 
3187 	iommu_flush_irt_and_complete(iommu, devid);
3188 
3189 	return 0;
3190 }
3191 
3192 static int modify_irte(struct amd_iommu *iommu,
3193 		       u16 devid, int index, union irte *irte)
3194 {
3195 	struct irq_remap_table *table;
3196 	unsigned long flags;
3197 
3198 	table = get_irq_table(iommu, devid);
3199 	if (!table)
3200 		return -ENOMEM;
3201 
3202 	raw_spin_lock_irqsave(&table->lock, flags);
3203 	table->table[index] = irte->val;
3204 	raw_spin_unlock_irqrestore(&table->lock, flags);
3205 
3206 	iommu_flush_irt_and_complete(iommu, devid);
3207 
3208 	return 0;
3209 }
3210 
3211 static void free_irte(struct amd_iommu *iommu, u16 devid, int index)
3212 {
3213 	struct irq_remap_table *table;
3214 	unsigned long flags;
3215 
3216 	table = get_irq_table(iommu, devid);
3217 	if (!table)
3218 		return;
3219 
3220 	raw_spin_lock_irqsave(&table->lock, flags);
3221 	iommu->irte_ops->clear_allocated(table, index);
3222 	raw_spin_unlock_irqrestore(&table->lock, flags);
3223 
3224 	iommu_flush_irt_and_complete(iommu, devid);
3225 }
3226 
3227 static void irte_prepare(void *entry,
3228 			 u32 delivery_mode, bool dest_mode,
3229 			 u8 vector, u32 dest_apicid, int devid)
3230 {
3231 	union irte *irte = (union irte *) entry;
3232 
3233 	irte->val                = 0;
3234 	irte->fields.vector      = vector;
3235 	irte->fields.int_type    = delivery_mode;
3236 	irte->fields.destination = dest_apicid;
3237 	irte->fields.dm          = dest_mode;
3238 	irte->fields.valid       = 1;
3239 }
3240 
3241 static void irte_ga_prepare(void *entry,
3242 			    u32 delivery_mode, bool dest_mode,
3243 			    u8 vector, u32 dest_apicid, int devid)
3244 {
3245 	struct irte_ga *irte = (struct irte_ga *) entry;
3246 
3247 	irte->lo.val                      = 0;
3248 	irte->hi.val                      = 0;
3249 	irte->lo.fields_remap.int_type    = delivery_mode;
3250 	irte->lo.fields_remap.dm          = dest_mode;
3251 	irte->hi.fields.vector            = vector;
3252 	irte->lo.fields_remap.destination = APICID_TO_IRTE_DEST_LO(dest_apicid);
3253 	irte->hi.fields.destination       = APICID_TO_IRTE_DEST_HI(dest_apicid);
3254 	irte->lo.fields_remap.valid       = 1;
3255 }
3256 
3257 static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
3258 {
3259 	union irte *irte = (union irte *) entry;
3260 
3261 	irte->fields.valid = 1;
3262 	modify_irte(iommu, devid, index, irte);
3263 }
3264 
3265 static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
3266 {
3267 	struct irte_ga *irte = (struct irte_ga *) entry;
3268 
3269 	irte->lo.fields_remap.valid = 1;
3270 	modify_irte_ga(iommu, devid, index, irte);
3271 }
3272 
3273 static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
3274 {
3275 	union irte *irte = (union irte *) entry;
3276 
3277 	irte->fields.valid = 0;
3278 	modify_irte(iommu, devid, index, irte);
3279 }
3280 
3281 static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
3282 {
3283 	struct irte_ga *irte = (struct irte_ga *) entry;
3284 
3285 	irte->lo.fields_remap.valid = 0;
3286 	modify_irte_ga(iommu, devid, index, irte);
3287 }
3288 
3289 static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,
3290 			      u8 vector, u32 dest_apicid)
3291 {
3292 	union irte *irte = (union irte *) entry;
3293 
3294 	irte->fields.vector = vector;
3295 	irte->fields.destination = dest_apicid;
3296 	modify_irte(iommu, devid, index, irte);
3297 }
3298 
3299 static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,
3300 				 u8 vector, u32 dest_apicid)
3301 {
3302 	struct irte_ga *irte = (struct irte_ga *) entry;
3303 
3304 	if (!irte->lo.fields_remap.guest_mode) {
3305 		irte->hi.fields.vector = vector;
3306 		irte->lo.fields_remap.destination =
3307 					APICID_TO_IRTE_DEST_LO(dest_apicid);
3308 		irte->hi.fields.destination =
3309 					APICID_TO_IRTE_DEST_HI(dest_apicid);
3310 		modify_irte_ga(iommu, devid, index, irte);
3311 	}
3312 }
3313 
3314 #define IRTE_ALLOCATED (~1U)
3315 static void irte_set_allocated(struct irq_remap_table *table, int index)
3316 {
3317 	table->table[index] = IRTE_ALLOCATED;
3318 }
3319 
3320 static void irte_ga_set_allocated(struct irq_remap_table *table, int index)
3321 {
3322 	struct irte_ga *ptr = (struct irte_ga *)table->table;
3323 	struct irte_ga *irte = &ptr[index];
3324 
3325 	memset(&irte->lo.val, 0, sizeof(u64));
3326 	memset(&irte->hi.val, 0, sizeof(u64));
3327 	irte->hi.fields.vector = 0xff;
3328 }
3329 
3330 static bool irte_is_allocated(struct irq_remap_table *table, int index)
3331 {
3332 	union irte *ptr = (union irte *)table->table;
3333 	union irte *irte = &ptr[index];
3334 
3335 	return irte->val != 0;
3336 }
3337 
3338 static bool irte_ga_is_allocated(struct irq_remap_table *table, int index)
3339 {
3340 	struct irte_ga *ptr = (struct irte_ga *)table->table;
3341 	struct irte_ga *irte = &ptr[index];
3342 
3343 	return irte->hi.fields.vector != 0;
3344 }
3345 
3346 static void irte_clear_allocated(struct irq_remap_table *table, int index)
3347 {
3348 	table->table[index] = 0;
3349 }
3350 
3351 static void irte_ga_clear_allocated(struct irq_remap_table *table, int index)
3352 {
3353 	struct irte_ga *ptr = (struct irte_ga *)table->table;
3354 	struct irte_ga *irte = &ptr[index];
3355 
3356 	memset(&irte->lo.val, 0, sizeof(u64));
3357 	memset(&irte->hi.val, 0, sizeof(u64));
3358 }
3359 
3360 static int get_devid(struct irq_alloc_info *info)
3361 {
3362 	switch (info->type) {
3363 	case X86_IRQ_ALLOC_TYPE_IOAPIC:
3364 		return get_ioapic_devid(info->devid);
3365 	case X86_IRQ_ALLOC_TYPE_HPET:
3366 		return get_hpet_devid(info->devid);
3367 	case X86_IRQ_ALLOC_TYPE_PCI_MSI:
3368 	case X86_IRQ_ALLOC_TYPE_PCI_MSIX:
3369 		return get_device_sbdf_id(msi_desc_to_dev(info->desc));
3370 	default:
3371 		WARN_ON_ONCE(1);
3372 		return -1;
3373 	}
3374 }
3375 
3376 struct irq_remap_ops amd_iommu_irq_ops = {
3377 	.prepare		= amd_iommu_prepare,
3378 	.enable			= amd_iommu_enable,
3379 	.disable		= amd_iommu_disable,
3380 	.reenable		= amd_iommu_reenable,
3381 	.enable_faulting	= amd_iommu_enable_faulting,
3382 };
3383 
3384 static void fill_msi_msg(struct msi_msg *msg, u32 index)
3385 {
3386 	msg->data = index;
3387 	msg->address_lo = 0;
3388 	msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;
3389 	msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;
3390 }
3391 
3392 static void irq_remapping_prepare_irte(struct amd_ir_data *data,
3393 				       struct irq_cfg *irq_cfg,
3394 				       struct irq_alloc_info *info,
3395 				       int devid, int index, int sub_handle)
3396 {
3397 	struct irq_2_irte *irte_info = &data->irq_2_irte;
3398 	struct amd_iommu *iommu = data->iommu;
3399 
3400 	if (!iommu)
3401 		return;
3402 
3403 	data->irq_2_irte.devid = devid;
3404 	data->irq_2_irte.index = index + sub_handle;
3405 	iommu->irte_ops->prepare(data->entry, APIC_DELIVERY_MODE_FIXED,
3406 				 apic->dest_mode_logical, irq_cfg->vector,
3407 				 irq_cfg->dest_apicid, devid);
3408 
3409 	switch (info->type) {
3410 	case X86_IRQ_ALLOC_TYPE_IOAPIC:
3411 	case X86_IRQ_ALLOC_TYPE_HPET:
3412 	case X86_IRQ_ALLOC_TYPE_PCI_MSI:
3413 	case X86_IRQ_ALLOC_TYPE_PCI_MSIX:
3414 		fill_msi_msg(&data->msi_entry, irte_info->index);
3415 		break;
3416 
3417 	default:
3418 		BUG_ON(1);
3419 		break;
3420 	}
3421 }
3422 
3423 struct amd_irte_ops irte_32_ops = {
3424 	.prepare = irte_prepare,
3425 	.activate = irte_activate,
3426 	.deactivate = irte_deactivate,
3427 	.set_affinity = irte_set_affinity,
3428 	.set_allocated = irte_set_allocated,
3429 	.is_allocated = irte_is_allocated,
3430 	.clear_allocated = irte_clear_allocated,
3431 };
3432 
3433 struct amd_irte_ops irte_128_ops = {
3434 	.prepare = irte_ga_prepare,
3435 	.activate = irte_ga_activate,
3436 	.deactivate = irte_ga_deactivate,
3437 	.set_affinity = irte_ga_set_affinity,
3438 	.set_allocated = irte_ga_set_allocated,
3439 	.is_allocated = irte_ga_is_allocated,
3440 	.clear_allocated = irte_ga_clear_allocated,
3441 };
3442 
3443 static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
3444 			       unsigned int nr_irqs, void *arg)
3445 {
3446 	struct irq_alloc_info *info = arg;
3447 	struct irq_data *irq_data;
3448 	struct amd_ir_data *data = NULL;
3449 	struct amd_iommu *iommu;
3450 	struct irq_cfg *cfg;
3451 	int i, ret, devid, seg, sbdf;
3452 	int index;
3453 
3454 	if (!info)
3455 		return -EINVAL;
3456 	if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_PCI_MSI)
3457 		return -EINVAL;
3458 
3459 	sbdf = get_devid(info);
3460 	if (sbdf < 0)
3461 		return -EINVAL;
3462 
3463 	seg = PCI_SBDF_TO_SEGID(sbdf);
3464 	devid = PCI_SBDF_TO_DEVID(sbdf);
3465 	iommu = __rlookup_amd_iommu(seg, devid);
3466 	if (!iommu)
3467 		return -EINVAL;
3468 
3469 	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
3470 	if (ret < 0)
3471 		return ret;
3472 
3473 	if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) {
3474 		struct irq_remap_table *table;
3475 
3476 		table = alloc_irq_table(iommu, devid, NULL);
3477 		if (table) {
3478 			if (!table->min_index) {
3479 				/*
3480 				 * Keep the first 32 indexes free for IOAPIC
3481 				 * interrupts.
3482 				 */
3483 				table->min_index = 32;
3484 				for (i = 0; i < 32; ++i)
3485 					iommu->irte_ops->set_allocated(table, i);
3486 			}
3487 			WARN_ON(table->min_index != 32);
3488 			index = info->ioapic.pin;
3489 		} else {
3490 			index = -ENOMEM;
3491 		}
3492 	} else if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI ||
3493 		   info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX) {
3494 		bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI);
3495 
3496 		index = alloc_irq_index(iommu, devid, nr_irqs, align,
3497 					msi_desc_to_pci_dev(info->desc));
3498 	} else {
3499 		index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL);
3500 	}
3501 
3502 	if (index < 0) {
3503 		pr_warn("Failed to allocate IRTE\n");
3504 		ret = index;
3505 		goto out_free_parent;
3506 	}
3507 
3508 	for (i = 0; i < nr_irqs; i++) {
3509 		irq_data = irq_domain_get_irq_data(domain, virq + i);
3510 		cfg = irq_data ? irqd_cfg(irq_data) : NULL;
3511 		if (!cfg) {
3512 			ret = -EINVAL;
3513 			goto out_free_data;
3514 		}
3515 
3516 		ret = -ENOMEM;
3517 		data = kzalloc(sizeof(*data), GFP_KERNEL);
3518 		if (!data)
3519 			goto out_free_data;
3520 
3521 		if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
3522 			data->entry = kzalloc(sizeof(union irte), GFP_KERNEL);
3523 		else
3524 			data->entry = kzalloc(sizeof(struct irte_ga),
3525 						     GFP_KERNEL);
3526 		if (!data->entry) {
3527 			kfree(data);
3528 			goto out_free_data;
3529 		}
3530 
3531 		data->iommu = iommu;
3532 		irq_data->hwirq = (devid << 16) + i;
3533 		irq_data->chip_data = data;
3534 		irq_data->chip = &amd_ir_chip;
3535 		irq_remapping_prepare_irte(data, cfg, info, devid, index, i);
3536 		irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT);
3537 	}
3538 
3539 	return 0;
3540 
3541 out_free_data:
3542 	for (i--; i >= 0; i--) {
3543 		irq_data = irq_domain_get_irq_data(domain, virq + i);
3544 		if (irq_data)
3545 			kfree(irq_data->chip_data);
3546 	}
3547 	for (i = 0; i < nr_irqs; i++)
3548 		free_irte(iommu, devid, index + i);
3549 out_free_parent:
3550 	irq_domain_free_irqs_common(domain, virq, nr_irqs);
3551 	return ret;
3552 }
3553 
3554 static void irq_remapping_free(struct irq_domain *domain, unsigned int virq,
3555 			       unsigned int nr_irqs)
3556 {
3557 	struct irq_2_irte *irte_info;
3558 	struct irq_data *irq_data;
3559 	struct amd_ir_data *data;
3560 	int i;
3561 
3562 	for (i = 0; i < nr_irqs; i++) {
3563 		irq_data = irq_domain_get_irq_data(domain, virq  + i);
3564 		if (irq_data && irq_data->chip_data) {
3565 			data = irq_data->chip_data;
3566 			irte_info = &data->irq_2_irte;
3567 			free_irte(data->iommu, irte_info->devid, irte_info->index);
3568 			kfree(data->entry);
3569 			kfree(data);
3570 		}
3571 	}
3572 	irq_domain_free_irqs_common(domain, virq, nr_irqs);
3573 }
3574 
3575 static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,
3576 			       struct amd_ir_data *ir_data,
3577 			       struct irq_2_irte *irte_info,
3578 			       struct irq_cfg *cfg);
3579 
3580 static int irq_remapping_activate(struct irq_domain *domain,
3581 				  struct irq_data *irq_data, bool reserve)
3582 {
3583 	struct amd_ir_data *data = irq_data->chip_data;
3584 	struct irq_2_irte *irte_info = &data->irq_2_irte;
3585 	struct amd_iommu *iommu = data->iommu;
3586 	struct irq_cfg *cfg = irqd_cfg(irq_data);
3587 
3588 	if (!iommu)
3589 		return 0;
3590 
3591 	iommu->irte_ops->activate(iommu, data->entry, irte_info->devid,
3592 				  irte_info->index);
3593 	amd_ir_update_irte(irq_data, iommu, data, irte_info, cfg);
3594 	return 0;
3595 }
3596 
3597 static void irq_remapping_deactivate(struct irq_domain *domain,
3598 				     struct irq_data *irq_data)
3599 {
3600 	struct amd_ir_data *data = irq_data->chip_data;
3601 	struct irq_2_irte *irte_info = &data->irq_2_irte;
3602 	struct amd_iommu *iommu = data->iommu;
3603 
3604 	if (iommu)
3605 		iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid,
3606 					    irte_info->index);
3607 }
3608 
3609 static int irq_remapping_select(struct irq_domain *d, struct irq_fwspec *fwspec,
3610 				enum irq_domain_bus_token bus_token)
3611 {
3612 	struct amd_iommu *iommu;
3613 	int devid = -1;
3614 
3615 	if (!amd_iommu_irq_remap)
3616 		return 0;
3617 
3618 	if (x86_fwspec_is_ioapic(fwspec))
3619 		devid = get_ioapic_devid(fwspec->param[0]);
3620 	else if (x86_fwspec_is_hpet(fwspec))
3621 		devid = get_hpet_devid(fwspec->param[0]);
3622 
3623 	if (devid < 0)
3624 		return 0;
3625 	iommu = __rlookup_amd_iommu((devid >> 16), (devid & 0xffff));
3626 
3627 	return iommu && iommu->ir_domain == d;
3628 }
3629 
3630 static const struct irq_domain_ops amd_ir_domain_ops = {
3631 	.select = irq_remapping_select,
3632 	.alloc = irq_remapping_alloc,
3633 	.free = irq_remapping_free,
3634 	.activate = irq_remapping_activate,
3635 	.deactivate = irq_remapping_deactivate,
3636 };
3637 
3638 int amd_iommu_activate_guest_mode(void *data)
3639 {
3640 	struct amd_ir_data *ir_data = (struct amd_ir_data *)data;
3641 	struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
3642 	u64 valid;
3643 
3644 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) || !entry)
3645 		return 0;
3646 
3647 	valid = entry->lo.fields_vapic.valid;
3648 
3649 	entry->lo.val = 0;
3650 	entry->hi.val = 0;
3651 
3652 	entry->lo.fields_vapic.valid       = valid;
3653 	entry->lo.fields_vapic.guest_mode  = 1;
3654 	entry->lo.fields_vapic.ga_log_intr = 1;
3655 	entry->hi.fields.ga_root_ptr       = ir_data->ga_root_ptr;
3656 	entry->hi.fields.vector            = ir_data->ga_vector;
3657 	entry->lo.fields_vapic.ga_tag      = ir_data->ga_tag;
3658 
3659 	return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,
3660 			      ir_data->irq_2_irte.index, entry);
3661 }
3662 EXPORT_SYMBOL(amd_iommu_activate_guest_mode);
3663 
3664 int amd_iommu_deactivate_guest_mode(void *data)
3665 {
3666 	struct amd_ir_data *ir_data = (struct amd_ir_data *)data;
3667 	struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
3668 	struct irq_cfg *cfg = ir_data->cfg;
3669 	u64 valid;
3670 
3671 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) ||
3672 	    !entry || !entry->lo.fields_vapic.guest_mode)
3673 		return 0;
3674 
3675 	valid = entry->lo.fields_remap.valid;
3676 
3677 	entry->lo.val = 0;
3678 	entry->hi.val = 0;
3679 
3680 	entry->lo.fields_remap.valid       = valid;
3681 	entry->lo.fields_remap.dm          = apic->dest_mode_logical;
3682 	entry->lo.fields_remap.int_type    = APIC_DELIVERY_MODE_FIXED;
3683 	entry->hi.fields.vector            = cfg->vector;
3684 	entry->lo.fields_remap.destination =
3685 				APICID_TO_IRTE_DEST_LO(cfg->dest_apicid);
3686 	entry->hi.fields.destination =
3687 				APICID_TO_IRTE_DEST_HI(cfg->dest_apicid);
3688 
3689 	return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,
3690 			      ir_data->irq_2_irte.index, entry);
3691 }
3692 EXPORT_SYMBOL(amd_iommu_deactivate_guest_mode);
3693 
3694 static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
3695 {
3696 	int ret;
3697 	struct amd_iommu_pi_data *pi_data = vcpu_info;
3698 	struct vcpu_data *vcpu_pi_info = pi_data->vcpu_data;
3699 	struct amd_ir_data *ir_data = data->chip_data;
3700 	struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
3701 	struct iommu_dev_data *dev_data;
3702 
3703 	if (ir_data->iommu == NULL)
3704 		return -EINVAL;
3705 
3706 	dev_data = search_dev_data(ir_data->iommu, irte_info->devid);
3707 
3708 	/* Note:
3709 	 * This device has never been set up for guest mode.
3710 	 * we should not modify the IRTE
3711 	 */
3712 	if (!dev_data || !dev_data->use_vapic)
3713 		return 0;
3714 
3715 	ir_data->cfg = irqd_cfg(data);
3716 	pi_data->ir_data = ir_data;
3717 
3718 	/* Note:
3719 	 * SVM tries to set up for VAPIC mode, but we are in
3720 	 * legacy mode. So, we force legacy mode instead.
3721 	 */
3722 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) {
3723 		pr_debug("%s: Fall back to using intr legacy remap\n",
3724 			 __func__);
3725 		pi_data->is_guest_mode = false;
3726 	}
3727 
3728 	pi_data->prev_ga_tag = ir_data->cached_ga_tag;
3729 	if (pi_data->is_guest_mode) {
3730 		ir_data->ga_root_ptr = (pi_data->base >> 12);
3731 		ir_data->ga_vector = vcpu_pi_info->vector;
3732 		ir_data->ga_tag = pi_data->ga_tag;
3733 		ret = amd_iommu_activate_guest_mode(ir_data);
3734 		if (!ret)
3735 			ir_data->cached_ga_tag = pi_data->ga_tag;
3736 	} else {
3737 		ret = amd_iommu_deactivate_guest_mode(ir_data);
3738 
3739 		/*
3740 		 * This communicates the ga_tag back to the caller
3741 		 * so that it can do all the necessary clean up.
3742 		 */
3743 		if (!ret)
3744 			ir_data->cached_ga_tag = 0;
3745 	}
3746 
3747 	return ret;
3748 }
3749 
3750 
3751 static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,
3752 			       struct amd_ir_data *ir_data,
3753 			       struct irq_2_irte *irte_info,
3754 			       struct irq_cfg *cfg)
3755 {
3756 
3757 	/*
3758 	 * Atomically updates the IRTE with the new destination, vector
3759 	 * and flushes the interrupt entry cache.
3760 	 */
3761 	iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid,
3762 				      irte_info->index, cfg->vector,
3763 				      cfg->dest_apicid);
3764 }
3765 
3766 static int amd_ir_set_affinity(struct irq_data *data,
3767 			       const struct cpumask *mask, bool force)
3768 {
3769 	struct amd_ir_data *ir_data = data->chip_data;
3770 	struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
3771 	struct irq_cfg *cfg = irqd_cfg(data);
3772 	struct irq_data *parent = data->parent_data;
3773 	struct amd_iommu *iommu = ir_data->iommu;
3774 	int ret;
3775 
3776 	if (!iommu)
3777 		return -ENODEV;
3778 
3779 	ret = parent->chip->irq_set_affinity(parent, mask, force);
3780 	if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
3781 		return ret;
3782 
3783 	amd_ir_update_irte(data, iommu, ir_data, irte_info, cfg);
3784 	/*
3785 	 * After this point, all the interrupts will start arriving
3786 	 * at the new destination. So, time to cleanup the previous
3787 	 * vector allocation.
3788 	 */
3789 	vector_schedule_cleanup(cfg);
3790 
3791 	return IRQ_SET_MASK_OK_DONE;
3792 }
3793 
3794 static void ir_compose_msi_msg(struct irq_data *irq_data, struct msi_msg *msg)
3795 {
3796 	struct amd_ir_data *ir_data = irq_data->chip_data;
3797 
3798 	*msg = ir_data->msi_entry;
3799 }
3800 
3801 static struct irq_chip amd_ir_chip = {
3802 	.name			= "AMD-IR",
3803 	.irq_ack		= apic_ack_irq,
3804 	.irq_set_affinity	= amd_ir_set_affinity,
3805 	.irq_set_vcpu_affinity	= amd_ir_set_vcpu_affinity,
3806 	.irq_compose_msi_msg	= ir_compose_msi_msg,
3807 };
3808 
3809 static const struct msi_parent_ops amdvi_msi_parent_ops = {
3810 	.supported_flags	= X86_VECTOR_MSI_FLAGS_SUPPORTED | MSI_FLAG_MULTI_PCI_MSI,
3811 	.prefix			= "IR-",
3812 	.init_dev_msi_info	= msi_parent_init_dev_msi_info,
3813 };
3814 
3815 int amd_iommu_create_irq_domain(struct amd_iommu *iommu)
3816 {
3817 	struct fwnode_handle *fn;
3818 
3819 	fn = irq_domain_alloc_named_id_fwnode("AMD-IR", iommu->index);
3820 	if (!fn)
3821 		return -ENOMEM;
3822 	iommu->ir_domain = irq_domain_create_hierarchy(arch_get_ir_parent_domain(), 0, 0,
3823 						       fn, &amd_ir_domain_ops, iommu);
3824 	if (!iommu->ir_domain) {
3825 		irq_domain_free_fwnode(fn);
3826 		return -ENOMEM;
3827 	}
3828 
3829 	irq_domain_update_bus_token(iommu->ir_domain,  DOMAIN_BUS_AMDVI);
3830 	iommu->ir_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT |
3831 				   IRQ_DOMAIN_FLAG_ISOLATED_MSI;
3832 	iommu->ir_domain->msi_parent_ops = &amdvi_msi_parent_ops;
3833 
3834 	return 0;
3835 }
3836 
3837 int amd_iommu_update_ga(int cpu, bool is_run, void *data)
3838 {
3839 	struct amd_ir_data *ir_data = (struct amd_ir_data *)data;
3840 	struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
3841 
3842 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) ||
3843 	    !entry || !entry->lo.fields_vapic.guest_mode)
3844 		return 0;
3845 
3846 	if (!ir_data->iommu)
3847 		return -ENODEV;
3848 
3849 	if (cpu >= 0) {
3850 		entry->lo.fields_vapic.destination =
3851 					APICID_TO_IRTE_DEST_LO(cpu);
3852 		entry->hi.fields.destination =
3853 					APICID_TO_IRTE_DEST_HI(cpu);
3854 	}
3855 	entry->lo.fields_vapic.is_run = is_run;
3856 
3857 	return __modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,
3858 				ir_data->irq_2_irte.index, entry);
3859 }
3860 EXPORT_SYMBOL(amd_iommu_update_ga);
3861 #endif
3862