xref: /linux/drivers/iommu/intel/iommu.c (revision c297aa7d3fb6755890b78b483e82c9cf07370d50)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright © 2006-2014 Intel Corporation.
4  *
5  * Authors: David Woodhouse <dwmw2@infradead.org>,
6  *          Ashok Raj <ashok.raj@intel.com>,
7  *          Shaohua Li <shaohua.li@intel.com>,
8  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
9  *          Fenghua Yu <fenghua.yu@intel.com>
10  *          Joerg Roedel <jroedel@suse.de>
11  */
12 
13 #define pr_fmt(fmt)     "DMAR: " fmt
14 #define dev_fmt(fmt)    pr_fmt(fmt)
15 
16 #include <linux/crash_dump.h>
17 #include <linux/dma-direct.h>
18 #include <linux/dmi.h>
19 #include <linux/memory.h>
20 #include <linux/pci.h>
21 #include <linux/pci-ats.h>
22 #include <linux/spinlock.h>
23 #include <linux/syscore_ops.h>
24 #include <linux/tboot.h>
25 #include <uapi/linux/iommufd.h>
26 
27 #include "iommu.h"
28 #include "../dma-iommu.h"
29 #include "../irq_remapping.h"
30 #include "../iommu-pages.h"
31 #include "pasid.h"
32 #include "cap_audit.h"
33 #include "perfmon.h"
34 
35 #define ROOT_SIZE		VTD_PAGE_SIZE
36 #define CONTEXT_SIZE		VTD_PAGE_SIZE
37 
38 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
39 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
40 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
41 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
42 
43 #define IOAPIC_RANGE_START	(0xfee00000)
44 #define IOAPIC_RANGE_END	(0xfeefffff)
45 #define IOVA_START_ADDR		(0x1000)
46 
47 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
48 
49 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << ((gaw) - VTD_PAGE_SHIFT)) - 1)
50 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << (gaw)) - 1)
51 
52 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
53    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
54 #define DOMAIN_MAX_PFN(gaw)	((unsigned long) min_t(uint64_t, \
55 				__DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
56 #define DOMAIN_MAX_ADDR(gaw)	(((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
57 
58 static void __init check_tylersburg_isoch(void);
59 static int rwbf_quirk;
60 
61 /*
62  * set to 1 to panic kernel if can't successfully enable VT-d
63  * (used when kernel is launched w/ TXT)
64  */
65 static int force_on = 0;
66 static int intel_iommu_tboot_noforce;
67 static int no_platform_optin;
68 
69 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
70 
71 /*
72  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
73  * if marked present.
74  */
75 static phys_addr_t root_entry_lctp(struct root_entry *re)
76 {
77 	if (!(re->lo & 1))
78 		return 0;
79 
80 	return re->lo & VTD_PAGE_MASK;
81 }
82 
83 /*
84  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
85  * if marked present.
86  */
87 static phys_addr_t root_entry_uctp(struct root_entry *re)
88 {
89 	if (!(re->hi & 1))
90 		return 0;
91 
92 	return re->hi & VTD_PAGE_MASK;
93 }
94 
95 static int device_rid_cmp_key(const void *key, const struct rb_node *node)
96 {
97 	struct device_domain_info *info =
98 		rb_entry(node, struct device_domain_info, node);
99 	const u16 *rid_lhs = key;
100 
101 	if (*rid_lhs < PCI_DEVID(info->bus, info->devfn))
102 		return -1;
103 
104 	if (*rid_lhs > PCI_DEVID(info->bus, info->devfn))
105 		return 1;
106 
107 	return 0;
108 }
109 
110 static int device_rid_cmp(struct rb_node *lhs, const struct rb_node *rhs)
111 {
112 	struct device_domain_info *info =
113 		rb_entry(lhs, struct device_domain_info, node);
114 	u16 key = PCI_DEVID(info->bus, info->devfn);
115 
116 	return device_rid_cmp_key(&key, rhs);
117 }
118 
119 /*
120  * Looks up an IOMMU-probed device using its source ID.
121  *
122  * Returns the pointer to the device if there is a match. Otherwise,
123  * returns NULL.
124  *
125  * Note that this helper doesn't guarantee that the device won't be
126  * released by the iommu subsystem after being returned. The caller
127  * should use its own synchronization mechanism to avoid the device
128  * being released during its use if its possibly the case.
129  */
130 struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid)
131 {
132 	struct device_domain_info *info = NULL;
133 	struct rb_node *node;
134 	unsigned long flags;
135 
136 	spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
137 	node = rb_find(&rid, &iommu->device_rbtree, device_rid_cmp_key);
138 	if (node)
139 		info = rb_entry(node, struct device_domain_info, node);
140 	spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
141 
142 	return info ? info->dev : NULL;
143 }
144 
145 static int device_rbtree_insert(struct intel_iommu *iommu,
146 				struct device_domain_info *info)
147 {
148 	struct rb_node *curr;
149 	unsigned long flags;
150 
151 	spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
152 	curr = rb_find_add(&info->node, &iommu->device_rbtree, device_rid_cmp);
153 	spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
154 	if (WARN_ON(curr))
155 		return -EEXIST;
156 
157 	return 0;
158 }
159 
160 static void device_rbtree_remove(struct device_domain_info *info)
161 {
162 	struct intel_iommu *iommu = info->iommu;
163 	unsigned long flags;
164 
165 	spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
166 	rb_erase(&info->node, &iommu->device_rbtree);
167 	spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
168 }
169 
170 struct dmar_rmrr_unit {
171 	struct list_head list;		/* list of rmrr units	*/
172 	struct acpi_dmar_header *hdr;	/* ACPI header		*/
173 	u64	base_address;		/* reserved base address*/
174 	u64	end_address;		/* reserved end address */
175 	struct dmar_dev_scope *devices;	/* target devices */
176 	int	devices_cnt;		/* target device count */
177 };
178 
179 struct dmar_atsr_unit {
180 	struct list_head list;		/* list of ATSR units */
181 	struct acpi_dmar_header *hdr;	/* ACPI header */
182 	struct dmar_dev_scope *devices;	/* target devices */
183 	int devices_cnt;		/* target device count */
184 	u8 include_all:1;		/* include all ports */
185 };
186 
187 struct dmar_satc_unit {
188 	struct list_head list;		/* list of SATC units */
189 	struct acpi_dmar_header *hdr;	/* ACPI header */
190 	struct dmar_dev_scope *devices;	/* target devices */
191 	struct intel_iommu *iommu;	/* the corresponding iommu */
192 	int devices_cnt;		/* target device count */
193 	u8 atc_required:1;		/* ATS is required */
194 };
195 
196 static LIST_HEAD(dmar_atsr_units);
197 static LIST_HEAD(dmar_rmrr_units);
198 static LIST_HEAD(dmar_satc_units);
199 
200 #define for_each_rmrr_units(rmrr) \
201 	list_for_each_entry(rmrr, &dmar_rmrr_units, list)
202 
203 static void intel_iommu_domain_free(struct iommu_domain *domain);
204 
205 int dmar_disabled = !IS_ENABLED(CONFIG_INTEL_IOMMU_DEFAULT_ON);
206 int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON);
207 
208 int intel_iommu_enabled = 0;
209 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
210 
211 static int intel_iommu_superpage = 1;
212 static int iommu_identity_mapping;
213 static int iommu_skip_te_disable;
214 static int disable_igfx_iommu;
215 
216 #define IDENTMAP_AZALIA		4
217 
218 const struct iommu_ops intel_iommu_ops;
219 static const struct iommu_dirty_ops intel_dirty_ops;
220 
221 static bool translation_pre_enabled(struct intel_iommu *iommu)
222 {
223 	return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
224 }
225 
226 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
227 {
228 	iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
229 }
230 
231 static void init_translation_status(struct intel_iommu *iommu)
232 {
233 	u32 gsts;
234 
235 	gsts = readl(iommu->reg + DMAR_GSTS_REG);
236 	if (gsts & DMA_GSTS_TES)
237 		iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
238 }
239 
240 static int __init intel_iommu_setup(char *str)
241 {
242 	if (!str)
243 		return -EINVAL;
244 
245 	while (*str) {
246 		if (!strncmp(str, "on", 2)) {
247 			dmar_disabled = 0;
248 			pr_info("IOMMU enabled\n");
249 		} else if (!strncmp(str, "off", 3)) {
250 			dmar_disabled = 1;
251 			no_platform_optin = 1;
252 			pr_info("IOMMU disabled\n");
253 		} else if (!strncmp(str, "igfx_off", 8)) {
254 			disable_igfx_iommu = 1;
255 			pr_info("Disable GFX device mapping\n");
256 		} else if (!strncmp(str, "forcedac", 8)) {
257 			pr_warn("intel_iommu=forcedac deprecated; use iommu.forcedac instead\n");
258 			iommu_dma_forcedac = true;
259 		} else if (!strncmp(str, "strict", 6)) {
260 			pr_warn("intel_iommu=strict deprecated; use iommu.strict=1 instead\n");
261 			iommu_set_dma_strict();
262 		} else if (!strncmp(str, "sp_off", 6)) {
263 			pr_info("Disable supported super page\n");
264 			intel_iommu_superpage = 0;
265 		} else if (!strncmp(str, "sm_on", 5)) {
266 			pr_info("Enable scalable mode if hardware supports\n");
267 			intel_iommu_sm = 1;
268 		} else if (!strncmp(str, "sm_off", 6)) {
269 			pr_info("Scalable mode is disallowed\n");
270 			intel_iommu_sm = 0;
271 		} else if (!strncmp(str, "tboot_noforce", 13)) {
272 			pr_info("Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
273 			intel_iommu_tboot_noforce = 1;
274 		} else {
275 			pr_notice("Unknown option - '%s'\n", str);
276 		}
277 
278 		str += strcspn(str, ",");
279 		while (*str == ',')
280 			str++;
281 	}
282 
283 	return 1;
284 }
285 __setup("intel_iommu=", intel_iommu_setup);
286 
287 static int domain_pfn_supported(struct dmar_domain *domain, unsigned long pfn)
288 {
289 	int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
290 
291 	return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
292 }
293 
294 /*
295  * Calculate the Supported Adjusted Guest Address Widths of an IOMMU.
296  * Refer to 11.4.2 of the VT-d spec for the encoding of each bit of
297  * the returned SAGAW.
298  */
299 static unsigned long __iommu_calculate_sagaw(struct intel_iommu *iommu)
300 {
301 	unsigned long fl_sagaw, sl_sagaw;
302 
303 	fl_sagaw = BIT(2) | (cap_fl5lp_support(iommu->cap) ? BIT(3) : 0);
304 	sl_sagaw = cap_sagaw(iommu->cap);
305 
306 	/* Second level only. */
307 	if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))
308 		return sl_sagaw;
309 
310 	/* First level only. */
311 	if (!ecap_slts(iommu->ecap))
312 		return fl_sagaw;
313 
314 	return fl_sagaw & sl_sagaw;
315 }
316 
317 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
318 {
319 	unsigned long sagaw;
320 	int agaw;
321 
322 	sagaw = __iommu_calculate_sagaw(iommu);
323 	for (agaw = width_to_agaw(max_gaw); agaw >= 0; agaw--) {
324 		if (test_bit(agaw, &sagaw))
325 			break;
326 	}
327 
328 	return agaw;
329 }
330 
331 /*
332  * Calculate max SAGAW for each iommu.
333  */
334 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
335 {
336 	return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
337 }
338 
339 /*
340  * calculate agaw for each iommu.
341  * "SAGAW" may be different across iommus, use a default agaw, and
342  * get a supported less agaw for iommus that don't support the default agaw.
343  */
344 int iommu_calculate_agaw(struct intel_iommu *iommu)
345 {
346 	return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
347 }
348 
349 static bool iommu_paging_structure_coherency(struct intel_iommu *iommu)
350 {
351 	return sm_supported(iommu) ?
352 			ecap_smpwc(iommu->ecap) : ecap_coherent(iommu->ecap);
353 }
354 
355 static void domain_update_iommu_coherency(struct dmar_domain *domain)
356 {
357 	struct iommu_domain_info *info;
358 	struct dmar_drhd_unit *drhd;
359 	struct intel_iommu *iommu;
360 	bool found = false;
361 	unsigned long i;
362 
363 	domain->iommu_coherency = true;
364 	xa_for_each(&domain->iommu_array, i, info) {
365 		found = true;
366 		if (!iommu_paging_structure_coherency(info->iommu)) {
367 			domain->iommu_coherency = false;
368 			break;
369 		}
370 	}
371 	if (found)
372 		return;
373 
374 	/* No hardware attached; use lowest common denominator */
375 	rcu_read_lock();
376 	for_each_active_iommu(iommu, drhd) {
377 		if (!iommu_paging_structure_coherency(iommu)) {
378 			domain->iommu_coherency = false;
379 			break;
380 		}
381 	}
382 	rcu_read_unlock();
383 }
384 
385 static int domain_update_iommu_superpage(struct dmar_domain *domain,
386 					 struct intel_iommu *skip)
387 {
388 	struct dmar_drhd_unit *drhd;
389 	struct intel_iommu *iommu;
390 	int mask = 0x3;
391 
392 	if (!intel_iommu_superpage)
393 		return 0;
394 
395 	/* set iommu_superpage to the smallest common denominator */
396 	rcu_read_lock();
397 	for_each_active_iommu(iommu, drhd) {
398 		if (iommu != skip) {
399 			if (domain && domain->use_first_level) {
400 				if (!cap_fl1gp_support(iommu->cap))
401 					mask = 0x1;
402 			} else {
403 				mask &= cap_super_page_val(iommu->cap);
404 			}
405 
406 			if (!mask)
407 				break;
408 		}
409 	}
410 	rcu_read_unlock();
411 
412 	return fls(mask);
413 }
414 
415 static int domain_update_device_node(struct dmar_domain *domain)
416 {
417 	struct device_domain_info *info;
418 	int nid = NUMA_NO_NODE;
419 	unsigned long flags;
420 
421 	spin_lock_irqsave(&domain->lock, flags);
422 	list_for_each_entry(info, &domain->devices, link) {
423 		/*
424 		 * There could possibly be multiple device numa nodes as devices
425 		 * within the same domain may sit behind different IOMMUs. There
426 		 * isn't perfect answer in such situation, so we select first
427 		 * come first served policy.
428 		 */
429 		nid = dev_to_node(info->dev);
430 		if (nid != NUMA_NO_NODE)
431 			break;
432 	}
433 	spin_unlock_irqrestore(&domain->lock, flags);
434 
435 	return nid;
436 }
437 
438 /* Return the super pagesize bitmap if supported. */
439 static unsigned long domain_super_pgsize_bitmap(struct dmar_domain *domain)
440 {
441 	unsigned long bitmap = 0;
442 
443 	/*
444 	 * 1-level super page supports page size of 2MiB, 2-level super page
445 	 * supports page size of both 2MiB and 1GiB.
446 	 */
447 	if (domain->iommu_superpage == 1)
448 		bitmap |= SZ_2M;
449 	else if (domain->iommu_superpage == 2)
450 		bitmap |= SZ_2M | SZ_1G;
451 
452 	return bitmap;
453 }
454 
455 /* Some capabilities may be different across iommus */
456 void domain_update_iommu_cap(struct dmar_domain *domain)
457 {
458 	domain_update_iommu_coherency(domain);
459 	domain->iommu_superpage = domain_update_iommu_superpage(domain, NULL);
460 
461 	/*
462 	 * If RHSA is missing, we should default to the device numa domain
463 	 * as fall back.
464 	 */
465 	if (domain->nid == NUMA_NO_NODE)
466 		domain->nid = domain_update_device_node(domain);
467 
468 	/*
469 	 * First-level translation restricts the input-address to a
470 	 * canonical address (i.e., address bits 63:N have the same
471 	 * value as address bit [N-1], where N is 48-bits with 4-level
472 	 * paging and 57-bits with 5-level paging). Hence, skip bit
473 	 * [N-1].
474 	 */
475 	if (domain->use_first_level)
476 		domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw - 1);
477 	else
478 		domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw);
479 
480 	domain->domain.pgsize_bitmap |= domain_super_pgsize_bitmap(domain);
481 }
482 
483 struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
484 					 u8 devfn, int alloc)
485 {
486 	struct root_entry *root = &iommu->root_entry[bus];
487 	struct context_entry *context;
488 	u64 *entry;
489 
490 	/*
491 	 * Except that the caller requested to allocate a new entry,
492 	 * returning a copied context entry makes no sense.
493 	 */
494 	if (!alloc && context_copied(iommu, bus, devfn))
495 		return NULL;
496 
497 	entry = &root->lo;
498 	if (sm_supported(iommu)) {
499 		if (devfn >= 0x80) {
500 			devfn -= 0x80;
501 			entry = &root->hi;
502 		}
503 		devfn *= 2;
504 	}
505 	if (*entry & 1)
506 		context = phys_to_virt(*entry & VTD_PAGE_MASK);
507 	else {
508 		unsigned long phy_addr;
509 		if (!alloc)
510 			return NULL;
511 
512 		context = iommu_alloc_page_node(iommu->node, GFP_ATOMIC);
513 		if (!context)
514 			return NULL;
515 
516 		__iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
517 		phy_addr = virt_to_phys((void *)context);
518 		*entry = phy_addr | 1;
519 		__iommu_flush_cache(iommu, entry, sizeof(*entry));
520 	}
521 	return &context[devfn];
522 }
523 
524 /**
525  * is_downstream_to_pci_bridge - test if a device belongs to the PCI
526  *				 sub-hierarchy of a candidate PCI-PCI bridge
527  * @dev: candidate PCI device belonging to @bridge PCI sub-hierarchy
528  * @bridge: the candidate PCI-PCI bridge
529  *
530  * Return: true if @dev belongs to @bridge PCI sub-hierarchy, else false.
531  */
532 static bool
533 is_downstream_to_pci_bridge(struct device *dev, struct device *bridge)
534 {
535 	struct pci_dev *pdev, *pbridge;
536 
537 	if (!dev_is_pci(dev) || !dev_is_pci(bridge))
538 		return false;
539 
540 	pdev = to_pci_dev(dev);
541 	pbridge = to_pci_dev(bridge);
542 
543 	if (pbridge->subordinate &&
544 	    pbridge->subordinate->number <= pdev->bus->number &&
545 	    pbridge->subordinate->busn_res.end >= pdev->bus->number)
546 		return true;
547 
548 	return false;
549 }
550 
551 static bool quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
552 {
553 	struct dmar_drhd_unit *drhd;
554 	u32 vtbar;
555 	int rc;
556 
557 	/* We know that this device on this chipset has its own IOMMU.
558 	 * If we find it under a different IOMMU, then the BIOS is lying
559 	 * to us. Hope that the IOMMU for this device is actually
560 	 * disabled, and it needs no translation...
561 	 */
562 	rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
563 	if (rc) {
564 		/* "can't" happen */
565 		dev_info(&pdev->dev, "failed to run vt-d quirk\n");
566 		return false;
567 	}
568 	vtbar &= 0xffff0000;
569 
570 	/* we know that the this iommu should be at offset 0xa000 from vtbar */
571 	drhd = dmar_find_matched_drhd_unit(pdev);
572 	if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {
573 		pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n");
574 		add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
575 		return true;
576 	}
577 
578 	return false;
579 }
580 
581 static bool iommu_is_dummy(struct intel_iommu *iommu, struct device *dev)
582 {
583 	if (!iommu || iommu->drhd->ignored)
584 		return true;
585 
586 	if (dev_is_pci(dev)) {
587 		struct pci_dev *pdev = to_pci_dev(dev);
588 
589 		if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
590 		    pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SNB &&
591 		    quirk_ioat_snb_local_iommu(pdev))
592 			return true;
593 	}
594 
595 	return false;
596 }
597 
598 static struct intel_iommu *device_lookup_iommu(struct device *dev, u8 *bus, u8 *devfn)
599 {
600 	struct dmar_drhd_unit *drhd = NULL;
601 	struct pci_dev *pdev = NULL;
602 	struct intel_iommu *iommu;
603 	struct device *tmp;
604 	u16 segment = 0;
605 	int i;
606 
607 	if (!dev)
608 		return NULL;
609 
610 	if (dev_is_pci(dev)) {
611 		struct pci_dev *pf_pdev;
612 
613 		pdev = pci_real_dma_dev(to_pci_dev(dev));
614 
615 		/* VFs aren't listed in scope tables; we need to look up
616 		 * the PF instead to find the IOMMU. */
617 		pf_pdev = pci_physfn(pdev);
618 		dev = &pf_pdev->dev;
619 		segment = pci_domain_nr(pdev->bus);
620 	} else if (has_acpi_companion(dev))
621 		dev = &ACPI_COMPANION(dev)->dev;
622 
623 	rcu_read_lock();
624 	for_each_iommu(iommu, drhd) {
625 		if (pdev && segment != drhd->segment)
626 			continue;
627 
628 		for_each_active_dev_scope(drhd->devices,
629 					  drhd->devices_cnt, i, tmp) {
630 			if (tmp == dev) {
631 				/* For a VF use its original BDF# not that of the PF
632 				 * which we used for the IOMMU lookup. Strictly speaking
633 				 * we could do this for all PCI devices; we only need to
634 				 * get the BDF# from the scope table for ACPI matches. */
635 				if (pdev && pdev->is_virtfn)
636 					goto got_pdev;
637 
638 				if (bus && devfn) {
639 					*bus = drhd->devices[i].bus;
640 					*devfn = drhd->devices[i].devfn;
641 				}
642 				goto out;
643 			}
644 
645 			if (is_downstream_to_pci_bridge(dev, tmp))
646 				goto got_pdev;
647 		}
648 
649 		if (pdev && drhd->include_all) {
650 got_pdev:
651 			if (bus && devfn) {
652 				*bus = pdev->bus->number;
653 				*devfn = pdev->devfn;
654 			}
655 			goto out;
656 		}
657 	}
658 	iommu = NULL;
659 out:
660 	if (iommu_is_dummy(iommu, dev))
661 		iommu = NULL;
662 
663 	rcu_read_unlock();
664 
665 	return iommu;
666 }
667 
668 static void domain_flush_cache(struct dmar_domain *domain,
669 			       void *addr, int size)
670 {
671 	if (!domain->iommu_coherency)
672 		clflush_cache_range(addr, size);
673 }
674 
675 static void free_context_table(struct intel_iommu *iommu)
676 {
677 	struct context_entry *context;
678 	int i;
679 
680 	if (!iommu->root_entry)
681 		return;
682 
683 	for (i = 0; i < ROOT_ENTRY_NR; i++) {
684 		context = iommu_context_addr(iommu, i, 0, 0);
685 		if (context)
686 			iommu_free_page(context);
687 
688 		if (!sm_supported(iommu))
689 			continue;
690 
691 		context = iommu_context_addr(iommu, i, 0x80, 0);
692 		if (context)
693 			iommu_free_page(context);
694 	}
695 
696 	iommu_free_page(iommu->root_entry);
697 	iommu->root_entry = NULL;
698 }
699 
700 #ifdef CONFIG_DMAR_DEBUG
701 static void pgtable_walk(struct intel_iommu *iommu, unsigned long pfn,
702 			 u8 bus, u8 devfn, struct dma_pte *parent, int level)
703 {
704 	struct dma_pte *pte;
705 	int offset;
706 
707 	while (1) {
708 		offset = pfn_level_offset(pfn, level);
709 		pte = &parent[offset];
710 		if (!pte || (dma_pte_superpage(pte) || !dma_pte_present(pte))) {
711 			pr_info("PTE not present at level %d\n", level);
712 			break;
713 		}
714 
715 		pr_info("pte level: %d, pte value: 0x%016llx\n", level, pte->val);
716 
717 		if (level == 1)
718 			break;
719 
720 		parent = phys_to_virt(dma_pte_addr(pte));
721 		level--;
722 	}
723 }
724 
725 void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,
726 			  unsigned long long addr, u32 pasid)
727 {
728 	struct pasid_dir_entry *dir, *pde;
729 	struct pasid_entry *entries, *pte;
730 	struct context_entry *ctx_entry;
731 	struct root_entry *rt_entry;
732 	int i, dir_index, index, level;
733 	u8 devfn = source_id & 0xff;
734 	u8 bus = source_id >> 8;
735 	struct dma_pte *pgtable;
736 
737 	pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr);
738 
739 	/* root entry dump */
740 	rt_entry = &iommu->root_entry[bus];
741 	if (!rt_entry) {
742 		pr_info("root table entry is not present\n");
743 		return;
744 	}
745 
746 	if (sm_supported(iommu))
747 		pr_info("scalable mode root entry: hi 0x%016llx, low 0x%016llx\n",
748 			rt_entry->hi, rt_entry->lo);
749 	else
750 		pr_info("root entry: 0x%016llx", rt_entry->lo);
751 
752 	/* context entry dump */
753 	ctx_entry = iommu_context_addr(iommu, bus, devfn, 0);
754 	if (!ctx_entry) {
755 		pr_info("context table entry is not present\n");
756 		return;
757 	}
758 
759 	pr_info("context entry: hi 0x%016llx, low 0x%016llx\n",
760 		ctx_entry->hi, ctx_entry->lo);
761 
762 	/* legacy mode does not require PASID entries */
763 	if (!sm_supported(iommu)) {
764 		level = agaw_to_level(ctx_entry->hi & 7);
765 		pgtable = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
766 		goto pgtable_walk;
767 	}
768 
769 	/* get the pointer to pasid directory entry */
770 	dir = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
771 	if (!dir) {
772 		pr_info("pasid directory entry is not present\n");
773 		return;
774 	}
775 	/* For request-without-pasid, get the pasid from context entry */
776 	if (intel_iommu_sm && pasid == IOMMU_PASID_INVALID)
777 		pasid = IOMMU_NO_PASID;
778 
779 	dir_index = pasid >> PASID_PDE_SHIFT;
780 	pde = &dir[dir_index];
781 	pr_info("pasid dir entry: 0x%016llx\n", pde->val);
782 
783 	/* get the pointer to the pasid table entry */
784 	entries = get_pasid_table_from_pde(pde);
785 	if (!entries) {
786 		pr_info("pasid table entry is not present\n");
787 		return;
788 	}
789 	index = pasid & PASID_PTE_MASK;
790 	pte = &entries[index];
791 	for (i = 0; i < ARRAY_SIZE(pte->val); i++)
792 		pr_info("pasid table entry[%d]: 0x%016llx\n", i, pte->val[i]);
793 
794 	if (pasid_pte_get_pgtt(pte) == PASID_ENTRY_PGTT_FL_ONLY) {
795 		level = pte->val[2] & BIT_ULL(2) ? 5 : 4;
796 		pgtable = phys_to_virt(pte->val[2] & VTD_PAGE_MASK);
797 	} else {
798 		level = agaw_to_level((pte->val[0] >> 2) & 0x7);
799 		pgtable = phys_to_virt(pte->val[0] & VTD_PAGE_MASK);
800 	}
801 
802 pgtable_walk:
803 	pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn, pgtable, level);
804 }
805 #endif
806 
807 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
808 				      unsigned long pfn, int *target_level,
809 				      gfp_t gfp)
810 {
811 	struct dma_pte *parent, *pte;
812 	int level = agaw_to_level(domain->agaw);
813 	int offset;
814 
815 	if (!domain_pfn_supported(domain, pfn))
816 		/* Address beyond IOMMU's addressing capabilities. */
817 		return NULL;
818 
819 	parent = domain->pgd;
820 
821 	while (1) {
822 		void *tmp_page;
823 
824 		offset = pfn_level_offset(pfn, level);
825 		pte = &parent[offset];
826 		if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
827 			break;
828 		if (level == *target_level)
829 			break;
830 
831 		if (!dma_pte_present(pte)) {
832 			uint64_t pteval, tmp;
833 
834 			tmp_page = iommu_alloc_page_node(domain->nid, gfp);
835 
836 			if (!tmp_page)
837 				return NULL;
838 
839 			domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
840 			pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
841 			if (domain->use_first_level)
842 				pteval |= DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
843 
844 			tmp = 0ULL;
845 			if (!try_cmpxchg64(&pte->val, &tmp, pteval))
846 				/* Someone else set it while we were thinking; use theirs. */
847 				iommu_free_page(tmp_page);
848 			else
849 				domain_flush_cache(domain, pte, sizeof(*pte));
850 		}
851 		if (level == 1)
852 			break;
853 
854 		parent = phys_to_virt(dma_pte_addr(pte));
855 		level--;
856 	}
857 
858 	if (!*target_level)
859 		*target_level = level;
860 
861 	return pte;
862 }
863 
864 /* return address's pte at specific level */
865 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
866 					 unsigned long pfn,
867 					 int level, int *large_page)
868 {
869 	struct dma_pte *parent, *pte;
870 	int total = agaw_to_level(domain->agaw);
871 	int offset;
872 
873 	parent = domain->pgd;
874 	while (level <= total) {
875 		offset = pfn_level_offset(pfn, total);
876 		pte = &parent[offset];
877 		if (level == total)
878 			return pte;
879 
880 		if (!dma_pte_present(pte)) {
881 			*large_page = total;
882 			break;
883 		}
884 
885 		if (dma_pte_superpage(pte)) {
886 			*large_page = total;
887 			return pte;
888 		}
889 
890 		parent = phys_to_virt(dma_pte_addr(pte));
891 		total--;
892 	}
893 	return NULL;
894 }
895 
896 /* clear last level pte, a tlb flush should be followed */
897 static void dma_pte_clear_range(struct dmar_domain *domain,
898 				unsigned long start_pfn,
899 				unsigned long last_pfn)
900 {
901 	unsigned int large_page;
902 	struct dma_pte *first_pte, *pte;
903 
904 	if (WARN_ON(!domain_pfn_supported(domain, last_pfn)) ||
905 	    WARN_ON(start_pfn > last_pfn))
906 		return;
907 
908 	/* we don't need lock here; nobody else touches the iova range */
909 	do {
910 		large_page = 1;
911 		first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
912 		if (!pte) {
913 			start_pfn = align_to_level(start_pfn + 1, large_page + 1);
914 			continue;
915 		}
916 		do {
917 			dma_clear_pte(pte);
918 			start_pfn += lvl_to_nr_pages(large_page);
919 			pte++;
920 		} while (start_pfn <= last_pfn && !first_pte_in_page(pte));
921 
922 		domain_flush_cache(domain, first_pte,
923 				   (void *)pte - (void *)first_pte);
924 
925 	} while (start_pfn && start_pfn <= last_pfn);
926 }
927 
928 static void dma_pte_free_level(struct dmar_domain *domain, int level,
929 			       int retain_level, struct dma_pte *pte,
930 			       unsigned long pfn, unsigned long start_pfn,
931 			       unsigned long last_pfn)
932 {
933 	pfn = max(start_pfn, pfn);
934 	pte = &pte[pfn_level_offset(pfn, level)];
935 
936 	do {
937 		unsigned long level_pfn;
938 		struct dma_pte *level_pte;
939 
940 		if (!dma_pte_present(pte) || dma_pte_superpage(pte))
941 			goto next;
942 
943 		level_pfn = pfn & level_mask(level);
944 		level_pte = phys_to_virt(dma_pte_addr(pte));
945 
946 		if (level > 2) {
947 			dma_pte_free_level(domain, level - 1, retain_level,
948 					   level_pte, level_pfn, start_pfn,
949 					   last_pfn);
950 		}
951 
952 		/*
953 		 * Free the page table if we're below the level we want to
954 		 * retain and the range covers the entire table.
955 		 */
956 		if (level < retain_level && !(start_pfn > level_pfn ||
957 		      last_pfn < level_pfn + level_size(level) - 1)) {
958 			dma_clear_pte(pte);
959 			domain_flush_cache(domain, pte, sizeof(*pte));
960 			iommu_free_page(level_pte);
961 		}
962 next:
963 		pfn += level_size(level);
964 	} while (!first_pte_in_page(++pte) && pfn <= last_pfn);
965 }
966 
967 /*
968  * clear last level (leaf) ptes and free page table pages below the
969  * level we wish to keep intact.
970  */
971 static void dma_pte_free_pagetable(struct dmar_domain *domain,
972 				   unsigned long start_pfn,
973 				   unsigned long last_pfn,
974 				   int retain_level)
975 {
976 	dma_pte_clear_range(domain, start_pfn, last_pfn);
977 
978 	/* We don't need lock here; nobody else touches the iova range */
979 	dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
980 			   domain->pgd, 0, start_pfn, last_pfn);
981 
982 	/* free pgd */
983 	if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
984 		iommu_free_page(domain->pgd);
985 		domain->pgd = NULL;
986 	}
987 }
988 
989 /* When a page at a given level is being unlinked from its parent, we don't
990    need to *modify* it at all. All we need to do is make a list of all the
991    pages which can be freed just as soon as we've flushed the IOTLB and we
992    know the hardware page-walk will no longer touch them.
993    The 'pte' argument is the *parent* PTE, pointing to the page that is to
994    be freed. */
995 static void dma_pte_list_pagetables(struct dmar_domain *domain,
996 				    int level, struct dma_pte *pte,
997 				    struct list_head *freelist)
998 {
999 	struct page *pg;
1000 
1001 	pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1002 	list_add_tail(&pg->lru, freelist);
1003 
1004 	if (level == 1)
1005 		return;
1006 
1007 	pte = page_address(pg);
1008 	do {
1009 		if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1010 			dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1011 		pte++;
1012 	} while (!first_pte_in_page(pte));
1013 }
1014 
1015 static void dma_pte_clear_level(struct dmar_domain *domain, int level,
1016 				struct dma_pte *pte, unsigned long pfn,
1017 				unsigned long start_pfn, unsigned long last_pfn,
1018 				struct list_head *freelist)
1019 {
1020 	struct dma_pte *first_pte = NULL, *last_pte = NULL;
1021 
1022 	pfn = max(start_pfn, pfn);
1023 	pte = &pte[pfn_level_offset(pfn, level)];
1024 
1025 	do {
1026 		unsigned long level_pfn = pfn & level_mask(level);
1027 
1028 		if (!dma_pte_present(pte))
1029 			goto next;
1030 
1031 		/* If range covers entire pagetable, free it */
1032 		if (start_pfn <= level_pfn &&
1033 		    last_pfn >= level_pfn + level_size(level) - 1) {
1034 			/* These suborbinate page tables are going away entirely. Don't
1035 			   bother to clear them; we're just going to *free* them. */
1036 			if (level > 1 && !dma_pte_superpage(pte))
1037 				dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1038 
1039 			dma_clear_pte(pte);
1040 			if (!first_pte)
1041 				first_pte = pte;
1042 			last_pte = pte;
1043 		} else if (level > 1) {
1044 			/* Recurse down into a level that isn't *entirely* obsolete */
1045 			dma_pte_clear_level(domain, level - 1,
1046 					    phys_to_virt(dma_pte_addr(pte)),
1047 					    level_pfn, start_pfn, last_pfn,
1048 					    freelist);
1049 		}
1050 next:
1051 		pfn = level_pfn + level_size(level);
1052 	} while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1053 
1054 	if (first_pte)
1055 		domain_flush_cache(domain, first_pte,
1056 				   (void *)++last_pte - (void *)first_pte);
1057 }
1058 
1059 /* We can't just free the pages because the IOMMU may still be walking
1060    the page tables, and may have cached the intermediate levels. The
1061    pages can only be freed after the IOTLB flush has been done. */
1062 static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn,
1063 			 unsigned long last_pfn, struct list_head *freelist)
1064 {
1065 	if (WARN_ON(!domain_pfn_supported(domain, last_pfn)) ||
1066 	    WARN_ON(start_pfn > last_pfn))
1067 		return;
1068 
1069 	/* we don't need lock here; nobody else touches the iova range */
1070 	dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1071 			    domain->pgd, 0, start_pfn, last_pfn, freelist);
1072 
1073 	/* free pgd */
1074 	if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1075 		struct page *pgd_page = virt_to_page(domain->pgd);
1076 		list_add_tail(&pgd_page->lru, freelist);
1077 		domain->pgd = NULL;
1078 	}
1079 }
1080 
1081 /* iommu handling */
1082 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1083 {
1084 	struct root_entry *root;
1085 
1086 	root = iommu_alloc_page_node(iommu->node, GFP_ATOMIC);
1087 	if (!root) {
1088 		pr_err("Allocating root entry for %s failed\n",
1089 			iommu->name);
1090 		return -ENOMEM;
1091 	}
1092 
1093 	__iommu_flush_cache(iommu, root, ROOT_SIZE);
1094 	iommu->root_entry = root;
1095 
1096 	return 0;
1097 }
1098 
1099 static void iommu_set_root_entry(struct intel_iommu *iommu)
1100 {
1101 	u64 addr;
1102 	u32 sts;
1103 	unsigned long flag;
1104 
1105 	addr = virt_to_phys(iommu->root_entry);
1106 	if (sm_supported(iommu))
1107 		addr |= DMA_RTADDR_SMT;
1108 
1109 	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1110 	dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1111 
1112 	writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1113 
1114 	/* Make sure hardware complete it */
1115 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1116 		      readl, (sts & DMA_GSTS_RTPS), sts);
1117 
1118 	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1119 
1120 	/*
1121 	 * Hardware invalidates all DMA remapping hardware translation
1122 	 * caches as part of SRTP flow.
1123 	 */
1124 	if (cap_esrtps(iommu->cap))
1125 		return;
1126 
1127 	iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1128 	if (sm_supported(iommu))
1129 		qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0);
1130 	iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1131 }
1132 
1133 void iommu_flush_write_buffer(struct intel_iommu *iommu)
1134 {
1135 	u32 val;
1136 	unsigned long flag;
1137 
1138 	if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1139 		return;
1140 
1141 	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1142 	writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1143 
1144 	/* Make sure hardware complete it */
1145 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1146 		      readl, (!(val & DMA_GSTS_WBFS)), val);
1147 
1148 	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1149 }
1150 
1151 /* return value determine if we need a write buffer flush */
1152 static void __iommu_flush_context(struct intel_iommu *iommu,
1153 				  u16 did, u16 source_id, u8 function_mask,
1154 				  u64 type)
1155 {
1156 	u64 val = 0;
1157 	unsigned long flag;
1158 
1159 	switch (type) {
1160 	case DMA_CCMD_GLOBAL_INVL:
1161 		val = DMA_CCMD_GLOBAL_INVL;
1162 		break;
1163 	case DMA_CCMD_DOMAIN_INVL:
1164 		val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1165 		break;
1166 	case DMA_CCMD_DEVICE_INVL:
1167 		val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1168 			| DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1169 		break;
1170 	default:
1171 		pr_warn("%s: Unexpected context-cache invalidation type 0x%llx\n",
1172 			iommu->name, type);
1173 		return;
1174 	}
1175 	val |= DMA_CCMD_ICC;
1176 
1177 	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1178 	dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1179 
1180 	/* Make sure hardware complete it */
1181 	IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1182 		dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1183 
1184 	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1185 }
1186 
1187 void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1188 			 unsigned int size_order, u64 type)
1189 {
1190 	int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1191 	u64 val = 0, val_iva = 0;
1192 	unsigned long flag;
1193 
1194 	switch (type) {
1195 	case DMA_TLB_GLOBAL_FLUSH:
1196 		/* global flush doesn't need set IVA_REG */
1197 		val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1198 		break;
1199 	case DMA_TLB_DSI_FLUSH:
1200 		val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1201 		break;
1202 	case DMA_TLB_PSI_FLUSH:
1203 		val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1204 		/* IH bit is passed in as part of address */
1205 		val_iva = size_order | addr;
1206 		break;
1207 	default:
1208 		pr_warn("%s: Unexpected iotlb invalidation type 0x%llx\n",
1209 			iommu->name, type);
1210 		return;
1211 	}
1212 
1213 	if (cap_write_drain(iommu->cap))
1214 		val |= DMA_TLB_WRITE_DRAIN;
1215 
1216 	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1217 	/* Note: Only uses first TLB reg currently */
1218 	if (val_iva)
1219 		dmar_writeq(iommu->reg + tlb_offset, val_iva);
1220 	dmar_writeq(iommu->reg + tlb_offset + 8, val);
1221 
1222 	/* Make sure hardware complete it */
1223 	IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1224 		dmar_readq, (!(val & DMA_TLB_IVT)), val);
1225 
1226 	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1227 
1228 	/* check IOTLB invalidation granularity */
1229 	if (DMA_TLB_IAIG(val) == 0)
1230 		pr_err("Flush IOTLB failed\n");
1231 	if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1232 		pr_debug("TLB flush request %Lx, actual %Lx\n",
1233 			(unsigned long long)DMA_TLB_IIRG(type),
1234 			(unsigned long long)DMA_TLB_IAIG(val));
1235 }
1236 
1237 static struct device_domain_info *
1238 domain_lookup_dev_info(struct dmar_domain *domain,
1239 		       struct intel_iommu *iommu, u8 bus, u8 devfn)
1240 {
1241 	struct device_domain_info *info;
1242 	unsigned long flags;
1243 
1244 	spin_lock_irqsave(&domain->lock, flags);
1245 	list_for_each_entry(info, &domain->devices, link) {
1246 		if (info->iommu == iommu && info->bus == bus &&
1247 		    info->devfn == devfn) {
1248 			spin_unlock_irqrestore(&domain->lock, flags);
1249 			return info;
1250 		}
1251 	}
1252 	spin_unlock_irqrestore(&domain->lock, flags);
1253 
1254 	return NULL;
1255 }
1256 
1257 /*
1258  * The extra devTLB flush quirk impacts those QAT devices with PCI device
1259  * IDs ranging from 0x4940 to 0x4943. It is exempted from risky_device()
1260  * check because it applies only to the built-in QAT devices and it doesn't
1261  * grant additional privileges.
1262  */
1263 #define BUGGY_QAT_DEVID_MASK 0x4940
1264 static bool dev_needs_extra_dtlb_flush(struct pci_dev *pdev)
1265 {
1266 	if (pdev->vendor != PCI_VENDOR_ID_INTEL)
1267 		return false;
1268 
1269 	if ((pdev->device & 0xfffc) != BUGGY_QAT_DEVID_MASK)
1270 		return false;
1271 
1272 	return true;
1273 }
1274 
1275 static void iommu_enable_pci_caps(struct device_domain_info *info)
1276 {
1277 	struct pci_dev *pdev;
1278 
1279 	if (!dev_is_pci(info->dev))
1280 		return;
1281 
1282 	pdev = to_pci_dev(info->dev);
1283 	if (info->ats_supported && pci_ats_page_aligned(pdev) &&
1284 	    !pci_enable_ats(pdev, VTD_PAGE_SHIFT))
1285 		info->ats_enabled = 1;
1286 }
1287 
1288 static void iommu_disable_pci_caps(struct device_domain_info *info)
1289 {
1290 	struct pci_dev *pdev;
1291 
1292 	if (!dev_is_pci(info->dev))
1293 		return;
1294 
1295 	pdev = to_pci_dev(info->dev);
1296 
1297 	if (info->ats_enabled) {
1298 		pci_disable_ats(pdev);
1299 		info->ats_enabled = 0;
1300 	}
1301 }
1302 
1303 static void intel_flush_iotlb_all(struct iommu_domain *domain)
1304 {
1305 	cache_tag_flush_all(to_dmar_domain(domain));
1306 }
1307 
1308 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1309 {
1310 	u32 pmen;
1311 	unsigned long flags;
1312 
1313 	if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1314 		return;
1315 
1316 	raw_spin_lock_irqsave(&iommu->register_lock, flags);
1317 	pmen = readl(iommu->reg + DMAR_PMEN_REG);
1318 	pmen &= ~DMA_PMEN_EPM;
1319 	writel(pmen, iommu->reg + DMAR_PMEN_REG);
1320 
1321 	/* wait for the protected region status bit to clear */
1322 	IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1323 		readl, !(pmen & DMA_PMEN_PRS), pmen);
1324 
1325 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1326 }
1327 
1328 static void iommu_enable_translation(struct intel_iommu *iommu)
1329 {
1330 	u32 sts;
1331 	unsigned long flags;
1332 
1333 	raw_spin_lock_irqsave(&iommu->register_lock, flags);
1334 	iommu->gcmd |= DMA_GCMD_TE;
1335 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1336 
1337 	/* Make sure hardware complete it */
1338 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1339 		      readl, (sts & DMA_GSTS_TES), sts);
1340 
1341 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1342 }
1343 
1344 static void iommu_disable_translation(struct intel_iommu *iommu)
1345 {
1346 	u32 sts;
1347 	unsigned long flag;
1348 
1349 	if (iommu_skip_te_disable && iommu->drhd->gfx_dedicated &&
1350 	    (cap_read_drain(iommu->cap) || cap_write_drain(iommu->cap)))
1351 		return;
1352 
1353 	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1354 	iommu->gcmd &= ~DMA_GCMD_TE;
1355 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1356 
1357 	/* Make sure hardware complete it */
1358 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1359 		      readl, (!(sts & DMA_GSTS_TES)), sts);
1360 
1361 	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1362 }
1363 
1364 static int iommu_init_domains(struct intel_iommu *iommu)
1365 {
1366 	u32 ndomains;
1367 
1368 	ndomains = cap_ndoms(iommu->cap);
1369 	pr_debug("%s: Number of Domains supported <%d>\n",
1370 		 iommu->name, ndomains);
1371 
1372 	spin_lock_init(&iommu->lock);
1373 
1374 	iommu->domain_ids = bitmap_zalloc(ndomains, GFP_KERNEL);
1375 	if (!iommu->domain_ids)
1376 		return -ENOMEM;
1377 
1378 	/*
1379 	 * If Caching mode is set, then invalid translations are tagged
1380 	 * with domain-id 0, hence we need to pre-allocate it. We also
1381 	 * use domain-id 0 as a marker for non-allocated domain-id, so
1382 	 * make sure it is not used for a real domain.
1383 	 */
1384 	set_bit(0, iommu->domain_ids);
1385 
1386 	/*
1387 	 * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
1388 	 * entry for first-level or pass-through translation modes should
1389 	 * be programmed with a domain id different from those used for
1390 	 * second-level or nested translation. We reserve a domain id for
1391 	 * this purpose. This domain id is also used for identity domain
1392 	 * in legacy mode.
1393 	 */
1394 	set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
1395 
1396 	return 0;
1397 }
1398 
1399 static void disable_dmar_iommu(struct intel_iommu *iommu)
1400 {
1401 	if (!iommu->domain_ids)
1402 		return;
1403 
1404 	/*
1405 	 * All iommu domains must have been detached from the devices,
1406 	 * hence there should be no domain IDs in use.
1407 	 */
1408 	if (WARN_ON(bitmap_weight(iommu->domain_ids, cap_ndoms(iommu->cap))
1409 		    > NUM_RESERVED_DID))
1410 		return;
1411 
1412 	if (iommu->gcmd & DMA_GCMD_TE)
1413 		iommu_disable_translation(iommu);
1414 }
1415 
1416 static void free_dmar_iommu(struct intel_iommu *iommu)
1417 {
1418 	if (iommu->domain_ids) {
1419 		bitmap_free(iommu->domain_ids);
1420 		iommu->domain_ids = NULL;
1421 	}
1422 
1423 	if (iommu->copied_tables) {
1424 		bitmap_free(iommu->copied_tables);
1425 		iommu->copied_tables = NULL;
1426 	}
1427 
1428 	/* free context mapping */
1429 	free_context_table(iommu);
1430 
1431 #ifdef CONFIG_INTEL_IOMMU_SVM
1432 	if (pasid_supported(iommu)) {
1433 		if (ecap_prs(iommu->ecap))
1434 			intel_svm_finish_prq(iommu);
1435 	}
1436 #endif
1437 }
1438 
1439 /*
1440  * Check and return whether first level is used by default for
1441  * DMA translation.
1442  */
1443 static bool first_level_by_default(unsigned int type)
1444 {
1445 	/* Only SL is available in legacy mode */
1446 	if (!scalable_mode_support())
1447 		return false;
1448 
1449 	/* Only level (either FL or SL) is available, just use it */
1450 	if (intel_cap_flts_sanity() ^ intel_cap_slts_sanity())
1451 		return intel_cap_flts_sanity();
1452 
1453 	/* Both levels are available, decide it based on domain type */
1454 	return type != IOMMU_DOMAIN_UNMANAGED;
1455 }
1456 
1457 static struct dmar_domain *alloc_domain(unsigned int type)
1458 {
1459 	struct dmar_domain *domain;
1460 
1461 	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
1462 	if (!domain)
1463 		return NULL;
1464 
1465 	domain->nid = NUMA_NO_NODE;
1466 	if (first_level_by_default(type))
1467 		domain->use_first_level = true;
1468 	INIT_LIST_HEAD(&domain->devices);
1469 	INIT_LIST_HEAD(&domain->dev_pasids);
1470 	INIT_LIST_HEAD(&domain->cache_tags);
1471 	spin_lock_init(&domain->lock);
1472 	spin_lock_init(&domain->cache_lock);
1473 	xa_init(&domain->iommu_array);
1474 
1475 	return domain;
1476 }
1477 
1478 int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)
1479 {
1480 	struct iommu_domain_info *info, *curr;
1481 	unsigned long ndomains;
1482 	int num, ret = -ENOSPC;
1483 
1484 	if (domain->domain.type == IOMMU_DOMAIN_SVA)
1485 		return 0;
1486 
1487 	info = kzalloc(sizeof(*info), GFP_KERNEL);
1488 	if (!info)
1489 		return -ENOMEM;
1490 
1491 	spin_lock(&iommu->lock);
1492 	curr = xa_load(&domain->iommu_array, iommu->seq_id);
1493 	if (curr) {
1494 		curr->refcnt++;
1495 		spin_unlock(&iommu->lock);
1496 		kfree(info);
1497 		return 0;
1498 	}
1499 
1500 	ndomains = cap_ndoms(iommu->cap);
1501 	num = find_first_zero_bit(iommu->domain_ids, ndomains);
1502 	if (num >= ndomains) {
1503 		pr_err("%s: No free domain ids\n", iommu->name);
1504 		goto err_unlock;
1505 	}
1506 
1507 	set_bit(num, iommu->domain_ids);
1508 	info->refcnt	= 1;
1509 	info->did	= num;
1510 	info->iommu	= iommu;
1511 	curr = xa_cmpxchg(&domain->iommu_array, iommu->seq_id,
1512 			  NULL, info, GFP_ATOMIC);
1513 	if (curr) {
1514 		ret = xa_err(curr) ? : -EBUSY;
1515 		goto err_clear;
1516 	}
1517 	domain_update_iommu_cap(domain);
1518 
1519 	spin_unlock(&iommu->lock);
1520 	return 0;
1521 
1522 err_clear:
1523 	clear_bit(info->did, iommu->domain_ids);
1524 err_unlock:
1525 	spin_unlock(&iommu->lock);
1526 	kfree(info);
1527 	return ret;
1528 }
1529 
1530 void domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)
1531 {
1532 	struct iommu_domain_info *info;
1533 
1534 	if (domain->domain.type == IOMMU_DOMAIN_SVA)
1535 		return;
1536 
1537 	spin_lock(&iommu->lock);
1538 	info = xa_load(&domain->iommu_array, iommu->seq_id);
1539 	if (--info->refcnt == 0) {
1540 		clear_bit(info->did, iommu->domain_ids);
1541 		xa_erase(&domain->iommu_array, iommu->seq_id);
1542 		domain->nid = NUMA_NO_NODE;
1543 		domain_update_iommu_cap(domain);
1544 		kfree(info);
1545 	}
1546 	spin_unlock(&iommu->lock);
1547 }
1548 
1549 static int guestwidth_to_adjustwidth(int gaw)
1550 {
1551 	int agaw;
1552 	int r = (gaw - 12) % 9;
1553 
1554 	if (r == 0)
1555 		agaw = gaw;
1556 	else
1557 		agaw = gaw + 9 - r;
1558 	if (agaw > 64)
1559 		agaw = 64;
1560 	return agaw;
1561 }
1562 
1563 static void domain_exit(struct dmar_domain *domain)
1564 {
1565 	if (domain->pgd) {
1566 		LIST_HEAD(freelist);
1567 
1568 		domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist);
1569 		iommu_put_pages_list(&freelist);
1570 	}
1571 
1572 	if (WARN_ON(!list_empty(&domain->devices)))
1573 		return;
1574 
1575 	kfree(domain->qi_batch);
1576 	kfree(domain);
1577 }
1578 
1579 /*
1580  * For kdump cases, old valid entries may be cached due to the
1581  * in-flight DMA and copied pgtable, but there is no unmapping
1582  * behaviour for them, thus we need an explicit cache flush for
1583  * the newly-mapped device. For kdump, at this point, the device
1584  * is supposed to finish reset at its driver probe stage, so no
1585  * in-flight DMA will exist, and we don't need to worry anymore
1586  * hereafter.
1587  */
1588 static void copied_context_tear_down(struct intel_iommu *iommu,
1589 				     struct context_entry *context,
1590 				     u8 bus, u8 devfn)
1591 {
1592 	u16 did_old;
1593 
1594 	if (!context_copied(iommu, bus, devfn))
1595 		return;
1596 
1597 	assert_spin_locked(&iommu->lock);
1598 
1599 	did_old = context_domain_id(context);
1600 	context_clear_entry(context);
1601 
1602 	if (did_old < cap_ndoms(iommu->cap)) {
1603 		iommu->flush.flush_context(iommu, did_old,
1604 					   (((u16)bus) << 8) | devfn,
1605 					   DMA_CCMD_MASK_NOBIT,
1606 					   DMA_CCMD_DEVICE_INVL);
1607 		iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
1608 					 DMA_TLB_DSI_FLUSH);
1609 	}
1610 
1611 	clear_context_copied(iommu, bus, devfn);
1612 }
1613 
1614 /*
1615  * It's a non-present to present mapping. If hardware doesn't cache
1616  * non-present entry we only need to flush the write-buffer. If the
1617  * _does_ cache non-present entries, then it does so in the special
1618  * domain #0, which we have to flush:
1619  */
1620 static void context_present_cache_flush(struct intel_iommu *iommu, u16 did,
1621 					u8 bus, u8 devfn)
1622 {
1623 	if (cap_caching_mode(iommu->cap)) {
1624 		iommu->flush.flush_context(iommu, 0,
1625 					   (((u16)bus) << 8) | devfn,
1626 					   DMA_CCMD_MASK_NOBIT,
1627 					   DMA_CCMD_DEVICE_INVL);
1628 		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1629 	} else {
1630 		iommu_flush_write_buffer(iommu);
1631 	}
1632 }
1633 
1634 static int domain_context_mapping_one(struct dmar_domain *domain,
1635 				      struct intel_iommu *iommu,
1636 				      u8 bus, u8 devfn)
1637 {
1638 	struct device_domain_info *info =
1639 			domain_lookup_dev_info(domain, iommu, bus, devfn);
1640 	u16 did = domain_id_iommu(domain, iommu);
1641 	int translation = CONTEXT_TT_MULTI_LEVEL;
1642 	struct dma_pte *pgd = domain->pgd;
1643 	struct context_entry *context;
1644 	int agaw, ret;
1645 
1646 	pr_debug("Set context mapping for %02x:%02x.%d\n",
1647 		bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1648 
1649 	spin_lock(&iommu->lock);
1650 	ret = -ENOMEM;
1651 	context = iommu_context_addr(iommu, bus, devfn, 1);
1652 	if (!context)
1653 		goto out_unlock;
1654 
1655 	ret = 0;
1656 	if (context_present(context) && !context_copied(iommu, bus, devfn))
1657 		goto out_unlock;
1658 
1659 	copied_context_tear_down(iommu, context, bus, devfn);
1660 	context_clear_entry(context);
1661 
1662 	context_set_domain_id(context, did);
1663 
1664 	/*
1665 	 * Skip top levels of page tables for iommu which has
1666 	 * less agaw than default. Unnecessary for PT mode.
1667 	 */
1668 	for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
1669 		ret = -ENOMEM;
1670 		pgd = phys_to_virt(dma_pte_addr(pgd));
1671 		if (!dma_pte_present(pgd))
1672 			goto out_unlock;
1673 	}
1674 
1675 	if (info && info->ats_supported)
1676 		translation = CONTEXT_TT_DEV_IOTLB;
1677 	else
1678 		translation = CONTEXT_TT_MULTI_LEVEL;
1679 
1680 	context_set_address_root(context, virt_to_phys(pgd));
1681 	context_set_address_width(context, agaw);
1682 	context_set_translation_type(context, translation);
1683 	context_set_fault_enable(context);
1684 	context_set_present(context);
1685 	if (!ecap_coherent(iommu->ecap))
1686 		clflush_cache_range(context, sizeof(*context));
1687 	context_present_cache_flush(iommu, did, bus, devfn);
1688 	ret = 0;
1689 
1690 out_unlock:
1691 	spin_unlock(&iommu->lock);
1692 
1693 	return ret;
1694 }
1695 
1696 static int domain_context_mapping_cb(struct pci_dev *pdev,
1697 				     u16 alias, void *opaque)
1698 {
1699 	struct device_domain_info *info = dev_iommu_priv_get(&pdev->dev);
1700 	struct intel_iommu *iommu = info->iommu;
1701 	struct dmar_domain *domain = opaque;
1702 
1703 	return domain_context_mapping_one(domain, iommu,
1704 					  PCI_BUS_NUM(alias), alias & 0xff);
1705 }
1706 
1707 static int
1708 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
1709 {
1710 	struct device_domain_info *info = dev_iommu_priv_get(dev);
1711 	struct intel_iommu *iommu = info->iommu;
1712 	u8 bus = info->bus, devfn = info->devfn;
1713 
1714 	if (!dev_is_pci(dev))
1715 		return domain_context_mapping_one(domain, iommu, bus, devfn);
1716 
1717 	return pci_for_each_dma_alias(to_pci_dev(dev),
1718 				      domain_context_mapping_cb, domain);
1719 }
1720 
1721 /* Return largest possible superpage level for a given mapping */
1722 static int hardware_largepage_caps(struct dmar_domain *domain, unsigned long iov_pfn,
1723 				   unsigned long phy_pfn, unsigned long pages)
1724 {
1725 	int support, level = 1;
1726 	unsigned long pfnmerge;
1727 
1728 	support = domain->iommu_superpage;
1729 
1730 	/* To use a large page, the virtual *and* physical addresses
1731 	   must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1732 	   of them will mean we have to use smaller pages. So just
1733 	   merge them and check both at once. */
1734 	pfnmerge = iov_pfn | phy_pfn;
1735 
1736 	while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1737 		pages >>= VTD_STRIDE_SHIFT;
1738 		if (!pages)
1739 			break;
1740 		pfnmerge >>= VTD_STRIDE_SHIFT;
1741 		level++;
1742 		support--;
1743 	}
1744 	return level;
1745 }
1746 
1747 /*
1748  * Ensure that old small page tables are removed to make room for superpage(s).
1749  * We're going to add new large pages, so make sure we don't remove their parent
1750  * tables. The IOTLB/devTLBs should be flushed if any PDE/PTEs are cleared.
1751  */
1752 static void switch_to_super_page(struct dmar_domain *domain,
1753 				 unsigned long start_pfn,
1754 				 unsigned long end_pfn, int level)
1755 {
1756 	unsigned long lvl_pages = lvl_to_nr_pages(level);
1757 	struct dma_pte *pte = NULL;
1758 
1759 	while (start_pfn <= end_pfn) {
1760 		if (!pte)
1761 			pte = pfn_to_dma_pte(domain, start_pfn, &level,
1762 					     GFP_ATOMIC);
1763 
1764 		if (dma_pte_present(pte)) {
1765 			dma_pte_free_pagetable(domain, start_pfn,
1766 					       start_pfn + lvl_pages - 1,
1767 					       level + 1);
1768 
1769 			cache_tag_flush_range(domain, start_pfn << VTD_PAGE_SHIFT,
1770 					      end_pfn << VTD_PAGE_SHIFT, 0);
1771 		}
1772 
1773 		pte++;
1774 		start_pfn += lvl_pages;
1775 		if (first_pte_in_page(pte))
1776 			pte = NULL;
1777 	}
1778 }
1779 
1780 static int
1781 __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1782 		 unsigned long phys_pfn, unsigned long nr_pages, int prot,
1783 		 gfp_t gfp)
1784 {
1785 	struct dma_pte *first_pte = NULL, *pte = NULL;
1786 	unsigned int largepage_lvl = 0;
1787 	unsigned long lvl_pages = 0;
1788 	phys_addr_t pteval;
1789 	u64 attr;
1790 
1791 	if (unlikely(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1)))
1792 		return -EINVAL;
1793 
1794 	if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1795 		return -EINVAL;
1796 
1797 	if (!(prot & DMA_PTE_WRITE) && domain->nested_parent) {
1798 		pr_err_ratelimited("Read-only mapping is disallowed on the domain which serves as the parent in a nested configuration, due to HW errata (ERRATA_772415_SPR17)\n");
1799 		return -EINVAL;
1800 	}
1801 
1802 	attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP);
1803 	attr |= DMA_FL_PTE_PRESENT;
1804 	if (domain->use_first_level) {
1805 		attr |= DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
1806 		if (prot & DMA_PTE_WRITE)
1807 			attr |= DMA_FL_PTE_DIRTY;
1808 	}
1809 
1810 	domain->has_mappings = true;
1811 
1812 	pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | attr;
1813 
1814 	while (nr_pages > 0) {
1815 		uint64_t tmp;
1816 
1817 		if (!pte) {
1818 			largepage_lvl = hardware_largepage_caps(domain, iov_pfn,
1819 					phys_pfn, nr_pages);
1820 
1821 			pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl,
1822 					     gfp);
1823 			if (!pte)
1824 				return -ENOMEM;
1825 			first_pte = pte;
1826 
1827 			lvl_pages = lvl_to_nr_pages(largepage_lvl);
1828 
1829 			/* It is large page*/
1830 			if (largepage_lvl > 1) {
1831 				unsigned long end_pfn;
1832 				unsigned long pages_to_remove;
1833 
1834 				pteval |= DMA_PTE_LARGE_PAGE;
1835 				pages_to_remove = min_t(unsigned long, nr_pages,
1836 							nr_pte_to_next_page(pte) * lvl_pages);
1837 				end_pfn = iov_pfn + pages_to_remove - 1;
1838 				switch_to_super_page(domain, iov_pfn, end_pfn, largepage_lvl);
1839 			} else {
1840 				pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
1841 			}
1842 
1843 		}
1844 		/* We don't need lock here, nobody else
1845 		 * touches the iova range
1846 		 */
1847 		tmp = 0ULL;
1848 		if (!try_cmpxchg64_local(&pte->val, &tmp, pteval)) {
1849 			static int dumps = 5;
1850 			pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1851 				iov_pfn, tmp, (unsigned long long)pteval);
1852 			if (dumps) {
1853 				dumps--;
1854 				debug_dma_dump_mappings(NULL);
1855 			}
1856 			WARN_ON(1);
1857 		}
1858 
1859 		nr_pages -= lvl_pages;
1860 		iov_pfn += lvl_pages;
1861 		phys_pfn += lvl_pages;
1862 		pteval += lvl_pages * VTD_PAGE_SIZE;
1863 
1864 		/* If the next PTE would be the first in a new page, then we
1865 		 * need to flush the cache on the entries we've just written.
1866 		 * And then we'll need to recalculate 'pte', so clear it and
1867 		 * let it get set again in the if (!pte) block above.
1868 		 *
1869 		 * If we're done (!nr_pages) we need to flush the cache too.
1870 		 *
1871 		 * Also if we've been setting superpages, we may need to
1872 		 * recalculate 'pte' and switch back to smaller pages for the
1873 		 * end of the mapping, if the trailing size is not enough to
1874 		 * use another superpage (i.e. nr_pages < lvl_pages).
1875 		 */
1876 		pte++;
1877 		if (!nr_pages || first_pte_in_page(pte) ||
1878 		    (largepage_lvl > 1 && nr_pages < lvl_pages)) {
1879 			domain_flush_cache(domain, first_pte,
1880 					   (void *)pte - (void *)first_pte);
1881 			pte = NULL;
1882 		}
1883 	}
1884 
1885 	return 0;
1886 }
1887 
1888 static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 devfn)
1889 {
1890 	struct intel_iommu *iommu = info->iommu;
1891 	struct context_entry *context;
1892 	u16 did;
1893 
1894 	spin_lock(&iommu->lock);
1895 	context = iommu_context_addr(iommu, bus, devfn, 0);
1896 	if (!context) {
1897 		spin_unlock(&iommu->lock);
1898 		return;
1899 	}
1900 
1901 	did = context_domain_id(context);
1902 	context_clear_entry(context);
1903 	__iommu_flush_cache(iommu, context, sizeof(*context));
1904 	spin_unlock(&iommu->lock);
1905 	intel_context_flush_present(info, context, did, true);
1906 }
1907 
1908 static int domain_setup_first_level(struct intel_iommu *iommu,
1909 				    struct dmar_domain *domain,
1910 				    struct device *dev,
1911 				    u32 pasid)
1912 {
1913 	struct dma_pte *pgd = domain->pgd;
1914 	int agaw, level;
1915 	int flags = 0;
1916 
1917 	/*
1918 	 * Skip top levels of page tables for iommu which has
1919 	 * less agaw than default. Unnecessary for PT mode.
1920 	 */
1921 	for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
1922 		pgd = phys_to_virt(dma_pte_addr(pgd));
1923 		if (!dma_pte_present(pgd))
1924 			return -ENOMEM;
1925 	}
1926 
1927 	level = agaw_to_level(agaw);
1928 	if (level != 4 && level != 5)
1929 		return -EINVAL;
1930 
1931 	if (level == 5)
1932 		flags |= PASID_FLAG_FL5LP;
1933 
1934 	if (domain->force_snooping)
1935 		flags |= PASID_FLAG_PAGE_SNOOP;
1936 
1937 	return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid,
1938 					     domain_id_iommu(domain, iommu),
1939 					     flags);
1940 }
1941 
1942 static bool dev_is_real_dma_subdevice(struct device *dev)
1943 {
1944 	return dev && dev_is_pci(dev) &&
1945 	       pci_real_dma_dev(to_pci_dev(dev)) != to_pci_dev(dev);
1946 }
1947 
1948 static int dmar_domain_attach_device(struct dmar_domain *domain,
1949 				     struct device *dev)
1950 {
1951 	struct device_domain_info *info = dev_iommu_priv_get(dev);
1952 	struct intel_iommu *iommu = info->iommu;
1953 	unsigned long flags;
1954 	int ret;
1955 
1956 	ret = domain_attach_iommu(domain, iommu);
1957 	if (ret)
1958 		return ret;
1959 
1960 	info->domain = domain;
1961 	spin_lock_irqsave(&domain->lock, flags);
1962 	list_add(&info->link, &domain->devices);
1963 	spin_unlock_irqrestore(&domain->lock, flags);
1964 
1965 	if (dev_is_real_dma_subdevice(dev))
1966 		return 0;
1967 
1968 	if (!sm_supported(iommu))
1969 		ret = domain_context_mapping(domain, dev);
1970 	else if (domain->use_first_level)
1971 		ret = domain_setup_first_level(iommu, domain, dev, IOMMU_NO_PASID);
1972 	else
1973 		ret = intel_pasid_setup_second_level(iommu, domain, dev, IOMMU_NO_PASID);
1974 
1975 	if (ret)
1976 		goto out_block_translation;
1977 
1978 	iommu_enable_pci_caps(info);
1979 
1980 	ret = cache_tag_assign_domain(domain, dev, IOMMU_NO_PASID);
1981 	if (ret)
1982 		goto out_block_translation;
1983 
1984 	return 0;
1985 
1986 out_block_translation:
1987 	device_block_translation(dev);
1988 	return ret;
1989 }
1990 
1991 /**
1992  * device_rmrr_is_relaxable - Test whether the RMRR of this device
1993  * is relaxable (ie. is allowed to be not enforced under some conditions)
1994  * @dev: device handle
1995  *
1996  * We assume that PCI USB devices with RMRRs have them largely
1997  * for historical reasons and that the RMRR space is not actively used post
1998  * boot.  This exclusion may change if vendors begin to abuse it.
1999  *
2000  * The same exception is made for graphics devices, with the requirement that
2001  * any use of the RMRR regions will be torn down before assigning the device
2002  * to a guest.
2003  *
2004  * Return: true if the RMRR is relaxable, false otherwise
2005  */
2006 static bool device_rmrr_is_relaxable(struct device *dev)
2007 {
2008 	struct pci_dev *pdev;
2009 
2010 	if (!dev_is_pci(dev))
2011 		return false;
2012 
2013 	pdev = to_pci_dev(dev);
2014 	if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2015 		return true;
2016 	else
2017 		return false;
2018 }
2019 
2020 static int device_def_domain_type(struct device *dev)
2021 {
2022 	struct device_domain_info *info = dev_iommu_priv_get(dev);
2023 	struct intel_iommu *iommu = info->iommu;
2024 
2025 	/*
2026 	 * Hardware does not support the passthrough translation mode.
2027 	 * Always use a dynamaic mapping domain.
2028 	 */
2029 	if (!ecap_pass_through(iommu->ecap))
2030 		return IOMMU_DOMAIN_DMA;
2031 
2032 	if (dev_is_pci(dev)) {
2033 		struct pci_dev *pdev = to_pci_dev(dev);
2034 
2035 		if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2036 			return IOMMU_DOMAIN_IDENTITY;
2037 	}
2038 
2039 	return 0;
2040 }
2041 
2042 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2043 {
2044 	/*
2045 	 * Start from the sane iommu hardware state.
2046 	 * If the queued invalidation is already initialized by us
2047 	 * (for example, while enabling interrupt-remapping) then
2048 	 * we got the things already rolling from a sane state.
2049 	 */
2050 	if (!iommu->qi) {
2051 		/*
2052 		 * Clear any previous faults.
2053 		 */
2054 		dmar_fault(-1, iommu);
2055 		/*
2056 		 * Disable queued invalidation if supported and already enabled
2057 		 * before OS handover.
2058 		 */
2059 		dmar_disable_qi(iommu);
2060 	}
2061 
2062 	if (dmar_enable_qi(iommu)) {
2063 		/*
2064 		 * Queued Invalidate not enabled, use Register Based Invalidate
2065 		 */
2066 		iommu->flush.flush_context = __iommu_flush_context;
2067 		iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2068 		pr_info("%s: Using Register based invalidation\n",
2069 			iommu->name);
2070 	} else {
2071 		iommu->flush.flush_context = qi_flush_context;
2072 		iommu->flush.flush_iotlb = qi_flush_iotlb;
2073 		pr_info("%s: Using Queued invalidation\n", iommu->name);
2074 	}
2075 }
2076 
2077 static int copy_context_table(struct intel_iommu *iommu,
2078 			      struct root_entry *old_re,
2079 			      struct context_entry **tbl,
2080 			      int bus, bool ext)
2081 {
2082 	int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2083 	struct context_entry *new_ce = NULL, ce;
2084 	struct context_entry *old_ce = NULL;
2085 	struct root_entry re;
2086 	phys_addr_t old_ce_phys;
2087 
2088 	tbl_idx = ext ? bus * 2 : bus;
2089 	memcpy(&re, old_re, sizeof(re));
2090 
2091 	for (devfn = 0; devfn < 256; devfn++) {
2092 		/* First calculate the correct index */
2093 		idx = (ext ? devfn * 2 : devfn) % 256;
2094 
2095 		if (idx == 0) {
2096 			/* First save what we may have and clean up */
2097 			if (new_ce) {
2098 				tbl[tbl_idx] = new_ce;
2099 				__iommu_flush_cache(iommu, new_ce,
2100 						    VTD_PAGE_SIZE);
2101 				pos = 1;
2102 			}
2103 
2104 			if (old_ce)
2105 				memunmap(old_ce);
2106 
2107 			ret = 0;
2108 			if (devfn < 0x80)
2109 				old_ce_phys = root_entry_lctp(&re);
2110 			else
2111 				old_ce_phys = root_entry_uctp(&re);
2112 
2113 			if (!old_ce_phys) {
2114 				if (ext && devfn == 0) {
2115 					/* No LCTP, try UCTP */
2116 					devfn = 0x7f;
2117 					continue;
2118 				} else {
2119 					goto out;
2120 				}
2121 			}
2122 
2123 			ret = -ENOMEM;
2124 			old_ce = memremap(old_ce_phys, PAGE_SIZE,
2125 					MEMREMAP_WB);
2126 			if (!old_ce)
2127 				goto out;
2128 
2129 			new_ce = iommu_alloc_page_node(iommu->node, GFP_KERNEL);
2130 			if (!new_ce)
2131 				goto out_unmap;
2132 
2133 			ret = 0;
2134 		}
2135 
2136 		/* Now copy the context entry */
2137 		memcpy(&ce, old_ce + idx, sizeof(ce));
2138 
2139 		if (!context_present(&ce))
2140 			continue;
2141 
2142 		did = context_domain_id(&ce);
2143 		if (did >= 0 && did < cap_ndoms(iommu->cap))
2144 			set_bit(did, iommu->domain_ids);
2145 
2146 		set_context_copied(iommu, bus, devfn);
2147 		new_ce[idx] = ce;
2148 	}
2149 
2150 	tbl[tbl_idx + pos] = new_ce;
2151 
2152 	__iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2153 
2154 out_unmap:
2155 	memunmap(old_ce);
2156 
2157 out:
2158 	return ret;
2159 }
2160 
2161 static int copy_translation_tables(struct intel_iommu *iommu)
2162 {
2163 	struct context_entry **ctxt_tbls;
2164 	struct root_entry *old_rt;
2165 	phys_addr_t old_rt_phys;
2166 	int ctxt_table_entries;
2167 	u64 rtaddr_reg;
2168 	int bus, ret;
2169 	bool new_ext, ext;
2170 
2171 	rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2172 	ext        = !!(rtaddr_reg & DMA_RTADDR_SMT);
2173 	new_ext    = !!sm_supported(iommu);
2174 
2175 	/*
2176 	 * The RTT bit can only be changed when translation is disabled,
2177 	 * but disabling translation means to open a window for data
2178 	 * corruption. So bail out and don't copy anything if we would
2179 	 * have to change the bit.
2180 	 */
2181 	if (new_ext != ext)
2182 		return -EINVAL;
2183 
2184 	iommu->copied_tables = bitmap_zalloc(BIT_ULL(16), GFP_KERNEL);
2185 	if (!iommu->copied_tables)
2186 		return -ENOMEM;
2187 
2188 	old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2189 	if (!old_rt_phys)
2190 		return -EINVAL;
2191 
2192 	old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
2193 	if (!old_rt)
2194 		return -ENOMEM;
2195 
2196 	/* This is too big for the stack - allocate it from slab */
2197 	ctxt_table_entries = ext ? 512 : 256;
2198 	ret = -ENOMEM;
2199 	ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
2200 	if (!ctxt_tbls)
2201 		goto out_unmap;
2202 
2203 	for (bus = 0; bus < 256; bus++) {
2204 		ret = copy_context_table(iommu, &old_rt[bus],
2205 					 ctxt_tbls, bus, ext);
2206 		if (ret) {
2207 			pr_err("%s: Failed to copy context table for bus %d\n",
2208 				iommu->name, bus);
2209 			continue;
2210 		}
2211 	}
2212 
2213 	spin_lock(&iommu->lock);
2214 
2215 	/* Context tables are copied, now write them to the root_entry table */
2216 	for (bus = 0; bus < 256; bus++) {
2217 		int idx = ext ? bus * 2 : bus;
2218 		u64 val;
2219 
2220 		if (ctxt_tbls[idx]) {
2221 			val = virt_to_phys(ctxt_tbls[idx]) | 1;
2222 			iommu->root_entry[bus].lo = val;
2223 		}
2224 
2225 		if (!ext || !ctxt_tbls[idx + 1])
2226 			continue;
2227 
2228 		val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2229 		iommu->root_entry[bus].hi = val;
2230 	}
2231 
2232 	spin_unlock(&iommu->lock);
2233 
2234 	kfree(ctxt_tbls);
2235 
2236 	__iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
2237 
2238 	ret = 0;
2239 
2240 out_unmap:
2241 	memunmap(old_rt);
2242 
2243 	return ret;
2244 }
2245 
2246 static int __init init_dmars(void)
2247 {
2248 	struct dmar_drhd_unit *drhd;
2249 	struct intel_iommu *iommu;
2250 	int ret;
2251 
2252 	ret = intel_cap_audit(CAP_AUDIT_STATIC_DMAR, NULL);
2253 	if (ret)
2254 		goto free_iommu;
2255 
2256 	for_each_iommu(iommu, drhd) {
2257 		if (drhd->ignored) {
2258 			iommu_disable_translation(iommu);
2259 			continue;
2260 		}
2261 
2262 		/*
2263 		 * Find the max pasid size of all IOMMU's in the system.
2264 		 * We need to ensure the system pasid table is no bigger
2265 		 * than the smallest supported.
2266 		 */
2267 		if (pasid_supported(iommu)) {
2268 			u32 temp = 2 << ecap_pss(iommu->ecap);
2269 
2270 			intel_pasid_max_id = min_t(u32, temp,
2271 						   intel_pasid_max_id);
2272 		}
2273 
2274 		intel_iommu_init_qi(iommu);
2275 
2276 		ret = iommu_init_domains(iommu);
2277 		if (ret)
2278 			goto free_iommu;
2279 
2280 		init_translation_status(iommu);
2281 
2282 		if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
2283 			iommu_disable_translation(iommu);
2284 			clear_translation_pre_enabled(iommu);
2285 			pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
2286 				iommu->name);
2287 		}
2288 
2289 		/*
2290 		 * TBD:
2291 		 * we could share the same root & context tables
2292 		 * among all IOMMU's. Need to Split it later.
2293 		 */
2294 		ret = iommu_alloc_root_entry(iommu);
2295 		if (ret)
2296 			goto free_iommu;
2297 
2298 		if (translation_pre_enabled(iommu)) {
2299 			pr_info("Translation already enabled - trying to copy translation structures\n");
2300 
2301 			ret = copy_translation_tables(iommu);
2302 			if (ret) {
2303 				/*
2304 				 * We found the IOMMU with translation
2305 				 * enabled - but failed to copy over the
2306 				 * old root-entry table. Try to proceed
2307 				 * by disabling translation now and
2308 				 * allocating a clean root-entry table.
2309 				 * This might cause DMAR faults, but
2310 				 * probably the dump will still succeed.
2311 				 */
2312 				pr_err("Failed to copy translation tables from previous kernel for %s\n",
2313 				       iommu->name);
2314 				iommu_disable_translation(iommu);
2315 				clear_translation_pre_enabled(iommu);
2316 			} else {
2317 				pr_info("Copied translation tables from previous kernel for %s\n",
2318 					iommu->name);
2319 			}
2320 		}
2321 
2322 		intel_svm_check(iommu);
2323 	}
2324 
2325 	/*
2326 	 * Now that qi is enabled on all iommus, set the root entry and flush
2327 	 * caches. This is required on some Intel X58 chipsets, otherwise the
2328 	 * flush_context function will loop forever and the boot hangs.
2329 	 */
2330 	for_each_active_iommu(iommu, drhd) {
2331 		iommu_flush_write_buffer(iommu);
2332 		iommu_set_root_entry(iommu);
2333 	}
2334 
2335 	check_tylersburg_isoch();
2336 
2337 	/*
2338 	 * for each drhd
2339 	 *   enable fault log
2340 	 *   global invalidate context cache
2341 	 *   global invalidate iotlb
2342 	 *   enable translation
2343 	 */
2344 	for_each_iommu(iommu, drhd) {
2345 		if (drhd->ignored) {
2346 			/*
2347 			 * we always have to disable PMRs or DMA may fail on
2348 			 * this device
2349 			 */
2350 			if (force_on)
2351 				iommu_disable_protect_mem_regions(iommu);
2352 			continue;
2353 		}
2354 
2355 		iommu_flush_write_buffer(iommu);
2356 
2357 #ifdef CONFIG_INTEL_IOMMU_SVM
2358 		if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
2359 			/*
2360 			 * Call dmar_alloc_hwirq() with dmar_global_lock held,
2361 			 * could cause possible lock race condition.
2362 			 */
2363 			up_write(&dmar_global_lock);
2364 			ret = intel_svm_enable_prq(iommu);
2365 			down_write(&dmar_global_lock);
2366 			if (ret)
2367 				goto free_iommu;
2368 		}
2369 #endif
2370 		ret = dmar_set_interrupt(iommu);
2371 		if (ret)
2372 			goto free_iommu;
2373 	}
2374 
2375 	return 0;
2376 
2377 free_iommu:
2378 	for_each_active_iommu(iommu, drhd) {
2379 		disable_dmar_iommu(iommu);
2380 		free_dmar_iommu(iommu);
2381 	}
2382 
2383 	return ret;
2384 }
2385 
2386 static void __init init_no_remapping_devices(void)
2387 {
2388 	struct dmar_drhd_unit *drhd;
2389 	struct device *dev;
2390 	int i;
2391 
2392 	for_each_drhd_unit(drhd) {
2393 		if (!drhd->include_all) {
2394 			for_each_active_dev_scope(drhd->devices,
2395 						  drhd->devices_cnt, i, dev)
2396 				break;
2397 			/* ignore DMAR unit if no devices exist */
2398 			if (i == drhd->devices_cnt)
2399 				drhd->ignored = 1;
2400 		}
2401 	}
2402 
2403 	for_each_active_drhd_unit(drhd) {
2404 		if (drhd->include_all)
2405 			continue;
2406 
2407 		for_each_active_dev_scope(drhd->devices,
2408 					  drhd->devices_cnt, i, dev)
2409 			if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
2410 				break;
2411 		if (i < drhd->devices_cnt)
2412 			continue;
2413 
2414 		/* This IOMMU has *only* gfx devices. Either bypass it or
2415 		   set the gfx_mapped flag, as appropriate */
2416 		drhd->gfx_dedicated = 1;
2417 		if (disable_igfx_iommu)
2418 			drhd->ignored = 1;
2419 	}
2420 }
2421 
2422 #ifdef CONFIG_SUSPEND
2423 static int init_iommu_hw(void)
2424 {
2425 	struct dmar_drhd_unit *drhd;
2426 	struct intel_iommu *iommu = NULL;
2427 	int ret;
2428 
2429 	for_each_active_iommu(iommu, drhd) {
2430 		if (iommu->qi) {
2431 			ret = dmar_reenable_qi(iommu);
2432 			if (ret)
2433 				return ret;
2434 		}
2435 	}
2436 
2437 	for_each_iommu(iommu, drhd) {
2438 		if (drhd->ignored) {
2439 			/*
2440 			 * we always have to disable PMRs or DMA may fail on
2441 			 * this device
2442 			 */
2443 			if (force_on)
2444 				iommu_disable_protect_mem_regions(iommu);
2445 			continue;
2446 		}
2447 
2448 		iommu_flush_write_buffer(iommu);
2449 		iommu_set_root_entry(iommu);
2450 		iommu_enable_translation(iommu);
2451 		iommu_disable_protect_mem_regions(iommu);
2452 	}
2453 
2454 	return 0;
2455 }
2456 
2457 static void iommu_flush_all(void)
2458 {
2459 	struct dmar_drhd_unit *drhd;
2460 	struct intel_iommu *iommu;
2461 
2462 	for_each_active_iommu(iommu, drhd) {
2463 		iommu->flush.flush_context(iommu, 0, 0, 0,
2464 					   DMA_CCMD_GLOBAL_INVL);
2465 		iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2466 					 DMA_TLB_GLOBAL_FLUSH);
2467 	}
2468 }
2469 
2470 static int iommu_suspend(void)
2471 {
2472 	struct dmar_drhd_unit *drhd;
2473 	struct intel_iommu *iommu = NULL;
2474 	unsigned long flag;
2475 
2476 	iommu_flush_all();
2477 
2478 	for_each_active_iommu(iommu, drhd) {
2479 		iommu_disable_translation(iommu);
2480 
2481 		raw_spin_lock_irqsave(&iommu->register_lock, flag);
2482 
2483 		iommu->iommu_state[SR_DMAR_FECTL_REG] =
2484 			readl(iommu->reg + DMAR_FECTL_REG);
2485 		iommu->iommu_state[SR_DMAR_FEDATA_REG] =
2486 			readl(iommu->reg + DMAR_FEDATA_REG);
2487 		iommu->iommu_state[SR_DMAR_FEADDR_REG] =
2488 			readl(iommu->reg + DMAR_FEADDR_REG);
2489 		iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
2490 			readl(iommu->reg + DMAR_FEUADDR_REG);
2491 
2492 		raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2493 	}
2494 	return 0;
2495 }
2496 
2497 static void iommu_resume(void)
2498 {
2499 	struct dmar_drhd_unit *drhd;
2500 	struct intel_iommu *iommu = NULL;
2501 	unsigned long flag;
2502 
2503 	if (init_iommu_hw()) {
2504 		if (force_on)
2505 			panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
2506 		else
2507 			WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
2508 		return;
2509 	}
2510 
2511 	for_each_active_iommu(iommu, drhd) {
2512 
2513 		raw_spin_lock_irqsave(&iommu->register_lock, flag);
2514 
2515 		writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
2516 			iommu->reg + DMAR_FECTL_REG);
2517 		writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
2518 			iommu->reg + DMAR_FEDATA_REG);
2519 		writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
2520 			iommu->reg + DMAR_FEADDR_REG);
2521 		writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
2522 			iommu->reg + DMAR_FEUADDR_REG);
2523 
2524 		raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2525 	}
2526 }
2527 
2528 static struct syscore_ops iommu_syscore_ops = {
2529 	.resume		= iommu_resume,
2530 	.suspend	= iommu_suspend,
2531 };
2532 
2533 static void __init init_iommu_pm_ops(void)
2534 {
2535 	register_syscore_ops(&iommu_syscore_ops);
2536 }
2537 
2538 #else
2539 static inline void init_iommu_pm_ops(void) {}
2540 #endif	/* CONFIG_PM */
2541 
2542 static int __init rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
2543 {
2544 	if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) ||
2545 	    !IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) ||
2546 	    rmrr->end_address <= rmrr->base_address ||
2547 	    arch_rmrr_sanity_check(rmrr))
2548 		return -EINVAL;
2549 
2550 	return 0;
2551 }
2552 
2553 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
2554 {
2555 	struct acpi_dmar_reserved_memory *rmrr;
2556 	struct dmar_rmrr_unit *rmrru;
2557 
2558 	rmrr = (struct acpi_dmar_reserved_memory *)header;
2559 	if (rmrr_sanity_check(rmrr)) {
2560 		pr_warn(FW_BUG
2561 			   "Your BIOS is broken; bad RMRR [%#018Lx-%#018Lx]\n"
2562 			   "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2563 			   rmrr->base_address, rmrr->end_address,
2564 			   dmi_get_system_info(DMI_BIOS_VENDOR),
2565 			   dmi_get_system_info(DMI_BIOS_VERSION),
2566 			   dmi_get_system_info(DMI_PRODUCT_VERSION));
2567 		add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
2568 	}
2569 
2570 	rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
2571 	if (!rmrru)
2572 		goto out;
2573 
2574 	rmrru->hdr = header;
2575 
2576 	rmrru->base_address = rmrr->base_address;
2577 	rmrru->end_address = rmrr->end_address;
2578 
2579 	rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
2580 				((void *)rmrr) + rmrr->header.length,
2581 				&rmrru->devices_cnt);
2582 	if (rmrru->devices_cnt && rmrru->devices == NULL)
2583 		goto free_rmrru;
2584 
2585 	list_add(&rmrru->list, &dmar_rmrr_units);
2586 
2587 	return 0;
2588 free_rmrru:
2589 	kfree(rmrru);
2590 out:
2591 	return -ENOMEM;
2592 }
2593 
2594 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
2595 {
2596 	struct dmar_atsr_unit *atsru;
2597 	struct acpi_dmar_atsr *tmp;
2598 
2599 	list_for_each_entry_rcu(atsru, &dmar_atsr_units, list,
2600 				dmar_rcu_check()) {
2601 		tmp = (struct acpi_dmar_atsr *)atsru->hdr;
2602 		if (atsr->segment != tmp->segment)
2603 			continue;
2604 		if (atsr->header.length != tmp->header.length)
2605 			continue;
2606 		if (memcmp(atsr, tmp, atsr->header.length) == 0)
2607 			return atsru;
2608 	}
2609 
2610 	return NULL;
2611 }
2612 
2613 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
2614 {
2615 	struct acpi_dmar_atsr *atsr;
2616 	struct dmar_atsr_unit *atsru;
2617 
2618 	if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
2619 		return 0;
2620 
2621 	atsr = container_of(hdr, struct acpi_dmar_atsr, header);
2622 	atsru = dmar_find_atsr(atsr);
2623 	if (atsru)
2624 		return 0;
2625 
2626 	atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
2627 	if (!atsru)
2628 		return -ENOMEM;
2629 
2630 	/*
2631 	 * If memory is allocated from slab by ACPI _DSM method, we need to
2632 	 * copy the memory content because the memory buffer will be freed
2633 	 * on return.
2634 	 */
2635 	atsru->hdr = (void *)(atsru + 1);
2636 	memcpy(atsru->hdr, hdr, hdr->length);
2637 	atsru->include_all = atsr->flags & 0x1;
2638 	if (!atsru->include_all) {
2639 		atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
2640 				(void *)atsr + atsr->header.length,
2641 				&atsru->devices_cnt);
2642 		if (atsru->devices_cnt && atsru->devices == NULL) {
2643 			kfree(atsru);
2644 			return -ENOMEM;
2645 		}
2646 	}
2647 
2648 	list_add_rcu(&atsru->list, &dmar_atsr_units);
2649 
2650 	return 0;
2651 }
2652 
2653 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
2654 {
2655 	dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
2656 	kfree(atsru);
2657 }
2658 
2659 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
2660 {
2661 	struct acpi_dmar_atsr *atsr;
2662 	struct dmar_atsr_unit *atsru;
2663 
2664 	atsr = container_of(hdr, struct acpi_dmar_atsr, header);
2665 	atsru = dmar_find_atsr(atsr);
2666 	if (atsru) {
2667 		list_del_rcu(&atsru->list);
2668 		synchronize_rcu();
2669 		intel_iommu_free_atsr(atsru);
2670 	}
2671 
2672 	return 0;
2673 }
2674 
2675 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
2676 {
2677 	int i;
2678 	struct device *dev;
2679 	struct acpi_dmar_atsr *atsr;
2680 	struct dmar_atsr_unit *atsru;
2681 
2682 	atsr = container_of(hdr, struct acpi_dmar_atsr, header);
2683 	atsru = dmar_find_atsr(atsr);
2684 	if (!atsru)
2685 		return 0;
2686 
2687 	if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
2688 		for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
2689 					  i, dev)
2690 			return -EBUSY;
2691 	}
2692 
2693 	return 0;
2694 }
2695 
2696 static struct dmar_satc_unit *dmar_find_satc(struct acpi_dmar_satc *satc)
2697 {
2698 	struct dmar_satc_unit *satcu;
2699 	struct acpi_dmar_satc *tmp;
2700 
2701 	list_for_each_entry_rcu(satcu, &dmar_satc_units, list,
2702 				dmar_rcu_check()) {
2703 		tmp = (struct acpi_dmar_satc *)satcu->hdr;
2704 		if (satc->segment != tmp->segment)
2705 			continue;
2706 		if (satc->header.length != tmp->header.length)
2707 			continue;
2708 		if (memcmp(satc, tmp, satc->header.length) == 0)
2709 			return satcu;
2710 	}
2711 
2712 	return NULL;
2713 }
2714 
2715 int dmar_parse_one_satc(struct acpi_dmar_header *hdr, void *arg)
2716 {
2717 	struct acpi_dmar_satc *satc;
2718 	struct dmar_satc_unit *satcu;
2719 
2720 	if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
2721 		return 0;
2722 
2723 	satc = container_of(hdr, struct acpi_dmar_satc, header);
2724 	satcu = dmar_find_satc(satc);
2725 	if (satcu)
2726 		return 0;
2727 
2728 	satcu = kzalloc(sizeof(*satcu) + hdr->length, GFP_KERNEL);
2729 	if (!satcu)
2730 		return -ENOMEM;
2731 
2732 	satcu->hdr = (void *)(satcu + 1);
2733 	memcpy(satcu->hdr, hdr, hdr->length);
2734 	satcu->atc_required = satc->flags & 0x1;
2735 	satcu->devices = dmar_alloc_dev_scope((void *)(satc + 1),
2736 					      (void *)satc + satc->header.length,
2737 					      &satcu->devices_cnt);
2738 	if (satcu->devices_cnt && !satcu->devices) {
2739 		kfree(satcu);
2740 		return -ENOMEM;
2741 	}
2742 	list_add_rcu(&satcu->list, &dmar_satc_units);
2743 
2744 	return 0;
2745 }
2746 
2747 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
2748 {
2749 	int sp, ret;
2750 	struct intel_iommu *iommu = dmaru->iommu;
2751 
2752 	ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_DMAR, iommu);
2753 	if (ret)
2754 		goto out;
2755 
2756 	sp = domain_update_iommu_superpage(NULL, iommu) - 1;
2757 	if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
2758 		pr_warn("%s: Doesn't support large page.\n",
2759 			iommu->name);
2760 		return -ENXIO;
2761 	}
2762 
2763 	/*
2764 	 * Disable translation if already enabled prior to OS handover.
2765 	 */
2766 	if (iommu->gcmd & DMA_GCMD_TE)
2767 		iommu_disable_translation(iommu);
2768 
2769 	ret = iommu_init_domains(iommu);
2770 	if (ret == 0)
2771 		ret = iommu_alloc_root_entry(iommu);
2772 	if (ret)
2773 		goto out;
2774 
2775 	intel_svm_check(iommu);
2776 
2777 	if (dmaru->ignored) {
2778 		/*
2779 		 * we always have to disable PMRs or DMA may fail on this device
2780 		 */
2781 		if (force_on)
2782 			iommu_disable_protect_mem_regions(iommu);
2783 		return 0;
2784 	}
2785 
2786 	intel_iommu_init_qi(iommu);
2787 	iommu_flush_write_buffer(iommu);
2788 
2789 #ifdef CONFIG_INTEL_IOMMU_SVM
2790 	if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
2791 		ret = intel_svm_enable_prq(iommu);
2792 		if (ret)
2793 			goto disable_iommu;
2794 	}
2795 #endif
2796 	ret = dmar_set_interrupt(iommu);
2797 	if (ret)
2798 		goto disable_iommu;
2799 
2800 	iommu_set_root_entry(iommu);
2801 	iommu_enable_translation(iommu);
2802 
2803 	iommu_disable_protect_mem_regions(iommu);
2804 	return 0;
2805 
2806 disable_iommu:
2807 	disable_dmar_iommu(iommu);
2808 out:
2809 	free_dmar_iommu(iommu);
2810 	return ret;
2811 }
2812 
2813 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
2814 {
2815 	int ret = 0;
2816 	struct intel_iommu *iommu = dmaru->iommu;
2817 
2818 	if (!intel_iommu_enabled)
2819 		return 0;
2820 	if (iommu == NULL)
2821 		return -EINVAL;
2822 
2823 	if (insert) {
2824 		ret = intel_iommu_add(dmaru);
2825 	} else {
2826 		disable_dmar_iommu(iommu);
2827 		free_dmar_iommu(iommu);
2828 	}
2829 
2830 	return ret;
2831 }
2832 
2833 static void intel_iommu_free_dmars(void)
2834 {
2835 	struct dmar_rmrr_unit *rmrru, *rmrr_n;
2836 	struct dmar_atsr_unit *atsru, *atsr_n;
2837 	struct dmar_satc_unit *satcu, *satc_n;
2838 
2839 	list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
2840 		list_del(&rmrru->list);
2841 		dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
2842 		kfree(rmrru);
2843 	}
2844 
2845 	list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
2846 		list_del(&atsru->list);
2847 		intel_iommu_free_atsr(atsru);
2848 	}
2849 	list_for_each_entry_safe(satcu, satc_n, &dmar_satc_units, list) {
2850 		list_del(&satcu->list);
2851 		dmar_free_dev_scope(&satcu->devices, &satcu->devices_cnt);
2852 		kfree(satcu);
2853 	}
2854 }
2855 
2856 static struct dmar_satc_unit *dmar_find_matched_satc_unit(struct pci_dev *dev)
2857 {
2858 	struct dmar_satc_unit *satcu;
2859 	struct acpi_dmar_satc *satc;
2860 	struct device *tmp;
2861 	int i;
2862 
2863 	dev = pci_physfn(dev);
2864 	rcu_read_lock();
2865 
2866 	list_for_each_entry_rcu(satcu, &dmar_satc_units, list) {
2867 		satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
2868 		if (satc->segment != pci_domain_nr(dev->bus))
2869 			continue;
2870 		for_each_dev_scope(satcu->devices, satcu->devices_cnt, i, tmp)
2871 			if (to_pci_dev(tmp) == dev)
2872 				goto out;
2873 	}
2874 	satcu = NULL;
2875 out:
2876 	rcu_read_unlock();
2877 	return satcu;
2878 }
2879 
2880 static int dmar_ats_supported(struct pci_dev *dev, struct intel_iommu *iommu)
2881 {
2882 	int i, ret = 1;
2883 	struct pci_bus *bus;
2884 	struct pci_dev *bridge = NULL;
2885 	struct device *tmp;
2886 	struct acpi_dmar_atsr *atsr;
2887 	struct dmar_atsr_unit *atsru;
2888 	struct dmar_satc_unit *satcu;
2889 
2890 	dev = pci_physfn(dev);
2891 	satcu = dmar_find_matched_satc_unit(dev);
2892 	if (satcu)
2893 		/*
2894 		 * This device supports ATS as it is in SATC table.
2895 		 * When IOMMU is in legacy mode, enabling ATS is done
2896 		 * automatically by HW for the device that requires
2897 		 * ATS, hence OS should not enable this device ATS
2898 		 * to avoid duplicated TLB invalidation.
2899 		 */
2900 		return !(satcu->atc_required && !sm_supported(iommu));
2901 
2902 	for (bus = dev->bus; bus; bus = bus->parent) {
2903 		bridge = bus->self;
2904 		/* If it's an integrated device, allow ATS */
2905 		if (!bridge)
2906 			return 1;
2907 		/* Connected via non-PCIe: no ATS */
2908 		if (!pci_is_pcie(bridge) ||
2909 		    pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
2910 			return 0;
2911 		/* If we found the root port, look it up in the ATSR */
2912 		if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
2913 			break;
2914 	}
2915 
2916 	rcu_read_lock();
2917 	list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
2918 		atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
2919 		if (atsr->segment != pci_domain_nr(dev->bus))
2920 			continue;
2921 
2922 		for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
2923 			if (tmp == &bridge->dev)
2924 				goto out;
2925 
2926 		if (atsru->include_all)
2927 			goto out;
2928 	}
2929 	ret = 0;
2930 out:
2931 	rcu_read_unlock();
2932 
2933 	return ret;
2934 }
2935 
2936 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
2937 {
2938 	int ret;
2939 	struct dmar_rmrr_unit *rmrru;
2940 	struct dmar_atsr_unit *atsru;
2941 	struct dmar_satc_unit *satcu;
2942 	struct acpi_dmar_atsr *atsr;
2943 	struct acpi_dmar_reserved_memory *rmrr;
2944 	struct acpi_dmar_satc *satc;
2945 
2946 	if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
2947 		return 0;
2948 
2949 	list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
2950 		rmrr = container_of(rmrru->hdr,
2951 				    struct acpi_dmar_reserved_memory, header);
2952 		if (info->event == BUS_NOTIFY_ADD_DEVICE) {
2953 			ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
2954 				((void *)rmrr) + rmrr->header.length,
2955 				rmrr->segment, rmrru->devices,
2956 				rmrru->devices_cnt);
2957 			if (ret < 0)
2958 				return ret;
2959 		} else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
2960 			dmar_remove_dev_scope(info, rmrr->segment,
2961 				rmrru->devices, rmrru->devices_cnt);
2962 		}
2963 	}
2964 
2965 	list_for_each_entry(atsru, &dmar_atsr_units, list) {
2966 		if (atsru->include_all)
2967 			continue;
2968 
2969 		atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
2970 		if (info->event == BUS_NOTIFY_ADD_DEVICE) {
2971 			ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
2972 					(void *)atsr + atsr->header.length,
2973 					atsr->segment, atsru->devices,
2974 					atsru->devices_cnt);
2975 			if (ret > 0)
2976 				break;
2977 			else if (ret < 0)
2978 				return ret;
2979 		} else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
2980 			if (dmar_remove_dev_scope(info, atsr->segment,
2981 					atsru->devices, atsru->devices_cnt))
2982 				break;
2983 		}
2984 	}
2985 	list_for_each_entry(satcu, &dmar_satc_units, list) {
2986 		satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
2987 		if (info->event == BUS_NOTIFY_ADD_DEVICE) {
2988 			ret = dmar_insert_dev_scope(info, (void *)(satc + 1),
2989 					(void *)satc + satc->header.length,
2990 					satc->segment, satcu->devices,
2991 					satcu->devices_cnt);
2992 			if (ret > 0)
2993 				break;
2994 			else if (ret < 0)
2995 				return ret;
2996 		} else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
2997 			if (dmar_remove_dev_scope(info, satc->segment,
2998 					satcu->devices, satcu->devices_cnt))
2999 				break;
3000 		}
3001 	}
3002 
3003 	return 0;
3004 }
3005 
3006 static void intel_disable_iommus(void)
3007 {
3008 	struct intel_iommu *iommu = NULL;
3009 	struct dmar_drhd_unit *drhd;
3010 
3011 	for_each_iommu(iommu, drhd)
3012 		iommu_disable_translation(iommu);
3013 }
3014 
3015 void intel_iommu_shutdown(void)
3016 {
3017 	struct dmar_drhd_unit *drhd;
3018 	struct intel_iommu *iommu = NULL;
3019 
3020 	if (no_iommu || dmar_disabled)
3021 		return;
3022 
3023 	down_write(&dmar_global_lock);
3024 
3025 	/* Disable PMRs explicitly here. */
3026 	for_each_iommu(iommu, drhd)
3027 		iommu_disable_protect_mem_regions(iommu);
3028 
3029 	/* Make sure the IOMMUs are switched off */
3030 	intel_disable_iommus();
3031 
3032 	up_write(&dmar_global_lock);
3033 }
3034 
3035 static struct intel_iommu *dev_to_intel_iommu(struct device *dev)
3036 {
3037 	struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
3038 
3039 	return container_of(iommu_dev, struct intel_iommu, iommu);
3040 }
3041 
3042 static ssize_t version_show(struct device *dev,
3043 			    struct device_attribute *attr, char *buf)
3044 {
3045 	struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3046 	u32 ver = readl(iommu->reg + DMAR_VER_REG);
3047 	return sysfs_emit(buf, "%d:%d\n",
3048 			  DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
3049 }
3050 static DEVICE_ATTR_RO(version);
3051 
3052 static ssize_t address_show(struct device *dev,
3053 			    struct device_attribute *attr, char *buf)
3054 {
3055 	struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3056 	return sysfs_emit(buf, "%llx\n", iommu->reg_phys);
3057 }
3058 static DEVICE_ATTR_RO(address);
3059 
3060 static ssize_t cap_show(struct device *dev,
3061 			struct device_attribute *attr, char *buf)
3062 {
3063 	struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3064 	return sysfs_emit(buf, "%llx\n", iommu->cap);
3065 }
3066 static DEVICE_ATTR_RO(cap);
3067 
3068 static ssize_t ecap_show(struct device *dev,
3069 			 struct device_attribute *attr, char *buf)
3070 {
3071 	struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3072 	return sysfs_emit(buf, "%llx\n", iommu->ecap);
3073 }
3074 static DEVICE_ATTR_RO(ecap);
3075 
3076 static ssize_t domains_supported_show(struct device *dev,
3077 				      struct device_attribute *attr, char *buf)
3078 {
3079 	struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3080 	return sysfs_emit(buf, "%ld\n", cap_ndoms(iommu->cap));
3081 }
3082 static DEVICE_ATTR_RO(domains_supported);
3083 
3084 static ssize_t domains_used_show(struct device *dev,
3085 				 struct device_attribute *attr, char *buf)
3086 {
3087 	struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3088 	return sysfs_emit(buf, "%d\n",
3089 			  bitmap_weight(iommu->domain_ids,
3090 					cap_ndoms(iommu->cap)));
3091 }
3092 static DEVICE_ATTR_RO(domains_used);
3093 
3094 static struct attribute *intel_iommu_attrs[] = {
3095 	&dev_attr_version.attr,
3096 	&dev_attr_address.attr,
3097 	&dev_attr_cap.attr,
3098 	&dev_attr_ecap.attr,
3099 	&dev_attr_domains_supported.attr,
3100 	&dev_attr_domains_used.attr,
3101 	NULL,
3102 };
3103 
3104 static struct attribute_group intel_iommu_group = {
3105 	.name = "intel-iommu",
3106 	.attrs = intel_iommu_attrs,
3107 };
3108 
3109 const struct attribute_group *intel_iommu_groups[] = {
3110 	&intel_iommu_group,
3111 	NULL,
3112 };
3113 
3114 static bool has_external_pci(void)
3115 {
3116 	struct pci_dev *pdev = NULL;
3117 
3118 	for_each_pci_dev(pdev)
3119 		if (pdev->external_facing) {
3120 			pci_dev_put(pdev);
3121 			return true;
3122 		}
3123 
3124 	return false;
3125 }
3126 
3127 static int __init platform_optin_force_iommu(void)
3128 {
3129 	if (!dmar_platform_optin() || no_platform_optin || !has_external_pci())
3130 		return 0;
3131 
3132 	if (no_iommu || dmar_disabled)
3133 		pr_info("Intel-IOMMU force enabled due to platform opt in\n");
3134 
3135 	/*
3136 	 * If Intel-IOMMU is disabled by default, we will apply identity
3137 	 * map for all devices except those marked as being untrusted.
3138 	 */
3139 	if (dmar_disabled)
3140 		iommu_set_default_passthrough(false);
3141 
3142 	dmar_disabled = 0;
3143 	no_iommu = 0;
3144 
3145 	return 1;
3146 }
3147 
3148 static int __init probe_acpi_namespace_devices(void)
3149 {
3150 	struct dmar_drhd_unit *drhd;
3151 	/* To avoid a -Wunused-but-set-variable warning. */
3152 	struct intel_iommu *iommu __maybe_unused;
3153 	struct device *dev;
3154 	int i, ret = 0;
3155 
3156 	for_each_active_iommu(iommu, drhd) {
3157 		for_each_active_dev_scope(drhd->devices,
3158 					  drhd->devices_cnt, i, dev) {
3159 			struct acpi_device_physical_node *pn;
3160 			struct acpi_device *adev;
3161 
3162 			if (dev->bus != &acpi_bus_type)
3163 				continue;
3164 
3165 			adev = to_acpi_device(dev);
3166 			mutex_lock(&adev->physical_node_lock);
3167 			list_for_each_entry(pn,
3168 					    &adev->physical_node_list, node) {
3169 				ret = iommu_probe_device(pn->dev);
3170 				if (ret)
3171 					break;
3172 			}
3173 			mutex_unlock(&adev->physical_node_lock);
3174 
3175 			if (ret)
3176 				return ret;
3177 		}
3178 	}
3179 
3180 	return 0;
3181 }
3182 
3183 static __init int tboot_force_iommu(void)
3184 {
3185 	if (!tboot_enabled())
3186 		return 0;
3187 
3188 	if (no_iommu || dmar_disabled)
3189 		pr_warn("Forcing Intel-IOMMU to enabled\n");
3190 
3191 	dmar_disabled = 0;
3192 	no_iommu = 0;
3193 
3194 	return 1;
3195 }
3196 
3197 int __init intel_iommu_init(void)
3198 {
3199 	int ret = -ENODEV;
3200 	struct dmar_drhd_unit *drhd;
3201 	struct intel_iommu *iommu;
3202 
3203 	/*
3204 	 * Intel IOMMU is required for a TXT/tboot launch or platform
3205 	 * opt in, so enforce that.
3206 	 */
3207 	force_on = (!intel_iommu_tboot_noforce && tboot_force_iommu()) ||
3208 		    platform_optin_force_iommu();
3209 
3210 	down_write(&dmar_global_lock);
3211 	if (dmar_table_init()) {
3212 		if (force_on)
3213 			panic("tboot: Failed to initialize DMAR table\n");
3214 		goto out_free_dmar;
3215 	}
3216 
3217 	if (dmar_dev_scope_init() < 0) {
3218 		if (force_on)
3219 			panic("tboot: Failed to initialize DMAR device scope\n");
3220 		goto out_free_dmar;
3221 	}
3222 
3223 	up_write(&dmar_global_lock);
3224 
3225 	/*
3226 	 * The bus notifier takes the dmar_global_lock, so lockdep will
3227 	 * complain later when we register it under the lock.
3228 	 */
3229 	dmar_register_bus_notifier();
3230 
3231 	down_write(&dmar_global_lock);
3232 
3233 	if (!no_iommu)
3234 		intel_iommu_debugfs_init();
3235 
3236 	if (no_iommu || dmar_disabled) {
3237 		/*
3238 		 * We exit the function here to ensure IOMMU's remapping and
3239 		 * mempool aren't setup, which means that the IOMMU's PMRs
3240 		 * won't be disabled via the call to init_dmars(). So disable
3241 		 * it explicitly here. The PMRs were setup by tboot prior to
3242 		 * calling SENTER, but the kernel is expected to reset/tear
3243 		 * down the PMRs.
3244 		 */
3245 		if (intel_iommu_tboot_noforce) {
3246 			for_each_iommu(iommu, drhd)
3247 				iommu_disable_protect_mem_regions(iommu);
3248 		}
3249 
3250 		/*
3251 		 * Make sure the IOMMUs are switched off, even when we
3252 		 * boot into a kexec kernel and the previous kernel left
3253 		 * them enabled
3254 		 */
3255 		intel_disable_iommus();
3256 		goto out_free_dmar;
3257 	}
3258 
3259 	if (list_empty(&dmar_rmrr_units))
3260 		pr_info("No RMRR found\n");
3261 
3262 	if (list_empty(&dmar_atsr_units))
3263 		pr_info("No ATSR found\n");
3264 
3265 	if (list_empty(&dmar_satc_units))
3266 		pr_info("No SATC found\n");
3267 
3268 	init_no_remapping_devices();
3269 
3270 	ret = init_dmars();
3271 	if (ret) {
3272 		if (force_on)
3273 			panic("tboot: Failed to initialize DMARs\n");
3274 		pr_err("Initialization failed\n");
3275 		goto out_free_dmar;
3276 	}
3277 	up_write(&dmar_global_lock);
3278 
3279 	init_iommu_pm_ops();
3280 
3281 	down_read(&dmar_global_lock);
3282 	for_each_active_iommu(iommu, drhd) {
3283 		/*
3284 		 * The flush queue implementation does not perform
3285 		 * page-selective invalidations that are required for efficient
3286 		 * TLB flushes in virtual environments.  The benefit of batching
3287 		 * is likely to be much lower than the overhead of synchronizing
3288 		 * the virtual and physical IOMMU page-tables.
3289 		 */
3290 		if (cap_caching_mode(iommu->cap) &&
3291 		    !first_level_by_default(IOMMU_DOMAIN_DMA)) {
3292 			pr_info_once("IOMMU batching disallowed due to virtualization\n");
3293 			iommu_set_dma_strict();
3294 		}
3295 		iommu_device_sysfs_add(&iommu->iommu, NULL,
3296 				       intel_iommu_groups,
3297 				       "%s", iommu->name);
3298 		iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);
3299 
3300 		iommu_pmu_register(iommu);
3301 	}
3302 
3303 	if (probe_acpi_namespace_devices())
3304 		pr_warn("ACPI name space devices didn't probe correctly\n");
3305 
3306 	/* Finally, we enable the DMA remapping hardware. */
3307 	for_each_iommu(iommu, drhd) {
3308 		if (!drhd->ignored && !translation_pre_enabled(iommu))
3309 			iommu_enable_translation(iommu);
3310 
3311 		iommu_disable_protect_mem_regions(iommu);
3312 	}
3313 	up_read(&dmar_global_lock);
3314 
3315 	pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
3316 
3317 	intel_iommu_enabled = 1;
3318 
3319 	return 0;
3320 
3321 out_free_dmar:
3322 	intel_iommu_free_dmars();
3323 	up_write(&dmar_global_lock);
3324 	return ret;
3325 }
3326 
3327 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
3328 {
3329 	struct device_domain_info *info = opaque;
3330 
3331 	domain_context_clear_one(info, PCI_BUS_NUM(alias), alias & 0xff);
3332 	return 0;
3333 }
3334 
3335 /*
3336  * NB - intel-iommu lacks any sort of reference counting for the users of
3337  * dependent devices.  If multiple endpoints have intersecting dependent
3338  * devices, unbinding the driver from any one of them will possibly leave
3339  * the others unable to operate.
3340  */
3341 static void domain_context_clear(struct device_domain_info *info)
3342 {
3343 	if (!dev_is_pci(info->dev))
3344 		domain_context_clear_one(info, info->bus, info->devfn);
3345 
3346 	pci_for_each_dma_alias(to_pci_dev(info->dev),
3347 			       &domain_context_clear_one_cb, info);
3348 }
3349 
3350 /*
3351  * Clear the page table pointer in context or pasid table entries so that
3352  * all DMA requests without PASID from the device are blocked. If the page
3353  * table has been set, clean up the data structures.
3354  */
3355 void device_block_translation(struct device *dev)
3356 {
3357 	struct device_domain_info *info = dev_iommu_priv_get(dev);
3358 	struct intel_iommu *iommu = info->iommu;
3359 	unsigned long flags;
3360 
3361 	iommu_disable_pci_caps(info);
3362 	if (!dev_is_real_dma_subdevice(dev)) {
3363 		if (sm_supported(iommu))
3364 			intel_pasid_tear_down_entry(iommu, dev,
3365 						    IOMMU_NO_PASID, false);
3366 		else
3367 			domain_context_clear(info);
3368 	}
3369 
3370 	if (!info->domain)
3371 		return;
3372 
3373 	spin_lock_irqsave(&info->domain->lock, flags);
3374 	list_del(&info->link);
3375 	spin_unlock_irqrestore(&info->domain->lock, flags);
3376 
3377 	cache_tag_unassign_domain(info->domain, dev, IOMMU_NO_PASID);
3378 	domain_detach_iommu(info->domain, iommu);
3379 	info->domain = NULL;
3380 }
3381 
3382 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3383 {
3384 	int adjust_width;
3385 
3386 	/* calculate AGAW */
3387 	domain->gaw = guest_width;
3388 	adjust_width = guestwidth_to_adjustwidth(guest_width);
3389 	domain->agaw = width_to_agaw(adjust_width);
3390 
3391 	domain->iommu_coherency = false;
3392 	domain->iommu_superpage = 0;
3393 	domain->max_addr = 0;
3394 
3395 	/* always allocate the top pgd */
3396 	domain->pgd = iommu_alloc_page_node(domain->nid, GFP_ATOMIC);
3397 	if (!domain->pgd)
3398 		return -ENOMEM;
3399 	domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3400 	return 0;
3401 }
3402 
3403 static int blocking_domain_attach_dev(struct iommu_domain *domain,
3404 				      struct device *dev)
3405 {
3406 	device_block_translation(dev);
3407 	return 0;
3408 }
3409 
3410 static struct iommu_domain blocking_domain = {
3411 	.type = IOMMU_DOMAIN_BLOCKED,
3412 	.ops = &(const struct iommu_domain_ops) {
3413 		.attach_dev	= blocking_domain_attach_dev,
3414 	}
3415 };
3416 
3417 static int iommu_superpage_capability(struct intel_iommu *iommu, bool first_stage)
3418 {
3419 	if (!intel_iommu_superpage)
3420 		return 0;
3421 
3422 	if (first_stage)
3423 		return cap_fl1gp_support(iommu->cap) ? 2 : 1;
3424 
3425 	return fls(cap_super_page_val(iommu->cap));
3426 }
3427 
3428 static struct dmar_domain *paging_domain_alloc(struct device *dev, bool first_stage)
3429 {
3430 	struct device_domain_info *info = dev_iommu_priv_get(dev);
3431 	struct intel_iommu *iommu = info->iommu;
3432 	struct dmar_domain *domain;
3433 	int addr_width;
3434 
3435 	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
3436 	if (!domain)
3437 		return ERR_PTR(-ENOMEM);
3438 
3439 	INIT_LIST_HEAD(&domain->devices);
3440 	INIT_LIST_HEAD(&domain->dev_pasids);
3441 	INIT_LIST_HEAD(&domain->cache_tags);
3442 	spin_lock_init(&domain->lock);
3443 	spin_lock_init(&domain->cache_lock);
3444 	xa_init(&domain->iommu_array);
3445 
3446 	domain->nid = dev_to_node(dev);
3447 	domain->use_first_level = first_stage;
3448 
3449 	/* calculate the address width */
3450 	addr_width = agaw_to_width(iommu->agaw);
3451 	if (addr_width > cap_mgaw(iommu->cap))
3452 		addr_width = cap_mgaw(iommu->cap);
3453 	domain->gaw = addr_width;
3454 	domain->agaw = iommu->agaw;
3455 	domain->max_addr = __DOMAIN_MAX_ADDR(addr_width);
3456 
3457 	/* iommu memory access coherency */
3458 	domain->iommu_coherency = iommu_paging_structure_coherency(iommu);
3459 
3460 	/* pagesize bitmap */
3461 	domain->domain.pgsize_bitmap = SZ_4K;
3462 	domain->iommu_superpage = iommu_superpage_capability(iommu, first_stage);
3463 	domain->domain.pgsize_bitmap |= domain_super_pgsize_bitmap(domain);
3464 
3465 	/*
3466 	 * IOVA aperture: First-level translation restricts the input-address
3467 	 * to a canonical address (i.e., address bits 63:N have the same value
3468 	 * as address bit [N-1], where N is 48-bits with 4-level paging and
3469 	 * 57-bits with 5-level paging). Hence, skip bit [N-1].
3470 	 */
3471 	domain->domain.geometry.force_aperture = true;
3472 	domain->domain.geometry.aperture_start = 0;
3473 	if (first_stage)
3474 		domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw - 1);
3475 	else
3476 		domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw);
3477 
3478 	/* always allocate the top pgd */
3479 	domain->pgd = iommu_alloc_page_node(domain->nid, GFP_KERNEL);
3480 	if (!domain->pgd) {
3481 		kfree(domain);
3482 		return ERR_PTR(-ENOMEM);
3483 	}
3484 	domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3485 
3486 	return domain;
3487 }
3488 
3489 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
3490 {
3491 	struct dmar_domain *dmar_domain;
3492 	struct iommu_domain *domain;
3493 
3494 	switch (type) {
3495 	case IOMMU_DOMAIN_DMA:
3496 	case IOMMU_DOMAIN_UNMANAGED:
3497 		dmar_domain = alloc_domain(type);
3498 		if (!dmar_domain) {
3499 			pr_err("Can't allocate dmar_domain\n");
3500 			return NULL;
3501 		}
3502 		if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3503 			pr_err("Domain initialization failed\n");
3504 			domain_exit(dmar_domain);
3505 			return NULL;
3506 		}
3507 
3508 		domain = &dmar_domain->domain;
3509 		domain->geometry.aperture_start = 0;
3510 		domain->geometry.aperture_end   =
3511 				__DOMAIN_MAX_ADDR(dmar_domain->gaw);
3512 		domain->geometry.force_aperture = true;
3513 
3514 		return domain;
3515 	default:
3516 		return NULL;
3517 	}
3518 
3519 	return NULL;
3520 }
3521 
3522 static struct iommu_domain *
3523 intel_iommu_domain_alloc_user(struct device *dev, u32 flags,
3524 			      struct iommu_domain *parent,
3525 			      const struct iommu_user_data *user_data)
3526 {
3527 	struct device_domain_info *info = dev_iommu_priv_get(dev);
3528 	bool dirty_tracking = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
3529 	bool nested_parent = flags & IOMMU_HWPT_ALLOC_NEST_PARENT;
3530 	struct intel_iommu *iommu = info->iommu;
3531 	struct dmar_domain *dmar_domain;
3532 	struct iommu_domain *domain;
3533 
3534 	/* Must be NESTING domain */
3535 	if (parent) {
3536 		if (!nested_supported(iommu) || flags)
3537 			return ERR_PTR(-EOPNOTSUPP);
3538 		return intel_nested_domain_alloc(parent, user_data);
3539 	}
3540 
3541 	if (flags &
3542 	    (~(IOMMU_HWPT_ALLOC_NEST_PARENT | IOMMU_HWPT_ALLOC_DIRTY_TRACKING)))
3543 		return ERR_PTR(-EOPNOTSUPP);
3544 	if (nested_parent && !nested_supported(iommu))
3545 		return ERR_PTR(-EOPNOTSUPP);
3546 	if (user_data || (dirty_tracking && !ssads_supported(iommu)))
3547 		return ERR_PTR(-EOPNOTSUPP);
3548 
3549 	/* Do not use first stage for user domain translation. */
3550 	dmar_domain = paging_domain_alloc(dev, false);
3551 	if (IS_ERR(dmar_domain))
3552 		return ERR_CAST(dmar_domain);
3553 	domain = &dmar_domain->domain;
3554 	domain->type = IOMMU_DOMAIN_UNMANAGED;
3555 	domain->owner = &intel_iommu_ops;
3556 	domain->ops = intel_iommu_ops.default_domain_ops;
3557 
3558 	if (nested_parent) {
3559 		dmar_domain->nested_parent = true;
3560 		INIT_LIST_HEAD(&dmar_domain->s1_domains);
3561 		spin_lock_init(&dmar_domain->s1_lock);
3562 	}
3563 
3564 	if (dirty_tracking) {
3565 		if (dmar_domain->use_first_level) {
3566 			iommu_domain_free(domain);
3567 			return ERR_PTR(-EOPNOTSUPP);
3568 		}
3569 		domain->dirty_ops = &intel_dirty_ops;
3570 	}
3571 
3572 	return domain;
3573 }
3574 
3575 static void intel_iommu_domain_free(struct iommu_domain *domain)
3576 {
3577 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3578 
3579 	WARN_ON(dmar_domain->nested_parent &&
3580 		!list_empty(&dmar_domain->s1_domains));
3581 	domain_exit(dmar_domain);
3582 }
3583 
3584 int prepare_domain_attach_device(struct iommu_domain *domain,
3585 				 struct device *dev)
3586 {
3587 	struct device_domain_info *info = dev_iommu_priv_get(dev);
3588 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3589 	struct intel_iommu *iommu = info->iommu;
3590 	int addr_width;
3591 
3592 	if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
3593 		return -EINVAL;
3594 
3595 	if (domain->dirty_ops && !ssads_supported(iommu))
3596 		return -EINVAL;
3597 
3598 	/* check if this iommu agaw is sufficient for max mapped address */
3599 	addr_width = agaw_to_width(iommu->agaw);
3600 	if (addr_width > cap_mgaw(iommu->cap))
3601 		addr_width = cap_mgaw(iommu->cap);
3602 
3603 	if (dmar_domain->max_addr > (1LL << addr_width))
3604 		return -EINVAL;
3605 	dmar_domain->gaw = addr_width;
3606 
3607 	/*
3608 	 * Knock out extra levels of page tables if necessary
3609 	 */
3610 	while (iommu->agaw < dmar_domain->agaw) {
3611 		struct dma_pte *pte;
3612 
3613 		pte = dmar_domain->pgd;
3614 		if (dma_pte_present(pte)) {
3615 			dmar_domain->pgd = phys_to_virt(dma_pte_addr(pte));
3616 			iommu_free_page(pte);
3617 		}
3618 		dmar_domain->agaw--;
3619 	}
3620 
3621 	if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) &&
3622 	    context_copied(iommu, info->bus, info->devfn))
3623 		return intel_pasid_setup_sm_context(dev);
3624 
3625 	return 0;
3626 }
3627 
3628 static int intel_iommu_attach_device(struct iommu_domain *domain,
3629 				     struct device *dev)
3630 {
3631 	int ret;
3632 
3633 	device_block_translation(dev);
3634 
3635 	ret = prepare_domain_attach_device(domain, dev);
3636 	if (ret)
3637 		return ret;
3638 
3639 	return dmar_domain_attach_device(to_dmar_domain(domain), dev);
3640 }
3641 
3642 static int intel_iommu_map(struct iommu_domain *domain,
3643 			   unsigned long iova, phys_addr_t hpa,
3644 			   size_t size, int iommu_prot, gfp_t gfp)
3645 {
3646 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3647 	u64 max_addr;
3648 	int prot = 0;
3649 
3650 	if (iommu_prot & IOMMU_READ)
3651 		prot |= DMA_PTE_READ;
3652 	if (iommu_prot & IOMMU_WRITE)
3653 		prot |= DMA_PTE_WRITE;
3654 	if (dmar_domain->set_pte_snp)
3655 		prot |= DMA_PTE_SNP;
3656 
3657 	max_addr = iova + size;
3658 	if (dmar_domain->max_addr < max_addr) {
3659 		u64 end;
3660 
3661 		/* check if minimum agaw is sufficient for mapped address */
3662 		end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
3663 		if (end < max_addr) {
3664 			pr_err("%s: iommu width (%d) is not "
3665 			       "sufficient for the mapped address (%llx)\n",
3666 			       __func__, dmar_domain->gaw, max_addr);
3667 			return -EFAULT;
3668 		}
3669 		dmar_domain->max_addr = max_addr;
3670 	}
3671 	/* Round up size to next multiple of PAGE_SIZE, if it and
3672 	   the low bits of hpa would take us onto the next page */
3673 	size = aligned_nrpages(hpa, size);
3674 	return __domain_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3675 				hpa >> VTD_PAGE_SHIFT, size, prot, gfp);
3676 }
3677 
3678 static int intel_iommu_map_pages(struct iommu_domain *domain,
3679 				 unsigned long iova, phys_addr_t paddr,
3680 				 size_t pgsize, size_t pgcount,
3681 				 int prot, gfp_t gfp, size_t *mapped)
3682 {
3683 	unsigned long pgshift = __ffs(pgsize);
3684 	size_t size = pgcount << pgshift;
3685 	int ret;
3686 
3687 	if (pgsize != SZ_4K && pgsize != SZ_2M && pgsize != SZ_1G)
3688 		return -EINVAL;
3689 
3690 	if (!IS_ALIGNED(iova | paddr, pgsize))
3691 		return -EINVAL;
3692 
3693 	ret = intel_iommu_map(domain, iova, paddr, size, prot, gfp);
3694 	if (!ret && mapped)
3695 		*mapped = size;
3696 
3697 	return ret;
3698 }
3699 
3700 static size_t intel_iommu_unmap(struct iommu_domain *domain,
3701 				unsigned long iova, size_t size,
3702 				struct iommu_iotlb_gather *gather)
3703 {
3704 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3705 	unsigned long start_pfn, last_pfn;
3706 	int level = 0;
3707 
3708 	/* Cope with horrid API which requires us to unmap more than the
3709 	   size argument if it happens to be a large-page mapping. */
3710 	if (unlikely(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT,
3711 				     &level, GFP_ATOMIC)))
3712 		return 0;
3713 
3714 	if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
3715 		size = VTD_PAGE_SIZE << level_to_offset_bits(level);
3716 
3717 	start_pfn = iova >> VTD_PAGE_SHIFT;
3718 	last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
3719 
3720 	domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist);
3721 
3722 	if (dmar_domain->max_addr == iova + size)
3723 		dmar_domain->max_addr = iova;
3724 
3725 	/*
3726 	 * We do not use page-selective IOTLB invalidation in flush queue,
3727 	 * so there is no need to track page and sync iotlb.
3728 	 */
3729 	if (!iommu_iotlb_gather_queued(gather))
3730 		iommu_iotlb_gather_add_page(domain, gather, iova, size);
3731 
3732 	return size;
3733 }
3734 
3735 static size_t intel_iommu_unmap_pages(struct iommu_domain *domain,
3736 				      unsigned long iova,
3737 				      size_t pgsize, size_t pgcount,
3738 				      struct iommu_iotlb_gather *gather)
3739 {
3740 	unsigned long pgshift = __ffs(pgsize);
3741 	size_t size = pgcount << pgshift;
3742 
3743 	return intel_iommu_unmap(domain, iova, size, gather);
3744 }
3745 
3746 static void intel_iommu_tlb_sync(struct iommu_domain *domain,
3747 				 struct iommu_iotlb_gather *gather)
3748 {
3749 	cache_tag_flush_range(to_dmar_domain(domain), gather->start,
3750 			      gather->end, list_empty(&gather->freelist));
3751 	iommu_put_pages_list(&gather->freelist);
3752 }
3753 
3754 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3755 					    dma_addr_t iova)
3756 {
3757 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3758 	struct dma_pte *pte;
3759 	int level = 0;
3760 	u64 phys = 0;
3761 
3762 	pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level,
3763 			     GFP_ATOMIC);
3764 	if (pte && dma_pte_present(pte))
3765 		phys = dma_pte_addr(pte) +
3766 			(iova & (BIT_MASK(level_to_offset_bits(level) +
3767 						VTD_PAGE_SHIFT) - 1));
3768 
3769 	return phys;
3770 }
3771 
3772 static bool domain_support_force_snooping(struct dmar_domain *domain)
3773 {
3774 	struct device_domain_info *info;
3775 	bool support = true;
3776 
3777 	assert_spin_locked(&domain->lock);
3778 	list_for_each_entry(info, &domain->devices, link) {
3779 		if (!ecap_sc_support(info->iommu->ecap)) {
3780 			support = false;
3781 			break;
3782 		}
3783 	}
3784 
3785 	return support;
3786 }
3787 
3788 static void domain_set_force_snooping(struct dmar_domain *domain)
3789 {
3790 	struct device_domain_info *info;
3791 
3792 	assert_spin_locked(&domain->lock);
3793 	/*
3794 	 * Second level page table supports per-PTE snoop control. The
3795 	 * iommu_map() interface will handle this by setting SNP bit.
3796 	 */
3797 	if (!domain->use_first_level) {
3798 		domain->set_pte_snp = true;
3799 		return;
3800 	}
3801 
3802 	list_for_each_entry(info, &domain->devices, link)
3803 		intel_pasid_setup_page_snoop_control(info->iommu, info->dev,
3804 						     IOMMU_NO_PASID);
3805 }
3806 
3807 static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain)
3808 {
3809 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3810 	unsigned long flags;
3811 
3812 	if (dmar_domain->force_snooping)
3813 		return true;
3814 
3815 	spin_lock_irqsave(&dmar_domain->lock, flags);
3816 	if (!domain_support_force_snooping(dmar_domain) ||
3817 	    (!dmar_domain->use_first_level && dmar_domain->has_mappings)) {
3818 		spin_unlock_irqrestore(&dmar_domain->lock, flags);
3819 		return false;
3820 	}
3821 
3822 	domain_set_force_snooping(dmar_domain);
3823 	dmar_domain->force_snooping = true;
3824 	spin_unlock_irqrestore(&dmar_domain->lock, flags);
3825 
3826 	return true;
3827 }
3828 
3829 static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap)
3830 {
3831 	struct device_domain_info *info = dev_iommu_priv_get(dev);
3832 
3833 	switch (cap) {
3834 	case IOMMU_CAP_CACHE_COHERENCY:
3835 	case IOMMU_CAP_DEFERRED_FLUSH:
3836 		return true;
3837 	case IOMMU_CAP_PRE_BOOT_PROTECTION:
3838 		return dmar_platform_optin();
3839 	case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
3840 		return ecap_sc_support(info->iommu->ecap);
3841 	case IOMMU_CAP_DIRTY_TRACKING:
3842 		return ssads_supported(info->iommu);
3843 	default:
3844 		return false;
3845 	}
3846 }
3847 
3848 static struct iommu_device *intel_iommu_probe_device(struct device *dev)
3849 {
3850 	struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL;
3851 	struct device_domain_info *info;
3852 	struct intel_iommu *iommu;
3853 	u8 bus, devfn;
3854 	int ret;
3855 
3856 	iommu = device_lookup_iommu(dev, &bus, &devfn);
3857 	if (!iommu || !iommu->iommu.ops)
3858 		return ERR_PTR(-ENODEV);
3859 
3860 	info = kzalloc(sizeof(*info), GFP_KERNEL);
3861 	if (!info)
3862 		return ERR_PTR(-ENOMEM);
3863 
3864 	if (dev_is_real_dma_subdevice(dev)) {
3865 		info->bus = pdev->bus->number;
3866 		info->devfn = pdev->devfn;
3867 		info->segment = pci_domain_nr(pdev->bus);
3868 	} else {
3869 		info->bus = bus;
3870 		info->devfn = devfn;
3871 		info->segment = iommu->segment;
3872 	}
3873 
3874 	info->dev = dev;
3875 	info->iommu = iommu;
3876 	if (dev_is_pci(dev)) {
3877 		if (ecap_dev_iotlb_support(iommu->ecap) &&
3878 		    pci_ats_supported(pdev) &&
3879 		    dmar_ats_supported(pdev, iommu)) {
3880 			info->ats_supported = 1;
3881 			info->dtlb_extra_inval = dev_needs_extra_dtlb_flush(pdev);
3882 
3883 			/*
3884 			 * For IOMMU that supports device IOTLB throttling
3885 			 * (DIT), we assign PFSID to the invalidation desc
3886 			 * of a VF such that IOMMU HW can gauge queue depth
3887 			 * at PF level. If DIT is not set, PFSID will be
3888 			 * treated as reserved, which should be set to 0.
3889 			 */
3890 			if (ecap_dit(iommu->ecap))
3891 				info->pfsid = pci_dev_id(pci_physfn(pdev));
3892 			info->ats_qdep = pci_ats_queue_depth(pdev);
3893 		}
3894 		if (sm_supported(iommu)) {
3895 			if (pasid_supported(iommu)) {
3896 				int features = pci_pasid_features(pdev);
3897 
3898 				if (features >= 0)
3899 					info->pasid_supported = features | 1;
3900 			}
3901 
3902 			if (info->ats_supported && ecap_prs(iommu->ecap) &&
3903 			    pci_pri_supported(pdev))
3904 				info->pri_supported = 1;
3905 		}
3906 	}
3907 
3908 	dev_iommu_priv_set(dev, info);
3909 	if (pdev && pci_ats_supported(pdev)) {
3910 		pci_prepare_ats(pdev, VTD_PAGE_SHIFT);
3911 		ret = device_rbtree_insert(iommu, info);
3912 		if (ret)
3913 			goto free;
3914 	}
3915 
3916 	if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
3917 		ret = intel_pasid_alloc_table(dev);
3918 		if (ret) {
3919 			dev_err(dev, "PASID table allocation failed\n");
3920 			goto clear_rbtree;
3921 		}
3922 
3923 		if (!context_copied(iommu, info->bus, info->devfn)) {
3924 			ret = intel_pasid_setup_sm_context(dev);
3925 			if (ret)
3926 				goto free_table;
3927 		}
3928 	}
3929 
3930 	intel_iommu_debugfs_create_dev(info);
3931 
3932 	/*
3933 	 * The PCIe spec, in its wisdom, declares that the behaviour of the
3934 	 * device is undefined if you enable PASID support after ATS support.
3935 	 * So always enable PASID support on devices which have it, even if
3936 	 * we can't yet know if we're ever going to use it.
3937 	 */
3938 	if (info->pasid_supported &&
3939 	    !pci_enable_pasid(pdev, info->pasid_supported & ~1))
3940 		info->pasid_enabled = 1;
3941 
3942 	return &iommu->iommu;
3943 free_table:
3944 	intel_pasid_free_table(dev);
3945 clear_rbtree:
3946 	device_rbtree_remove(info);
3947 free:
3948 	kfree(info);
3949 
3950 	return ERR_PTR(ret);
3951 }
3952 
3953 static void intel_iommu_release_device(struct device *dev)
3954 {
3955 	struct device_domain_info *info = dev_iommu_priv_get(dev);
3956 	struct intel_iommu *iommu = info->iommu;
3957 
3958 	if (info->pasid_enabled) {
3959 		pci_disable_pasid(to_pci_dev(dev));
3960 		info->pasid_enabled = 0;
3961 	}
3962 
3963 	mutex_lock(&iommu->iopf_lock);
3964 	if (dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev)))
3965 		device_rbtree_remove(info);
3966 	mutex_unlock(&iommu->iopf_lock);
3967 
3968 	if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) &&
3969 	    !context_copied(iommu, info->bus, info->devfn))
3970 		intel_pasid_teardown_sm_context(dev);
3971 
3972 	intel_pasid_free_table(dev);
3973 	intel_iommu_debugfs_remove_dev(info);
3974 	kfree(info);
3975 	set_dma_ops(dev, NULL);
3976 }
3977 
3978 static void intel_iommu_get_resv_regions(struct device *device,
3979 					 struct list_head *head)
3980 {
3981 	int prot = DMA_PTE_READ | DMA_PTE_WRITE;
3982 	struct iommu_resv_region *reg;
3983 	struct dmar_rmrr_unit *rmrr;
3984 	struct device *i_dev;
3985 	int i;
3986 
3987 	rcu_read_lock();
3988 	for_each_rmrr_units(rmrr) {
3989 		for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3990 					  i, i_dev) {
3991 			struct iommu_resv_region *resv;
3992 			enum iommu_resv_type type;
3993 			size_t length;
3994 
3995 			if (i_dev != device &&
3996 			    !is_downstream_to_pci_bridge(device, i_dev))
3997 				continue;
3998 
3999 			length = rmrr->end_address - rmrr->base_address + 1;
4000 
4001 			type = device_rmrr_is_relaxable(device) ?
4002 				IOMMU_RESV_DIRECT_RELAXABLE : IOMMU_RESV_DIRECT;
4003 
4004 			resv = iommu_alloc_resv_region(rmrr->base_address,
4005 						       length, prot, type,
4006 						       GFP_ATOMIC);
4007 			if (!resv)
4008 				break;
4009 
4010 			list_add_tail(&resv->list, head);
4011 		}
4012 	}
4013 	rcu_read_unlock();
4014 
4015 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
4016 	if (dev_is_pci(device)) {
4017 		struct pci_dev *pdev = to_pci_dev(device);
4018 
4019 		if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) {
4020 			reg = iommu_alloc_resv_region(0, 1UL << 24, prot,
4021 					IOMMU_RESV_DIRECT_RELAXABLE,
4022 					GFP_KERNEL);
4023 			if (reg)
4024 				list_add_tail(&reg->list, head);
4025 		}
4026 	}
4027 #endif /* CONFIG_INTEL_IOMMU_FLOPPY_WA */
4028 
4029 	reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
4030 				      IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
4031 				      0, IOMMU_RESV_MSI, GFP_KERNEL);
4032 	if (!reg)
4033 		return;
4034 	list_add_tail(&reg->list, head);
4035 }
4036 
4037 static struct iommu_group *intel_iommu_device_group(struct device *dev)
4038 {
4039 	if (dev_is_pci(dev))
4040 		return pci_device_group(dev);
4041 	return generic_device_group(dev);
4042 }
4043 
4044 static int intel_iommu_enable_sva(struct device *dev)
4045 {
4046 	struct device_domain_info *info = dev_iommu_priv_get(dev);
4047 	struct intel_iommu *iommu;
4048 
4049 	if (!info || dmar_disabled)
4050 		return -EINVAL;
4051 
4052 	iommu = info->iommu;
4053 	if (!iommu)
4054 		return -EINVAL;
4055 
4056 	if (!(iommu->flags & VTD_FLAG_SVM_CAPABLE))
4057 		return -ENODEV;
4058 
4059 	if (!info->pasid_enabled || !info->ats_enabled)
4060 		return -EINVAL;
4061 
4062 	/*
4063 	 * Devices having device-specific I/O fault handling should not
4064 	 * support PCI/PRI. The IOMMU side has no means to check the
4065 	 * capability of device-specific IOPF.  Therefore, IOMMU can only
4066 	 * default that if the device driver enables SVA on a non-PRI
4067 	 * device, it will handle IOPF in its own way.
4068 	 */
4069 	if (!info->pri_supported)
4070 		return 0;
4071 
4072 	/* Devices supporting PRI should have it enabled. */
4073 	if (!info->pri_enabled)
4074 		return -EINVAL;
4075 
4076 	return 0;
4077 }
4078 
4079 static int context_flip_pri(struct device_domain_info *info, bool enable)
4080 {
4081 	struct intel_iommu *iommu = info->iommu;
4082 	u8 bus = info->bus, devfn = info->devfn;
4083 	struct context_entry *context;
4084 	u16 did;
4085 
4086 	spin_lock(&iommu->lock);
4087 	if (context_copied(iommu, bus, devfn)) {
4088 		spin_unlock(&iommu->lock);
4089 		return -EINVAL;
4090 	}
4091 
4092 	context = iommu_context_addr(iommu, bus, devfn, false);
4093 	if (!context || !context_present(context)) {
4094 		spin_unlock(&iommu->lock);
4095 		return -ENODEV;
4096 	}
4097 	did = context_domain_id(context);
4098 
4099 	if (enable)
4100 		context_set_sm_pre(context);
4101 	else
4102 		context_clear_sm_pre(context);
4103 
4104 	if (!ecap_coherent(iommu->ecap))
4105 		clflush_cache_range(context, sizeof(*context));
4106 	intel_context_flush_present(info, context, did, true);
4107 	spin_unlock(&iommu->lock);
4108 
4109 	return 0;
4110 }
4111 
4112 static int intel_iommu_enable_iopf(struct device *dev)
4113 {
4114 	struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL;
4115 	struct device_domain_info *info = dev_iommu_priv_get(dev);
4116 	struct intel_iommu *iommu;
4117 	int ret;
4118 
4119 	if (!pdev || !info || !info->ats_enabled || !info->pri_supported)
4120 		return -ENODEV;
4121 
4122 	if (info->pri_enabled)
4123 		return -EBUSY;
4124 
4125 	iommu = info->iommu;
4126 	if (!iommu)
4127 		return -EINVAL;
4128 
4129 	/* PASID is required in PRG Response Message. */
4130 	if (info->pasid_enabled && !pci_prg_resp_pasid_required(pdev))
4131 		return -EINVAL;
4132 
4133 	ret = pci_reset_pri(pdev);
4134 	if (ret)
4135 		return ret;
4136 
4137 	ret = iopf_queue_add_device(iommu->iopf_queue, dev);
4138 	if (ret)
4139 		return ret;
4140 
4141 	ret = context_flip_pri(info, true);
4142 	if (ret)
4143 		goto err_remove_device;
4144 
4145 	ret = pci_enable_pri(pdev, PRQ_DEPTH);
4146 	if (ret)
4147 		goto err_clear_pri;
4148 
4149 	info->pri_enabled = 1;
4150 
4151 	return 0;
4152 err_clear_pri:
4153 	context_flip_pri(info, false);
4154 err_remove_device:
4155 	iopf_queue_remove_device(iommu->iopf_queue, dev);
4156 
4157 	return ret;
4158 }
4159 
4160 static int intel_iommu_disable_iopf(struct device *dev)
4161 {
4162 	struct device_domain_info *info = dev_iommu_priv_get(dev);
4163 	struct intel_iommu *iommu = info->iommu;
4164 
4165 	if (!info->pri_enabled)
4166 		return -EINVAL;
4167 
4168 	/* Disable new PRI reception: */
4169 	context_flip_pri(info, false);
4170 
4171 	/*
4172 	 * Remove device from fault queue and acknowledge all outstanding
4173 	 * PRQs to the device:
4174 	 */
4175 	iopf_queue_remove_device(iommu->iopf_queue, dev);
4176 
4177 	/*
4178 	 * PCIe spec states that by clearing PRI enable bit, the Page
4179 	 * Request Interface will not issue new page requests, but has
4180 	 * outstanding page requests that have been transmitted or are
4181 	 * queued for transmission. This is supposed to be called after
4182 	 * the device driver has stopped DMA, all PASIDs have been
4183 	 * unbound and the outstanding PRQs have been drained.
4184 	 */
4185 	pci_disable_pri(to_pci_dev(dev));
4186 	info->pri_enabled = 0;
4187 
4188 	return 0;
4189 }
4190 
4191 static int
4192 intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
4193 {
4194 	switch (feat) {
4195 	case IOMMU_DEV_FEAT_IOPF:
4196 		return intel_iommu_enable_iopf(dev);
4197 
4198 	case IOMMU_DEV_FEAT_SVA:
4199 		return intel_iommu_enable_sva(dev);
4200 
4201 	default:
4202 		return -ENODEV;
4203 	}
4204 }
4205 
4206 static int
4207 intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
4208 {
4209 	switch (feat) {
4210 	case IOMMU_DEV_FEAT_IOPF:
4211 		return intel_iommu_disable_iopf(dev);
4212 
4213 	case IOMMU_DEV_FEAT_SVA:
4214 		return 0;
4215 
4216 	default:
4217 		return -ENODEV;
4218 	}
4219 }
4220 
4221 static bool intel_iommu_is_attach_deferred(struct device *dev)
4222 {
4223 	struct device_domain_info *info = dev_iommu_priv_get(dev);
4224 
4225 	return translation_pre_enabled(info->iommu) && !info->domain;
4226 }
4227 
4228 /*
4229  * Check that the device does not live on an external facing PCI port that is
4230  * marked as untrusted. Such devices should not be able to apply quirks and
4231  * thus not be able to bypass the IOMMU restrictions.
4232  */
4233 static bool risky_device(struct pci_dev *pdev)
4234 {
4235 	if (pdev->untrusted) {
4236 		pci_info(pdev,
4237 			 "Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n",
4238 			 pdev->vendor, pdev->device);
4239 		pci_info(pdev, "Please check with your BIOS/Platform vendor about this\n");
4240 		return true;
4241 	}
4242 	return false;
4243 }
4244 
4245 static int intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
4246 				      unsigned long iova, size_t size)
4247 {
4248 	cache_tag_flush_range_np(to_dmar_domain(domain), iova, iova + size - 1);
4249 
4250 	return 0;
4251 }
4252 
4253 static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
4254 					 struct iommu_domain *domain)
4255 {
4256 	struct device_domain_info *info = dev_iommu_priv_get(dev);
4257 	struct dev_pasid_info *curr, *dev_pasid = NULL;
4258 	struct intel_iommu *iommu = info->iommu;
4259 	struct dmar_domain *dmar_domain;
4260 	unsigned long flags;
4261 
4262 	if (domain->type == IOMMU_DOMAIN_IDENTITY) {
4263 		intel_pasid_tear_down_entry(iommu, dev, pasid, false);
4264 		return;
4265 	}
4266 
4267 	dmar_domain = to_dmar_domain(domain);
4268 	spin_lock_irqsave(&dmar_domain->lock, flags);
4269 	list_for_each_entry(curr, &dmar_domain->dev_pasids, link_domain) {
4270 		if (curr->dev == dev && curr->pasid == pasid) {
4271 			list_del(&curr->link_domain);
4272 			dev_pasid = curr;
4273 			break;
4274 		}
4275 	}
4276 	WARN_ON_ONCE(!dev_pasid);
4277 	spin_unlock_irqrestore(&dmar_domain->lock, flags);
4278 
4279 	cache_tag_unassign_domain(dmar_domain, dev, pasid);
4280 	domain_detach_iommu(dmar_domain, iommu);
4281 	intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
4282 	kfree(dev_pasid);
4283 	intel_pasid_tear_down_entry(iommu, dev, pasid, false);
4284 	intel_drain_pasid_prq(dev, pasid);
4285 }
4286 
4287 static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
4288 				     struct device *dev, ioasid_t pasid)
4289 {
4290 	struct device_domain_info *info = dev_iommu_priv_get(dev);
4291 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4292 	struct intel_iommu *iommu = info->iommu;
4293 	struct dev_pasid_info *dev_pasid;
4294 	unsigned long flags;
4295 	int ret;
4296 
4297 	if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))
4298 		return -EOPNOTSUPP;
4299 
4300 	if (domain->dirty_ops)
4301 		return -EINVAL;
4302 
4303 	if (context_copied(iommu, info->bus, info->devfn))
4304 		return -EBUSY;
4305 
4306 	ret = prepare_domain_attach_device(domain, dev);
4307 	if (ret)
4308 		return ret;
4309 
4310 	dev_pasid = kzalloc(sizeof(*dev_pasid), GFP_KERNEL);
4311 	if (!dev_pasid)
4312 		return -ENOMEM;
4313 
4314 	ret = domain_attach_iommu(dmar_domain, iommu);
4315 	if (ret)
4316 		goto out_free;
4317 
4318 	ret = cache_tag_assign_domain(dmar_domain, dev, pasid);
4319 	if (ret)
4320 		goto out_detach_iommu;
4321 
4322 	if (dmar_domain->use_first_level)
4323 		ret = domain_setup_first_level(iommu, dmar_domain,
4324 					       dev, pasid);
4325 	else
4326 		ret = intel_pasid_setup_second_level(iommu, dmar_domain,
4327 						     dev, pasid);
4328 	if (ret)
4329 		goto out_unassign_tag;
4330 
4331 	dev_pasid->dev = dev;
4332 	dev_pasid->pasid = pasid;
4333 	spin_lock_irqsave(&dmar_domain->lock, flags);
4334 	list_add(&dev_pasid->link_domain, &dmar_domain->dev_pasids);
4335 	spin_unlock_irqrestore(&dmar_domain->lock, flags);
4336 
4337 	if (domain->type & __IOMMU_DOMAIN_PAGING)
4338 		intel_iommu_debugfs_create_dev_pasid(dev_pasid);
4339 
4340 	return 0;
4341 out_unassign_tag:
4342 	cache_tag_unassign_domain(dmar_domain, dev, pasid);
4343 out_detach_iommu:
4344 	domain_detach_iommu(dmar_domain, iommu);
4345 out_free:
4346 	kfree(dev_pasid);
4347 	return ret;
4348 }
4349 
4350 static void *intel_iommu_hw_info(struct device *dev, u32 *length, u32 *type)
4351 {
4352 	struct device_domain_info *info = dev_iommu_priv_get(dev);
4353 	struct intel_iommu *iommu = info->iommu;
4354 	struct iommu_hw_info_vtd *vtd;
4355 
4356 	vtd = kzalloc(sizeof(*vtd), GFP_KERNEL);
4357 	if (!vtd)
4358 		return ERR_PTR(-ENOMEM);
4359 
4360 	vtd->flags = IOMMU_HW_INFO_VTD_ERRATA_772415_SPR17;
4361 	vtd->cap_reg = iommu->cap;
4362 	vtd->ecap_reg = iommu->ecap;
4363 	*length = sizeof(*vtd);
4364 	*type = IOMMU_HW_INFO_TYPE_INTEL_VTD;
4365 	return vtd;
4366 }
4367 
4368 /*
4369  * Set dirty tracking for the device list of a domain. The caller must
4370  * hold the domain->lock when calling it.
4371  */
4372 static int device_set_dirty_tracking(struct list_head *devices, bool enable)
4373 {
4374 	struct device_domain_info *info;
4375 	int ret = 0;
4376 
4377 	list_for_each_entry(info, devices, link) {
4378 		ret = intel_pasid_setup_dirty_tracking(info->iommu, info->dev,
4379 						       IOMMU_NO_PASID, enable);
4380 		if (ret)
4381 			break;
4382 	}
4383 
4384 	return ret;
4385 }
4386 
4387 static int parent_domain_set_dirty_tracking(struct dmar_domain *domain,
4388 					    bool enable)
4389 {
4390 	struct dmar_domain *s1_domain;
4391 	unsigned long flags;
4392 	int ret;
4393 
4394 	spin_lock(&domain->s1_lock);
4395 	list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) {
4396 		spin_lock_irqsave(&s1_domain->lock, flags);
4397 		ret = device_set_dirty_tracking(&s1_domain->devices, enable);
4398 		spin_unlock_irqrestore(&s1_domain->lock, flags);
4399 		if (ret)
4400 			goto err_unwind;
4401 	}
4402 	spin_unlock(&domain->s1_lock);
4403 	return 0;
4404 
4405 err_unwind:
4406 	list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) {
4407 		spin_lock_irqsave(&s1_domain->lock, flags);
4408 		device_set_dirty_tracking(&s1_domain->devices,
4409 					  domain->dirty_tracking);
4410 		spin_unlock_irqrestore(&s1_domain->lock, flags);
4411 	}
4412 	spin_unlock(&domain->s1_lock);
4413 	return ret;
4414 }
4415 
4416 static int intel_iommu_set_dirty_tracking(struct iommu_domain *domain,
4417 					  bool enable)
4418 {
4419 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4420 	int ret;
4421 
4422 	spin_lock(&dmar_domain->lock);
4423 	if (dmar_domain->dirty_tracking == enable)
4424 		goto out_unlock;
4425 
4426 	ret = device_set_dirty_tracking(&dmar_domain->devices, enable);
4427 	if (ret)
4428 		goto err_unwind;
4429 
4430 	if (dmar_domain->nested_parent) {
4431 		ret = parent_domain_set_dirty_tracking(dmar_domain, enable);
4432 		if (ret)
4433 			goto err_unwind;
4434 	}
4435 
4436 	dmar_domain->dirty_tracking = enable;
4437 out_unlock:
4438 	spin_unlock(&dmar_domain->lock);
4439 
4440 	return 0;
4441 
4442 err_unwind:
4443 	device_set_dirty_tracking(&dmar_domain->devices,
4444 				  dmar_domain->dirty_tracking);
4445 	spin_unlock(&dmar_domain->lock);
4446 	return ret;
4447 }
4448 
4449 static int intel_iommu_read_and_clear_dirty(struct iommu_domain *domain,
4450 					    unsigned long iova, size_t size,
4451 					    unsigned long flags,
4452 					    struct iommu_dirty_bitmap *dirty)
4453 {
4454 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4455 	unsigned long end = iova + size - 1;
4456 	unsigned long pgsize;
4457 
4458 	/*
4459 	 * IOMMUFD core calls into a dirty tracking disabled domain without an
4460 	 * IOVA bitmap set in order to clean dirty bits in all PTEs that might
4461 	 * have occurred when we stopped dirty tracking. This ensures that we
4462 	 * never inherit dirtied bits from a previous cycle.
4463 	 */
4464 	if (!dmar_domain->dirty_tracking && dirty->bitmap)
4465 		return -EINVAL;
4466 
4467 	do {
4468 		struct dma_pte *pte;
4469 		int lvl = 0;
4470 
4471 		pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &lvl,
4472 				     GFP_ATOMIC);
4473 		pgsize = level_size(lvl) << VTD_PAGE_SHIFT;
4474 		if (!pte || !dma_pte_present(pte)) {
4475 			iova += pgsize;
4476 			continue;
4477 		}
4478 
4479 		if (dma_sl_pte_test_and_clear_dirty(pte, flags))
4480 			iommu_dirty_bitmap_record(dirty, iova, pgsize);
4481 		iova += pgsize;
4482 	} while (iova < end);
4483 
4484 	return 0;
4485 }
4486 
4487 static const struct iommu_dirty_ops intel_dirty_ops = {
4488 	.set_dirty_tracking = intel_iommu_set_dirty_tracking,
4489 	.read_and_clear_dirty = intel_iommu_read_and_clear_dirty,
4490 };
4491 
4492 static int context_setup_pass_through(struct device *dev, u8 bus, u8 devfn)
4493 {
4494 	struct device_domain_info *info = dev_iommu_priv_get(dev);
4495 	struct intel_iommu *iommu = info->iommu;
4496 	struct context_entry *context;
4497 
4498 	spin_lock(&iommu->lock);
4499 	context = iommu_context_addr(iommu, bus, devfn, 1);
4500 	if (!context) {
4501 		spin_unlock(&iommu->lock);
4502 		return -ENOMEM;
4503 	}
4504 
4505 	if (context_present(context) && !context_copied(iommu, bus, devfn)) {
4506 		spin_unlock(&iommu->lock);
4507 		return 0;
4508 	}
4509 
4510 	copied_context_tear_down(iommu, context, bus, devfn);
4511 	context_clear_entry(context);
4512 	context_set_domain_id(context, FLPT_DEFAULT_DID);
4513 
4514 	/*
4515 	 * In pass through mode, AW must be programmed to indicate the largest
4516 	 * AGAW value supported by hardware. And ASR is ignored by hardware.
4517 	 */
4518 	context_set_address_width(context, iommu->msagaw);
4519 	context_set_translation_type(context, CONTEXT_TT_PASS_THROUGH);
4520 	context_set_fault_enable(context);
4521 	context_set_present(context);
4522 	if (!ecap_coherent(iommu->ecap))
4523 		clflush_cache_range(context, sizeof(*context));
4524 	context_present_cache_flush(iommu, FLPT_DEFAULT_DID, bus, devfn);
4525 	spin_unlock(&iommu->lock);
4526 
4527 	return 0;
4528 }
4529 
4530 static int context_setup_pass_through_cb(struct pci_dev *pdev, u16 alias, void *data)
4531 {
4532 	struct device *dev = data;
4533 
4534 	if (dev != &pdev->dev)
4535 		return 0;
4536 
4537 	return context_setup_pass_through(dev, PCI_BUS_NUM(alias), alias & 0xff);
4538 }
4539 
4540 static int device_setup_pass_through(struct device *dev)
4541 {
4542 	struct device_domain_info *info = dev_iommu_priv_get(dev);
4543 
4544 	if (!dev_is_pci(dev))
4545 		return context_setup_pass_through(dev, info->bus, info->devfn);
4546 
4547 	return pci_for_each_dma_alias(to_pci_dev(dev),
4548 				      context_setup_pass_through_cb, dev);
4549 }
4550 
4551 static int identity_domain_attach_dev(struct iommu_domain *domain, struct device *dev)
4552 {
4553 	struct device_domain_info *info = dev_iommu_priv_get(dev);
4554 	struct intel_iommu *iommu = info->iommu;
4555 	int ret;
4556 
4557 	device_block_translation(dev);
4558 
4559 	if (dev_is_real_dma_subdevice(dev))
4560 		return 0;
4561 
4562 	if (sm_supported(iommu)) {
4563 		ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID);
4564 		if (!ret)
4565 			iommu_enable_pci_caps(info);
4566 	} else {
4567 		ret = device_setup_pass_through(dev);
4568 	}
4569 
4570 	return ret;
4571 }
4572 
4573 static int identity_domain_set_dev_pasid(struct iommu_domain *domain,
4574 					 struct device *dev, ioasid_t pasid)
4575 {
4576 	struct device_domain_info *info = dev_iommu_priv_get(dev);
4577 	struct intel_iommu *iommu = info->iommu;
4578 
4579 	if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))
4580 		return -EOPNOTSUPP;
4581 
4582 	return intel_pasid_setup_pass_through(iommu, dev, pasid);
4583 }
4584 
4585 static struct iommu_domain identity_domain = {
4586 	.type = IOMMU_DOMAIN_IDENTITY,
4587 	.ops = &(const struct iommu_domain_ops) {
4588 		.attach_dev	= identity_domain_attach_dev,
4589 		.set_dev_pasid	= identity_domain_set_dev_pasid,
4590 	},
4591 };
4592 
4593 const struct iommu_ops intel_iommu_ops = {
4594 	.blocked_domain		= &blocking_domain,
4595 	.release_domain		= &blocking_domain,
4596 	.identity_domain	= &identity_domain,
4597 	.capable		= intel_iommu_capable,
4598 	.hw_info		= intel_iommu_hw_info,
4599 	.domain_alloc		= intel_iommu_domain_alloc,
4600 	.domain_alloc_user	= intel_iommu_domain_alloc_user,
4601 	.domain_alloc_sva	= intel_svm_domain_alloc,
4602 	.probe_device		= intel_iommu_probe_device,
4603 	.release_device		= intel_iommu_release_device,
4604 	.get_resv_regions	= intel_iommu_get_resv_regions,
4605 	.device_group		= intel_iommu_device_group,
4606 	.dev_enable_feat	= intel_iommu_dev_enable_feat,
4607 	.dev_disable_feat	= intel_iommu_dev_disable_feat,
4608 	.is_attach_deferred	= intel_iommu_is_attach_deferred,
4609 	.def_domain_type	= device_def_domain_type,
4610 	.remove_dev_pasid	= intel_iommu_remove_dev_pasid,
4611 	.pgsize_bitmap		= SZ_4K,
4612 #ifdef CONFIG_INTEL_IOMMU_SVM
4613 	.page_response		= intel_svm_page_response,
4614 #endif
4615 	.default_domain_ops = &(const struct iommu_domain_ops) {
4616 		.attach_dev		= intel_iommu_attach_device,
4617 		.set_dev_pasid		= intel_iommu_set_dev_pasid,
4618 		.map_pages		= intel_iommu_map_pages,
4619 		.unmap_pages		= intel_iommu_unmap_pages,
4620 		.iotlb_sync_map		= intel_iommu_iotlb_sync_map,
4621 		.flush_iotlb_all        = intel_flush_iotlb_all,
4622 		.iotlb_sync		= intel_iommu_tlb_sync,
4623 		.iova_to_phys		= intel_iommu_iova_to_phys,
4624 		.free			= intel_iommu_domain_free,
4625 		.enforce_cache_coherency = intel_iommu_enforce_cache_coherency,
4626 	}
4627 };
4628 
4629 static void quirk_iommu_igfx(struct pci_dev *dev)
4630 {
4631 	if (risky_device(dev))
4632 		return;
4633 
4634 	pci_info(dev, "Disabling IOMMU for graphics on this chipset\n");
4635 	disable_igfx_iommu = 1;
4636 }
4637 
4638 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4639 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_igfx);
4640 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_igfx);
4641 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_igfx);
4642 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_igfx);
4643 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_igfx);
4644 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_igfx);
4645 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_igfx);
4646 
4647 /* Broadwell igfx malfunctions with dmar */
4648 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1606, quirk_iommu_igfx);
4649 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160B, quirk_iommu_igfx);
4650 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160E, quirk_iommu_igfx);
4651 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1602, quirk_iommu_igfx);
4652 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160A, quirk_iommu_igfx);
4653 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160D, quirk_iommu_igfx);
4654 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1616, quirk_iommu_igfx);
4655 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161B, quirk_iommu_igfx);
4656 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161E, quirk_iommu_igfx);
4657 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1612, quirk_iommu_igfx);
4658 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161A, quirk_iommu_igfx);
4659 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161D, quirk_iommu_igfx);
4660 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1626, quirk_iommu_igfx);
4661 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162B, quirk_iommu_igfx);
4662 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162E, quirk_iommu_igfx);
4663 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1622, quirk_iommu_igfx);
4664 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162A, quirk_iommu_igfx);
4665 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162D, quirk_iommu_igfx);
4666 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1636, quirk_iommu_igfx);
4667 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163B, quirk_iommu_igfx);
4668 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163E, quirk_iommu_igfx);
4669 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1632, quirk_iommu_igfx);
4670 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163A, quirk_iommu_igfx);
4671 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163D, quirk_iommu_igfx);
4672 
4673 static void quirk_iommu_rwbf(struct pci_dev *dev)
4674 {
4675 	if (risky_device(dev))
4676 		return;
4677 
4678 	/*
4679 	 * Mobile 4 Series Chipset neglects to set RWBF capability,
4680 	 * but needs it. Same seems to hold for the desktop versions.
4681 	 */
4682 	pci_info(dev, "Forcing write-buffer flush capability\n");
4683 	rwbf_quirk = 1;
4684 }
4685 
4686 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4687 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4688 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4689 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4690 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4691 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4692 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4693 
4694 #define GGC 0x52
4695 #define GGC_MEMORY_SIZE_MASK	(0xf << 8)
4696 #define GGC_MEMORY_SIZE_NONE	(0x0 << 8)
4697 #define GGC_MEMORY_SIZE_1M	(0x1 << 8)
4698 #define GGC_MEMORY_SIZE_2M	(0x3 << 8)
4699 #define GGC_MEMORY_VT_ENABLED	(0x8 << 8)
4700 #define GGC_MEMORY_SIZE_2M_VT	(0x9 << 8)
4701 #define GGC_MEMORY_SIZE_3M_VT	(0xa << 8)
4702 #define GGC_MEMORY_SIZE_4M_VT	(0xb << 8)
4703 
4704 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4705 {
4706 	unsigned short ggc;
4707 
4708 	if (risky_device(dev))
4709 		return;
4710 
4711 	if (pci_read_config_word(dev, GGC, &ggc))
4712 		return;
4713 
4714 	if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4715 		pci_info(dev, "BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4716 		disable_igfx_iommu = 1;
4717 	} else if (!disable_igfx_iommu) {
4718 		/* we have to ensure the gfx device is idle before we flush */
4719 		pci_info(dev, "Disabling batched IOTLB flush on Ironlake\n");
4720 		iommu_set_dma_strict();
4721 	}
4722 }
4723 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4724 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4725 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4726 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4727 
4728 static void quirk_igfx_skip_te_disable(struct pci_dev *dev)
4729 {
4730 	unsigned short ver;
4731 
4732 	if (!IS_GFX_DEVICE(dev))
4733 		return;
4734 
4735 	ver = (dev->device >> 8) & 0xff;
4736 	if (ver != 0x45 && ver != 0x46 && ver != 0x4c &&
4737 	    ver != 0x4e && ver != 0x8a && ver != 0x98 &&
4738 	    ver != 0x9a && ver != 0xa7 && ver != 0x7d)
4739 		return;
4740 
4741 	if (risky_device(dev))
4742 		return;
4743 
4744 	pci_info(dev, "Skip IOMMU disabling for graphics\n");
4745 	iommu_skip_te_disable = 1;
4746 }
4747 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, quirk_igfx_skip_te_disable);
4748 
4749 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4750    ISOCH DMAR unit for the Azalia sound device, but not give it any
4751    TLB entries, which causes it to deadlock. Check for that.  We do
4752    this in a function called from init_dmars(), instead of in a PCI
4753    quirk, because we don't want to print the obnoxious "BIOS broken"
4754    message if VT-d is actually disabled.
4755 */
4756 static void __init check_tylersburg_isoch(void)
4757 {
4758 	struct pci_dev *pdev;
4759 	uint32_t vtisochctrl;
4760 
4761 	/* If there's no Azalia in the system anyway, forget it. */
4762 	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4763 	if (!pdev)
4764 		return;
4765 
4766 	if (risky_device(pdev)) {
4767 		pci_dev_put(pdev);
4768 		return;
4769 	}
4770 
4771 	pci_dev_put(pdev);
4772 
4773 	/* System Management Registers. Might be hidden, in which case
4774 	   we can't do the sanity check. But that's OK, because the
4775 	   known-broken BIOSes _don't_ actually hide it, so far. */
4776 	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4777 	if (!pdev)
4778 		return;
4779 
4780 	if (risky_device(pdev)) {
4781 		pci_dev_put(pdev);
4782 		return;
4783 	}
4784 
4785 	if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4786 		pci_dev_put(pdev);
4787 		return;
4788 	}
4789 
4790 	pci_dev_put(pdev);
4791 
4792 	/* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4793 	if (vtisochctrl & 1)
4794 		return;
4795 
4796 	/* Drop all bits other than the number of TLB entries */
4797 	vtisochctrl &= 0x1c;
4798 
4799 	/* If we have the recommended number of TLB entries (16), fine. */
4800 	if (vtisochctrl == 0x10)
4801 		return;
4802 
4803 	/* Zero TLB entries? You get to ride the short bus to school. */
4804 	if (!vtisochctrl) {
4805 		WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4806 		     "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4807 		     dmi_get_system_info(DMI_BIOS_VENDOR),
4808 		     dmi_get_system_info(DMI_BIOS_VERSION),
4809 		     dmi_get_system_info(DMI_PRODUCT_VERSION));
4810 		iommu_identity_mapping |= IDENTMAP_AZALIA;
4811 		return;
4812 	}
4813 
4814 	pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4815 	       vtisochctrl);
4816 }
4817 
4818 /*
4819  * Here we deal with a device TLB defect where device may inadvertently issue ATS
4820  * invalidation completion before posted writes initiated with translated address
4821  * that utilized translations matching the invalidation address range, violating
4822  * the invalidation completion ordering.
4823  * Therefore, any use cases that cannot guarantee DMA is stopped before unmap is
4824  * vulnerable to this defect. In other words, any dTLB invalidation initiated not
4825  * under the control of the trusted/privileged host device driver must use this
4826  * quirk.
4827  * Device TLBs are invalidated under the following six conditions:
4828  * 1. Device driver does DMA API unmap IOVA
4829  * 2. Device driver unbind a PASID from a process, sva_unbind_device()
4830  * 3. PASID is torn down, after PASID cache is flushed. e.g. process
4831  *    exit_mmap() due to crash
4832  * 4. Under SVA usage, called by mmu_notifier.invalidate_range() where
4833  *    VM has to free pages that were unmapped
4834  * 5. Userspace driver unmaps a DMA buffer
4835  * 6. Cache invalidation in vSVA usage (upcoming)
4836  *
4837  * For #1 and #2, device drivers are responsible for stopping DMA traffic
4838  * before unmap/unbind. For #3, iommu driver gets mmu_notifier to
4839  * invalidate TLB the same way as normal user unmap which will use this quirk.
4840  * The dTLB invalidation after PASID cache flush does not need this quirk.
4841  *
4842  * As a reminder, #6 will *NEED* this quirk as we enable nested translation.
4843  */
4844 void quirk_extra_dev_tlb_flush(struct device_domain_info *info,
4845 			       unsigned long address, unsigned long mask,
4846 			       u32 pasid, u16 qdep)
4847 {
4848 	u16 sid;
4849 
4850 	if (likely(!info->dtlb_extra_inval))
4851 		return;
4852 
4853 	sid = PCI_DEVID(info->bus, info->devfn);
4854 	if (pasid == IOMMU_NO_PASID) {
4855 		qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
4856 				   qdep, address, mask);
4857 	} else {
4858 		qi_flush_dev_iotlb_pasid(info->iommu, sid, info->pfsid,
4859 					 pasid, qdep, address, mask);
4860 	}
4861 }
4862 
4863 #define ecmd_get_status_code(res)	(((res) & 0xff) >> 1)
4864 
4865 /*
4866  * Function to submit a command to the enhanced command interface. The
4867  * valid enhanced command descriptions are defined in Table 47 of the
4868  * VT-d spec. The VT-d hardware implementation may support some but not
4869  * all commands, which can be determined by checking the Enhanced
4870  * Command Capability Register.
4871  *
4872  * Return values:
4873  *  - 0: Command successful without any error;
4874  *  - Negative: software error value;
4875  *  - Nonzero positive: failure status code defined in Table 48.
4876  */
4877 int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob)
4878 {
4879 	unsigned long flags;
4880 	u64 res;
4881 	int ret;
4882 
4883 	if (!cap_ecmds(iommu->cap))
4884 		return -ENODEV;
4885 
4886 	raw_spin_lock_irqsave(&iommu->register_lock, flags);
4887 
4888 	res = dmar_readq(iommu->reg + DMAR_ECRSP_REG);
4889 	if (res & DMA_ECMD_ECRSP_IP) {
4890 		ret = -EBUSY;
4891 		goto err;
4892 	}
4893 
4894 	/*
4895 	 * Unconditionally write the operand B, because
4896 	 * - There is no side effect if an ecmd doesn't require an
4897 	 *   operand B, but we set the register to some value.
4898 	 * - It's not invoked in any critical path. The extra MMIO
4899 	 *   write doesn't bring any performance concerns.
4900 	 */
4901 	dmar_writeq(iommu->reg + DMAR_ECEO_REG, ob);
4902 	dmar_writeq(iommu->reg + DMAR_ECMD_REG, ecmd | (oa << DMA_ECMD_OA_SHIFT));
4903 
4904 	IOMMU_WAIT_OP(iommu, DMAR_ECRSP_REG, dmar_readq,
4905 		      !(res & DMA_ECMD_ECRSP_IP), res);
4906 
4907 	if (res & DMA_ECMD_ECRSP_IP) {
4908 		ret = -ETIMEDOUT;
4909 		goto err;
4910 	}
4911 
4912 	ret = ecmd_get_status_code(res);
4913 err:
4914 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
4915 
4916 	return ret;
4917 }
4918