Lines Matching +full:pcie +full:- +full:ob
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright © 2006-2014 Intel Corporation.
17 #include <linux/dma-direct.h>
21 #include <linux/pci-ats.h>
28 #include "../dma-iommu.h"
30 #include "../iommu-pages.h"
38 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
39 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
40 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
48 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << ((gaw) - VTD_PAGE_SHIFT)) - 1)
49 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << (gaw)) - 1)
54 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
60 #define rwbf_required(iommu) (rwbf_quirk || cap_rwbf((iommu)->cap))
63 * set to 1 to panic kernel if can't successfully enable VT-d
78 if (!(re->lo & 1))
81 return re->lo & VTD_PAGE_MASK;
90 if (!(re->hi & 1))
93 return re->hi & VTD_PAGE_MASK;
102 if (*rid_lhs < PCI_DEVID(info->bus, info->devfn))
103 return -1;
105 if (*rid_lhs > PCI_DEVID(info->bus, info->devfn))
115 u16 key = PCI_DEVID(info->bus, info->devfn);
121 * Looks up an IOMMU-probed device using its source ID.
137 spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
138 node = rb_find(&rid, &iommu->device_rbtree, device_rid_cmp_key);
141 spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
143 return info ? info->dev : NULL;
152 spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
153 curr = rb_find_add(&info->node, &iommu->device_rbtree, device_rid_cmp);
154 spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
156 return -EEXIST;
163 struct intel_iommu *iommu = info->iommu;
166 spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
167 rb_erase(&info->node, &iommu->device_rbtree);
168 spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
224 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
229 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
236 gsts = readl(iommu->reg + DMAR_GSTS_REG);
238 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
244 return -EINVAL;
273 pr_info("Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
276 pr_notice("Unknown option - '%s'\n", str);
290 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
297 * Refer to 11.4.2 of the VT-d spec for the encoding of each bit of
304 fl_sagaw = BIT(2) | (cap_fl5lp_support(iommu->cap) ? BIT(3) : 0);
305 sl_sagaw = cap_sagaw(iommu->cap);
308 if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))
312 if (!ecap_slts(iommu->ecap))
324 for (agaw = width_to_agaw(max_gaw); agaw >= 0; agaw--) {
353 ecap_smpwc(iommu->ecap) : ecap_coherent(iommu->ecap);
362 * 1-level super page supports page size of 2MiB, 2-level super page
365 if (domain->iommu_superpage == 1)
367 else if (domain->iommu_superpage == 2)
376 struct root_entry *root = &iommu->root_entry[bus];
387 entry = &root->lo;
390 devfn -= 0x80;
391 entry = &root->hi;
402 context = iommu_alloc_pages_node_sz(iommu->node, GFP_ATOMIC,
416 * is_downstream_to_pci_bridge - test if a device belongs to the PCI
417 * sub-hierarchy of a candidate PCI-PCI bridge
418 * @dev: candidate PCI device belonging to @bridge PCI sub-hierarchy
419 * @bridge: the candidate PCI-PCI bridge
421 * Return: true if @dev belongs to @bridge PCI sub-hierarchy, else false.
434 if (pbridge->subordinate &&
435 pbridge->subordinate->number <= pdev->bus->number &&
436 pbridge->subordinate->busn_res.end >= pdev->bus->number)
453 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
456 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
463 if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {
464 pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n");
474 if (!iommu || iommu->drhd->ignored)
480 if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
481 pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SNB &&
509 dev = &pf_pdev->dev;
510 segment = pci_domain_nr(pdev->bus);
512 dev = &ACPI_COMPANION(dev)->dev;
516 if (pdev && segment != drhd->segment)
519 for_each_active_dev_scope(drhd->devices,
520 drhd->devices_cnt, i, tmp) {
526 if (pdev && pdev->is_virtfn)
530 *bus = drhd->devices[i].bus;
531 *devfn = drhd->devices[i].devfn;
540 if (pdev && drhd->include_all) {
543 *bus = pdev->bus->number;
544 *devfn = pdev->devfn;
562 if (!domain->iommu_coherency)
571 if (!iommu->root_entry)
587 iommu_free_pages(iommu->root_entry);
588 iommu->root_entry = NULL;
602 pr_info("pte level: %d, pte value: 0x%016llx\n", level, pte->val);
605 pr_info("page table not present at level %d\n", level - 1);
613 level--;
629 pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr);
632 if (!iommu->root_entry) {
636 rt_entry = &iommu->root_entry[bus];
640 rt_entry->hi, rt_entry->lo);
642 pr_info("root entry: 0x%016llx", rt_entry->lo);
652 ctx_entry->hi, ctx_entry->lo);
660 level = agaw_to_level(ctx_entry->hi & 7);
661 pgtable = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
671 dir = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
673 /* For request-without-pasid, get the pasid from context entry */
679 pr_info("pasid dir entry: 0x%016llx\n", pde->val);
689 for (i = 0; i < ARRAY_SIZE(pte->val); i++)
690 pr_info("pasid table entry[%d]: 0x%016llx\n", i, pte->val[i]);
698 level = pte->val[2] & BIT_ULL(2) ? 5 : 4;
699 pgtable = phys_to_virt(pte->val[2] & VTD_PAGE_MASK);
701 level = agaw_to_level((pte->val[0] >> 2) & 0x7);
702 pgtable = phys_to_virt(pte->val[0] & VTD_PAGE_MASK);
715 int level = agaw_to_level(domain->agaw);
722 parent = domain->pgd;
737 tmp_page = iommu_alloc_pages_node_sz(domain->nid, gfp,
746 if (domain->use_first_level)
750 if (!try_cmpxchg64(&pte->val, &tmp, pteval))
760 level--;
775 int total = agaw_to_level(domain->agaw);
778 parent = domain->pgd;
796 total--;
828 (void *)pte - (void *)first_pte);
852 dma_pte_free_level(domain, level - 1, retain_level,
862 last_pfn < level_pfn + level_size(level) - 1)) {
884 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
885 domain->pgd, 0, start_pfn, last_pfn);
888 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
889 iommu_free_pages(domain->pgd);
890 domain->pgd = NULL;
897 know the hardware page-walk will no longer touch them.
913 dma_pte_list_pagetables(domain, level - 1, pte, freelist);
936 last_pfn >= level_pfn + level_size(level) - 1) {
940 dma_pte_list_pagetables(domain, level - 1, pte, freelist);
948 dma_pte_clear_level(domain, level - 1,
959 (void *)++last_pte - (void *)first_pte);
974 dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
975 domain->pgd, 0, start_pfn, last_pfn, freelist);
978 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
979 iommu_pages_list_add(freelist, domain->pgd);
980 domain->pgd = NULL;
989 root = iommu_alloc_pages_node_sz(iommu->node, GFP_ATOMIC, SZ_4K);
992 iommu->name);
993 return -ENOMEM;
997 iommu->root_entry = root;
1008 addr = virt_to_phys(iommu->root_entry);
1012 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1013 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1015 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1021 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1027 if (cap_esrtps(iommu->cap))
1030 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1033 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1041 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1044 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1045 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1051 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1074 pr_warn("%s: Unexpected context-cache invalidation type 0x%llx\n",
1075 iommu->name, type);
1080 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1081 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1087 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1093 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1112 iommu->name, type);
1116 if (cap_write_drain(iommu->cap))
1119 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1122 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1123 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1129 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1147 spin_lock_irqsave(&domain->lock, flags);
1148 list_for_each_entry(info, &domain->devices, link) {
1149 if (info->iommu == iommu && info->bus == bus &&
1150 info->devfn == devfn) {
1151 spin_unlock_irqrestore(&domain->lock, flags);
1155 spin_unlock_irqrestore(&domain->lock, flags);
1163 * check because it applies only to the built-in QAT devices and it doesn't
1169 if (pdev->vendor != PCI_VENDOR_ID_INTEL)
1172 if ((pdev->device & 0xfffc) != BUGGY_QAT_DEVID_MASK)
1182 if (!info->ats_supported)
1185 pdev = to_pci_dev(info->dev);
1190 info->ats_enabled = 1;
1195 if (!info->ats_enabled)
1198 pci_disable_ats(to_pci_dev(info->dev));
1199 info->ats_enabled = 0;
1206 if (!info->ats_enabled || !info->pri_supported)
1209 pdev = to_pci_dev(info->dev);
1211 if (info->pasid_enabled && !pci_prg_resp_pasid_required(pdev))
1218 info->pri_enabled = 1;
1223 if (!info->pri_enabled)
1226 if (WARN_ON(info->iopf_refcount))
1227 iopf_queue_remove_device(info->iommu->iopf_queue, info->dev);
1229 pci_disable_pri(to_pci_dev(info->dev));
1230 info->pri_enabled = 0;
1243 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1246 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1247 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1249 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1255 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1263 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1264 iommu->gcmd |= DMA_GCMD_TE;
1265 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1271 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1279 if (iommu_skip_te_disable && iommu->drhd->gfx_dedicated &&
1280 (cap_read_drain(iommu->cap) || cap_write_drain(iommu->cap)))
1283 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1284 iommu->gcmd &= ~DMA_GCMD_TE;
1285 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1291 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1300 if (WARN_ON(!ida_is_empty(&iommu->domain_ida)))
1303 if (iommu->gcmd & DMA_GCMD_TE)
1309 if (iommu->copied_tables) {
1310 bitmap_free(iommu->copied_tables);
1311 iommu->copied_tables = NULL;
1317 if (ecap_prs(iommu->ecap))
1332 if (ecap_flts(iommu->ecap) ^ ecap_slts(iommu->ecap))
1333 return ecap_flts(iommu->ecap);
1341 int num, ret = -ENOSPC;
1343 if (domain->domain.type == IOMMU_DOMAIN_SVA)
1348 return -ENOMEM;
1350 guard(mutex)(&iommu->did_lock);
1351 curr = xa_load(&domain->iommu_array, iommu->seq_id);
1353 curr->refcnt++;
1358 num = ida_alloc_range(&iommu->domain_ida, IDA_START_DID,
1359 cap_ndoms(iommu->cap) - 1, GFP_KERNEL);
1361 pr_err("%s: No free domain ids\n", iommu->name);
1365 info->refcnt = 1;
1366 info->did = num;
1367 info->iommu = iommu;
1368 curr = xa_cmpxchg(&domain->iommu_array, iommu->seq_id,
1371 ret = xa_err(curr) ? : -EBUSY;
1378 ida_free(&iommu->domain_ida, info->did);
1388 if (domain->domain.type == IOMMU_DOMAIN_SVA)
1391 guard(mutex)(&iommu->did_lock);
1392 info = xa_load(&domain->iommu_array, iommu->seq_id);
1393 if (--info->refcnt == 0) {
1394 ida_free(&iommu->domain_ida, info->did);
1395 xa_erase(&domain->iommu_array, iommu->seq_id);
1402 * in-flight DMA and copied pgtable, but there is no unmapping
1404 * the newly-mapped device. For kdump, at this point, the device
1406 * in-flight DMA will exist, and we don't need to worry anymore
1418 assert_spin_locked(&iommu->lock);
1423 if (did_old < cap_ndoms(iommu->cap)) {
1424 iommu->flush.flush_context(iommu, did_old,
1428 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
1436 * It's a non-present to present mapping. If hardware doesn't cache
1437 * non-present entry we only need to flush the write-buffer. If the
1438 * _does_ cache non-present entries, then it does so in the special
1444 if (cap_caching_mode(iommu->cap)) {
1445 iommu->flush.flush_context(iommu, 0,
1449 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1463 struct dma_pte *pgd = domain->pgd;
1468 return -EINVAL;
1473 spin_lock(&iommu->lock);
1474 ret = -ENOMEM;
1487 if (info && info->ats_supported)
1493 context_set_address_width(context, domain->agaw);
1497 if (!ecap_coherent(iommu->ecap))
1503 spin_unlock(&iommu->lock);
1511 struct device_domain_info *info = dev_iommu_priv_get(&pdev->dev);
1512 struct intel_iommu *iommu = info->iommu;
1523 struct intel_iommu *iommu = info->iommu;
1524 u8 bus = info->bus, devfn = info->devfn;
1547 support = domain->iommu_superpage;
1561 support--;
1589 start_pfn + lvl_pages - 1,
1614 if (unlikely(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1)))
1615 return -EINVAL;
1618 return -EINVAL;
1620 if (!(prot & DMA_PTE_WRITE) && domain->nested_parent) {
1621 pr_err_ratelimited("Read-only mapping is disallowed on the domain which serves as the parent in a nested configuration, due to HW errata (ERRATA_772415_SPR17)\n");
1622 return -EINVAL;
1626 if (domain->use_first_level) {
1632 domain->has_mappings = true;
1646 return -ENOMEM;
1660 end_pfn = iov_pfn + pages_to_remove - 1;
1671 if (!try_cmpxchg64_local(&pte->val, &tmp, pteval)) {
1676 dumps--;
1682 nr_pages -= lvl_pages;
1703 (void *)pte - (void *)first_pte);
1713 struct intel_iommu *iommu = info->iommu;
1717 spin_lock(&iommu->lock);
1720 spin_unlock(&iommu->lock);
1727 spin_unlock(&iommu->lock);
1772 struct dma_pte *pgd = domain->pgd;
1775 level = agaw_to_level(domain->agaw);
1777 return -EINVAL;
1782 if (domain->force_snooping)
1794 struct intel_iommu *iommu = info->iommu;
1802 info->domain = domain;
1803 info->domain_attached = true;
1804 spin_lock_irqsave(&domain->lock, flags);
1805 list_add(&info->link, &domain->devices);
1806 spin_unlock_irqrestore(&domain->lock, flags);
1820 ret = -EINVAL;
1837 * device_rmrr_is_relaxable - Test whether the RMRR of this device
1868 struct intel_iommu *iommu = info->iommu;
1874 if (!ecap_pass_through(iommu->ecap))
1892 * (for example, while enabling interrupt-remapping) then
1895 if (!iommu->qi) {
1899 dmar_fault(-1, iommu);
1911 iommu->flush.flush_context = __iommu_flush_context;
1912 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
1914 iommu->name);
1916 iommu->flush.flush_context = qi_flush_context;
1917 iommu->flush.flush_iotlb = qi_flush_iotlb;
1918 pr_info("%s: Using Queued invalidation\n", iommu->name);
1968 ret = -ENOMEM;
1974 new_ce = iommu_alloc_pages_node_sz(iommu->node,
1989 if (did >= 0 && did < cap_ndoms(iommu->cap))
1990 ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL);
2017 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2028 return -EINVAL;
2030 iommu->copied_tables = bitmap_zalloc(BIT_ULL(16), GFP_KERNEL);
2031 if (!iommu->copied_tables)
2032 return -ENOMEM;
2036 return -EINVAL;
2040 return -ENOMEM;
2042 /* This is too big for the stack - allocate it from slab */
2044 ret = -ENOMEM;
2054 iommu->name, bus);
2059 spin_lock(&iommu->lock);
2068 iommu->root_entry[bus].lo = val;
2075 iommu->root_entry[bus].hi = val;
2078 spin_unlock(&iommu->lock);
2082 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
2099 if (drhd->ignored) {
2110 u32 temp = 2 << ecap_pss(iommu->ecap);
2123 iommu->name);
2136 pr_info("Translation already enabled - trying to copy translation structures\n");
2142 * enabled - but failed to copy over the
2143 * old root-entry table. Try to proceed
2145 * allocating a clean root-entry table.
2150 iommu->name);
2155 iommu->name);
2182 if (drhd->ignored) {
2194 if (ecap_prs(iommu->ecap)) {
2229 if (!drhd->include_all) {
2230 for_each_active_dev_scope(drhd->devices,
2231 drhd->devices_cnt, i, dev)
2234 if (i == drhd->devices_cnt)
2235 drhd->ignored = 1;
2240 if (drhd->include_all)
2243 for_each_active_dev_scope(drhd->devices,
2244 drhd->devices_cnt, i, dev)
2247 if (i < drhd->devices_cnt)
2252 drhd->gfx_dedicated = 1;
2254 drhd->ignored = 1;
2266 if (iommu->qi) {
2274 if (drhd->ignored) {
2299 iommu->flush.flush_context(iommu, 0, 0, 0,
2301 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2317 raw_spin_lock_irqsave(&iommu->register_lock, flag);
2319 iommu->iommu_state[SR_DMAR_FECTL_REG] =
2320 readl(iommu->reg + DMAR_FECTL_REG);
2321 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
2322 readl(iommu->reg + DMAR_FEDATA_REG);
2323 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
2324 readl(iommu->reg + DMAR_FEADDR_REG);
2325 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
2326 readl(iommu->reg + DMAR_FEUADDR_REG);
2328 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2349 raw_spin_lock_irqsave(&iommu->register_lock, flag);
2351 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
2352 iommu->reg + DMAR_FECTL_REG);
2353 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
2354 iommu->reg + DMAR_FEDATA_REG);
2355 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
2356 iommu->reg + DMAR_FEADDR_REG);
2357 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
2358 iommu->reg + DMAR_FEUADDR_REG);
2360 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2380 if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) ||
2381 !IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) ||
2382 rmrr->end_address <= rmrr->base_address ||
2384 return -EINVAL;
2397 "Your BIOS is broken; bad RMRR [%#018Lx-%#018Lx]\n"
2399 rmrr->base_address, rmrr->end_address,
2410 rmrru->hdr = header;
2412 rmrru->base_address = rmrr->base_address;
2413 rmrru->end_address = rmrr->end_address;
2415 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
2416 ((void *)rmrr) + rmrr->header.length,
2417 &rmrru->devices_cnt);
2418 if (rmrru->devices_cnt && rmrru->devices == NULL)
2421 list_add(&rmrru->list, &dmar_rmrr_units);
2427 return -ENOMEM;
2437 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
2438 if (atsr->segment != tmp->segment)
2440 if (atsr->header.length != tmp->header.length)
2442 if (memcmp(atsr, tmp, atsr->header.length) == 0)
2462 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
2464 return -ENOMEM;
2471 atsru->hdr = (void *)(atsru + 1);
2472 memcpy(atsru->hdr, hdr, hdr->length);
2473 atsru->include_all = atsr->flags & 0x1;
2474 if (!atsru->include_all) {
2475 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
2476 (void *)atsr + atsr->header.length,
2477 &atsru->devices_cnt);
2478 if (atsru->devices_cnt && atsru->devices == NULL) {
2480 return -ENOMEM;
2484 list_add_rcu(&atsru->list, &dmar_atsr_units);
2491 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
2503 list_del_rcu(&atsru->list);
2523 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
2524 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
2526 return -EBUSY;
2539 tmp = (struct acpi_dmar_satc *)satcu->hdr;
2540 if (satc->segment != tmp->segment)
2542 if (satc->header.length != tmp->header.length)
2544 if (memcmp(satc, tmp, satc->header.length) == 0)
2564 satcu = kzalloc(sizeof(*satcu) + hdr->length, GFP_KERNEL);
2566 return -ENOMEM;
2568 satcu->hdr = (void *)(satcu + 1);
2569 memcpy(satcu->hdr, hdr, hdr->length);
2570 satcu->atc_required = satc->flags & 0x1;
2571 satcu->devices = dmar_alloc_dev_scope((void *)(satc + 1),
2572 (void *)satc + satc->header.length,
2573 &satcu->devices_cnt);
2574 if (satcu->devices_cnt && !satcu->devices) {
2576 return -ENOMEM;
2578 list_add_rcu(&satcu->list, &dmar_satc_units);
2585 struct intel_iommu *iommu = dmaru->iommu;
2591 if (iommu->gcmd & DMA_GCMD_TE)
2600 if (dmaru->ignored) {
2612 if (ecap_prs(iommu->ecap)) {
2638 struct intel_iommu *iommu = dmaru->iommu;
2643 return -EINVAL;
2662 list_del(&rmrru->list);
2663 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
2668 list_del(&atsru->list);
2672 list_del(&satcu->list);
2673 dmar_free_dev_scope(&satcu->devices, &satcu->devices_cnt);
2688 satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
2689 if (satc->segment != pci_domain_nr(dev->bus))
2691 for_each_dev_scope(satcu->devices, satcu->devices_cnt, i, tmp)
2722 return !(satcu->atc_required && !sm_supported(iommu));
2724 for (bus = dev->bus; bus; bus = bus->parent) {
2725 bridge = bus->self;
2729 /* Connected via non-PCIe: no ATS */
2740 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
2741 if (atsr->segment != pci_domain_nr(dev->bus))
2744 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
2745 if (tmp == &bridge->dev)
2748 if (atsru->include_all)
2772 rmrr = container_of(rmrru->hdr,
2774 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
2776 ((void *)rmrr) + rmrr->header.length,
2777 rmrr->segment, rmrru->devices,
2778 rmrru->devices_cnt);
2781 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
2782 dmar_remove_dev_scope(info, rmrr->segment,
2783 rmrru->devices, rmrru->devices_cnt);
2788 if (atsru->include_all)
2791 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
2792 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
2794 (void *)atsr + atsr->header.length,
2795 atsr->segment, atsru->devices,
2796 atsru->devices_cnt);
2801 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
2802 if (dmar_remove_dev_scope(info, atsr->segment,
2803 atsru->devices, atsru->devices_cnt))
2808 satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
2809 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
2811 (void *)satc + satc->header.length,
2812 satc->segment, satcu->devices,
2813 satcu->devices_cnt);
2818 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
2819 if (dmar_remove_dev_scope(info, satc->segment,
2820 satcu->devices, satcu->devices_cnt))
2850 iommu = drhd->iommu;
2871 u32 ver = readl(iommu->reg + DMAR_VER_REG);
2881 return sysfs_emit(buf, "%llx\n", iommu->reg_phys);
2889 return sysfs_emit(buf, "%llx\n", iommu->cap);
2897 return sysfs_emit(buf, "%llx\n", iommu->ecap);
2905 return sysfs_emit(buf, "%ld\n", cap_ndoms(iommu->cap));
2916 for (id = 0; id < cap_ndoms(iommu->cap); id++)
2917 if (ida_exists(&iommu->domain_ida, id))
2935 .name = "intel-iommu",
2949 if (pdev->external_facing) {
2963 pr_info("Intel-IOMMU force enabled due to platform opt in\n");
2966 * If Intel-IOMMU is disabled by default, we will apply identity
2981 /* To avoid a -Wunused-but-set-variable warning. */
2987 for_each_active_dev_scope(drhd->devices,
2988 drhd->devices_cnt, i, dev) {
2992 if (dev->bus != &acpi_bus_type)
2997 mutex_lock(&adev->physical_node_lock);
2999 &adev->physical_node_list, node) {
3000 ret = iommu_probe_device(pn->dev);
3004 mutex_unlock(&adev->physical_node_lock);
3021 pr_warn("Forcing Intel-IOMMU to enabled\n");
3031 int ret = -ENODEV;
3117 * page-selective invalidations that are required for efficient
3120 * the virtual and physical IOMMU page-tables.
3122 if (cap_caching_mode(iommu->cap) &&
3127 iommu_device_sysfs_add(&iommu->iommu, NULL,
3129 "%s", iommu->name);
3136 iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);
3147 if (!drhd->ignored && !translation_pre_enabled(iommu))
3175 * NB - intel-iommu lacks any sort of reference counting for the users of
3182 if (!dev_is_pci(info->dev)) {
3183 domain_context_clear_one(info, info->bus, info->devfn);
3187 pci_for_each_dma_alias(to_pci_dev(info->dev),
3200 struct intel_iommu *iommu = info->iommu;
3204 if (!info->domain_attached)
3207 if (info->domain)
3208 cache_tag_unassign_domain(info->domain, dev, IOMMU_NO_PASID);
3219 info->domain_attached = false;
3221 if (!info->domain)
3224 spin_lock_irqsave(&info->domain->lock, flags);
3225 list_del(&info->link);
3226 spin_unlock_irqrestore(&info->domain->lock, flags);
3228 domain_detach_iommu(info->domain, iommu);
3229 info->domain = NULL;
3237 iopf_for_domain_remove(info->domain ? &info->domain->domain : NULL, dev);
3260 return cap_fl1gp_support(iommu->cap) ? 2 : 1;
3262 return fls(cap_super_page_val(iommu->cap));
3268 struct intel_iommu *iommu = info->iommu;
3274 return ERR_PTR(-ENOMEM);
3276 INIT_LIST_HEAD(&domain->devices);
3277 INIT_LIST_HEAD(&domain->dev_pasids);
3278 INIT_LIST_HEAD(&domain->cache_tags);
3279 spin_lock_init(&domain->lock);
3280 spin_lock_init(&domain->cache_lock);
3281 xa_init(&domain->iommu_array);
3282 INIT_LIST_HEAD(&domain->s1_domains);
3283 spin_lock_init(&domain->s1_lock);
3285 domain->nid = dev_to_node(dev);
3286 domain->use_first_level = first_stage;
3288 domain->domain.type = IOMMU_DOMAIN_UNMANAGED;
3291 addr_width = agaw_to_width(iommu->agaw);
3292 if (addr_width > cap_mgaw(iommu->cap))
3293 addr_width = cap_mgaw(iommu->cap);
3294 domain->gaw = addr_width;
3295 domain->agaw = iommu->agaw;
3296 domain->max_addr = __DOMAIN_MAX_ADDR(addr_width);
3299 domain->iommu_coherency = iommu_paging_structure_coherency(iommu);
3302 domain->domain.pgsize_bitmap = SZ_4K;
3303 domain->iommu_superpage = iommu_superpage_capability(iommu, first_stage);
3304 domain->domain.pgsize_bitmap |= domain_super_pgsize_bitmap(domain);
3307 * IOVA aperture: First-level translation restricts the input-address
3309 * as address bit [N-1], where N is 48-bits with 4-level paging and
3310 * 57-bits with 5-level paging). Hence, skip bit [N-1].
3312 domain->domain.geometry.force_aperture = true;
3313 domain->domain.geometry.aperture_start = 0;
3315 domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw - 1);
3317 domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw);
3320 domain->pgd = iommu_alloc_pages_node_sz(domain->nid, GFP_KERNEL, SZ_4K);
3321 if (!domain->pgd) {
3323 return ERR_PTR(-ENOMEM);
3325 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3337 return ERR_PTR(-EOPNOTSUPP);
3340 if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))
3341 return ERR_PTR(-EOPNOTSUPP);
3347 dmar_domain->domain.ops = &intel_fs_paging_domain_ops;
3354 dmar_domain->iotlb_sync_map = true;
3356 return &dmar_domain->domain;
3368 return ERR_PTR(-EOPNOTSUPP);
3374 return ERR_PTR(-EOPNOTSUPP);
3377 if (sm_supported(iommu) && !ecap_slts(iommu->ecap))
3378 return ERR_PTR(-EOPNOTSUPP);
3384 dmar_domain->domain.ops = &intel_ss_paging_domain_ops;
3385 dmar_domain->nested_parent = flags & IOMMU_HWPT_ALLOC_NEST_PARENT;
3388 dmar_domain->domain.dirty_ops = &intel_dirty_ops;
3395 if (rwbf_required(iommu) || cap_caching_mode(iommu->cap))
3396 dmar_domain->iotlb_sync_map = true;
3398 return &dmar_domain->domain;
3406 struct intel_iommu *iommu = info->iommu;
3410 return ERR_PTR(-EOPNOTSUPP);
3414 if (domain != ERR_PTR(-EOPNOTSUPP))
3423 if (WARN_ON(dmar_domain->nested_parent &&
3424 !list_empty(&dmar_domain->s1_domains)))
3427 if (WARN_ON(!list_empty(&dmar_domain->devices)))
3430 if (dmar_domain->pgd) {
3434 domain_unmap(dmar_domain, 0, DOMAIN_MAX_PFN(dmar_domain->gaw),
3439 kfree(dmar_domain->qi_batch);
3446 if (WARN_ON(dmar_domain->domain.dirty_ops ||
3447 dmar_domain->nested_parent))
3448 return -EINVAL;
3451 if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))
3452 return -EINVAL;
3455 if (!cap_fl1gp_support(iommu->cap) &&
3456 (dmar_domain->domain.pgsize_bitmap & SZ_1G))
3457 return -EINVAL;
3460 if ((rwbf_required(iommu)) && !dmar_domain->iotlb_sync_map)
3461 return -EINVAL;
3470 unsigned int sslps = cap_super_page_val(iommu->cap);
3472 if (dmar_domain->domain.dirty_ops && !ssads_supported(iommu))
3473 return -EINVAL;
3474 if (dmar_domain->nested_parent && !nested_supported(iommu))
3475 return -EINVAL;
3478 if (sm_supported(iommu) && !ecap_slts(iommu->ecap))
3479 return -EINVAL;
3482 if (!(sslps & BIT(0)) && (dmar_domain->domain.pgsize_bitmap & SZ_2M))
3483 return -EINVAL;
3484 if (!(sslps & BIT(1)) && (dmar_domain->domain.pgsize_bitmap & SZ_1G))
3485 return -EINVAL;
3488 if ((rwbf_required(iommu) || cap_caching_mode(iommu->cap)) &&
3489 !dmar_domain->iotlb_sync_map)
3490 return -EINVAL;
3499 struct intel_iommu *iommu = info->iommu;
3500 int ret = -EINVAL;
3508 ret = -EINVAL;
3514 * dmar_domain->lock
3516 if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
3517 return -EINVAL;
3519 if (dmar_domain->iommu_coherency !=
3521 return -EINVAL;
3525 addr_width = agaw_to_width(iommu->agaw);
3526 if (addr_width > cap_mgaw(iommu->cap))
3527 addr_width = cap_mgaw(iommu->cap);
3529 if (dmar_domain->gaw > addr_width || dmar_domain->agaw > iommu->agaw)
3530 return -EINVAL;
3533 context_copied(iommu, info->bus, info->devfn))
3573 if (dmar_domain->set_pte_snp)
3577 if (dmar_domain->max_addr < max_addr) {
3581 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
3585 __func__, dmar_domain->gaw, max_addr);
3586 return -EFAULT;
3588 dmar_domain->max_addr = max_addr;
3607 return -EINVAL;
3610 return -EINVAL;
3628 size argument if it happens to be a large-page mapping. */
3637 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
3639 domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist);
3641 if (dmar_domain->max_addr == iova + size)
3642 dmar_domain->max_addr = iova;
3645 * We do not use page-selective IOTLB invalidation in flush queue,
3668 cache_tag_flush_range(to_dmar_domain(domain), gather->start,
3669 gather->end,
3670 iommu_pages_list_empty(&gather->freelist));
3671 iommu_put_pages_list(&gather->freelist);
3687 VTD_PAGE_SHIFT) - 1));
3697 assert_spin_locked(&domain->lock);
3698 list_for_each_entry(info, &domain->devices, link) {
3699 if (!ecap_sc_support(info->iommu->ecap)) {
3713 guard(spinlock_irqsave)(&dmar_domain->lock);
3715 if (dmar_domain->force_snooping)
3721 dmar_domain->force_snooping = true;
3722 list_for_each_entry(info, &dmar_domain->devices, link)
3723 intel_pasid_setup_page_snoop_control(info->iommu, info->dev,
3732 guard(spinlock_irqsave)(&dmar_domain->lock);
3734 dmar_domain->has_mappings)
3738 * Second level page table supports per-PTE snoop control. The
3741 dmar_domain->set_pte_snp = true;
3742 dmar_domain->force_snooping = true;
3757 return ecap_sc_support(info->iommu->ecap);
3759 return ssads_supported(info->iommu);
3774 if (!iommu || !iommu->iommu.ops)
3775 return ERR_PTR(-ENODEV);
3779 return ERR_PTR(-ENOMEM);
3782 info->bus = pdev->bus->number;
3783 info->devfn = pdev->devfn;
3784 info->segment = pci_domain_nr(pdev->bus);
3786 info->bus = bus;
3787 info->devfn = devfn;
3788 info->segment = iommu->segment;
3791 info->dev = dev;
3792 info->iommu = iommu;
3794 if (ecap_dev_iotlb_support(iommu->ecap) &&
3797 info->ats_supported = 1;
3798 info->dtlb_extra_inval = dev_needs_extra_dtlb_flush(pdev);
3807 if (ecap_dit(iommu->ecap))
3808 info->pfsid = pci_dev_id(pci_physfn(pdev));
3809 info->ats_qdep = pci_ats_queue_depth(pdev);
3816 info->pasid_supported = features | 1;
3819 if (info->ats_supported && ecap_prs(iommu->ecap) &&
3821 info->pri_supported = 1;
3840 if (!context_copied(iommu, info->bus, info->devfn)) {
3849 return &iommu->iommu;
3863 struct intel_iommu *iommu = info->iommu;
3866 * The PCIe spec, in its wisdom, declares that the behaviour of the
3871 if (info->pasid_supported &&
3872 !pci_enable_pasid(to_pci_dev(dev), info->pasid_supported & ~1))
3873 info->pasid_enabled = 1;
3878 if (info->ats_enabled && info->domain) {
3879 u16 did = domain_id_iommu(info->domain, iommu);
3881 if (cache_tag_assign(info->domain, did, dev,
3892 struct intel_iommu *iommu = info->iommu;
3897 if (info->pasid_enabled) {
3899 info->pasid_enabled = 0;
3902 mutex_lock(&iommu->iopf_lock);
3905 mutex_unlock(&iommu->iopf_lock);
3908 !context_copied(iommu, info->bus, info->devfn))
3927 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3937 length = rmrr->end_address - rmrr->base_address + 1;
3942 resv = iommu_alloc_resv_region(rmrr->base_address,
3948 list_add_tail(&resv->list, head);
3957 if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) {
3962 list_add_tail(®->list, head);
3968 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
3972 list_add_tail(®->list, head);
3985 struct intel_iommu *iommu = info->iommu;
3988 if (!info->pri_enabled)
3989 return -ENODEV;
3993 if (info->iopf_refcount) {
3994 info->iopf_refcount++;
3998 ret = iopf_queue_add_device(iommu->iopf_queue, dev);
4002 info->iopf_refcount = 1;
4010 struct intel_iommu *iommu = info->iommu;
4012 if (WARN_ON(!info->pri_enabled || !info->iopf_refcount))
4016 if (--info->iopf_refcount)
4019 iopf_queue_remove_device(iommu->iopf_queue, dev);
4026 return translation_pre_enabled(info->iommu) && !info->domain;
4036 if (pdev->untrusted) {
4039 pdev->vendor, pdev->device);
4051 if (dmar_domain->iotlb_sync_map)
4052 cache_tag_flush_range_np(dmar_domain, iova, iova + size - 1);
4062 struct intel_iommu *iommu = info->iommu;
4070 if (domain->type == IOMMU_DOMAIN_IDENTITY)
4074 spin_lock_irqsave(&dmar_domain->lock, flags);
4075 list_for_each_entry(curr, &dmar_domain->dev_pasids, link_domain) {
4076 if (curr->dev == dev && curr->pasid == pasid) {
4077 list_del(&curr->link_domain);
4082 spin_unlock_irqrestore(&dmar_domain->lock, flags);
4098 intel_pasid_tear_down_entry(info->iommu, dev, pasid, false);
4111 struct intel_iommu *iommu = info->iommu;
4118 return ERR_PTR(-ENOMEM);
4128 dev_pasid->dev = dev;
4129 dev_pasid->pasid = pasid;
4130 spin_lock_irqsave(&dmar_domain->lock, flags);
4131 list_add(&dev_pasid->link_domain, &dmar_domain->dev_pasids);
4132 spin_unlock_irqrestore(&dmar_domain->lock, flags);
4148 struct intel_iommu *iommu = info->iommu;
4152 if (WARN_ON_ONCE(!(domain->type & __IOMMU_DOMAIN_PAGING)))
4153 return -EINVAL;
4156 return -EOPNOTSUPP;
4158 if (domain->dirty_ops)
4159 return -EINVAL;
4161 if (context_copied(iommu, info->bus, info->devfn))
4162 return -EBUSY;
4183 ret = -EINVAL;
4205 struct intel_iommu *iommu = info->iommu;
4210 return ERR_PTR(-EOPNOTSUPP);
4214 return ERR_PTR(-ENOMEM);
4216 vtd->flags = IOMMU_HW_INFO_VTD_ERRATA_772415_SPR17;
4217 vtd->cap_reg = iommu->cap;
4218 vtd->ecap_reg = iommu->ecap;
4226 * hold the domain->lock when calling it.
4234 ret = intel_pasid_setup_dirty_tracking(info->iommu, info->dev,
4250 spin_lock(&domain->s1_lock);
4251 list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) {
4252 spin_lock_irqsave(&s1_domain->lock, flags);
4253 ret = device_set_dirty_tracking(&s1_domain->devices, enable);
4254 spin_unlock_irqrestore(&s1_domain->lock, flags);
4258 spin_unlock(&domain->s1_lock);
4262 list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) {
4263 spin_lock_irqsave(&s1_domain->lock, flags);
4264 device_set_dirty_tracking(&s1_domain->devices,
4265 domain->dirty_tracking);
4266 spin_unlock_irqrestore(&s1_domain->lock, flags);
4268 spin_unlock(&domain->s1_lock);
4278 spin_lock(&dmar_domain->lock);
4279 if (dmar_domain->dirty_tracking == enable)
4282 ret = device_set_dirty_tracking(&dmar_domain->devices, enable);
4286 if (dmar_domain->nested_parent) {
4292 dmar_domain->dirty_tracking = enable;
4294 spin_unlock(&dmar_domain->lock);
4299 device_set_dirty_tracking(&dmar_domain->devices,
4300 dmar_domain->dirty_tracking);
4301 spin_unlock(&dmar_domain->lock);
4311 unsigned long end = iova + size - 1;
4320 if (!dmar_domain->dirty_tracking && dirty->bitmap)
4321 return -EINVAL;
4351 struct intel_iommu *iommu = info->iommu;
4354 spin_lock(&iommu->lock);
4357 spin_unlock(&iommu->lock);
4358 return -ENOMEM;
4362 spin_unlock(&iommu->lock);
4374 context_set_address_width(context, iommu->msagaw);
4378 if (!ecap_coherent(iommu->ecap))
4381 spin_unlock(&iommu->lock);
4398 return context_setup_pass_through(dev, info->bus, info->devfn);
4407 struct intel_iommu *iommu = info->iommu;
4426 info->domain_attached = true;
4436 struct intel_iommu *iommu = info->iommu;
4440 return -EOPNOTSUPP;
4565 pci_info(dev, "Forcing write-buffer flush capability\n");
4617 ver = (dev->device >> 8) & 0xff;
4636 message if VT-d is actually disabled.
4657 known-broken BIOSes _don't_ actually hide it, so far. */
4674 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4732 if (likely(!info->dtlb_extra_inval))
4735 sid = PCI_DEVID(info->bus, info->devfn);
4737 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
4740 qi_flush_dev_iotlb_pasid(info->iommu, sid, info->pfsid,
4750 * VT-d spec. The VT-d hardware implementation may support some but not
4755 * - 0: Command successful without any error;
4756 * - Negative: software error value;
4757 * - Nonzero positive: failure status code defined in Table 48.
4759 int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob)
4765 if (!cap_ecmds(iommu->cap))
4766 return -ENODEV;
4768 raw_spin_lock_irqsave(&iommu->register_lock, flags);
4770 res = dmar_readq(iommu->reg + DMAR_ECRSP_REG);
4772 ret = -EBUSY;
4778 * - There is no side effect if an ecmd doesn't require an
4780 * - It's not invoked in any critical path. The extra MMIO
4783 dmar_writeq(iommu->reg + DMAR_ECEO_REG, ob);
4784 dmar_writeq(iommu->reg + DMAR_ECMD_REG, ecmd | (oa << DMA_ECMD_OA_SHIFT));
4790 ret = -ETIMEDOUT;
4796 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);