1672cf6dfSJoerg Roedel // SPDX-License-Identifier: GPL-2.0-only 2672cf6dfSJoerg Roedel /* 3672cf6dfSJoerg Roedel * Copyright © 2015 Intel Corporation. 4672cf6dfSJoerg Roedel * 5672cf6dfSJoerg Roedel * Authors: David Woodhouse <dwmw2@infradead.org> 6672cf6dfSJoerg Roedel */ 7672cf6dfSJoerg Roedel 8672cf6dfSJoerg Roedel #include <linux/mmu_notifier.h> 9672cf6dfSJoerg Roedel #include <linux/sched.h> 10672cf6dfSJoerg Roedel #include <linux/sched/mm.h> 11672cf6dfSJoerg Roedel #include <linux/slab.h> 12672cf6dfSJoerg Roedel #include <linux/rculist.h> 13672cf6dfSJoerg Roedel #include <linux/pci.h> 14672cf6dfSJoerg Roedel #include <linux/pci-ats.h> 15672cf6dfSJoerg Roedel #include <linux/dmar.h> 16672cf6dfSJoerg Roedel #include <linux/interrupt.h> 17672cf6dfSJoerg Roedel #include <linux/mm_types.h> 18100b8a14SLu Baolu #include <linux/xarray.h> 19672cf6dfSJoerg Roedel #include <asm/page.h> 2020f0afd1SFenghua Yu #include <asm/fpu/api.h> 21672cf6dfSJoerg Roedel 222585a279SLu Baolu #include "iommu.h" 2302f3effdSLu Baolu #include "pasid.h" 240f4834abSLu Baolu #include "perf.h" 2506c37505SPasha Tatashin #include "../iommu-pages.h" 26933ab6d3SLu Baolu #include "trace.h" 27672cf6dfSJoerg Roedel 28672cf6dfSJoerg Roedel static irqreturn_t prq_event_thread(int irq, void *d); 29672cf6dfSJoerg Roedel 30672cf6dfSJoerg Roedel int intel_svm_enable_prq(struct intel_iommu *iommu) 31672cf6dfSJoerg Roedel { 324c82b886SLu Baolu struct iopf_queue *iopfq; 33672cf6dfSJoerg Roedel int irq, ret; 34672cf6dfSJoerg Roedel 3506c37505SPasha Tatashin iommu->prq = iommu_alloc_pages_node(iommu->node, GFP_KERNEL, PRQ_ORDER); 3606c37505SPasha Tatashin if (!iommu->prq) { 37672cf6dfSJoerg Roedel pr_warn("IOMMU: %s: Failed to allocate page request queue\n", 38672cf6dfSJoerg Roedel iommu->name); 39672cf6dfSJoerg Roedel return -ENOMEM; 40672cf6dfSJoerg Roedel } 41672cf6dfSJoerg Roedel 424a0d4265SKan Liang irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PRQ + iommu->seq_id, iommu->node, iommu); 43672cf6dfSJoerg Roedel if (irq <= 0) { 44672cf6dfSJoerg Roedel pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n", 45672cf6dfSJoerg Roedel iommu->name); 46672cf6dfSJoerg Roedel ret = -EINVAL; 474c82b886SLu Baolu goto free_prq; 48672cf6dfSJoerg Roedel } 49672cf6dfSJoerg Roedel iommu->pr_irq = irq; 50672cf6dfSJoerg Roedel 514c82b886SLu Baolu snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name), 524c82b886SLu Baolu "dmar%d-iopfq", iommu->seq_id); 534c82b886SLu Baolu iopfq = iopf_queue_alloc(iommu->iopfq_name); 544c82b886SLu Baolu if (!iopfq) { 554c82b886SLu Baolu pr_err("IOMMU: %s: Failed to allocate iopf queue\n", iommu->name); 564c82b886SLu Baolu ret = -ENOMEM; 574c82b886SLu Baolu goto free_hwirq; 584c82b886SLu Baolu } 594c82b886SLu Baolu iommu->iopf_queue = iopfq; 604c82b886SLu Baolu 61672cf6dfSJoerg Roedel snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id); 62672cf6dfSJoerg Roedel 63672cf6dfSJoerg Roedel ret = request_threaded_irq(irq, NULL, prq_event_thread, IRQF_ONESHOT, 64672cf6dfSJoerg Roedel iommu->prq_name, iommu); 65672cf6dfSJoerg Roedel if (ret) { 66672cf6dfSJoerg Roedel pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n", 67672cf6dfSJoerg Roedel iommu->name); 684c82b886SLu Baolu goto free_iopfq; 69672cf6dfSJoerg Roedel } 70672cf6dfSJoerg Roedel dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 71672cf6dfSJoerg Roedel dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 72672cf6dfSJoerg Roedel dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER); 73672cf6dfSJoerg Roedel 74672cf6dfSJoerg Roedel init_completion(&iommu->prq_complete); 75672cf6dfSJoerg Roedel 76672cf6dfSJoerg Roedel return 0; 774c82b886SLu Baolu 784c82b886SLu Baolu free_iopfq: 794c82b886SLu Baolu iopf_queue_free(iommu->iopf_queue); 804c82b886SLu Baolu iommu->iopf_queue = NULL; 814c82b886SLu Baolu free_hwirq: 824c82b886SLu Baolu dmar_free_hwirq(irq); 834c82b886SLu Baolu iommu->pr_irq = 0; 844c82b886SLu Baolu free_prq: 8506c37505SPasha Tatashin iommu_free_pages(iommu->prq, PRQ_ORDER); 864c82b886SLu Baolu iommu->prq = NULL; 874c82b886SLu Baolu 884c82b886SLu Baolu return ret; 89672cf6dfSJoerg Roedel } 90672cf6dfSJoerg Roedel 91672cf6dfSJoerg Roedel int intel_svm_finish_prq(struct intel_iommu *iommu) 92672cf6dfSJoerg Roedel { 93672cf6dfSJoerg Roedel dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 94672cf6dfSJoerg Roedel dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 95672cf6dfSJoerg Roedel dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL); 96672cf6dfSJoerg Roedel 97672cf6dfSJoerg Roedel if (iommu->pr_irq) { 98672cf6dfSJoerg Roedel free_irq(iommu->pr_irq, iommu); 99672cf6dfSJoerg Roedel dmar_free_hwirq(iommu->pr_irq); 100672cf6dfSJoerg Roedel iommu->pr_irq = 0; 101672cf6dfSJoerg Roedel } 102672cf6dfSJoerg Roedel 1034c82b886SLu Baolu if (iommu->iopf_queue) { 1044c82b886SLu Baolu iopf_queue_free(iommu->iopf_queue); 1054c82b886SLu Baolu iommu->iopf_queue = NULL; 1064c82b886SLu Baolu } 1074c82b886SLu Baolu 10806c37505SPasha Tatashin iommu_free_pages(iommu->prq, PRQ_ORDER); 109672cf6dfSJoerg Roedel iommu->prq = NULL; 110672cf6dfSJoerg Roedel 111672cf6dfSJoerg Roedel return 0; 112672cf6dfSJoerg Roedel } 113672cf6dfSJoerg Roedel 114672cf6dfSJoerg Roedel void intel_svm_check(struct intel_iommu *iommu) 115672cf6dfSJoerg Roedel { 116672cf6dfSJoerg Roedel if (!pasid_supported(iommu)) 117672cf6dfSJoerg Roedel return; 118672cf6dfSJoerg Roedel 119672cf6dfSJoerg Roedel if (cpu_feature_enabled(X86_FEATURE_GBPAGES) && 120672cf6dfSJoerg Roedel !cap_fl1gp_support(iommu->cap)) { 121672cf6dfSJoerg Roedel pr_err("%s SVM disabled, incompatible 1GB page capability\n", 122672cf6dfSJoerg Roedel iommu->name); 123672cf6dfSJoerg Roedel return; 124672cf6dfSJoerg Roedel } 125672cf6dfSJoerg Roedel 126672cf6dfSJoerg Roedel if (cpu_feature_enabled(X86_FEATURE_LA57) && 127b722cb32SYi Liu !cap_fl5lp_support(iommu->cap)) { 128672cf6dfSJoerg Roedel pr_err("%s SVM disabled, incompatible paging mode\n", 129672cf6dfSJoerg Roedel iommu->name); 130672cf6dfSJoerg Roedel return; 131672cf6dfSJoerg Roedel } 132672cf6dfSJoerg Roedel 133672cf6dfSJoerg Roedel iommu->flags |= VTD_FLAG_SVM_CAPABLE; 134672cf6dfSJoerg Roedel } 135672cf6dfSJoerg Roedel 136672cf6dfSJoerg Roedel /* Pages have been freed at this point */ 1371af5a810SAlistair Popple static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, 138672cf6dfSJoerg Roedel struct mm_struct *mm, 139672cf6dfSJoerg Roedel unsigned long start, unsigned long end) 140672cf6dfSJoerg Roedel { 141*886f816cSLu Baolu struct dmar_domain *domain = container_of(mn, struct dmar_domain, notifier); 142672cf6dfSJoerg Roedel 143*886f816cSLu Baolu if (start == 0 && end == ULONG_MAX) { 1444f609dbfSLu Baolu cache_tag_flush_all(domain); 145e7ad6c2aSLu Baolu return; 146e7ad6c2aSLu Baolu } 147e7ad6c2aSLu Baolu 1484f609dbfSLu Baolu /* 1494f609dbfSLu Baolu * The mm_types defines vm_end as the first byte after the end address, 1504f609dbfSLu Baolu * different from IOMMU subsystem using the last address of an address 1514f609dbfSLu Baolu * range. 1524f609dbfSLu Baolu */ 1534f609dbfSLu Baolu cache_tag_flush_range(domain, start, end - 1, 0); 154672cf6dfSJoerg Roedel } 155672cf6dfSJoerg Roedel 156672cf6dfSJoerg Roedel static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 157672cf6dfSJoerg Roedel { 158*886f816cSLu Baolu struct dmar_domain *domain = container_of(mn, struct dmar_domain, notifier); 159deda9a7bSLu Baolu struct dev_pasid_info *dev_pasid; 160deda9a7bSLu Baolu struct device_domain_info *info; 161deda9a7bSLu Baolu unsigned long flags; 162672cf6dfSJoerg Roedel 163672cf6dfSJoerg Roedel /* This might end up being called from exit_mmap(), *before* the page 164672cf6dfSJoerg Roedel * tables are cleared. And __mmu_notifier_release() will delete us from 165672cf6dfSJoerg Roedel * the list of notifiers so that our invalidate_range() callback doesn't 166672cf6dfSJoerg Roedel * get called when the page tables are cleared. So we need to protect 167672cf6dfSJoerg Roedel * against hardware accessing those page tables. 168672cf6dfSJoerg Roedel * 169672cf6dfSJoerg Roedel * We do it by clearing the entry in the PASID table and then flushing 170672cf6dfSJoerg Roedel * the IOTLB and the PASID table caches. This might upset hardware; 171672cf6dfSJoerg Roedel * perhaps we'll want to point the PASID to a dummy PGD (like the zero 172672cf6dfSJoerg Roedel * page) so that we end up taking a fault that the hardware really 173672cf6dfSJoerg Roedel * *has* to handle gracefully without affecting other processes. 174672cf6dfSJoerg Roedel */ 175deda9a7bSLu Baolu spin_lock_irqsave(&domain->lock, flags); 176deda9a7bSLu Baolu list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) { 177deda9a7bSLu Baolu info = dev_iommu_priv_get(dev_pasid->dev); 178deda9a7bSLu Baolu intel_pasid_tear_down_entry(info->iommu, dev_pasid->dev, 179deda9a7bSLu Baolu dev_pasid->pasid, true); 180deda9a7bSLu Baolu } 181deda9a7bSLu Baolu spin_unlock_irqrestore(&domain->lock, flags); 182672cf6dfSJoerg Roedel 183672cf6dfSJoerg Roedel } 184672cf6dfSJoerg Roedel 185*886f816cSLu Baolu static void intel_mm_free_notifier(struct mmu_notifier *mn) 186*886f816cSLu Baolu { 187*886f816cSLu Baolu kfree(container_of(mn, struct dmar_domain, notifier)); 188*886f816cSLu Baolu } 189*886f816cSLu Baolu 190672cf6dfSJoerg Roedel static const struct mmu_notifier_ops intel_mmuops = { 191672cf6dfSJoerg Roedel .release = intel_mm_release, 1921af5a810SAlistair Popple .arch_invalidate_secondary_tlbs = intel_arch_invalidate_secondary_tlbs, 193*886f816cSLu Baolu .free_notifier = intel_mm_free_notifier, 194672cf6dfSJoerg Roedel }; 195672cf6dfSJoerg Roedel 1968ca918cbSTina Zhang static int intel_svm_set_dev_pasid(struct iommu_domain *domain, 1978ca918cbSTina Zhang struct device *dev, ioasid_t pasid) 19840483774SLu Baolu { 199586081d3SLu Baolu struct device_domain_info *info = dev_iommu_priv_get(dev); 200deda9a7bSLu Baolu struct dmar_domain *dmar_domain = to_dmar_domain(domain); 2018ca918cbSTina Zhang struct intel_iommu *iommu = info->iommu; 2025c79705dSTina Zhang struct mm_struct *mm = domain->mm; 203deda9a7bSLu Baolu struct dev_pasid_info *dev_pasid; 204ffd5869dSLu Baolu unsigned long sflags; 205deda9a7bSLu Baolu unsigned long flags; 20640483774SLu Baolu int ret = 0; 207672cf6dfSJoerg Roedel 208deda9a7bSLu Baolu dev_pasid = kzalloc(sizeof(*dev_pasid), GFP_KERNEL); 209deda9a7bSLu Baolu if (!dev_pasid) 210ec9ab12dSLu Baolu return -ENOMEM; 211672cf6dfSJoerg Roedel 212deda9a7bSLu Baolu dev_pasid->dev = dev; 213deda9a7bSLu Baolu dev_pasid->pasid = pasid; 214672cf6dfSJoerg Roedel 2153b1d9e2bSLu Baolu ret = cache_tag_assign_domain(to_dmar_domain(domain), dev, pasid); 2163b1d9e2bSLu Baolu if (ret) 217deda9a7bSLu Baolu goto free_dev_pasid; 218672cf6dfSJoerg Roedel 21940483774SLu Baolu /* Setup the pasid table: */ 220942fd543SLu Baolu sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; 2215c79705dSTina Zhang ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, pasid, 22240483774SLu Baolu FLPT_DEFAULT_DID, sflags); 22340483774SLu Baolu if (ret) 2243b1d9e2bSLu Baolu goto unassign_tag; 22540483774SLu Baolu 226deda9a7bSLu Baolu spin_lock_irqsave(&dmar_domain->lock, flags); 227deda9a7bSLu Baolu list_add(&dev_pasid->link_domain, &dmar_domain->dev_pasids); 228deda9a7bSLu Baolu spin_unlock_irqrestore(&dmar_domain->lock, flags); 22949cab9d2SLu Baolu 230ec9ab12dSLu Baolu return 0; 23140483774SLu Baolu 2323b1d9e2bSLu Baolu unassign_tag: 2333b1d9e2bSLu Baolu cache_tag_unassign_domain(to_dmar_domain(domain), dev, pasid); 234deda9a7bSLu Baolu free_dev_pasid: 235deda9a7bSLu Baolu kfree(dev_pasid); 23640483774SLu Baolu 237ec9ab12dSLu Baolu return ret; 238672cf6dfSJoerg Roedel } 239672cf6dfSJoerg Roedel 240672cf6dfSJoerg Roedel /* Page request queue descriptor */ 241672cf6dfSJoerg Roedel struct page_req_dsc { 242672cf6dfSJoerg Roedel union { 243672cf6dfSJoerg Roedel struct { 244672cf6dfSJoerg Roedel u64 type:8; 245672cf6dfSJoerg Roedel u64 pasid_present:1; 246621b7e54SJingqi Liu u64 rsvd:7; 247672cf6dfSJoerg Roedel u64 rid:16; 248672cf6dfSJoerg Roedel u64 pasid:20; 249672cf6dfSJoerg Roedel u64 exe_req:1; 250672cf6dfSJoerg Roedel u64 pm_req:1; 251672cf6dfSJoerg Roedel u64 rsvd2:10; 252672cf6dfSJoerg Roedel }; 253672cf6dfSJoerg Roedel u64 qw_0; 254672cf6dfSJoerg Roedel }; 255672cf6dfSJoerg Roedel union { 256672cf6dfSJoerg Roedel struct { 257672cf6dfSJoerg Roedel u64 rd_req:1; 258672cf6dfSJoerg Roedel u64 wr_req:1; 259672cf6dfSJoerg Roedel u64 lpig:1; 260672cf6dfSJoerg Roedel u64 prg_index:9; 261672cf6dfSJoerg Roedel u64 addr:52; 262672cf6dfSJoerg Roedel }; 263672cf6dfSJoerg Roedel u64 qw_1; 264672cf6dfSJoerg Roedel }; 265621b7e54SJingqi Liu u64 qw_2; 266621b7e54SJingqi Liu u64 qw_3; 267672cf6dfSJoerg Roedel }; 268672cf6dfSJoerg Roedel 269672cf6dfSJoerg Roedel static bool is_canonical_address(u64 addr) 270672cf6dfSJoerg Roedel { 271672cf6dfSJoerg Roedel int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1); 272672cf6dfSJoerg Roedel long saddr = (long) addr; 273672cf6dfSJoerg Roedel 274672cf6dfSJoerg Roedel return (((saddr << shift) >> shift) == saddr); 275672cf6dfSJoerg Roedel } 276672cf6dfSJoerg Roedel 277672cf6dfSJoerg Roedel /** 27815478623SLu Baolu * intel_drain_pasid_prq - Drain page requests and responses for a pasid 279672cf6dfSJoerg Roedel * @dev: target device 280672cf6dfSJoerg Roedel * @pasid: pasid for draining 281672cf6dfSJoerg Roedel * 282672cf6dfSJoerg Roedel * Drain all pending page requests and responses related to @pasid in both 283672cf6dfSJoerg Roedel * software and hardware. This is supposed to be called after the device 284672cf6dfSJoerg Roedel * driver has stopped DMA, the pasid entry has been cleared, and both IOTLB 285672cf6dfSJoerg Roedel * and DevTLB have been invalidated. 286672cf6dfSJoerg Roedel * 287672cf6dfSJoerg Roedel * It waits until all pending page requests for @pasid in the page fault 288672cf6dfSJoerg Roedel * queue are completed by the prq handling thread. Then follow the steps 289672cf6dfSJoerg Roedel * described in VT-d spec CH7.10 to drain all page requests and page 290672cf6dfSJoerg Roedel * responses pending in the hardware. 291672cf6dfSJoerg Roedel */ 29215478623SLu Baolu void intel_drain_pasid_prq(struct device *dev, u32 pasid) 293672cf6dfSJoerg Roedel { 294672cf6dfSJoerg Roedel struct device_domain_info *info; 295672cf6dfSJoerg Roedel struct dmar_domain *domain; 296672cf6dfSJoerg Roedel struct intel_iommu *iommu; 297672cf6dfSJoerg Roedel struct qi_desc desc[3]; 298672cf6dfSJoerg Roedel struct pci_dev *pdev; 299672cf6dfSJoerg Roedel int head, tail; 300672cf6dfSJoerg Roedel u16 sid, did; 301672cf6dfSJoerg Roedel int qdep; 302672cf6dfSJoerg Roedel 303586081d3SLu Baolu info = dev_iommu_priv_get(dev); 304672cf6dfSJoerg Roedel if (WARN_ON(!info || !dev_is_pci(dev))) 305672cf6dfSJoerg Roedel return; 306672cf6dfSJoerg Roedel 307672cf6dfSJoerg Roedel if (!info->pri_enabled) 308672cf6dfSJoerg Roedel return; 309672cf6dfSJoerg Roedel 310672cf6dfSJoerg Roedel iommu = info->iommu; 311672cf6dfSJoerg Roedel domain = info->domain; 312672cf6dfSJoerg Roedel pdev = to_pci_dev(dev); 313672cf6dfSJoerg Roedel sid = PCI_DEVID(info->bus, info->devfn); 314ba949f4cSLu Baolu did = domain_id_iommu(domain, iommu); 315672cf6dfSJoerg Roedel qdep = pci_ats_queue_depth(pdev); 316672cf6dfSJoerg Roedel 317672cf6dfSJoerg Roedel /* 318672cf6dfSJoerg Roedel * Check and wait until all pending page requests in the queue are 319672cf6dfSJoerg Roedel * handled by the prq handling thread. 320672cf6dfSJoerg Roedel */ 321672cf6dfSJoerg Roedel prq_retry: 322672cf6dfSJoerg Roedel reinit_completion(&iommu->prq_complete); 323672cf6dfSJoerg Roedel tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 324672cf6dfSJoerg Roedel head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 325672cf6dfSJoerg Roedel while (head != tail) { 326672cf6dfSJoerg Roedel struct page_req_dsc *req; 327672cf6dfSJoerg Roedel 328672cf6dfSJoerg Roedel req = &iommu->prq[head / sizeof(*req)]; 329672cf6dfSJoerg Roedel if (!req->pasid_present || req->pasid != pasid) { 330672cf6dfSJoerg Roedel head = (head + sizeof(*req)) & PRQ_RING_MASK; 331672cf6dfSJoerg Roedel continue; 332672cf6dfSJoerg Roedel } 333672cf6dfSJoerg Roedel 334672cf6dfSJoerg Roedel wait_for_completion(&iommu->prq_complete); 335672cf6dfSJoerg Roedel goto prq_retry; 336672cf6dfSJoerg Roedel } 337672cf6dfSJoerg Roedel 338d5b9e4bfSLu Baolu iopf_queue_flush_dev(dev); 339d5b9e4bfSLu Baolu 340672cf6dfSJoerg Roedel /* 341672cf6dfSJoerg Roedel * Perform steps described in VT-d spec CH7.10 to drain page 342672cf6dfSJoerg Roedel * requests and responses in hardware. 343672cf6dfSJoerg Roedel */ 344672cf6dfSJoerg Roedel memset(desc, 0, sizeof(desc)); 345672cf6dfSJoerg Roedel desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) | 346672cf6dfSJoerg Roedel QI_IWD_FENCE | 347672cf6dfSJoerg Roedel QI_IWD_TYPE; 348672cf6dfSJoerg Roedel desc[1].qw0 = QI_EIOTLB_PASID(pasid) | 349672cf6dfSJoerg Roedel QI_EIOTLB_DID(did) | 350672cf6dfSJoerg Roedel QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | 351672cf6dfSJoerg Roedel QI_EIOTLB_TYPE; 352672cf6dfSJoerg Roedel desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) | 353672cf6dfSJoerg Roedel QI_DEV_EIOTLB_SID(sid) | 354672cf6dfSJoerg Roedel QI_DEV_EIOTLB_QDEP(qdep) | 355672cf6dfSJoerg Roedel QI_DEIOTLB_TYPE | 356672cf6dfSJoerg Roedel QI_DEV_IOTLB_PFSID(info->pfsid); 357672cf6dfSJoerg Roedel qi_retry: 358672cf6dfSJoerg Roedel reinit_completion(&iommu->prq_complete); 359672cf6dfSJoerg Roedel qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN); 360672cf6dfSJoerg Roedel if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 361672cf6dfSJoerg Roedel wait_for_completion(&iommu->prq_complete); 362672cf6dfSJoerg Roedel goto qi_retry; 363672cf6dfSJoerg Roedel } 364672cf6dfSJoerg Roedel } 365672cf6dfSJoerg Roedel 366eb8d93eaSLu Baolu static int prq_to_iommu_prot(struct page_req_dsc *req) 367eb8d93eaSLu Baolu { 368eb8d93eaSLu Baolu int prot = 0; 369eb8d93eaSLu Baolu 370eb8d93eaSLu Baolu if (req->rd_req) 371eb8d93eaSLu Baolu prot |= IOMMU_FAULT_PERM_READ; 372eb8d93eaSLu Baolu if (req->wr_req) 373eb8d93eaSLu Baolu prot |= IOMMU_FAULT_PERM_WRITE; 374eb8d93eaSLu Baolu if (req->exe_req) 375eb8d93eaSLu Baolu prot |= IOMMU_FAULT_PERM_EXEC; 376eb8d93eaSLu Baolu if (req->pm_req) 377eb8d93eaSLu Baolu prot |= IOMMU_FAULT_PERM_PRIV; 378eb8d93eaSLu Baolu 379eb8d93eaSLu Baolu return prot; 380eb8d93eaSLu Baolu } 381eb8d93eaSLu Baolu 3823dfa64aeSLu Baolu static void intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, 3830f4834abSLu Baolu struct page_req_dsc *desc) 384eb8d93eaSLu Baolu { 3853f02a9dcSLu Baolu struct iopf_fault event = { }; 386eb8d93eaSLu Baolu 387eb8d93eaSLu Baolu /* Fill in event data for device specific processing */ 388eb8d93eaSLu Baolu event.fault.type = IOMMU_FAULT_PAGE_REQ; 38903d20509SLu Baolu event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT; 390eb8d93eaSLu Baolu event.fault.prm.pasid = desc->pasid; 391eb8d93eaSLu Baolu event.fault.prm.grpid = desc->prg_index; 392eb8d93eaSLu Baolu event.fault.prm.perm = prq_to_iommu_prot(desc); 393eb8d93eaSLu Baolu 394eb8d93eaSLu Baolu if (desc->lpig) 395eb8d93eaSLu Baolu event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 396eb8d93eaSLu Baolu if (desc->pasid_present) { 397eb8d93eaSLu Baolu event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 398eb8d93eaSLu Baolu event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; 399eb8d93eaSLu Baolu } 400eb8d93eaSLu Baolu 4013dfa64aeSLu Baolu iommu_report_device_fault(dev, &event); 402eb8d93eaSLu Baolu } 403eb8d93eaSLu Baolu 404ae7f09b1SLu Baolu static void handle_bad_prq_event(struct intel_iommu *iommu, 405ae7f09b1SLu Baolu struct page_req_dsc *req, int result) 406672cf6dfSJoerg Roedel { 407621b7e54SJingqi Liu struct qi_desc desc = { }; 408672cf6dfSJoerg Roedel 409ae7f09b1SLu Baolu pr_err("%s: Invalid page request: %08llx %08llx\n", 410672cf6dfSJoerg Roedel iommu->name, ((unsigned long long *)req)[0], 411672cf6dfSJoerg Roedel ((unsigned long long *)req)[1]); 412672cf6dfSJoerg Roedel 413621b7e54SJingqi Liu if (!req->lpig) 414ae7f09b1SLu Baolu return; 415ae7f09b1SLu Baolu 416ae7f09b1SLu Baolu desc.qw0 = QI_PGRP_PASID(req->pasid) | 417ae7f09b1SLu Baolu QI_PGRP_DID(req->rid) | 418ae7f09b1SLu Baolu QI_PGRP_PASID_P(req->pasid_present) | 419ae7f09b1SLu Baolu QI_PGRP_RESP_CODE(result) | 420ae7f09b1SLu Baolu QI_PGRP_RESP_TYPE; 421ae7f09b1SLu Baolu desc.qw1 = QI_PGRP_IDX(req->prg_index) | 422ae7f09b1SLu Baolu QI_PGRP_LPIG(req->lpig); 423606636dcSGustavo A. R. Silva 424ae7f09b1SLu Baolu qi_submit_sync(iommu, &desc, 1, 0); 425eb8d93eaSLu Baolu } 426eb8d93eaSLu Baolu 427ae7f09b1SLu Baolu static irqreturn_t prq_event_thread(int irq, void *d) 428ae7f09b1SLu Baolu { 429ae7f09b1SLu Baolu struct intel_iommu *iommu = d; 430ae7f09b1SLu Baolu struct page_req_dsc *req; 431ae7f09b1SLu Baolu int head, tail, handled; 432def054b0SLu Baolu struct device *dev; 433ae7f09b1SLu Baolu u64 address; 434ae7f09b1SLu Baolu 435ae7f09b1SLu Baolu /* 436ae7f09b1SLu Baolu * Clear PPR bit before reading head/tail registers, to ensure that 437ae7f09b1SLu Baolu * we get a new interrupt if needed. 438ae7f09b1SLu Baolu */ 439ae7f09b1SLu Baolu writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG); 440ae7f09b1SLu Baolu 441ae7f09b1SLu Baolu tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 442ae7f09b1SLu Baolu head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 443ae7f09b1SLu Baolu handled = (head != tail); 444ae7f09b1SLu Baolu while (head != tail) { 445ae7f09b1SLu Baolu req = &iommu->prq[head / sizeof(*req)]; 446ae7f09b1SLu Baolu address = (u64)req->addr << VTD_PAGE_SHIFT; 447ae7f09b1SLu Baolu 448ae7f09b1SLu Baolu if (unlikely(!req->pasid_present)) { 449ae7f09b1SLu Baolu pr_err("IOMMU: %s: Page request without PASID\n", 450ae7f09b1SLu Baolu iommu->name); 451ae7f09b1SLu Baolu bad_req: 452ae7f09b1SLu Baolu handle_bad_prq_event(iommu, req, QI_RESP_INVALID); 453ae7f09b1SLu Baolu goto prq_advance; 454ae7f09b1SLu Baolu } 455ae7f09b1SLu Baolu 456ae7f09b1SLu Baolu if (unlikely(!is_canonical_address(address))) { 457ae7f09b1SLu Baolu pr_err("IOMMU: %s: Address is not canonical\n", 458ae7f09b1SLu Baolu iommu->name); 459ae7f09b1SLu Baolu goto bad_req; 460ae7f09b1SLu Baolu } 461ae7f09b1SLu Baolu 462ae7f09b1SLu Baolu if (unlikely(req->pm_req && (req->rd_req | req->wr_req))) { 463ae7f09b1SLu Baolu pr_err("IOMMU: %s: Page request in Privilege Mode\n", 464ae7f09b1SLu Baolu iommu->name); 465ae7f09b1SLu Baolu goto bad_req; 466ae7f09b1SLu Baolu } 467ae7f09b1SLu Baolu 468ae7f09b1SLu Baolu if (unlikely(req->exe_req && req->rd_req)) { 469ae7f09b1SLu Baolu pr_err("IOMMU: %s: Execution request not supported\n", 470ae7f09b1SLu Baolu iommu->name); 471ae7f09b1SLu Baolu goto bad_req; 472ae7f09b1SLu Baolu } 473ae7f09b1SLu Baolu 474da8669ffSLu Baolu /* Drop Stop Marker message. No need for a response. */ 475da8669ffSLu Baolu if (unlikely(req->lpig && !req->rd_req && !req->wr_req)) 476da8669ffSLu Baolu goto prq_advance; 477da8669ffSLu Baolu 478ae7f09b1SLu Baolu /* 479ae7f09b1SLu Baolu * If prq is to be handled outside iommu driver via receiver of 480ae7f09b1SLu Baolu * the fault notifiers, we skip the page response here. 481ae7f09b1SLu Baolu */ 482def054b0SLu Baolu mutex_lock(&iommu->iopf_lock); 483def054b0SLu Baolu dev = device_rbtree_find(iommu, req->rid); 484def054b0SLu Baolu if (!dev) { 485def054b0SLu Baolu mutex_unlock(&iommu->iopf_lock); 4866927d352SYang Yingliang goto bad_req; 487def054b0SLu Baolu } 488e93a67f5SLu Baolu 489f379a7e9SJoerg Roedel intel_svm_prq_report(iommu, dev, req); 490def054b0SLu Baolu trace_prq_report(iommu, dev, req->qw_0, req->qw_1, 491621b7e54SJingqi Liu req->qw_2, req->qw_3, 49206f4b8d0SLu Baolu iommu->prq_seq_number++); 493def054b0SLu Baolu mutex_unlock(&iommu->iopf_lock); 494eb8d93eaSLu Baolu prq_advance: 495672cf6dfSJoerg Roedel head = (head + sizeof(*req)) & PRQ_RING_MASK; 496672cf6dfSJoerg Roedel } 497672cf6dfSJoerg Roedel 498672cf6dfSJoerg Roedel dmar_writeq(iommu->reg + DMAR_PQH_REG, tail); 499672cf6dfSJoerg Roedel 500672cf6dfSJoerg Roedel /* 501672cf6dfSJoerg Roedel * Clear the page request overflow bit and wake up all threads that 502672cf6dfSJoerg Roedel * are waiting for the completion of this handling. 503672cf6dfSJoerg Roedel */ 50428a77185SLu Baolu if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 50528a77185SLu Baolu pr_info_ratelimited("IOMMU: %s: PRQ overflow detected\n", 50628a77185SLu Baolu iommu->name); 50728a77185SLu Baolu head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 50828a77185SLu Baolu tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 50928a77185SLu Baolu if (head == tail) { 510d5b9e4bfSLu Baolu iopf_queue_discard_partial(iommu->iopf_queue); 511672cf6dfSJoerg Roedel writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG); 51228a77185SLu Baolu pr_info_ratelimited("IOMMU: %s: PRQ overflow cleared", 51328a77185SLu Baolu iommu->name); 51428a77185SLu Baolu } 51528a77185SLu Baolu } 516672cf6dfSJoerg Roedel 517672cf6dfSJoerg Roedel if (!completion_done(&iommu->prq_complete)) 518672cf6dfSJoerg Roedel complete(&iommu->prq_complete); 519672cf6dfSJoerg Roedel 520672cf6dfSJoerg Roedel return IRQ_RETVAL(handled); 521672cf6dfSJoerg Roedel } 522672cf6dfSJoerg Roedel 523b554e396SLu Baolu void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, 5248b737121SLu Baolu struct iommu_page_response *msg) 5258b737121SLu Baolu { 5261903ef8fSLu Baolu struct device_domain_info *info = dev_iommu_priv_get(dev); 5271903ef8fSLu Baolu struct intel_iommu *iommu = info->iommu; 5281903ef8fSLu Baolu u8 bus = info->bus, devfn = info->devfn; 5298b737121SLu Baolu struct iommu_fault_page_request *prm; 530621b7e54SJingqi Liu struct qi_desc desc; 5318b737121SLu Baolu bool pasid_present; 5328b737121SLu Baolu bool last_page; 5338b737121SLu Baolu u16 sid; 5348b737121SLu Baolu 5358b737121SLu Baolu prm = &evt->fault.prm; 5368b737121SLu Baolu sid = PCI_DEVID(bus, devfn); 5378b737121SLu Baolu pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 5388b737121SLu Baolu last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 5398b737121SLu Baolu 5408b737121SLu Baolu desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) | 5418b737121SLu Baolu QI_PGRP_PASID_P(pasid_present) | 5428b737121SLu Baolu QI_PGRP_RESP_CODE(msg->code) | 5438b737121SLu Baolu QI_PGRP_RESP_TYPE; 5448b737121SLu Baolu desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page); 5458b737121SLu Baolu desc.qw2 = 0; 5468b737121SLu Baolu desc.qw3 = 0; 547606636dcSGustavo A. R. Silva 5488b737121SLu Baolu qi_submit_sync(iommu, &desc, 1, 0); 5498b737121SLu Baolu } 550eaca8889SLu Baolu 551eaca8889SLu Baolu static void intel_svm_domain_free(struct iommu_domain *domain) 552eaca8889SLu Baolu { 553*886f816cSLu Baolu struct dmar_domain *dmar_domain = to_dmar_domain(domain); 554*886f816cSLu Baolu 555*886f816cSLu Baolu /* dmar_domain free is deferred to the mmu free_notifier callback. */ 556*886f816cSLu Baolu mmu_notifier_put(&dmar_domain->notifier); 557eaca8889SLu Baolu } 558eaca8889SLu Baolu 559eaca8889SLu Baolu static const struct iommu_domain_ops intel_svm_domain_ops = { 560eaca8889SLu Baolu .set_dev_pasid = intel_svm_set_dev_pasid, 561eaca8889SLu Baolu .free = intel_svm_domain_free 562eaca8889SLu Baolu }; 563eaca8889SLu Baolu 564*886f816cSLu Baolu struct iommu_domain *intel_svm_domain_alloc(struct device *dev, 565*886f816cSLu Baolu struct mm_struct *mm) 566eaca8889SLu Baolu { 567eaca8889SLu Baolu struct dmar_domain *domain; 568*886f816cSLu Baolu int ret; 569eaca8889SLu Baolu 570eaca8889SLu Baolu domain = kzalloc(sizeof(*domain), GFP_KERNEL); 571eaca8889SLu Baolu if (!domain) 572*886f816cSLu Baolu return ERR_PTR(-ENOMEM); 573*886f816cSLu Baolu 574eaca8889SLu Baolu domain->domain.ops = &intel_svm_domain_ops; 5754f609dbfSLu Baolu domain->use_first_level = true; 576deda9a7bSLu Baolu INIT_LIST_HEAD(&domain->dev_pasids); 5773b1d9e2bSLu Baolu INIT_LIST_HEAD(&domain->cache_tags); 5783b1d9e2bSLu Baolu spin_lock_init(&domain->cache_lock); 579deda9a7bSLu Baolu spin_lock_init(&domain->lock); 580eaca8889SLu Baolu 581*886f816cSLu Baolu domain->notifier.ops = &intel_mmuops; 582*886f816cSLu Baolu ret = mmu_notifier_register(&domain->notifier, mm); 583*886f816cSLu Baolu if (ret) { 584*886f816cSLu Baolu kfree(domain); 585*886f816cSLu Baolu return ERR_PTR(ret); 586*886f816cSLu Baolu } 587eaca8889SLu Baolu 588eaca8889SLu Baolu return &domain->domain; 589eaca8889SLu Baolu } 590