1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright © 2015 Intel Corporation. 4 * 5 * Authors: David Woodhouse <dwmw2@infradead.org> 6 */ 7 8 #include <linux/intel-iommu.h> 9 #include <linux/mmu_notifier.h> 10 #include <linux/sched.h> 11 #include <linux/sched/mm.h> 12 #include <linux/slab.h> 13 #include <linux/intel-svm.h> 14 #include <linux/rculist.h> 15 #include <linux/pci.h> 16 #include <linux/pci-ats.h> 17 #include <linux/dmar.h> 18 #include <linux/interrupt.h> 19 #include <linux/mm_types.h> 20 #include <linux/xarray.h> 21 #include <linux/ioasid.h> 22 #include <asm/page.h> 23 #include <asm/fpu/api.h> 24 #include <trace/events/intel_iommu.h> 25 26 #include "pasid.h" 27 #include "perf.h" 28 #include "../iommu-sva-lib.h" 29 30 static irqreturn_t prq_event_thread(int irq, void *d); 31 static void intel_svm_drain_prq(struct device *dev, u32 pasid); 32 #define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva) 33 34 static DEFINE_XARRAY_ALLOC(pasid_private_array); 35 static int pasid_private_add(ioasid_t pasid, void *priv) 36 { 37 return xa_alloc(&pasid_private_array, &pasid, priv, 38 XA_LIMIT(pasid, pasid), GFP_ATOMIC); 39 } 40 41 static void pasid_private_remove(ioasid_t pasid) 42 { 43 xa_erase(&pasid_private_array, pasid); 44 } 45 46 static void *pasid_private_find(ioasid_t pasid) 47 { 48 return xa_load(&pasid_private_array, pasid); 49 } 50 51 static struct intel_svm_dev * 52 svm_lookup_device_by_sid(struct intel_svm *svm, u16 sid) 53 { 54 struct intel_svm_dev *sdev = NULL, *t; 55 56 rcu_read_lock(); 57 list_for_each_entry_rcu(t, &svm->devs, list) { 58 if (t->sid == sid) { 59 sdev = t; 60 break; 61 } 62 } 63 rcu_read_unlock(); 64 65 return sdev; 66 } 67 68 static struct intel_svm_dev * 69 svm_lookup_device_by_dev(struct intel_svm *svm, struct device *dev) 70 { 71 struct intel_svm_dev *sdev = NULL, *t; 72 73 rcu_read_lock(); 74 list_for_each_entry_rcu(t, &svm->devs, list) { 75 if (t->dev == dev) { 76 sdev = t; 77 break; 78 } 79 } 80 rcu_read_unlock(); 81 82 return sdev; 83 } 84 85 int intel_svm_enable_prq(struct intel_iommu *iommu) 86 { 87 struct iopf_queue *iopfq; 88 struct page *pages; 89 int irq, ret; 90 91 pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, PRQ_ORDER); 92 if (!pages) { 93 pr_warn("IOMMU: %s: Failed to allocate page request queue\n", 94 iommu->name); 95 return -ENOMEM; 96 } 97 iommu->prq = page_address(pages); 98 99 irq = dmar_alloc_hwirq(DMAR_UNITS_SUPPORTED + iommu->seq_id, iommu->node, iommu); 100 if (irq <= 0) { 101 pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n", 102 iommu->name); 103 ret = -EINVAL; 104 goto free_prq; 105 } 106 iommu->pr_irq = irq; 107 108 snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name), 109 "dmar%d-iopfq", iommu->seq_id); 110 iopfq = iopf_queue_alloc(iommu->iopfq_name); 111 if (!iopfq) { 112 pr_err("IOMMU: %s: Failed to allocate iopf queue\n", iommu->name); 113 ret = -ENOMEM; 114 goto free_hwirq; 115 } 116 iommu->iopf_queue = iopfq; 117 118 snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id); 119 120 ret = request_threaded_irq(irq, NULL, prq_event_thread, IRQF_ONESHOT, 121 iommu->prq_name, iommu); 122 if (ret) { 123 pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n", 124 iommu->name); 125 goto free_iopfq; 126 } 127 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 128 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 129 dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER); 130 131 init_completion(&iommu->prq_complete); 132 133 return 0; 134 135 free_iopfq: 136 iopf_queue_free(iommu->iopf_queue); 137 iommu->iopf_queue = NULL; 138 free_hwirq: 139 dmar_free_hwirq(irq); 140 iommu->pr_irq = 0; 141 free_prq: 142 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 143 iommu->prq = NULL; 144 145 return ret; 146 } 147 148 int intel_svm_finish_prq(struct intel_iommu *iommu) 149 { 150 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 151 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 152 dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL); 153 154 if (iommu->pr_irq) { 155 free_irq(iommu->pr_irq, iommu); 156 dmar_free_hwirq(iommu->pr_irq); 157 iommu->pr_irq = 0; 158 } 159 160 if (iommu->iopf_queue) { 161 iopf_queue_free(iommu->iopf_queue); 162 iommu->iopf_queue = NULL; 163 } 164 165 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 166 iommu->prq = NULL; 167 168 return 0; 169 } 170 171 static inline bool intel_svm_capable(struct intel_iommu *iommu) 172 { 173 return iommu->flags & VTD_FLAG_SVM_CAPABLE; 174 } 175 176 void intel_svm_check(struct intel_iommu *iommu) 177 { 178 if (!pasid_supported(iommu)) 179 return; 180 181 if (cpu_feature_enabled(X86_FEATURE_GBPAGES) && 182 !cap_fl1gp_support(iommu->cap)) { 183 pr_err("%s SVM disabled, incompatible 1GB page capability\n", 184 iommu->name); 185 return; 186 } 187 188 if (cpu_feature_enabled(X86_FEATURE_LA57) && 189 !cap_5lp_support(iommu->cap)) { 190 pr_err("%s SVM disabled, incompatible paging mode\n", 191 iommu->name); 192 return; 193 } 194 195 iommu->flags |= VTD_FLAG_SVM_CAPABLE; 196 } 197 198 static void __flush_svm_range_dev(struct intel_svm *svm, 199 struct intel_svm_dev *sdev, 200 unsigned long address, 201 unsigned long pages, int ih) 202 { 203 struct device_domain_info *info = get_domain_info(sdev->dev); 204 205 if (WARN_ON(!pages)) 206 return; 207 208 qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih); 209 if (info->ats_enabled) 210 qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, 211 svm->pasid, sdev->qdep, address, 212 order_base_2(pages)); 213 } 214 215 static void intel_flush_svm_range_dev(struct intel_svm *svm, 216 struct intel_svm_dev *sdev, 217 unsigned long address, 218 unsigned long pages, int ih) 219 { 220 unsigned long shift = ilog2(__roundup_pow_of_two(pages)); 221 unsigned long align = (1ULL << (VTD_PAGE_SHIFT + shift)); 222 unsigned long start = ALIGN_DOWN(address, align); 223 unsigned long end = ALIGN(address + (pages << VTD_PAGE_SHIFT), align); 224 225 while (start < end) { 226 __flush_svm_range_dev(svm, sdev, start, align >> VTD_PAGE_SHIFT, ih); 227 start += align; 228 } 229 } 230 231 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, 232 unsigned long pages, int ih) 233 { 234 struct intel_svm_dev *sdev; 235 236 rcu_read_lock(); 237 list_for_each_entry_rcu(sdev, &svm->devs, list) 238 intel_flush_svm_range_dev(svm, sdev, address, pages, ih); 239 rcu_read_unlock(); 240 } 241 242 /* Pages have been freed at this point */ 243 static void intel_invalidate_range(struct mmu_notifier *mn, 244 struct mm_struct *mm, 245 unsigned long start, unsigned long end) 246 { 247 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 248 249 intel_flush_svm_range(svm, start, 250 (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0); 251 } 252 253 static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 254 { 255 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 256 struct intel_svm_dev *sdev; 257 258 /* This might end up being called from exit_mmap(), *before* the page 259 * tables are cleared. And __mmu_notifier_release() will delete us from 260 * the list of notifiers so that our invalidate_range() callback doesn't 261 * get called when the page tables are cleared. So we need to protect 262 * against hardware accessing those page tables. 263 * 264 * We do it by clearing the entry in the PASID table and then flushing 265 * the IOTLB and the PASID table caches. This might upset hardware; 266 * perhaps we'll want to point the PASID to a dummy PGD (like the zero 267 * page) so that we end up taking a fault that the hardware really 268 * *has* to handle gracefully without affecting other processes. 269 */ 270 rcu_read_lock(); 271 list_for_each_entry_rcu(sdev, &svm->devs, list) 272 intel_pasid_tear_down_entry(sdev->iommu, sdev->dev, 273 svm->pasid, true); 274 rcu_read_unlock(); 275 276 } 277 278 static const struct mmu_notifier_ops intel_mmuops = { 279 .release = intel_mm_release, 280 .invalidate_range = intel_invalidate_range, 281 }; 282 283 static DEFINE_MUTEX(pasid_mutex); 284 285 static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid, 286 struct intel_svm **rsvm, 287 struct intel_svm_dev **rsdev) 288 { 289 struct intel_svm_dev *sdev = NULL; 290 struct intel_svm *svm; 291 292 /* The caller should hold the pasid_mutex lock */ 293 if (WARN_ON(!mutex_is_locked(&pasid_mutex))) 294 return -EINVAL; 295 296 if (pasid == INVALID_IOASID || pasid >= PASID_MAX) 297 return -EINVAL; 298 299 svm = pasid_private_find(pasid); 300 if (IS_ERR(svm)) 301 return PTR_ERR(svm); 302 303 if (!svm) 304 goto out; 305 306 /* 307 * If we found svm for the PASID, there must be at least one device 308 * bond. 309 */ 310 if (WARN_ON(list_empty(&svm->devs))) 311 return -EINVAL; 312 sdev = svm_lookup_device_by_dev(svm, dev); 313 314 out: 315 *rsvm = svm; 316 *rsdev = sdev; 317 318 return 0; 319 } 320 321 int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, 322 struct iommu_gpasid_bind_data *data) 323 { 324 struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); 325 struct intel_svm_dev *sdev = NULL; 326 struct dmar_domain *dmar_domain; 327 struct device_domain_info *info; 328 struct intel_svm *svm = NULL; 329 unsigned long iflags; 330 int ret = 0; 331 332 if (WARN_ON(!iommu) || !data) 333 return -EINVAL; 334 335 if (data->format != IOMMU_PASID_FORMAT_INTEL_VTD) 336 return -EINVAL; 337 338 /* IOMMU core ensures argsz is more than the start of the union */ 339 if (data->argsz < offsetofend(struct iommu_gpasid_bind_data, vendor.vtd)) 340 return -EINVAL; 341 342 /* Make sure no undefined flags are used in vendor data */ 343 if (data->vendor.vtd.flags & ~(IOMMU_SVA_VTD_GPASID_LAST - 1)) 344 return -EINVAL; 345 346 if (!dev_is_pci(dev)) 347 return -ENOTSUPP; 348 349 /* VT-d supports devices with full 20 bit PASIDs only */ 350 if (pci_max_pasids(to_pci_dev(dev)) != PASID_MAX) 351 return -EINVAL; 352 353 /* 354 * We only check host PASID range, we have no knowledge to check 355 * guest PASID range. 356 */ 357 if (data->hpasid <= 0 || data->hpasid >= PASID_MAX) 358 return -EINVAL; 359 360 info = get_domain_info(dev); 361 if (!info) 362 return -EINVAL; 363 364 dmar_domain = to_dmar_domain(domain); 365 366 mutex_lock(&pasid_mutex); 367 ret = pasid_to_svm_sdev(dev, data->hpasid, &svm, &sdev); 368 if (ret) 369 goto out; 370 371 if (sdev) { 372 /* 373 * Do not allow multiple bindings of the same device-PASID since 374 * there is only one SL page tables per PASID. We may revisit 375 * once sharing PGD across domains are supported. 376 */ 377 dev_warn_ratelimited(dev, "Already bound with PASID %u\n", 378 svm->pasid); 379 ret = -EBUSY; 380 goto out; 381 } 382 383 if (!svm) { 384 /* We come here when PASID has never been bond to a device. */ 385 svm = kzalloc(sizeof(*svm), GFP_KERNEL); 386 if (!svm) { 387 ret = -ENOMEM; 388 goto out; 389 } 390 /* REVISIT: upper layer/VFIO can track host process that bind 391 * the PASID. ioasid_set = mm might be sufficient for vfio to 392 * check pasid VMM ownership. We can drop the following line 393 * once VFIO and IOASID set check is in place. 394 */ 395 svm->mm = get_task_mm(current); 396 svm->pasid = data->hpasid; 397 if (data->flags & IOMMU_SVA_GPASID_VAL) { 398 svm->gpasid = data->gpasid; 399 svm->flags |= SVM_FLAG_GUEST_PASID; 400 } 401 pasid_private_add(data->hpasid, svm); 402 INIT_LIST_HEAD_RCU(&svm->devs); 403 mmput(svm->mm); 404 } 405 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); 406 if (!sdev) { 407 ret = -ENOMEM; 408 goto out; 409 } 410 sdev->dev = dev; 411 sdev->sid = PCI_DEVID(info->bus, info->devfn); 412 sdev->iommu = iommu; 413 414 /* Only count users if device has aux domains */ 415 if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX)) 416 sdev->users = 1; 417 418 /* Set up device context entry for PASID if not enabled already */ 419 ret = intel_iommu_enable_pasid(iommu, sdev->dev); 420 if (ret) { 421 dev_err_ratelimited(dev, "Failed to enable PASID capability\n"); 422 kfree(sdev); 423 goto out; 424 } 425 426 /* 427 * PASID table is per device for better security. Therefore, for 428 * each bind of a new device even with an existing PASID, we need to 429 * call the nested mode setup function here. 430 */ 431 spin_lock_irqsave(&iommu->lock, iflags); 432 ret = intel_pasid_setup_nested(iommu, dev, 433 (pgd_t *)(uintptr_t)data->gpgd, 434 data->hpasid, &data->vendor.vtd, dmar_domain, 435 data->addr_width); 436 spin_unlock_irqrestore(&iommu->lock, iflags); 437 if (ret) { 438 dev_err_ratelimited(dev, "Failed to set up PASID %llu in nested mode, Err %d\n", 439 data->hpasid, ret); 440 /* 441 * PASID entry should be in cleared state if nested mode 442 * set up failed. So we only need to clear IOASID tracking 443 * data such that free call will succeed. 444 */ 445 kfree(sdev); 446 goto out; 447 } 448 449 svm->flags |= SVM_FLAG_GUEST_MODE; 450 451 init_rcu_head(&sdev->rcu); 452 list_add_rcu(&sdev->list, &svm->devs); 453 out: 454 if (!IS_ERR_OR_NULL(svm) && list_empty(&svm->devs)) { 455 pasid_private_remove(data->hpasid); 456 kfree(svm); 457 } 458 459 mutex_unlock(&pasid_mutex); 460 return ret; 461 } 462 463 int intel_svm_unbind_gpasid(struct device *dev, u32 pasid) 464 { 465 struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); 466 struct intel_svm_dev *sdev; 467 struct intel_svm *svm; 468 int ret; 469 470 if (WARN_ON(!iommu)) 471 return -EINVAL; 472 473 mutex_lock(&pasid_mutex); 474 ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev); 475 if (ret) 476 goto out; 477 478 if (sdev) { 479 if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX)) 480 sdev->users--; 481 if (!sdev->users) { 482 list_del_rcu(&sdev->list); 483 intel_pasid_tear_down_entry(iommu, dev, 484 svm->pasid, false); 485 intel_svm_drain_prq(dev, svm->pasid); 486 kfree_rcu(sdev, rcu); 487 488 if (list_empty(&svm->devs)) { 489 /* 490 * We do not free the IOASID here in that 491 * IOMMU driver did not allocate it. 492 * Unlike native SVM, IOASID for guest use was 493 * allocated prior to the bind call. 494 * In any case, if the free call comes before 495 * the unbind, IOMMU driver will get notified 496 * and perform cleanup. 497 */ 498 pasid_private_remove(pasid); 499 kfree(svm); 500 } 501 } 502 } 503 out: 504 mutex_unlock(&pasid_mutex); 505 return ret; 506 } 507 508 static void _load_pasid(void *unused) 509 { 510 update_pasid(); 511 } 512 513 static void load_pasid(struct mm_struct *mm, u32 pasid) 514 { 515 mutex_lock(&mm->context.lock); 516 517 /* Synchronize with READ_ONCE in update_pasid(). */ 518 smp_store_release(&mm->pasid, pasid); 519 520 /* Update PASID MSR on all CPUs running the mm's tasks. */ 521 on_each_cpu_mask(mm_cpumask(mm), _load_pasid, NULL, true); 522 523 mutex_unlock(&mm->context.lock); 524 } 525 526 static int intel_svm_alloc_pasid(struct device *dev, struct mm_struct *mm, 527 unsigned int flags) 528 { 529 ioasid_t max_pasid = dev_is_pci(dev) ? 530 pci_max_pasids(to_pci_dev(dev)) : intel_pasid_max_id; 531 532 return iommu_sva_alloc_pasid(mm, PASID_MIN, max_pasid - 1); 533 } 534 535 static void intel_svm_free_pasid(struct mm_struct *mm) 536 { 537 iommu_sva_free_pasid(mm); 538 } 539 540 static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu, 541 struct device *dev, 542 struct mm_struct *mm, 543 unsigned int flags) 544 { 545 struct device_domain_info *info = get_domain_info(dev); 546 unsigned long iflags, sflags; 547 struct intel_svm_dev *sdev; 548 struct intel_svm *svm; 549 int ret = 0; 550 551 svm = pasid_private_find(mm->pasid); 552 if (!svm) { 553 svm = kzalloc(sizeof(*svm), GFP_KERNEL); 554 if (!svm) 555 return ERR_PTR(-ENOMEM); 556 557 svm->pasid = mm->pasid; 558 svm->mm = mm; 559 svm->flags = flags; 560 INIT_LIST_HEAD_RCU(&svm->devs); 561 562 if (!(flags & SVM_FLAG_SUPERVISOR_MODE)) { 563 svm->notifier.ops = &intel_mmuops; 564 ret = mmu_notifier_register(&svm->notifier, mm); 565 if (ret) { 566 kfree(svm); 567 return ERR_PTR(ret); 568 } 569 } 570 571 ret = pasid_private_add(svm->pasid, svm); 572 if (ret) { 573 if (svm->notifier.ops) 574 mmu_notifier_unregister(&svm->notifier, mm); 575 kfree(svm); 576 return ERR_PTR(ret); 577 } 578 } 579 580 /* Find the matching device in svm list */ 581 sdev = svm_lookup_device_by_dev(svm, dev); 582 if (sdev) { 583 sdev->users++; 584 goto success; 585 } 586 587 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); 588 if (!sdev) { 589 ret = -ENOMEM; 590 goto free_svm; 591 } 592 593 sdev->dev = dev; 594 sdev->iommu = iommu; 595 sdev->did = FLPT_DEFAULT_DID; 596 sdev->sid = PCI_DEVID(info->bus, info->devfn); 597 sdev->users = 1; 598 sdev->pasid = svm->pasid; 599 sdev->sva.dev = dev; 600 init_rcu_head(&sdev->rcu); 601 if (info->ats_enabled) { 602 sdev->dev_iotlb = 1; 603 sdev->qdep = info->ats_qdep; 604 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS) 605 sdev->qdep = 0; 606 } 607 608 /* Setup the pasid table: */ 609 sflags = (flags & SVM_FLAG_SUPERVISOR_MODE) ? 610 PASID_FLAG_SUPERVISOR_MODE : 0; 611 sflags |= cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; 612 spin_lock_irqsave(&iommu->lock, iflags); 613 ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, mm->pasid, 614 FLPT_DEFAULT_DID, sflags); 615 spin_unlock_irqrestore(&iommu->lock, iflags); 616 617 if (ret) 618 goto free_sdev; 619 620 /* The newly allocated pasid is loaded to the mm. */ 621 if (!(flags & SVM_FLAG_SUPERVISOR_MODE) && list_empty(&svm->devs)) 622 load_pasid(mm, svm->pasid); 623 624 list_add_rcu(&sdev->list, &svm->devs); 625 success: 626 return &sdev->sva; 627 628 free_sdev: 629 kfree(sdev); 630 free_svm: 631 if (list_empty(&svm->devs)) { 632 if (svm->notifier.ops) 633 mmu_notifier_unregister(&svm->notifier, mm); 634 pasid_private_remove(mm->pasid); 635 kfree(svm); 636 } 637 638 return ERR_PTR(ret); 639 } 640 641 /* Caller must hold pasid_mutex */ 642 static int intel_svm_unbind_mm(struct device *dev, u32 pasid) 643 { 644 struct intel_svm_dev *sdev; 645 struct intel_iommu *iommu; 646 struct intel_svm *svm; 647 struct mm_struct *mm; 648 int ret = -EINVAL; 649 650 iommu = device_to_iommu(dev, NULL, NULL); 651 if (!iommu) 652 goto out; 653 654 ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev); 655 if (ret) 656 goto out; 657 mm = svm->mm; 658 659 if (sdev) { 660 sdev->users--; 661 if (!sdev->users) { 662 list_del_rcu(&sdev->list); 663 /* Flush the PASID cache and IOTLB for this device. 664 * Note that we do depend on the hardware *not* using 665 * the PASID any more. Just as we depend on other 666 * devices never using PASIDs that they have no right 667 * to use. We have a *shared* PASID table, because it's 668 * large and has to be physically contiguous. So it's 669 * hard to be as defensive as we might like. */ 670 intel_pasid_tear_down_entry(iommu, dev, 671 svm->pasid, false); 672 intel_svm_drain_prq(dev, svm->pasid); 673 kfree_rcu(sdev, rcu); 674 675 if (list_empty(&svm->devs)) { 676 if (svm->notifier.ops) { 677 mmu_notifier_unregister(&svm->notifier, mm); 678 /* Clear mm's pasid. */ 679 load_pasid(mm, PASID_DISABLED); 680 } 681 pasid_private_remove(svm->pasid); 682 /* We mandate that no page faults may be outstanding 683 * for the PASID when intel_svm_unbind_mm() is called. 684 * If that is not obeyed, subtle errors will happen. 685 * Let's make them less subtle... */ 686 memset(svm, 0x6b, sizeof(*svm)); 687 kfree(svm); 688 } 689 } 690 /* Drop a PASID reference and free it if no reference. */ 691 intel_svm_free_pasid(mm); 692 } 693 out: 694 return ret; 695 } 696 697 /* Page request queue descriptor */ 698 struct page_req_dsc { 699 union { 700 struct { 701 u64 type:8; 702 u64 pasid_present:1; 703 u64 priv_data_present:1; 704 u64 rsvd:6; 705 u64 rid:16; 706 u64 pasid:20; 707 u64 exe_req:1; 708 u64 pm_req:1; 709 u64 rsvd2:10; 710 }; 711 u64 qw_0; 712 }; 713 union { 714 struct { 715 u64 rd_req:1; 716 u64 wr_req:1; 717 u64 lpig:1; 718 u64 prg_index:9; 719 u64 addr:52; 720 }; 721 u64 qw_1; 722 }; 723 u64 priv_data[2]; 724 }; 725 726 static bool is_canonical_address(u64 addr) 727 { 728 int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1); 729 long saddr = (long) addr; 730 731 return (((saddr << shift) >> shift) == saddr); 732 } 733 734 /** 735 * intel_svm_drain_prq - Drain page requests and responses for a pasid 736 * @dev: target device 737 * @pasid: pasid for draining 738 * 739 * Drain all pending page requests and responses related to @pasid in both 740 * software and hardware. This is supposed to be called after the device 741 * driver has stopped DMA, the pasid entry has been cleared, and both IOTLB 742 * and DevTLB have been invalidated. 743 * 744 * It waits until all pending page requests for @pasid in the page fault 745 * queue are completed by the prq handling thread. Then follow the steps 746 * described in VT-d spec CH7.10 to drain all page requests and page 747 * responses pending in the hardware. 748 */ 749 static void intel_svm_drain_prq(struct device *dev, u32 pasid) 750 { 751 struct device_domain_info *info; 752 struct dmar_domain *domain; 753 struct intel_iommu *iommu; 754 struct qi_desc desc[3]; 755 struct pci_dev *pdev; 756 int head, tail; 757 u16 sid, did; 758 int qdep; 759 760 info = get_domain_info(dev); 761 if (WARN_ON(!info || !dev_is_pci(dev))) 762 return; 763 764 if (!info->pri_enabled) 765 return; 766 767 iommu = info->iommu; 768 domain = info->domain; 769 pdev = to_pci_dev(dev); 770 sid = PCI_DEVID(info->bus, info->devfn); 771 did = domain->iommu_did[iommu->seq_id]; 772 qdep = pci_ats_queue_depth(pdev); 773 774 /* 775 * Check and wait until all pending page requests in the queue are 776 * handled by the prq handling thread. 777 */ 778 prq_retry: 779 reinit_completion(&iommu->prq_complete); 780 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 781 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 782 while (head != tail) { 783 struct page_req_dsc *req; 784 785 req = &iommu->prq[head / sizeof(*req)]; 786 if (!req->pasid_present || req->pasid != pasid) { 787 head = (head + sizeof(*req)) & PRQ_RING_MASK; 788 continue; 789 } 790 791 wait_for_completion(&iommu->prq_complete); 792 goto prq_retry; 793 } 794 795 iopf_queue_flush_dev(dev); 796 797 /* 798 * Perform steps described in VT-d spec CH7.10 to drain page 799 * requests and responses in hardware. 800 */ 801 memset(desc, 0, sizeof(desc)); 802 desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) | 803 QI_IWD_FENCE | 804 QI_IWD_TYPE; 805 desc[1].qw0 = QI_EIOTLB_PASID(pasid) | 806 QI_EIOTLB_DID(did) | 807 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | 808 QI_EIOTLB_TYPE; 809 desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) | 810 QI_DEV_EIOTLB_SID(sid) | 811 QI_DEV_EIOTLB_QDEP(qdep) | 812 QI_DEIOTLB_TYPE | 813 QI_DEV_IOTLB_PFSID(info->pfsid); 814 qi_retry: 815 reinit_completion(&iommu->prq_complete); 816 qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN); 817 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 818 wait_for_completion(&iommu->prq_complete); 819 goto qi_retry; 820 } 821 } 822 823 static int prq_to_iommu_prot(struct page_req_dsc *req) 824 { 825 int prot = 0; 826 827 if (req->rd_req) 828 prot |= IOMMU_FAULT_PERM_READ; 829 if (req->wr_req) 830 prot |= IOMMU_FAULT_PERM_WRITE; 831 if (req->exe_req) 832 prot |= IOMMU_FAULT_PERM_EXEC; 833 if (req->pm_req) 834 prot |= IOMMU_FAULT_PERM_PRIV; 835 836 return prot; 837 } 838 839 static int intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, 840 struct page_req_dsc *desc) 841 { 842 struct iommu_fault_event event; 843 844 if (!dev || !dev_is_pci(dev)) 845 return -ENODEV; 846 847 /* Fill in event data for device specific processing */ 848 memset(&event, 0, sizeof(struct iommu_fault_event)); 849 event.fault.type = IOMMU_FAULT_PAGE_REQ; 850 event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT; 851 event.fault.prm.pasid = desc->pasid; 852 event.fault.prm.grpid = desc->prg_index; 853 event.fault.prm.perm = prq_to_iommu_prot(desc); 854 855 if (desc->lpig) 856 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 857 if (desc->pasid_present) { 858 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 859 event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; 860 } 861 if (desc->priv_data_present) { 862 /* 863 * Set last page in group bit if private data is present, 864 * page response is required as it does for LPIG. 865 * iommu_report_device_fault() doesn't understand this vendor 866 * specific requirement thus we set last_page as a workaround. 867 */ 868 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 869 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; 870 event.fault.prm.private_data[0] = desc->priv_data[0]; 871 event.fault.prm.private_data[1] = desc->priv_data[1]; 872 } else if (dmar_latency_enabled(iommu, DMAR_LATENCY_PRQ)) { 873 /* 874 * If the private data fields are not used by hardware, use it 875 * to monitor the prq handle latency. 876 */ 877 event.fault.prm.private_data[0] = ktime_to_ns(ktime_get()); 878 } 879 880 return iommu_report_device_fault(dev, &event); 881 } 882 883 static void handle_bad_prq_event(struct intel_iommu *iommu, 884 struct page_req_dsc *req, int result) 885 { 886 struct qi_desc desc; 887 888 pr_err("%s: Invalid page request: %08llx %08llx\n", 889 iommu->name, ((unsigned long long *)req)[0], 890 ((unsigned long long *)req)[1]); 891 892 /* 893 * Per VT-d spec. v3.0 ch7.7, system software must 894 * respond with page group response if private data 895 * is present (PDP) or last page in group (LPIG) bit 896 * is set. This is an additional VT-d feature beyond 897 * PCI ATS spec. 898 */ 899 if (!req->lpig && !req->priv_data_present) 900 return; 901 902 desc.qw0 = QI_PGRP_PASID(req->pasid) | 903 QI_PGRP_DID(req->rid) | 904 QI_PGRP_PASID_P(req->pasid_present) | 905 QI_PGRP_PDP(req->priv_data_present) | 906 QI_PGRP_RESP_CODE(result) | 907 QI_PGRP_RESP_TYPE; 908 desc.qw1 = QI_PGRP_IDX(req->prg_index) | 909 QI_PGRP_LPIG(req->lpig); 910 911 if (req->priv_data_present) { 912 desc.qw2 = req->priv_data[0]; 913 desc.qw3 = req->priv_data[1]; 914 } else { 915 desc.qw2 = 0; 916 desc.qw3 = 0; 917 } 918 919 qi_submit_sync(iommu, &desc, 1, 0); 920 } 921 922 static irqreturn_t prq_event_thread(int irq, void *d) 923 { 924 struct intel_svm_dev *sdev = NULL; 925 struct intel_iommu *iommu = d; 926 struct intel_svm *svm = NULL; 927 struct page_req_dsc *req; 928 int head, tail, handled; 929 u64 address; 930 931 /* 932 * Clear PPR bit before reading head/tail registers, to ensure that 933 * we get a new interrupt if needed. 934 */ 935 writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG); 936 937 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 938 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 939 handled = (head != tail); 940 while (head != tail) { 941 req = &iommu->prq[head / sizeof(*req)]; 942 address = (u64)req->addr << VTD_PAGE_SHIFT; 943 944 if (unlikely(!req->pasid_present)) { 945 pr_err("IOMMU: %s: Page request without PASID\n", 946 iommu->name); 947 bad_req: 948 svm = NULL; 949 sdev = NULL; 950 handle_bad_prq_event(iommu, req, QI_RESP_INVALID); 951 goto prq_advance; 952 } 953 954 if (unlikely(!is_canonical_address(address))) { 955 pr_err("IOMMU: %s: Address is not canonical\n", 956 iommu->name); 957 goto bad_req; 958 } 959 960 if (unlikely(req->pm_req && (req->rd_req | req->wr_req))) { 961 pr_err("IOMMU: %s: Page request in Privilege Mode\n", 962 iommu->name); 963 goto bad_req; 964 } 965 966 if (unlikely(req->exe_req && req->rd_req)) { 967 pr_err("IOMMU: %s: Execution request not supported\n", 968 iommu->name); 969 goto bad_req; 970 } 971 972 if (!svm || svm->pasid != req->pasid) { 973 /* 974 * It can't go away, because the driver is not permitted 975 * to unbind the mm while any page faults are outstanding. 976 */ 977 svm = pasid_private_find(req->pasid); 978 if (IS_ERR_OR_NULL(svm) || (svm->flags & SVM_FLAG_SUPERVISOR_MODE)) 979 goto bad_req; 980 } 981 982 if (!sdev || sdev->sid != req->rid) { 983 sdev = svm_lookup_device_by_sid(svm, req->rid); 984 if (!sdev) 985 goto bad_req; 986 } 987 988 sdev->prq_seq_number++; 989 990 /* 991 * If prq is to be handled outside iommu driver via receiver of 992 * the fault notifiers, we skip the page response here. 993 */ 994 if (intel_svm_prq_report(iommu, sdev->dev, req)) 995 handle_bad_prq_event(iommu, req, QI_RESP_INVALID); 996 997 trace_prq_report(iommu, sdev->dev, req->qw_0, req->qw_1, 998 req->priv_data[0], req->priv_data[1], 999 sdev->prq_seq_number); 1000 prq_advance: 1001 head = (head + sizeof(*req)) & PRQ_RING_MASK; 1002 } 1003 1004 dmar_writeq(iommu->reg + DMAR_PQH_REG, tail); 1005 1006 /* 1007 * Clear the page request overflow bit and wake up all threads that 1008 * are waiting for the completion of this handling. 1009 */ 1010 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 1011 pr_info_ratelimited("IOMMU: %s: PRQ overflow detected\n", 1012 iommu->name); 1013 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 1014 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 1015 if (head == tail) { 1016 iopf_queue_discard_partial(iommu->iopf_queue); 1017 writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG); 1018 pr_info_ratelimited("IOMMU: %s: PRQ overflow cleared", 1019 iommu->name); 1020 } 1021 } 1022 1023 if (!completion_done(&iommu->prq_complete)) 1024 complete(&iommu->prq_complete); 1025 1026 return IRQ_RETVAL(handled); 1027 } 1028 1029 struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata) 1030 { 1031 struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); 1032 unsigned int flags = 0; 1033 struct iommu_sva *sva; 1034 int ret; 1035 1036 if (drvdata) 1037 flags = *(unsigned int *)drvdata; 1038 1039 if (flags & SVM_FLAG_SUPERVISOR_MODE) { 1040 if (!ecap_srs(iommu->ecap)) { 1041 dev_err(dev, "%s: Supervisor PASID not supported\n", 1042 iommu->name); 1043 return ERR_PTR(-EOPNOTSUPP); 1044 } 1045 1046 if (mm) { 1047 dev_err(dev, "%s: Supervisor PASID with user provided mm\n", 1048 iommu->name); 1049 return ERR_PTR(-EINVAL); 1050 } 1051 1052 mm = &init_mm; 1053 } 1054 1055 mutex_lock(&pasid_mutex); 1056 ret = intel_svm_alloc_pasid(dev, mm, flags); 1057 if (ret) { 1058 mutex_unlock(&pasid_mutex); 1059 return ERR_PTR(ret); 1060 } 1061 1062 sva = intel_svm_bind_mm(iommu, dev, mm, flags); 1063 if (IS_ERR_OR_NULL(sva)) 1064 intel_svm_free_pasid(mm); 1065 mutex_unlock(&pasid_mutex); 1066 1067 return sva; 1068 } 1069 1070 void intel_svm_unbind(struct iommu_sva *sva) 1071 { 1072 struct intel_svm_dev *sdev = to_intel_svm_dev(sva); 1073 1074 mutex_lock(&pasid_mutex); 1075 intel_svm_unbind_mm(sdev->dev, sdev->pasid); 1076 mutex_unlock(&pasid_mutex); 1077 } 1078 1079 u32 intel_svm_get_pasid(struct iommu_sva *sva) 1080 { 1081 struct intel_svm_dev *sdev; 1082 u32 pasid; 1083 1084 mutex_lock(&pasid_mutex); 1085 sdev = to_intel_svm_dev(sva); 1086 pasid = sdev->pasid; 1087 mutex_unlock(&pasid_mutex); 1088 1089 return pasid; 1090 } 1091 1092 int intel_svm_page_response(struct device *dev, 1093 struct iommu_fault_event *evt, 1094 struct iommu_page_response *msg) 1095 { 1096 struct iommu_fault_page_request *prm; 1097 struct intel_svm_dev *sdev = NULL; 1098 struct intel_svm *svm = NULL; 1099 struct intel_iommu *iommu; 1100 bool private_present; 1101 bool pasid_present; 1102 bool last_page; 1103 u8 bus, devfn; 1104 int ret = 0; 1105 u16 sid; 1106 1107 if (!dev || !dev_is_pci(dev)) 1108 return -ENODEV; 1109 1110 iommu = device_to_iommu(dev, &bus, &devfn); 1111 if (!iommu) 1112 return -ENODEV; 1113 1114 if (!msg || !evt) 1115 return -EINVAL; 1116 1117 mutex_lock(&pasid_mutex); 1118 1119 prm = &evt->fault.prm; 1120 sid = PCI_DEVID(bus, devfn); 1121 pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 1122 private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; 1123 last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 1124 1125 if (!pasid_present) { 1126 ret = -EINVAL; 1127 goto out; 1128 } 1129 1130 if (prm->pasid == 0 || prm->pasid >= PASID_MAX) { 1131 ret = -EINVAL; 1132 goto out; 1133 } 1134 1135 ret = pasid_to_svm_sdev(dev, prm->pasid, &svm, &sdev); 1136 if (ret || !sdev) { 1137 ret = -ENODEV; 1138 goto out; 1139 } 1140 1141 /* 1142 * For responses from userspace, need to make sure that the 1143 * pasid has been bound to its mm. 1144 */ 1145 if (svm->flags & SVM_FLAG_GUEST_MODE) { 1146 struct mm_struct *mm; 1147 1148 mm = get_task_mm(current); 1149 if (!mm) { 1150 ret = -EINVAL; 1151 goto out; 1152 } 1153 1154 if (mm != svm->mm) { 1155 ret = -ENODEV; 1156 mmput(mm); 1157 goto out; 1158 } 1159 1160 mmput(mm); 1161 } 1162 1163 /* 1164 * Per VT-d spec. v3.0 ch7.7, system software must respond 1165 * with page group response if private data is present (PDP) 1166 * or last page in group (LPIG) bit is set. This is an 1167 * additional VT-d requirement beyond PCI ATS spec. 1168 */ 1169 if (last_page || private_present) { 1170 struct qi_desc desc; 1171 1172 desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) | 1173 QI_PGRP_PASID_P(pasid_present) | 1174 QI_PGRP_PDP(private_present) | 1175 QI_PGRP_RESP_CODE(msg->code) | 1176 QI_PGRP_RESP_TYPE; 1177 desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page); 1178 desc.qw2 = 0; 1179 desc.qw3 = 0; 1180 1181 if (private_present) { 1182 desc.qw2 = prm->private_data[0]; 1183 desc.qw3 = prm->private_data[1]; 1184 } else if (prm->private_data[0]) { 1185 dmar_latency_update(iommu, DMAR_LATENCY_PRQ, 1186 ktime_to_ns(ktime_get()) - prm->private_data[0]); 1187 } 1188 1189 qi_submit_sync(iommu, &desc, 1, 0); 1190 } 1191 out: 1192 mutex_unlock(&pasid_mutex); 1193 return ret; 1194 } 1195