1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <jroedel@suse.de> 5 * Leo Duran <leo.duran@amd.com> 6 */ 7 8 #define pr_fmt(fmt) "AMD-Vi: " fmt 9 #define dev_fmt(fmt) pr_fmt(fmt) 10 11 #include <linux/ratelimit.h> 12 #include <linux/pci.h> 13 #include <linux/acpi.h> 14 #include <linux/pci-ats.h> 15 #include <linux/bitmap.h> 16 #include <linux/slab.h> 17 #include <linux/debugfs.h> 18 #include <linux/scatterlist.h> 19 #include <linux/dma-map-ops.h> 20 #include <linux/dma-direct.h> 21 #include <linux/iommu-helper.h> 22 #include <linux/delay.h> 23 #include <linux/amd-iommu.h> 24 #include <linux/notifier.h> 25 #include <linux/export.h> 26 #include <linux/irq.h> 27 #include <linux/msi.h> 28 #include <linux/irqdomain.h> 29 #include <linux/percpu.h> 30 #include <linux/io-pgtable.h> 31 #include <linux/cc_platform.h> 32 #include <asm/irq_remapping.h> 33 #include <asm/io_apic.h> 34 #include <asm/apic.h> 35 #include <asm/hw_irq.h> 36 #include <asm/proto.h> 37 #include <asm/iommu.h> 38 #include <asm/gart.h> 39 #include <asm/dma.h> 40 #include <uapi/linux/iommufd.h> 41 42 #include "amd_iommu.h" 43 #include "../dma-iommu.h" 44 #include "../irq_remapping.h" 45 #include "../iommu-pages.h" 46 47 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) 48 49 /* Reserved IOVA ranges */ 50 #define MSI_RANGE_START (0xfee00000) 51 #define MSI_RANGE_END (0xfeefffff) 52 #define HT_RANGE_START (0xfd00000000ULL) 53 #define HT_RANGE_END (0xffffffffffULL) 54 55 static DEFINE_SPINLOCK(pd_bitmap_lock); 56 57 LIST_HEAD(ioapic_map); 58 LIST_HEAD(hpet_map); 59 LIST_HEAD(acpihid_map); 60 61 const struct iommu_ops amd_iommu_ops; 62 static const struct iommu_dirty_ops amd_dirty_ops; 63 64 int amd_iommu_max_glx_val = -1; 65 66 /* 67 * general struct to manage commands send to an IOMMU 68 */ 69 struct iommu_cmd { 70 u32 data[4]; 71 }; 72 73 struct kmem_cache *amd_iommu_irq_cache; 74 75 static void detach_device(struct device *dev); 76 77 static void set_dte_entry(struct amd_iommu *iommu, 78 struct iommu_dev_data *dev_data); 79 80 /**************************************************************************** 81 * 82 * Helper functions 83 * 84 ****************************************************************************/ 85 86 static inline bool pdom_is_v2_pgtbl_mode(struct protection_domain *pdom) 87 { 88 return (pdom && (pdom->pd_mode == PD_MODE_V2)); 89 } 90 91 static inline bool pdom_is_in_pt_mode(struct protection_domain *pdom) 92 { 93 return (pdom->domain.type == IOMMU_DOMAIN_IDENTITY); 94 } 95 96 /* 97 * We cannot support PASID w/ existing v1 page table in the same domain 98 * since it will be nested. However, existing domain w/ v2 page table 99 * or passthrough mode can be used for PASID. 100 */ 101 static inline bool pdom_is_sva_capable(struct protection_domain *pdom) 102 { 103 return pdom_is_v2_pgtbl_mode(pdom) || pdom_is_in_pt_mode(pdom); 104 } 105 106 static inline int get_acpihid_device_id(struct device *dev, 107 struct acpihid_map_entry **entry) 108 { 109 struct acpi_device *adev = ACPI_COMPANION(dev); 110 struct acpihid_map_entry *p; 111 112 if (!adev) 113 return -ENODEV; 114 115 list_for_each_entry(p, &acpihid_map, list) { 116 if (acpi_dev_hid_uid_match(adev, p->hid, 117 p->uid[0] ? p->uid : NULL)) { 118 if (entry) 119 *entry = p; 120 return p->devid; 121 } 122 } 123 return -EINVAL; 124 } 125 126 static inline int get_device_sbdf_id(struct device *dev) 127 { 128 int sbdf; 129 130 if (dev_is_pci(dev)) 131 sbdf = get_pci_sbdf_id(to_pci_dev(dev)); 132 else 133 sbdf = get_acpihid_device_id(dev, NULL); 134 135 return sbdf; 136 } 137 138 struct dev_table_entry *get_dev_table(struct amd_iommu *iommu) 139 { 140 struct dev_table_entry *dev_table; 141 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 142 143 BUG_ON(pci_seg == NULL); 144 dev_table = pci_seg->dev_table; 145 BUG_ON(dev_table == NULL); 146 147 return dev_table; 148 } 149 150 static inline u16 get_device_segment(struct device *dev) 151 { 152 u16 seg; 153 154 if (dev_is_pci(dev)) { 155 struct pci_dev *pdev = to_pci_dev(dev); 156 157 seg = pci_domain_nr(pdev->bus); 158 } else { 159 u32 devid = get_acpihid_device_id(dev, NULL); 160 161 seg = PCI_SBDF_TO_SEGID(devid); 162 } 163 164 return seg; 165 } 166 167 /* Writes the specific IOMMU for a device into the PCI segment rlookup table */ 168 void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid) 169 { 170 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 171 172 pci_seg->rlookup_table[devid] = iommu; 173 } 174 175 static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid) 176 { 177 struct amd_iommu_pci_seg *pci_seg; 178 179 for_each_pci_segment(pci_seg) { 180 if (pci_seg->id == seg) 181 return pci_seg->rlookup_table[devid]; 182 } 183 return NULL; 184 } 185 186 static struct amd_iommu *rlookup_amd_iommu(struct device *dev) 187 { 188 u16 seg = get_device_segment(dev); 189 int devid = get_device_sbdf_id(dev); 190 191 if (devid < 0) 192 return NULL; 193 return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid)); 194 } 195 196 static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid) 197 { 198 struct iommu_dev_data *dev_data; 199 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 200 201 dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); 202 if (!dev_data) 203 return NULL; 204 205 spin_lock_init(&dev_data->lock); 206 dev_data->devid = devid; 207 ratelimit_default_init(&dev_data->rs); 208 209 llist_add(&dev_data->dev_data_list, &pci_seg->dev_data_list); 210 return dev_data; 211 } 212 213 static struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid) 214 { 215 struct iommu_dev_data *dev_data; 216 struct llist_node *node; 217 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 218 219 if (llist_empty(&pci_seg->dev_data_list)) 220 return NULL; 221 222 node = pci_seg->dev_data_list.first; 223 llist_for_each_entry(dev_data, node, dev_data_list) { 224 if (dev_data->devid == devid) 225 return dev_data; 226 } 227 228 return NULL; 229 } 230 231 static int clone_alias(struct pci_dev *pdev, u16 alias, void *data) 232 { 233 struct amd_iommu *iommu; 234 struct dev_table_entry *dev_table; 235 u16 devid = pci_dev_id(pdev); 236 237 if (devid == alias) 238 return 0; 239 240 iommu = rlookup_amd_iommu(&pdev->dev); 241 if (!iommu) 242 return 0; 243 244 amd_iommu_set_rlookup_table(iommu, alias); 245 dev_table = get_dev_table(iommu); 246 memcpy(dev_table[alias].data, 247 dev_table[devid].data, 248 sizeof(dev_table[alias].data)); 249 250 return 0; 251 } 252 253 static void clone_aliases(struct amd_iommu *iommu, struct device *dev) 254 { 255 struct pci_dev *pdev; 256 257 if (!dev_is_pci(dev)) 258 return; 259 pdev = to_pci_dev(dev); 260 261 /* 262 * The IVRS alias stored in the alias table may not be 263 * part of the PCI DMA aliases if it's bus differs 264 * from the original device. 265 */ 266 clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL); 267 268 pci_for_each_dma_alias(pdev, clone_alias, NULL); 269 } 270 271 static void setup_aliases(struct amd_iommu *iommu, struct device *dev) 272 { 273 struct pci_dev *pdev = to_pci_dev(dev); 274 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 275 u16 ivrs_alias; 276 277 /* For ACPI HID devices, there are no aliases */ 278 if (!dev_is_pci(dev)) 279 return; 280 281 /* 282 * Add the IVRS alias to the pci aliases if it is on the same 283 * bus. The IVRS table may know about a quirk that we don't. 284 */ 285 ivrs_alias = pci_seg->alias_table[pci_dev_id(pdev)]; 286 if (ivrs_alias != pci_dev_id(pdev) && 287 PCI_BUS_NUM(ivrs_alias) == pdev->bus->number) 288 pci_add_dma_alias(pdev, ivrs_alias & 0xff, 1); 289 290 clone_aliases(iommu, dev); 291 } 292 293 static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid) 294 { 295 struct iommu_dev_data *dev_data; 296 297 dev_data = search_dev_data(iommu, devid); 298 299 if (dev_data == NULL) { 300 dev_data = alloc_dev_data(iommu, devid); 301 if (!dev_data) 302 return NULL; 303 304 if (translation_pre_enabled(iommu)) 305 dev_data->defer_attach = true; 306 } 307 308 return dev_data; 309 } 310 311 /* 312 * Find or create an IOMMU group for a acpihid device. 313 */ 314 static struct iommu_group *acpihid_device_group(struct device *dev) 315 { 316 struct acpihid_map_entry *p, *entry = NULL; 317 int devid; 318 319 devid = get_acpihid_device_id(dev, &entry); 320 if (devid < 0) 321 return ERR_PTR(devid); 322 323 list_for_each_entry(p, &acpihid_map, list) { 324 if ((devid == p->devid) && p->group) 325 entry->group = p->group; 326 } 327 328 if (!entry->group) 329 entry->group = generic_device_group(dev); 330 else 331 iommu_group_ref_get(entry->group); 332 333 return entry->group; 334 } 335 336 static inline bool pdev_pasid_supported(struct iommu_dev_data *dev_data) 337 { 338 return (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP); 339 } 340 341 static u32 pdev_get_caps(struct pci_dev *pdev) 342 { 343 int features; 344 u32 flags = 0; 345 346 if (pci_ats_supported(pdev)) 347 flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP; 348 349 if (pci_pri_supported(pdev)) 350 flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP; 351 352 features = pci_pasid_features(pdev); 353 if (features >= 0) { 354 flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP; 355 356 if (features & PCI_PASID_CAP_EXEC) 357 flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP; 358 359 if (features & PCI_PASID_CAP_PRIV) 360 flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP; 361 } 362 363 return flags; 364 } 365 366 static inline int pdev_enable_cap_ats(struct pci_dev *pdev) 367 { 368 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 369 int ret = -EINVAL; 370 371 if (dev_data->ats_enabled) 372 return 0; 373 374 if (amd_iommu_iotlb_sup && 375 (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP)) { 376 ret = pci_enable_ats(pdev, PAGE_SHIFT); 377 if (!ret) { 378 dev_data->ats_enabled = 1; 379 dev_data->ats_qdep = pci_ats_queue_depth(pdev); 380 } 381 } 382 383 return ret; 384 } 385 386 static inline void pdev_disable_cap_ats(struct pci_dev *pdev) 387 { 388 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 389 390 if (dev_data->ats_enabled) { 391 pci_disable_ats(pdev); 392 dev_data->ats_enabled = 0; 393 } 394 } 395 396 static inline int pdev_enable_cap_pri(struct pci_dev *pdev) 397 { 398 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 399 int ret = -EINVAL; 400 401 if (dev_data->pri_enabled) 402 return 0; 403 404 if (!dev_data->ats_enabled) 405 return 0; 406 407 if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) { 408 /* 409 * First reset the PRI state of the device. 410 * FIXME: Hardcode number of outstanding requests for now 411 */ 412 if (!pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32)) { 413 dev_data->pri_enabled = 1; 414 dev_data->pri_tlp = pci_prg_resp_pasid_required(pdev); 415 416 ret = 0; 417 } 418 } 419 420 return ret; 421 } 422 423 static inline void pdev_disable_cap_pri(struct pci_dev *pdev) 424 { 425 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 426 427 if (dev_data->pri_enabled) { 428 pci_disable_pri(pdev); 429 dev_data->pri_enabled = 0; 430 } 431 } 432 433 static inline int pdev_enable_cap_pasid(struct pci_dev *pdev) 434 { 435 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 436 int ret = -EINVAL; 437 438 if (dev_data->pasid_enabled) 439 return 0; 440 441 if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) { 442 /* Only allow access to user-accessible pages */ 443 ret = pci_enable_pasid(pdev, 0); 444 if (!ret) 445 dev_data->pasid_enabled = 1; 446 } 447 448 return ret; 449 } 450 451 static inline void pdev_disable_cap_pasid(struct pci_dev *pdev) 452 { 453 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 454 455 if (dev_data->pasid_enabled) { 456 pci_disable_pasid(pdev); 457 dev_data->pasid_enabled = 0; 458 } 459 } 460 461 static void pdev_enable_caps(struct pci_dev *pdev) 462 { 463 pdev_enable_cap_ats(pdev); 464 pdev_enable_cap_pasid(pdev); 465 pdev_enable_cap_pri(pdev); 466 } 467 468 static void pdev_disable_caps(struct pci_dev *pdev) 469 { 470 pdev_disable_cap_ats(pdev); 471 pdev_disable_cap_pasid(pdev); 472 pdev_disable_cap_pri(pdev); 473 } 474 475 /* 476 * This function checks if the driver got a valid device from the caller to 477 * avoid dereferencing invalid pointers. 478 */ 479 static bool check_device(struct device *dev) 480 { 481 struct amd_iommu_pci_seg *pci_seg; 482 struct amd_iommu *iommu; 483 int devid, sbdf; 484 485 if (!dev) 486 return false; 487 488 sbdf = get_device_sbdf_id(dev); 489 if (sbdf < 0) 490 return false; 491 devid = PCI_SBDF_TO_DEVID(sbdf); 492 493 iommu = rlookup_amd_iommu(dev); 494 if (!iommu) 495 return false; 496 497 /* Out of our scope? */ 498 pci_seg = iommu->pci_seg; 499 if (devid > pci_seg->last_bdf) 500 return false; 501 502 return true; 503 } 504 505 static int iommu_init_device(struct amd_iommu *iommu, struct device *dev) 506 { 507 struct iommu_dev_data *dev_data; 508 int devid, sbdf; 509 510 if (dev_iommu_priv_get(dev)) 511 return 0; 512 513 sbdf = get_device_sbdf_id(dev); 514 if (sbdf < 0) 515 return sbdf; 516 517 devid = PCI_SBDF_TO_DEVID(sbdf); 518 dev_data = find_dev_data(iommu, devid); 519 if (!dev_data) 520 return -ENOMEM; 521 522 dev_data->dev = dev; 523 setup_aliases(iommu, dev); 524 525 /* 526 * By default we use passthrough mode for IOMMUv2 capable device. 527 * But if amd_iommu=force_isolation is set (e.g. to debug DMA to 528 * invalid address), we ignore the capability for the device so 529 * it'll be forced to go into translation mode. 530 */ 531 if ((iommu_default_passthrough() || !amd_iommu_force_isolation) && 532 dev_is_pci(dev) && amd_iommu_gt_ppr_supported()) { 533 dev_data->flags = pdev_get_caps(to_pci_dev(dev)); 534 } 535 536 dev_iommu_priv_set(dev, dev_data); 537 538 return 0; 539 } 540 541 static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev) 542 { 543 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 544 struct dev_table_entry *dev_table = get_dev_table(iommu); 545 int devid, sbdf; 546 547 sbdf = get_device_sbdf_id(dev); 548 if (sbdf < 0) 549 return; 550 551 devid = PCI_SBDF_TO_DEVID(sbdf); 552 pci_seg->rlookup_table[devid] = NULL; 553 memset(&dev_table[devid], 0, sizeof(struct dev_table_entry)); 554 555 setup_aliases(iommu, dev); 556 } 557 558 static void amd_iommu_uninit_device(struct device *dev) 559 { 560 struct iommu_dev_data *dev_data; 561 562 dev_data = dev_iommu_priv_get(dev); 563 if (!dev_data) 564 return; 565 566 if (dev_data->domain) 567 detach_device(dev); 568 569 /* 570 * We keep dev_data around for unplugged devices and reuse it when the 571 * device is re-plugged - not doing so would introduce a ton of races. 572 */ 573 } 574 575 /**************************************************************************** 576 * 577 * Interrupt handling functions 578 * 579 ****************************************************************************/ 580 581 static void dump_dte_entry(struct amd_iommu *iommu, u16 devid) 582 { 583 int i; 584 struct dev_table_entry *dev_table = get_dev_table(iommu); 585 586 for (i = 0; i < 4; ++i) 587 pr_err("DTE[%d]: %016llx\n", i, dev_table[devid].data[i]); 588 } 589 590 static void dump_command(unsigned long phys_addr) 591 { 592 struct iommu_cmd *cmd = iommu_phys_to_virt(phys_addr); 593 int i; 594 595 for (i = 0; i < 4; ++i) 596 pr_err("CMD[%d]: %08x\n", i, cmd->data[i]); 597 } 598 599 static void amd_iommu_report_rmp_hw_error(struct amd_iommu *iommu, volatile u32 *event) 600 { 601 struct iommu_dev_data *dev_data = NULL; 602 int devid, vmg_tag, flags; 603 struct pci_dev *pdev; 604 u64 spa; 605 606 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK; 607 vmg_tag = (event[1]) & 0xFFFF; 608 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; 609 spa = ((u64)event[3] << 32) | (event[2] & 0xFFFFFFF8); 610 611 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), 612 devid & 0xff); 613 if (pdev) 614 dev_data = dev_iommu_priv_get(&pdev->dev); 615 616 if (dev_data) { 617 if (__ratelimit(&dev_data->rs)) { 618 pci_err(pdev, "Event logged [RMP_HW_ERROR vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n", 619 vmg_tag, spa, flags); 620 } 621 } else { 622 pr_err_ratelimited("Event logged [RMP_HW_ERROR device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n", 623 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 624 vmg_tag, spa, flags); 625 } 626 627 if (pdev) 628 pci_dev_put(pdev); 629 } 630 631 static void amd_iommu_report_rmp_fault(struct amd_iommu *iommu, volatile u32 *event) 632 { 633 struct iommu_dev_data *dev_data = NULL; 634 int devid, flags_rmp, vmg_tag, flags; 635 struct pci_dev *pdev; 636 u64 gpa; 637 638 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK; 639 flags_rmp = (event[0] >> EVENT_FLAGS_SHIFT) & 0xFF; 640 vmg_tag = (event[1]) & 0xFFFF; 641 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; 642 gpa = ((u64)event[3] << 32) | event[2]; 643 644 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), 645 devid & 0xff); 646 if (pdev) 647 dev_data = dev_iommu_priv_get(&pdev->dev); 648 649 if (dev_data) { 650 if (__ratelimit(&dev_data->rs)) { 651 pci_err(pdev, "Event logged [RMP_PAGE_FAULT vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n", 652 vmg_tag, gpa, flags_rmp, flags); 653 } 654 } else { 655 pr_err_ratelimited("Event logged [RMP_PAGE_FAULT device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n", 656 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 657 vmg_tag, gpa, flags_rmp, flags); 658 } 659 660 if (pdev) 661 pci_dev_put(pdev); 662 } 663 664 #define IS_IOMMU_MEM_TRANSACTION(flags) \ 665 (((flags) & EVENT_FLAG_I) == 0) 666 667 #define IS_WRITE_REQUEST(flags) \ 668 ((flags) & EVENT_FLAG_RW) 669 670 static void amd_iommu_report_page_fault(struct amd_iommu *iommu, 671 u16 devid, u16 domain_id, 672 u64 address, int flags) 673 { 674 struct iommu_dev_data *dev_data = NULL; 675 struct pci_dev *pdev; 676 677 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), 678 devid & 0xff); 679 if (pdev) 680 dev_data = dev_iommu_priv_get(&pdev->dev); 681 682 if (dev_data) { 683 /* 684 * If this is a DMA fault (for which the I(nterrupt) 685 * bit will be unset), allow report_iommu_fault() to 686 * prevent logging it. 687 */ 688 if (IS_IOMMU_MEM_TRANSACTION(flags)) { 689 /* Device not attached to domain properly */ 690 if (dev_data->domain == NULL) { 691 pr_err_ratelimited("Event logged [Device not attached to domain properly]\n"); 692 pr_err_ratelimited(" device=%04x:%02x:%02x.%x domain=0x%04x\n", 693 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), 694 PCI_FUNC(devid), domain_id); 695 goto out; 696 } 697 698 if (!report_iommu_fault(&dev_data->domain->domain, 699 &pdev->dev, address, 700 IS_WRITE_REQUEST(flags) ? 701 IOMMU_FAULT_WRITE : 702 IOMMU_FAULT_READ)) 703 goto out; 704 } 705 706 if (__ratelimit(&dev_data->rs)) { 707 pci_err(pdev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n", 708 domain_id, address, flags); 709 } 710 } else { 711 pr_err_ratelimited("Event logged [IO_PAGE_FAULT device=%04x:%02x:%02x.%x domain=0x%04x address=0x%llx flags=0x%04x]\n", 712 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 713 domain_id, address, flags); 714 } 715 716 out: 717 if (pdev) 718 pci_dev_put(pdev); 719 } 720 721 static void iommu_print_event(struct amd_iommu *iommu, void *__evt) 722 { 723 struct device *dev = iommu->iommu.dev; 724 int type, devid, flags, tag; 725 volatile u32 *event = __evt; 726 int count = 0; 727 u64 address; 728 u32 pasid; 729 730 retry: 731 type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK; 732 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK; 733 pasid = (event[0] & EVENT_DOMID_MASK_HI) | 734 (event[1] & EVENT_DOMID_MASK_LO); 735 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; 736 address = (u64)(((u64)event[3]) << 32) | event[2]; 737 738 if (type == 0) { 739 /* Did we hit the erratum? */ 740 if (++count == LOOP_TIMEOUT) { 741 pr_err("No event written to event log\n"); 742 return; 743 } 744 udelay(1); 745 goto retry; 746 } 747 748 if (type == EVENT_TYPE_IO_FAULT) { 749 amd_iommu_report_page_fault(iommu, devid, pasid, address, flags); 750 return; 751 } 752 753 switch (type) { 754 case EVENT_TYPE_ILL_DEV: 755 dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", 756 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 757 pasid, address, flags); 758 dump_dte_entry(iommu, devid); 759 break; 760 case EVENT_TYPE_DEV_TAB_ERR: 761 dev_err(dev, "Event logged [DEV_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x " 762 "address=0x%llx flags=0x%04x]\n", 763 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 764 address, flags); 765 break; 766 case EVENT_TYPE_PAGE_TAB_ERR: 767 dev_err(dev, "Event logged [PAGE_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x pasid=0x%04x address=0x%llx flags=0x%04x]\n", 768 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 769 pasid, address, flags); 770 break; 771 case EVENT_TYPE_ILL_CMD: 772 dev_err(dev, "Event logged [ILLEGAL_COMMAND_ERROR address=0x%llx]\n", address); 773 dump_command(address); 774 break; 775 case EVENT_TYPE_CMD_HARD_ERR: 776 dev_err(dev, "Event logged [COMMAND_HARDWARE_ERROR address=0x%llx flags=0x%04x]\n", 777 address, flags); 778 break; 779 case EVENT_TYPE_IOTLB_INV_TO: 780 dev_err(dev, "Event logged [IOTLB_INV_TIMEOUT device=%04x:%02x:%02x.%x address=0x%llx]\n", 781 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 782 address); 783 break; 784 case EVENT_TYPE_INV_DEV_REQ: 785 dev_err(dev, "Event logged [INVALID_DEVICE_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", 786 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 787 pasid, address, flags); 788 break; 789 case EVENT_TYPE_RMP_FAULT: 790 amd_iommu_report_rmp_fault(iommu, event); 791 break; 792 case EVENT_TYPE_RMP_HW_ERR: 793 amd_iommu_report_rmp_hw_error(iommu, event); 794 break; 795 case EVENT_TYPE_INV_PPR_REQ: 796 pasid = PPR_PASID(*((u64 *)__evt)); 797 tag = event[1] & 0x03FF; 798 dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x tag=0x%03x]\n", 799 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 800 pasid, address, flags, tag); 801 break; 802 default: 803 dev_err(dev, "Event logged [UNKNOWN event[0]=0x%08x event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n", 804 event[0], event[1], event[2], event[3]); 805 } 806 807 /* 808 * To detect the hardware errata 732 we need to clear the 809 * entry back to zero. This issue does not exist on SNP 810 * enabled system. Also this buffer is not writeable on 811 * SNP enabled system. 812 */ 813 if (!amd_iommu_snp_en) 814 memset(__evt, 0, 4 * sizeof(u32)); 815 } 816 817 static void iommu_poll_events(struct amd_iommu *iommu) 818 { 819 u32 head, tail; 820 821 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); 822 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); 823 824 while (head != tail) { 825 iommu_print_event(iommu, iommu->evt_buf + head); 826 827 /* Update head pointer of hardware ring-buffer */ 828 head = (head + EVENT_ENTRY_SIZE) % EVT_BUFFER_SIZE; 829 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); 830 } 831 832 } 833 834 #ifdef CONFIG_IRQ_REMAP 835 static int (*iommu_ga_log_notifier)(u32); 836 837 int amd_iommu_register_ga_log_notifier(int (*notifier)(u32)) 838 { 839 iommu_ga_log_notifier = notifier; 840 841 return 0; 842 } 843 EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier); 844 845 static void iommu_poll_ga_log(struct amd_iommu *iommu) 846 { 847 u32 head, tail; 848 849 if (iommu->ga_log == NULL) 850 return; 851 852 head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET); 853 tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET); 854 855 while (head != tail) { 856 volatile u64 *raw; 857 u64 log_entry; 858 859 raw = (u64 *)(iommu->ga_log + head); 860 861 /* Avoid memcpy function-call overhead */ 862 log_entry = *raw; 863 864 /* Update head pointer of hardware ring-buffer */ 865 head = (head + GA_ENTRY_SIZE) % GA_LOG_SIZE; 866 writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); 867 868 /* Handle GA entry */ 869 switch (GA_REQ_TYPE(log_entry)) { 870 case GA_GUEST_NR: 871 if (!iommu_ga_log_notifier) 872 break; 873 874 pr_debug("%s: devid=%#x, ga_tag=%#x\n", 875 __func__, GA_DEVID(log_entry), 876 GA_TAG(log_entry)); 877 878 if (iommu_ga_log_notifier(GA_TAG(log_entry)) != 0) 879 pr_err("GA log notifier failed.\n"); 880 break; 881 default: 882 break; 883 } 884 } 885 } 886 887 static void 888 amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) 889 { 890 if (!irq_remapping_enabled || !dev_is_pci(dev) || 891 !pci_dev_has_default_msi_parent_domain(to_pci_dev(dev))) 892 return; 893 894 dev_set_msi_domain(dev, iommu->ir_domain); 895 } 896 897 #else /* CONFIG_IRQ_REMAP */ 898 static inline void 899 amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) { } 900 #endif /* !CONFIG_IRQ_REMAP */ 901 902 static void amd_iommu_handle_irq(void *data, const char *evt_type, 903 u32 int_mask, u32 overflow_mask, 904 void (*int_handler)(struct amd_iommu *), 905 void (*overflow_handler)(struct amd_iommu *)) 906 { 907 struct amd_iommu *iommu = (struct amd_iommu *) data; 908 u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 909 u32 mask = int_mask | overflow_mask; 910 911 while (status & mask) { 912 /* Enable interrupt sources again */ 913 writel(mask, iommu->mmio_base + MMIO_STATUS_OFFSET); 914 915 if (int_handler) { 916 pr_devel("Processing IOMMU (ivhd%d) %s Log\n", 917 iommu->index, evt_type); 918 int_handler(iommu); 919 } 920 921 if ((status & overflow_mask) && overflow_handler) 922 overflow_handler(iommu); 923 924 /* 925 * Hardware bug: ERBT1312 926 * When re-enabling interrupt (by writing 1 927 * to clear the bit), the hardware might also try to set 928 * the interrupt bit in the event status register. 929 * In this scenario, the bit will be set, and disable 930 * subsequent interrupts. 931 * 932 * Workaround: The IOMMU driver should read back the 933 * status register and check if the interrupt bits are cleared. 934 * If not, driver will need to go through the interrupt handler 935 * again and re-clear the bits 936 */ 937 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 938 } 939 } 940 941 irqreturn_t amd_iommu_int_thread_evtlog(int irq, void *data) 942 { 943 amd_iommu_handle_irq(data, "Evt", MMIO_STATUS_EVT_INT_MASK, 944 MMIO_STATUS_EVT_OVERFLOW_MASK, 945 iommu_poll_events, amd_iommu_restart_event_logging); 946 947 return IRQ_HANDLED; 948 } 949 950 irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data) 951 { 952 amd_iommu_handle_irq(data, "PPR", MMIO_STATUS_PPR_INT_MASK, 953 MMIO_STATUS_PPR_OVERFLOW_MASK, 954 amd_iommu_poll_ppr_log, amd_iommu_restart_ppr_log); 955 956 return IRQ_HANDLED; 957 } 958 959 irqreturn_t amd_iommu_int_thread_galog(int irq, void *data) 960 { 961 #ifdef CONFIG_IRQ_REMAP 962 amd_iommu_handle_irq(data, "GA", MMIO_STATUS_GALOG_INT_MASK, 963 MMIO_STATUS_GALOG_OVERFLOW_MASK, 964 iommu_poll_ga_log, amd_iommu_restart_ga_log); 965 #endif 966 967 return IRQ_HANDLED; 968 } 969 970 irqreturn_t amd_iommu_int_thread(int irq, void *data) 971 { 972 amd_iommu_int_thread_evtlog(irq, data); 973 amd_iommu_int_thread_pprlog(irq, data); 974 amd_iommu_int_thread_galog(irq, data); 975 976 return IRQ_HANDLED; 977 } 978 979 irqreturn_t amd_iommu_int_handler(int irq, void *data) 980 { 981 return IRQ_WAKE_THREAD; 982 } 983 984 /**************************************************************************** 985 * 986 * IOMMU command queuing functions 987 * 988 ****************************************************************************/ 989 990 static int wait_on_sem(struct amd_iommu *iommu, u64 data) 991 { 992 int i = 0; 993 994 while (*iommu->cmd_sem != data && i < LOOP_TIMEOUT) { 995 udelay(1); 996 i += 1; 997 } 998 999 if (i == LOOP_TIMEOUT) { 1000 pr_alert("Completion-Wait loop timed out\n"); 1001 return -EIO; 1002 } 1003 1004 return 0; 1005 } 1006 1007 static void copy_cmd_to_buffer(struct amd_iommu *iommu, 1008 struct iommu_cmd *cmd) 1009 { 1010 u8 *target; 1011 u32 tail; 1012 1013 /* Copy command to buffer */ 1014 tail = iommu->cmd_buf_tail; 1015 target = iommu->cmd_buf + tail; 1016 memcpy(target, cmd, sizeof(*cmd)); 1017 1018 tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE; 1019 iommu->cmd_buf_tail = tail; 1020 1021 /* Tell the IOMMU about it */ 1022 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); 1023 } 1024 1025 static void build_completion_wait(struct iommu_cmd *cmd, 1026 struct amd_iommu *iommu, 1027 u64 data) 1028 { 1029 u64 paddr = iommu_virt_to_phys((void *)iommu->cmd_sem); 1030 1031 memset(cmd, 0, sizeof(*cmd)); 1032 cmd->data[0] = lower_32_bits(paddr) | CMD_COMPL_WAIT_STORE_MASK; 1033 cmd->data[1] = upper_32_bits(paddr); 1034 cmd->data[2] = lower_32_bits(data); 1035 cmd->data[3] = upper_32_bits(data); 1036 CMD_SET_TYPE(cmd, CMD_COMPL_WAIT); 1037 } 1038 1039 static void build_inv_dte(struct iommu_cmd *cmd, u16 devid) 1040 { 1041 memset(cmd, 0, sizeof(*cmd)); 1042 cmd->data[0] = devid; 1043 CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY); 1044 } 1045 1046 /* 1047 * Builds an invalidation address which is suitable for one page or multiple 1048 * pages. Sets the size bit (S) as needed is more than one page is flushed. 1049 */ 1050 static inline u64 build_inv_address(u64 address, size_t size) 1051 { 1052 u64 pages, end, msb_diff; 1053 1054 pages = iommu_num_pages(address, size, PAGE_SIZE); 1055 1056 if (pages == 1) 1057 return address & PAGE_MASK; 1058 1059 end = address + size - 1; 1060 1061 /* 1062 * msb_diff would hold the index of the most significant bit that 1063 * flipped between the start and end. 1064 */ 1065 msb_diff = fls64(end ^ address) - 1; 1066 1067 /* 1068 * Bits 63:52 are sign extended. If for some reason bit 51 is different 1069 * between the start and the end, invalidate everything. 1070 */ 1071 if (unlikely(msb_diff > 51)) { 1072 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; 1073 } else { 1074 /* 1075 * The msb-bit must be clear on the address. Just set all the 1076 * lower bits. 1077 */ 1078 address |= (1ull << msb_diff) - 1; 1079 } 1080 1081 /* Clear bits 11:0 */ 1082 address &= PAGE_MASK; 1083 1084 /* Set the size bit - we flush more than one 4kb page */ 1085 return address | CMD_INV_IOMMU_PAGES_SIZE_MASK; 1086 } 1087 1088 static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, 1089 size_t size, u16 domid, 1090 ioasid_t pasid, bool gn) 1091 { 1092 u64 inv_address = build_inv_address(address, size); 1093 1094 memset(cmd, 0, sizeof(*cmd)); 1095 1096 cmd->data[1] |= domid; 1097 cmd->data[2] = lower_32_bits(inv_address); 1098 cmd->data[3] = upper_32_bits(inv_address); 1099 /* PDE bit - we want to flush everything, not only the PTEs */ 1100 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK; 1101 if (gn) { 1102 cmd->data[0] |= pasid; 1103 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK; 1104 } 1105 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES); 1106 } 1107 1108 static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep, 1109 u64 address, size_t size, 1110 ioasid_t pasid, bool gn) 1111 { 1112 u64 inv_address = build_inv_address(address, size); 1113 1114 memset(cmd, 0, sizeof(*cmd)); 1115 1116 cmd->data[0] = devid; 1117 cmd->data[0] |= (qdep & 0xff) << 24; 1118 cmd->data[1] = devid; 1119 cmd->data[2] = lower_32_bits(inv_address); 1120 cmd->data[3] = upper_32_bits(inv_address); 1121 if (gn) { 1122 cmd->data[0] |= ((pasid >> 8) & 0xff) << 16; 1123 cmd->data[1] |= (pasid & 0xff) << 16; 1124 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK; 1125 } 1126 1127 CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES); 1128 } 1129 1130 static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, u32 pasid, 1131 int status, int tag, u8 gn) 1132 { 1133 memset(cmd, 0, sizeof(*cmd)); 1134 1135 cmd->data[0] = devid; 1136 if (gn) { 1137 cmd->data[1] = pasid; 1138 cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK; 1139 } 1140 cmd->data[3] = tag & 0x1ff; 1141 cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT; 1142 1143 CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR); 1144 } 1145 1146 static void build_inv_all(struct iommu_cmd *cmd) 1147 { 1148 memset(cmd, 0, sizeof(*cmd)); 1149 CMD_SET_TYPE(cmd, CMD_INV_ALL); 1150 } 1151 1152 static void build_inv_irt(struct iommu_cmd *cmd, u16 devid) 1153 { 1154 memset(cmd, 0, sizeof(*cmd)); 1155 cmd->data[0] = devid; 1156 CMD_SET_TYPE(cmd, CMD_INV_IRT); 1157 } 1158 1159 /* 1160 * Writes the command to the IOMMUs command buffer and informs the 1161 * hardware about the new command. 1162 */ 1163 static int __iommu_queue_command_sync(struct amd_iommu *iommu, 1164 struct iommu_cmd *cmd, 1165 bool sync) 1166 { 1167 unsigned int count = 0; 1168 u32 left, next_tail; 1169 1170 next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE; 1171 again: 1172 left = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE; 1173 1174 if (left <= 0x20) { 1175 /* Skip udelay() the first time around */ 1176 if (count++) { 1177 if (count == LOOP_TIMEOUT) { 1178 pr_err("Command buffer timeout\n"); 1179 return -EIO; 1180 } 1181 1182 udelay(1); 1183 } 1184 1185 /* Update head and recheck remaining space */ 1186 iommu->cmd_buf_head = readl(iommu->mmio_base + 1187 MMIO_CMD_HEAD_OFFSET); 1188 1189 goto again; 1190 } 1191 1192 copy_cmd_to_buffer(iommu, cmd); 1193 1194 /* Do we need to make sure all commands are processed? */ 1195 iommu->need_sync = sync; 1196 1197 return 0; 1198 } 1199 1200 static int iommu_queue_command_sync(struct amd_iommu *iommu, 1201 struct iommu_cmd *cmd, 1202 bool sync) 1203 { 1204 unsigned long flags; 1205 int ret; 1206 1207 raw_spin_lock_irqsave(&iommu->lock, flags); 1208 ret = __iommu_queue_command_sync(iommu, cmd, sync); 1209 raw_spin_unlock_irqrestore(&iommu->lock, flags); 1210 1211 return ret; 1212 } 1213 1214 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) 1215 { 1216 return iommu_queue_command_sync(iommu, cmd, true); 1217 } 1218 1219 /* 1220 * This function queues a completion wait command into the command 1221 * buffer of an IOMMU 1222 */ 1223 static int iommu_completion_wait(struct amd_iommu *iommu) 1224 { 1225 struct iommu_cmd cmd; 1226 unsigned long flags; 1227 int ret; 1228 u64 data; 1229 1230 if (!iommu->need_sync) 1231 return 0; 1232 1233 data = atomic64_add_return(1, &iommu->cmd_sem_val); 1234 build_completion_wait(&cmd, iommu, data); 1235 1236 raw_spin_lock_irqsave(&iommu->lock, flags); 1237 1238 ret = __iommu_queue_command_sync(iommu, &cmd, false); 1239 if (ret) 1240 goto out_unlock; 1241 1242 ret = wait_on_sem(iommu, data); 1243 1244 out_unlock: 1245 raw_spin_unlock_irqrestore(&iommu->lock, flags); 1246 1247 return ret; 1248 } 1249 1250 static void domain_flush_complete(struct protection_domain *domain) 1251 { 1252 int i; 1253 1254 for (i = 0; i < amd_iommu_get_num_iommus(); ++i) { 1255 if (domain && !domain->dev_iommu[i]) 1256 continue; 1257 1258 /* 1259 * Devices of this domain are behind this IOMMU 1260 * We need to wait for completion of all commands. 1261 */ 1262 iommu_completion_wait(amd_iommus[i]); 1263 } 1264 } 1265 1266 static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid) 1267 { 1268 struct iommu_cmd cmd; 1269 1270 build_inv_dte(&cmd, devid); 1271 1272 return iommu_queue_command(iommu, &cmd); 1273 } 1274 1275 static void amd_iommu_flush_dte_all(struct amd_iommu *iommu) 1276 { 1277 u32 devid; 1278 u16 last_bdf = iommu->pci_seg->last_bdf; 1279 1280 for (devid = 0; devid <= last_bdf; ++devid) 1281 iommu_flush_dte(iommu, devid); 1282 1283 iommu_completion_wait(iommu); 1284 } 1285 1286 /* 1287 * This function uses heavy locking and may disable irqs for some time. But 1288 * this is no issue because it is only called during resume. 1289 */ 1290 static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu) 1291 { 1292 u32 dom_id; 1293 u16 last_bdf = iommu->pci_seg->last_bdf; 1294 1295 for (dom_id = 0; dom_id <= last_bdf; ++dom_id) { 1296 struct iommu_cmd cmd; 1297 build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1298 dom_id, IOMMU_NO_PASID, false); 1299 iommu_queue_command(iommu, &cmd); 1300 } 1301 1302 iommu_completion_wait(iommu); 1303 } 1304 1305 static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id) 1306 { 1307 struct iommu_cmd cmd; 1308 1309 build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1310 dom_id, IOMMU_NO_PASID, false); 1311 iommu_queue_command(iommu, &cmd); 1312 1313 iommu_completion_wait(iommu); 1314 } 1315 1316 static void amd_iommu_flush_all(struct amd_iommu *iommu) 1317 { 1318 struct iommu_cmd cmd; 1319 1320 build_inv_all(&cmd); 1321 1322 iommu_queue_command(iommu, &cmd); 1323 iommu_completion_wait(iommu); 1324 } 1325 1326 static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid) 1327 { 1328 struct iommu_cmd cmd; 1329 1330 build_inv_irt(&cmd, devid); 1331 1332 iommu_queue_command(iommu, &cmd); 1333 } 1334 1335 static void amd_iommu_flush_irt_all(struct amd_iommu *iommu) 1336 { 1337 u32 devid; 1338 u16 last_bdf = iommu->pci_seg->last_bdf; 1339 1340 if (iommu->irtcachedis_enabled) 1341 return; 1342 1343 for (devid = 0; devid <= last_bdf; devid++) 1344 iommu_flush_irt(iommu, devid); 1345 1346 iommu_completion_wait(iommu); 1347 } 1348 1349 void amd_iommu_flush_all_caches(struct amd_iommu *iommu) 1350 { 1351 if (check_feature(FEATURE_IA)) { 1352 amd_iommu_flush_all(iommu); 1353 } else { 1354 amd_iommu_flush_dte_all(iommu); 1355 amd_iommu_flush_irt_all(iommu); 1356 amd_iommu_flush_tlb_all(iommu); 1357 } 1358 } 1359 1360 /* 1361 * Command send function for flushing on-device TLB 1362 */ 1363 static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address, 1364 size_t size, ioasid_t pasid, bool gn) 1365 { 1366 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 1367 struct iommu_cmd cmd; 1368 int qdep = dev_data->ats_qdep; 1369 1370 build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, 1371 size, pasid, gn); 1372 1373 return iommu_queue_command(iommu, &cmd); 1374 } 1375 1376 static int device_flush_dte_alias(struct pci_dev *pdev, u16 alias, void *data) 1377 { 1378 struct amd_iommu *iommu = data; 1379 1380 return iommu_flush_dte(iommu, alias); 1381 } 1382 1383 /* 1384 * Command send function for invalidating a device table entry 1385 */ 1386 static int device_flush_dte(struct iommu_dev_data *dev_data) 1387 { 1388 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 1389 struct pci_dev *pdev = NULL; 1390 struct amd_iommu_pci_seg *pci_seg; 1391 u16 alias; 1392 int ret; 1393 1394 if (dev_is_pci(dev_data->dev)) 1395 pdev = to_pci_dev(dev_data->dev); 1396 1397 if (pdev) 1398 ret = pci_for_each_dma_alias(pdev, 1399 device_flush_dte_alias, iommu); 1400 else 1401 ret = iommu_flush_dte(iommu, dev_data->devid); 1402 if (ret) 1403 return ret; 1404 1405 pci_seg = iommu->pci_seg; 1406 alias = pci_seg->alias_table[dev_data->devid]; 1407 if (alias != dev_data->devid) { 1408 ret = iommu_flush_dte(iommu, alias); 1409 if (ret) 1410 return ret; 1411 } 1412 1413 if (dev_data->ats_enabled) { 1414 /* Invalidate the entire contents of an IOTLB */ 1415 ret = device_flush_iotlb(dev_data, 0, ~0UL, 1416 IOMMU_NO_PASID, false); 1417 } 1418 1419 return ret; 1420 } 1421 1422 static int domain_flush_pages_v2(struct protection_domain *pdom, 1423 u64 address, size_t size) 1424 { 1425 struct iommu_dev_data *dev_data; 1426 struct iommu_cmd cmd; 1427 int ret = 0; 1428 1429 list_for_each_entry(dev_data, &pdom->dev_list, list) { 1430 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); 1431 u16 domid = dev_data->gcr3_info.domid; 1432 1433 build_inv_iommu_pages(&cmd, address, size, 1434 domid, IOMMU_NO_PASID, true); 1435 1436 ret |= iommu_queue_command(iommu, &cmd); 1437 } 1438 1439 return ret; 1440 } 1441 1442 static int domain_flush_pages_v1(struct protection_domain *pdom, 1443 u64 address, size_t size) 1444 { 1445 struct iommu_cmd cmd; 1446 int ret = 0, i; 1447 1448 build_inv_iommu_pages(&cmd, address, size, 1449 pdom->id, IOMMU_NO_PASID, false); 1450 1451 for (i = 0; i < amd_iommu_get_num_iommus(); ++i) { 1452 if (!pdom->dev_iommu[i]) 1453 continue; 1454 1455 /* 1456 * Devices of this domain are behind this IOMMU 1457 * We need a TLB flush 1458 */ 1459 ret |= iommu_queue_command(amd_iommus[i], &cmd); 1460 } 1461 1462 return ret; 1463 } 1464 1465 /* 1466 * TLB invalidation function which is called from the mapping functions. 1467 * It flushes range of PTEs of the domain. 1468 */ 1469 static void __domain_flush_pages(struct protection_domain *domain, 1470 u64 address, size_t size) 1471 { 1472 struct iommu_dev_data *dev_data; 1473 int ret = 0; 1474 ioasid_t pasid = IOMMU_NO_PASID; 1475 bool gn = false; 1476 1477 if (pdom_is_v2_pgtbl_mode(domain)) { 1478 gn = true; 1479 ret = domain_flush_pages_v2(domain, address, size); 1480 } else { 1481 ret = domain_flush_pages_v1(domain, address, size); 1482 } 1483 1484 list_for_each_entry(dev_data, &domain->dev_list, list) { 1485 1486 if (!dev_data->ats_enabled) 1487 continue; 1488 1489 ret |= device_flush_iotlb(dev_data, address, size, pasid, gn); 1490 } 1491 1492 WARN_ON(ret); 1493 } 1494 1495 void amd_iommu_domain_flush_pages(struct protection_domain *domain, 1496 u64 address, size_t size) 1497 { 1498 if (likely(!amd_iommu_np_cache)) { 1499 __domain_flush_pages(domain, address, size); 1500 1501 /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ 1502 domain_flush_complete(domain); 1503 1504 return; 1505 } 1506 1507 /* 1508 * When NpCache is on, we infer that we run in a VM and use a vIOMMU. 1509 * In such setups it is best to avoid flushes of ranges which are not 1510 * naturally aligned, since it would lead to flushes of unmodified 1511 * PTEs. Such flushes would require the hypervisor to do more work than 1512 * necessary. Therefore, perform repeated flushes of aligned ranges 1513 * until you cover the range. Each iteration flushes the smaller 1514 * between the natural alignment of the address that we flush and the 1515 * greatest naturally aligned region that fits in the range. 1516 */ 1517 while (size != 0) { 1518 int addr_alignment = __ffs(address); 1519 int size_alignment = __fls(size); 1520 int min_alignment; 1521 size_t flush_size; 1522 1523 /* 1524 * size is always non-zero, but address might be zero, causing 1525 * addr_alignment to be negative. As the casting of the 1526 * argument in __ffs(address) to long might trim the high bits 1527 * of the address on x86-32, cast to long when doing the check. 1528 */ 1529 if (likely((unsigned long)address != 0)) 1530 min_alignment = min(addr_alignment, size_alignment); 1531 else 1532 min_alignment = size_alignment; 1533 1534 flush_size = 1ul << min_alignment; 1535 1536 __domain_flush_pages(domain, address, flush_size); 1537 address += flush_size; 1538 size -= flush_size; 1539 } 1540 1541 /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ 1542 domain_flush_complete(domain); 1543 } 1544 1545 /* Flush the whole IO/TLB for a given protection domain - including PDE */ 1546 static void amd_iommu_domain_flush_all(struct protection_domain *domain) 1547 { 1548 amd_iommu_domain_flush_pages(domain, 0, 1549 CMD_INV_IOMMU_ALL_PAGES_ADDRESS); 1550 } 1551 1552 void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, 1553 ioasid_t pasid, u64 address, size_t size) 1554 { 1555 struct iommu_cmd cmd; 1556 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); 1557 1558 build_inv_iommu_pages(&cmd, address, size, 1559 dev_data->gcr3_info.domid, pasid, true); 1560 iommu_queue_command(iommu, &cmd); 1561 1562 if (dev_data->ats_enabled) 1563 device_flush_iotlb(dev_data, address, size, pasid, true); 1564 1565 iommu_completion_wait(iommu); 1566 } 1567 1568 static void dev_flush_pasid_all(struct iommu_dev_data *dev_data, 1569 ioasid_t pasid) 1570 { 1571 amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0, 1572 CMD_INV_IOMMU_ALL_PAGES_ADDRESS); 1573 } 1574 1575 /* Flush the not present cache if it exists */ 1576 static void domain_flush_np_cache(struct protection_domain *domain, 1577 dma_addr_t iova, size_t size) 1578 { 1579 if (unlikely(amd_iommu_np_cache)) { 1580 unsigned long flags; 1581 1582 spin_lock_irqsave(&domain->lock, flags); 1583 amd_iommu_domain_flush_pages(domain, iova, size); 1584 spin_unlock_irqrestore(&domain->lock, flags); 1585 } 1586 } 1587 1588 1589 /* 1590 * This function flushes the DTEs for all devices in domain 1591 */ 1592 void amd_iommu_update_and_flush_device_table(struct protection_domain *domain) 1593 { 1594 struct iommu_dev_data *dev_data; 1595 1596 list_for_each_entry(dev_data, &domain->dev_list, list) { 1597 struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev); 1598 1599 set_dte_entry(iommu, dev_data); 1600 clone_aliases(iommu, dev_data->dev); 1601 } 1602 1603 list_for_each_entry(dev_data, &domain->dev_list, list) 1604 device_flush_dte(dev_data); 1605 1606 domain_flush_complete(domain); 1607 } 1608 1609 void amd_iommu_domain_update(struct protection_domain *domain) 1610 { 1611 /* Update device table */ 1612 amd_iommu_update_and_flush_device_table(domain); 1613 1614 /* Flush domain TLB(s) and wait for completion */ 1615 amd_iommu_domain_flush_all(domain); 1616 } 1617 1618 int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag) 1619 { 1620 struct iommu_dev_data *dev_data; 1621 struct amd_iommu *iommu; 1622 struct iommu_cmd cmd; 1623 1624 dev_data = dev_iommu_priv_get(dev); 1625 iommu = get_amd_iommu_from_dev(dev); 1626 1627 build_complete_ppr(&cmd, dev_data->devid, pasid, status, 1628 tag, dev_data->pri_tlp); 1629 1630 return iommu_queue_command(iommu, &cmd); 1631 } 1632 1633 /**************************************************************************** 1634 * 1635 * The next functions belong to the domain allocation. A domain is 1636 * allocated for every IOMMU as the default domain. If device isolation 1637 * is enabled, every device get its own domain. The most important thing 1638 * about domains is the page table mapping the DMA address space they 1639 * contain. 1640 * 1641 ****************************************************************************/ 1642 1643 static u16 domain_id_alloc(void) 1644 { 1645 unsigned long flags; 1646 int id; 1647 1648 spin_lock_irqsave(&pd_bitmap_lock, flags); 1649 id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID); 1650 BUG_ON(id == 0); 1651 if (id > 0 && id < MAX_DOMAIN_ID) 1652 __set_bit(id, amd_iommu_pd_alloc_bitmap); 1653 else 1654 id = 0; 1655 spin_unlock_irqrestore(&pd_bitmap_lock, flags); 1656 1657 return id; 1658 } 1659 1660 static void domain_id_free(int id) 1661 { 1662 unsigned long flags; 1663 1664 spin_lock_irqsave(&pd_bitmap_lock, flags); 1665 if (id > 0 && id < MAX_DOMAIN_ID) 1666 __clear_bit(id, amd_iommu_pd_alloc_bitmap); 1667 spin_unlock_irqrestore(&pd_bitmap_lock, flags); 1668 } 1669 1670 static void free_gcr3_tbl_level1(u64 *tbl) 1671 { 1672 u64 *ptr; 1673 int i; 1674 1675 for (i = 0; i < 512; ++i) { 1676 if (!(tbl[i] & GCR3_VALID)) 1677 continue; 1678 1679 ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK); 1680 1681 iommu_free_page(ptr); 1682 } 1683 } 1684 1685 static void free_gcr3_tbl_level2(u64 *tbl) 1686 { 1687 u64 *ptr; 1688 int i; 1689 1690 for (i = 0; i < 512; ++i) { 1691 if (!(tbl[i] & GCR3_VALID)) 1692 continue; 1693 1694 ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK); 1695 1696 free_gcr3_tbl_level1(ptr); 1697 } 1698 } 1699 1700 static void free_gcr3_table(struct gcr3_tbl_info *gcr3_info) 1701 { 1702 if (gcr3_info->glx == 2) 1703 free_gcr3_tbl_level2(gcr3_info->gcr3_tbl); 1704 else if (gcr3_info->glx == 1) 1705 free_gcr3_tbl_level1(gcr3_info->gcr3_tbl); 1706 else 1707 WARN_ON_ONCE(gcr3_info->glx != 0); 1708 1709 gcr3_info->glx = 0; 1710 1711 /* Free per device domain ID */ 1712 domain_id_free(gcr3_info->domid); 1713 1714 iommu_free_page(gcr3_info->gcr3_tbl); 1715 gcr3_info->gcr3_tbl = NULL; 1716 } 1717 1718 /* 1719 * Number of GCR3 table levels required. Level must be 4-Kbyte 1720 * page and can contain up to 512 entries. 1721 */ 1722 static int get_gcr3_levels(int pasids) 1723 { 1724 int levels; 1725 1726 if (pasids == -1) 1727 return amd_iommu_max_glx_val; 1728 1729 levels = get_count_order(pasids); 1730 1731 return levels ? (DIV_ROUND_UP(levels, 9) - 1) : levels; 1732 } 1733 1734 static int setup_gcr3_table(struct gcr3_tbl_info *gcr3_info, 1735 struct amd_iommu *iommu, int pasids) 1736 { 1737 int levels = get_gcr3_levels(pasids); 1738 int nid = iommu ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; 1739 1740 if (levels > amd_iommu_max_glx_val) 1741 return -EINVAL; 1742 1743 if (gcr3_info->gcr3_tbl) 1744 return -EBUSY; 1745 1746 /* Allocate per device domain ID */ 1747 gcr3_info->domid = domain_id_alloc(); 1748 1749 gcr3_info->gcr3_tbl = iommu_alloc_page_node(nid, GFP_ATOMIC); 1750 if (gcr3_info->gcr3_tbl == NULL) { 1751 domain_id_free(gcr3_info->domid); 1752 return -ENOMEM; 1753 } 1754 1755 gcr3_info->glx = levels; 1756 1757 return 0; 1758 } 1759 1760 static u64 *__get_gcr3_pte(struct gcr3_tbl_info *gcr3_info, 1761 ioasid_t pasid, bool alloc) 1762 { 1763 int index; 1764 u64 *pte; 1765 u64 *root = gcr3_info->gcr3_tbl; 1766 int level = gcr3_info->glx; 1767 1768 while (true) { 1769 1770 index = (pasid >> (9 * level)) & 0x1ff; 1771 pte = &root[index]; 1772 1773 if (level == 0) 1774 break; 1775 1776 if (!(*pte & GCR3_VALID)) { 1777 if (!alloc) 1778 return NULL; 1779 1780 root = (void *)get_zeroed_page(GFP_ATOMIC); 1781 if (root == NULL) 1782 return NULL; 1783 1784 *pte = iommu_virt_to_phys(root) | GCR3_VALID; 1785 } 1786 1787 root = iommu_phys_to_virt(*pte & PAGE_MASK); 1788 1789 level -= 1; 1790 } 1791 1792 return pte; 1793 } 1794 1795 static int update_gcr3(struct iommu_dev_data *dev_data, 1796 ioasid_t pasid, unsigned long gcr3, bool set) 1797 { 1798 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 1799 u64 *pte; 1800 1801 pte = __get_gcr3_pte(gcr3_info, pasid, true); 1802 if (pte == NULL) 1803 return -ENOMEM; 1804 1805 if (set) 1806 *pte = (gcr3 & PAGE_MASK) | GCR3_VALID; 1807 else 1808 *pte = 0; 1809 1810 dev_flush_pasid_all(dev_data, pasid); 1811 return 0; 1812 } 1813 1814 int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid, 1815 unsigned long gcr3) 1816 { 1817 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 1818 int ret; 1819 1820 iommu_group_mutex_assert(dev_data->dev); 1821 1822 ret = update_gcr3(dev_data, pasid, gcr3, true); 1823 if (ret) 1824 return ret; 1825 1826 gcr3_info->pasid_cnt++; 1827 return ret; 1828 } 1829 1830 int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid) 1831 { 1832 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 1833 int ret; 1834 1835 iommu_group_mutex_assert(dev_data->dev); 1836 1837 ret = update_gcr3(dev_data, pasid, 0, false); 1838 if (ret) 1839 return ret; 1840 1841 gcr3_info->pasid_cnt--; 1842 return ret; 1843 } 1844 1845 static void set_dte_entry(struct amd_iommu *iommu, 1846 struct iommu_dev_data *dev_data) 1847 { 1848 u64 pte_root = 0; 1849 u64 flags = 0; 1850 u32 old_domid; 1851 u16 devid = dev_data->devid; 1852 u16 domid; 1853 struct protection_domain *domain = dev_data->domain; 1854 struct dev_table_entry *dev_table = get_dev_table(iommu); 1855 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 1856 1857 if (gcr3_info && gcr3_info->gcr3_tbl) 1858 domid = dev_data->gcr3_info.domid; 1859 else 1860 domid = domain->id; 1861 1862 if (domain->iop.mode != PAGE_MODE_NONE) 1863 pte_root = iommu_virt_to_phys(domain->iop.root); 1864 1865 pte_root |= (domain->iop.mode & DEV_ENTRY_MODE_MASK) 1866 << DEV_ENTRY_MODE_SHIFT; 1867 1868 pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V; 1869 1870 /* 1871 * When SNP is enabled, Only set TV bit when IOMMU 1872 * page translation is in use. 1873 */ 1874 if (!amd_iommu_snp_en || (domid != 0)) 1875 pte_root |= DTE_FLAG_TV; 1876 1877 flags = dev_table[devid].data[1]; 1878 1879 if (dev_data->ats_enabled) 1880 flags |= DTE_FLAG_IOTLB; 1881 1882 if (dev_data->ppr) 1883 pte_root |= 1ULL << DEV_ENTRY_PPR; 1884 1885 if (domain->dirty_tracking) 1886 pte_root |= DTE_FLAG_HAD; 1887 1888 if (gcr3_info && gcr3_info->gcr3_tbl) { 1889 u64 gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl); 1890 u64 glx = gcr3_info->glx; 1891 u64 tmp; 1892 1893 pte_root |= DTE_FLAG_GV; 1894 pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT; 1895 1896 /* First mask out possible old values for GCR3 table */ 1897 tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B; 1898 flags &= ~tmp; 1899 1900 tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C; 1901 flags &= ~tmp; 1902 1903 /* Encode GCR3 table into DTE */ 1904 tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A; 1905 pte_root |= tmp; 1906 1907 tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B; 1908 flags |= tmp; 1909 1910 tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C; 1911 flags |= tmp; 1912 1913 if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) { 1914 dev_table[devid].data[2] |= 1915 ((u64)GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT); 1916 } 1917 1918 /* GIOV is supported with V2 page table mode only */ 1919 if (pdom_is_v2_pgtbl_mode(domain)) 1920 pte_root |= DTE_FLAG_GIOV; 1921 } 1922 1923 flags &= ~DEV_DOMID_MASK; 1924 flags |= domid; 1925 1926 old_domid = dev_table[devid].data[1] & DEV_DOMID_MASK; 1927 dev_table[devid].data[1] = flags; 1928 dev_table[devid].data[0] = pte_root; 1929 1930 /* 1931 * A kdump kernel might be replacing a domain ID that was copied from 1932 * the previous kernel--if so, it needs to flush the translation cache 1933 * entries for the old domain ID that is being overwritten 1934 */ 1935 if (old_domid) { 1936 amd_iommu_flush_tlb_domid(iommu, old_domid); 1937 } 1938 } 1939 1940 static void clear_dte_entry(struct amd_iommu *iommu, u16 devid) 1941 { 1942 struct dev_table_entry *dev_table = get_dev_table(iommu); 1943 1944 /* remove entry from the device table seen by the hardware */ 1945 dev_table[devid].data[0] = DTE_FLAG_V; 1946 1947 if (!amd_iommu_snp_en) 1948 dev_table[devid].data[0] |= DTE_FLAG_TV; 1949 1950 dev_table[devid].data[1] &= DTE_FLAG_MASK; 1951 1952 amd_iommu_apply_erratum_63(iommu, devid); 1953 } 1954 1955 /* Update and flush DTE for the given device */ 1956 static void dev_update_dte(struct iommu_dev_data *dev_data, bool set) 1957 { 1958 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); 1959 1960 if (set) 1961 set_dte_entry(iommu, dev_data); 1962 else 1963 clear_dte_entry(iommu, dev_data->devid); 1964 1965 clone_aliases(iommu, dev_data->dev); 1966 device_flush_dte(dev_data); 1967 iommu_completion_wait(iommu); 1968 } 1969 1970 /* 1971 * If domain is SVA capable then initialize GCR3 table. Also if domain is 1972 * in v2 page table mode then update GCR3[0]. 1973 */ 1974 static int init_gcr3_table(struct iommu_dev_data *dev_data, 1975 struct protection_domain *pdom) 1976 { 1977 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 1978 int max_pasids = dev_data->max_pasids; 1979 int ret = 0; 1980 1981 /* 1982 * If domain is in pt mode then setup GCR3 table only if device 1983 * is PASID capable 1984 */ 1985 if (pdom_is_in_pt_mode(pdom) && !pdev_pasid_supported(dev_data)) 1986 return ret; 1987 1988 /* 1989 * By default, setup GCR3 table to support MAX PASIDs 1990 * supported by the device/IOMMU. 1991 */ 1992 ret = setup_gcr3_table(&dev_data->gcr3_info, iommu, 1993 max_pasids > 0 ? max_pasids : 1); 1994 if (ret) 1995 return ret; 1996 1997 /* Setup GCR3[0] only if domain is setup with v2 page table mode */ 1998 if (!pdom_is_v2_pgtbl_mode(pdom)) 1999 return ret; 2000 2001 ret = update_gcr3(dev_data, 0, iommu_virt_to_phys(pdom->iop.pgd), true); 2002 if (ret) 2003 free_gcr3_table(&dev_data->gcr3_info); 2004 2005 return ret; 2006 } 2007 2008 static void destroy_gcr3_table(struct iommu_dev_data *dev_data, 2009 struct protection_domain *pdom) 2010 { 2011 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 2012 2013 if (pdom_is_v2_pgtbl_mode(pdom)) 2014 update_gcr3(dev_data, 0, 0, false); 2015 2016 if (gcr3_info->gcr3_tbl == NULL) 2017 return; 2018 2019 free_gcr3_table(gcr3_info); 2020 } 2021 2022 static int do_attach(struct iommu_dev_data *dev_data, 2023 struct protection_domain *domain) 2024 { 2025 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 2026 struct io_pgtable_cfg *cfg = &domain->iop.pgtbl.cfg; 2027 int ret = 0; 2028 2029 /* Update data structures */ 2030 dev_data->domain = domain; 2031 list_add(&dev_data->list, &domain->dev_list); 2032 2033 /* Update NUMA Node ID */ 2034 if (cfg->amd.nid == NUMA_NO_NODE) 2035 cfg->amd.nid = dev_to_node(dev_data->dev); 2036 2037 /* Do reference counting */ 2038 domain->dev_iommu[iommu->index] += 1; 2039 domain->dev_cnt += 1; 2040 2041 /* Setup GCR3 table */ 2042 if (pdom_is_sva_capable(domain)) { 2043 ret = init_gcr3_table(dev_data, domain); 2044 if (ret) 2045 return ret; 2046 } 2047 2048 return ret; 2049 } 2050 2051 static void do_detach(struct iommu_dev_data *dev_data) 2052 { 2053 struct protection_domain *domain = dev_data->domain; 2054 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 2055 2056 /* Clear DTE and flush the entry */ 2057 dev_update_dte(dev_data, false); 2058 2059 /* Flush IOTLB and wait for the flushes to finish */ 2060 amd_iommu_domain_flush_all(domain); 2061 2062 /* Clear GCR3 table */ 2063 if (pdom_is_sva_capable(domain)) 2064 destroy_gcr3_table(dev_data, domain); 2065 2066 /* Update data structures */ 2067 dev_data->domain = NULL; 2068 list_del(&dev_data->list); 2069 2070 /* decrease reference counters - needs to happen after the flushes */ 2071 domain->dev_iommu[iommu->index] -= 1; 2072 domain->dev_cnt -= 1; 2073 } 2074 2075 /* 2076 * If a device is not yet associated with a domain, this function makes the 2077 * device visible in the domain 2078 */ 2079 static int attach_device(struct device *dev, 2080 struct protection_domain *domain) 2081 { 2082 struct iommu_dev_data *dev_data; 2083 unsigned long flags; 2084 int ret = 0; 2085 2086 spin_lock_irqsave(&domain->lock, flags); 2087 2088 dev_data = dev_iommu_priv_get(dev); 2089 2090 spin_lock(&dev_data->lock); 2091 2092 if (dev_data->domain != NULL) { 2093 ret = -EBUSY; 2094 goto out; 2095 } 2096 2097 ret = do_attach(dev_data, domain); 2098 2099 out: 2100 spin_unlock(&dev_data->lock); 2101 2102 spin_unlock_irqrestore(&domain->lock, flags); 2103 2104 return ret; 2105 } 2106 2107 /* 2108 * Removes a device from a protection domain (with devtable_lock held) 2109 */ 2110 static void detach_device(struct device *dev) 2111 { 2112 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2113 struct protection_domain *domain = dev_data->domain; 2114 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 2115 unsigned long flags; 2116 bool ppr = dev_data->ppr; 2117 2118 spin_lock_irqsave(&domain->lock, flags); 2119 2120 spin_lock(&dev_data->lock); 2121 2122 /* 2123 * First check if the device is still attached. It might already 2124 * be detached from its domain because the generic 2125 * iommu_detach_group code detached it and we try again here in 2126 * our alias handling. 2127 */ 2128 if (WARN_ON(!dev_data->domain)) 2129 goto out; 2130 2131 if (ppr) { 2132 iopf_queue_flush_dev(dev); 2133 2134 /* Updated here so that it gets reflected in DTE */ 2135 dev_data->ppr = false; 2136 } 2137 2138 do_detach(dev_data); 2139 2140 out: 2141 spin_unlock(&dev_data->lock); 2142 2143 spin_unlock_irqrestore(&domain->lock, flags); 2144 2145 /* Remove IOPF handler */ 2146 if (ppr) 2147 amd_iommu_iopf_remove_device(iommu, dev_data); 2148 2149 if (dev_is_pci(dev)) 2150 pdev_disable_caps(to_pci_dev(dev)); 2151 2152 } 2153 2154 static struct iommu_device *amd_iommu_probe_device(struct device *dev) 2155 { 2156 struct iommu_device *iommu_dev; 2157 struct amd_iommu *iommu; 2158 struct iommu_dev_data *dev_data; 2159 int ret; 2160 2161 if (!check_device(dev)) 2162 return ERR_PTR(-ENODEV); 2163 2164 iommu = rlookup_amd_iommu(dev); 2165 if (!iommu) 2166 return ERR_PTR(-ENODEV); 2167 2168 /* Not registered yet? */ 2169 if (!iommu->iommu.ops) 2170 return ERR_PTR(-ENODEV); 2171 2172 if (dev_iommu_priv_get(dev)) 2173 return &iommu->iommu; 2174 2175 ret = iommu_init_device(iommu, dev); 2176 if (ret) { 2177 dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); 2178 iommu_dev = ERR_PTR(ret); 2179 iommu_ignore_device(iommu, dev); 2180 goto out_err; 2181 } 2182 2183 amd_iommu_set_pci_msi_domain(dev, iommu); 2184 iommu_dev = &iommu->iommu; 2185 2186 /* 2187 * If IOMMU and device supports PASID then it will contain max 2188 * supported PASIDs, else it will be zero. 2189 */ 2190 dev_data = dev_iommu_priv_get(dev); 2191 if (amd_iommu_pasid_supported() && dev_is_pci(dev) && 2192 pdev_pasid_supported(dev_data)) { 2193 dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids, 2194 pci_max_pasids(to_pci_dev(dev))); 2195 } 2196 2197 out_err: 2198 iommu_completion_wait(iommu); 2199 2200 if (dev_is_pci(dev)) 2201 pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT); 2202 2203 return iommu_dev; 2204 } 2205 2206 static void amd_iommu_release_device(struct device *dev) 2207 { 2208 struct amd_iommu *iommu; 2209 2210 if (!check_device(dev)) 2211 return; 2212 2213 iommu = rlookup_amd_iommu(dev); 2214 if (!iommu) 2215 return; 2216 2217 amd_iommu_uninit_device(dev); 2218 iommu_completion_wait(iommu); 2219 } 2220 2221 static struct iommu_group *amd_iommu_device_group(struct device *dev) 2222 { 2223 if (dev_is_pci(dev)) 2224 return pci_device_group(dev); 2225 2226 return acpihid_device_group(dev); 2227 } 2228 2229 /***************************************************************************** 2230 * 2231 * The following functions belong to the exported interface of AMD IOMMU 2232 * 2233 * This interface allows access to lower level functions of the IOMMU 2234 * like protection domain handling and assignement of devices to domains 2235 * which is not possible with the dma_ops interface. 2236 * 2237 *****************************************************************************/ 2238 2239 static void cleanup_domain(struct protection_domain *domain) 2240 { 2241 struct iommu_dev_data *entry; 2242 2243 lockdep_assert_held(&domain->lock); 2244 2245 if (!domain->dev_cnt) 2246 return; 2247 2248 while (!list_empty(&domain->dev_list)) { 2249 entry = list_first_entry(&domain->dev_list, 2250 struct iommu_dev_data, list); 2251 BUG_ON(!entry->domain); 2252 do_detach(entry); 2253 } 2254 WARN_ON(domain->dev_cnt != 0); 2255 } 2256 2257 void protection_domain_free(struct protection_domain *domain) 2258 { 2259 WARN_ON(!list_empty(&domain->dev_list)); 2260 if (domain->domain.type & __IOMMU_DOMAIN_PAGING) 2261 free_io_pgtable_ops(&domain->iop.pgtbl.ops); 2262 domain_id_free(domain->id); 2263 kfree(domain); 2264 } 2265 2266 struct protection_domain *protection_domain_alloc(unsigned int type, int nid) 2267 { 2268 struct io_pgtable_ops *pgtbl_ops; 2269 struct protection_domain *domain; 2270 int pgtable; 2271 2272 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 2273 if (!domain) 2274 return NULL; 2275 2276 domain->id = domain_id_alloc(); 2277 if (!domain->id) 2278 goto err_free; 2279 2280 spin_lock_init(&domain->lock); 2281 INIT_LIST_HEAD(&domain->dev_list); 2282 INIT_LIST_HEAD(&domain->dev_data_list); 2283 domain->iop.pgtbl.cfg.amd.nid = nid; 2284 2285 switch (type) { 2286 /* No need to allocate io pgtable ops in passthrough mode */ 2287 case IOMMU_DOMAIN_IDENTITY: 2288 case IOMMU_DOMAIN_SVA: 2289 return domain; 2290 case IOMMU_DOMAIN_DMA: 2291 pgtable = amd_iommu_pgtable; 2292 break; 2293 /* 2294 * Force IOMMU v1 page table when allocating 2295 * domain for pass-through devices. 2296 */ 2297 case IOMMU_DOMAIN_UNMANAGED: 2298 pgtable = AMD_IOMMU_V1; 2299 break; 2300 default: 2301 goto err_id; 2302 } 2303 2304 switch (pgtable) { 2305 case AMD_IOMMU_V1: 2306 domain->pd_mode = PD_MODE_V1; 2307 break; 2308 case AMD_IOMMU_V2: 2309 domain->pd_mode = PD_MODE_V2; 2310 break; 2311 default: 2312 goto err_id; 2313 } 2314 2315 pgtbl_ops = 2316 alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl.cfg, domain); 2317 if (!pgtbl_ops) 2318 goto err_id; 2319 2320 return domain; 2321 err_id: 2322 domain_id_free(domain->id); 2323 err_free: 2324 kfree(domain); 2325 return NULL; 2326 } 2327 2328 static inline u64 dma_max_address(void) 2329 { 2330 if (amd_iommu_pgtable == AMD_IOMMU_V1) 2331 return ~0ULL; 2332 2333 /* V2 with 4/5 level page table */ 2334 return ((1ULL << PM_LEVEL_SHIFT(amd_iommu_gpt_level)) - 1); 2335 } 2336 2337 static bool amd_iommu_hd_support(struct amd_iommu *iommu) 2338 { 2339 return iommu && (iommu->features & FEATURE_HDSUP); 2340 } 2341 2342 static struct iommu_domain *do_iommu_domain_alloc(unsigned int type, 2343 struct device *dev, u32 flags) 2344 { 2345 bool dirty_tracking = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING; 2346 struct protection_domain *domain; 2347 struct amd_iommu *iommu = NULL; 2348 2349 if (dev) 2350 iommu = get_amd_iommu_from_dev(dev); 2351 2352 /* 2353 * Since DTE[Mode]=0 is prohibited on SNP-enabled system, 2354 * default to use IOMMU_DOMAIN_DMA[_FQ]. 2355 */ 2356 if (amd_iommu_snp_en && (type == IOMMU_DOMAIN_IDENTITY)) 2357 return ERR_PTR(-EINVAL); 2358 2359 if (dirty_tracking && !amd_iommu_hd_support(iommu)) 2360 return ERR_PTR(-EOPNOTSUPP); 2361 2362 domain = protection_domain_alloc(type, 2363 dev ? dev_to_node(dev) : NUMA_NO_NODE); 2364 if (!domain) 2365 return ERR_PTR(-ENOMEM); 2366 2367 domain->domain.geometry.aperture_start = 0; 2368 domain->domain.geometry.aperture_end = dma_max_address(); 2369 domain->domain.geometry.force_aperture = true; 2370 domain->domain.pgsize_bitmap = domain->iop.pgtbl.cfg.pgsize_bitmap; 2371 2372 if (iommu) { 2373 domain->domain.type = type; 2374 domain->domain.ops = iommu->iommu.ops->default_domain_ops; 2375 2376 if (dirty_tracking) 2377 domain->domain.dirty_ops = &amd_dirty_ops; 2378 } 2379 2380 return &domain->domain; 2381 } 2382 2383 static struct iommu_domain *amd_iommu_domain_alloc(unsigned int type) 2384 { 2385 struct iommu_domain *domain; 2386 2387 domain = do_iommu_domain_alloc(type, NULL, 0); 2388 if (IS_ERR(domain)) 2389 return NULL; 2390 2391 return domain; 2392 } 2393 2394 static struct iommu_domain * 2395 amd_iommu_domain_alloc_user(struct device *dev, u32 flags, 2396 struct iommu_domain *parent, 2397 const struct iommu_user_data *user_data) 2398 2399 { 2400 unsigned int type = IOMMU_DOMAIN_UNMANAGED; 2401 2402 if ((flags & ~IOMMU_HWPT_ALLOC_DIRTY_TRACKING) || parent || user_data) 2403 return ERR_PTR(-EOPNOTSUPP); 2404 2405 return do_iommu_domain_alloc(type, dev, flags); 2406 } 2407 2408 void amd_iommu_domain_free(struct iommu_domain *dom) 2409 { 2410 struct protection_domain *domain; 2411 unsigned long flags; 2412 2413 domain = to_pdomain(dom); 2414 2415 spin_lock_irqsave(&domain->lock, flags); 2416 2417 cleanup_domain(domain); 2418 2419 spin_unlock_irqrestore(&domain->lock, flags); 2420 2421 protection_domain_free(domain); 2422 } 2423 2424 static int blocked_domain_attach_device(struct iommu_domain *domain, 2425 struct device *dev) 2426 { 2427 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2428 2429 if (dev_data->domain) 2430 detach_device(dev); 2431 2432 /* Clear DTE and flush the entry */ 2433 spin_lock(&dev_data->lock); 2434 dev_update_dte(dev_data, false); 2435 spin_unlock(&dev_data->lock); 2436 2437 return 0; 2438 } 2439 2440 static struct iommu_domain blocked_domain = { 2441 .type = IOMMU_DOMAIN_BLOCKED, 2442 .ops = &(const struct iommu_domain_ops) { 2443 .attach_dev = blocked_domain_attach_device, 2444 } 2445 }; 2446 2447 static int amd_iommu_attach_device(struct iommu_domain *dom, 2448 struct device *dev) 2449 { 2450 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2451 struct protection_domain *domain = to_pdomain(dom); 2452 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); 2453 struct pci_dev *pdev; 2454 int ret; 2455 2456 /* 2457 * Skip attach device to domain if new domain is same as 2458 * devices current domain 2459 */ 2460 if (dev_data->domain == domain) 2461 return 0; 2462 2463 dev_data->defer_attach = false; 2464 2465 /* 2466 * Restrict to devices with compatible IOMMU hardware support 2467 * when enforcement of dirty tracking is enabled. 2468 */ 2469 if (dom->dirty_ops && !amd_iommu_hd_support(iommu)) 2470 return -EINVAL; 2471 2472 if (dev_data->domain) 2473 detach_device(dev); 2474 2475 ret = attach_device(dev, domain); 2476 2477 #ifdef CONFIG_IRQ_REMAP 2478 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) { 2479 if (dom->type == IOMMU_DOMAIN_UNMANAGED) 2480 dev_data->use_vapic = 1; 2481 else 2482 dev_data->use_vapic = 0; 2483 } 2484 #endif 2485 2486 pdev = dev_is_pci(dev_data->dev) ? to_pci_dev(dev_data->dev) : NULL; 2487 if (pdev && pdom_is_sva_capable(domain)) { 2488 pdev_enable_caps(pdev); 2489 2490 /* 2491 * Device can continue to function even if IOPF 2492 * enablement failed. Hence in error path just 2493 * disable device PRI support. 2494 */ 2495 if (amd_iommu_iopf_add_device(iommu, dev_data)) 2496 pdev_disable_cap_pri(pdev); 2497 } else if (pdev) { 2498 pdev_enable_cap_ats(pdev); 2499 } 2500 2501 /* Update device table */ 2502 dev_update_dte(dev_data, true); 2503 2504 return ret; 2505 } 2506 2507 static int amd_iommu_iotlb_sync_map(struct iommu_domain *dom, 2508 unsigned long iova, size_t size) 2509 { 2510 struct protection_domain *domain = to_pdomain(dom); 2511 struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops; 2512 2513 if (ops->map_pages) 2514 domain_flush_np_cache(domain, iova, size); 2515 return 0; 2516 } 2517 2518 static int amd_iommu_map_pages(struct iommu_domain *dom, unsigned long iova, 2519 phys_addr_t paddr, size_t pgsize, size_t pgcount, 2520 int iommu_prot, gfp_t gfp, size_t *mapped) 2521 { 2522 struct protection_domain *domain = to_pdomain(dom); 2523 struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops; 2524 int prot = 0; 2525 int ret = -EINVAL; 2526 2527 if ((domain->pd_mode == PD_MODE_V1) && 2528 (domain->iop.mode == PAGE_MODE_NONE)) 2529 return -EINVAL; 2530 2531 if (iommu_prot & IOMMU_READ) 2532 prot |= IOMMU_PROT_IR; 2533 if (iommu_prot & IOMMU_WRITE) 2534 prot |= IOMMU_PROT_IW; 2535 2536 if (ops->map_pages) { 2537 ret = ops->map_pages(ops, iova, paddr, pgsize, 2538 pgcount, prot, gfp, mapped); 2539 } 2540 2541 return ret; 2542 } 2543 2544 static void amd_iommu_iotlb_gather_add_page(struct iommu_domain *domain, 2545 struct iommu_iotlb_gather *gather, 2546 unsigned long iova, size_t size) 2547 { 2548 /* 2549 * AMD's IOMMU can flush as many pages as necessary in a single flush. 2550 * Unless we run in a virtual machine, which can be inferred according 2551 * to whether "non-present cache" is on, it is probably best to prefer 2552 * (potentially) too extensive TLB flushing (i.e., more misses) over 2553 * mutliple TLB flushes (i.e., more flushes). For virtual machines the 2554 * hypervisor needs to synchronize the host IOMMU PTEs with those of 2555 * the guest, and the trade-off is different: unnecessary TLB flushes 2556 * should be avoided. 2557 */ 2558 if (amd_iommu_np_cache && 2559 iommu_iotlb_gather_is_disjoint(gather, iova, size)) 2560 iommu_iotlb_sync(domain, gather); 2561 2562 iommu_iotlb_gather_add_range(gather, iova, size); 2563 } 2564 2565 static size_t amd_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova, 2566 size_t pgsize, size_t pgcount, 2567 struct iommu_iotlb_gather *gather) 2568 { 2569 struct protection_domain *domain = to_pdomain(dom); 2570 struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops; 2571 size_t r; 2572 2573 if ((domain->pd_mode == PD_MODE_V1) && 2574 (domain->iop.mode == PAGE_MODE_NONE)) 2575 return 0; 2576 2577 r = (ops->unmap_pages) ? ops->unmap_pages(ops, iova, pgsize, pgcount, NULL) : 0; 2578 2579 if (r) 2580 amd_iommu_iotlb_gather_add_page(dom, gather, iova, r); 2581 2582 return r; 2583 } 2584 2585 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, 2586 dma_addr_t iova) 2587 { 2588 struct protection_domain *domain = to_pdomain(dom); 2589 struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops; 2590 2591 return ops->iova_to_phys(ops, iova); 2592 } 2593 2594 static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) 2595 { 2596 switch (cap) { 2597 case IOMMU_CAP_CACHE_COHERENCY: 2598 return true; 2599 case IOMMU_CAP_NOEXEC: 2600 return false; 2601 case IOMMU_CAP_PRE_BOOT_PROTECTION: 2602 return amdr_ivrs_remap_support; 2603 case IOMMU_CAP_ENFORCE_CACHE_COHERENCY: 2604 return true; 2605 case IOMMU_CAP_DEFERRED_FLUSH: 2606 return true; 2607 case IOMMU_CAP_DIRTY_TRACKING: { 2608 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); 2609 2610 return amd_iommu_hd_support(iommu); 2611 } 2612 default: 2613 break; 2614 } 2615 2616 return false; 2617 } 2618 2619 static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain, 2620 bool enable) 2621 { 2622 struct protection_domain *pdomain = to_pdomain(domain); 2623 struct dev_table_entry *dev_table; 2624 struct iommu_dev_data *dev_data; 2625 bool domain_flush = false; 2626 struct amd_iommu *iommu; 2627 unsigned long flags; 2628 u64 pte_root; 2629 2630 spin_lock_irqsave(&pdomain->lock, flags); 2631 if (!(pdomain->dirty_tracking ^ enable)) { 2632 spin_unlock_irqrestore(&pdomain->lock, flags); 2633 return 0; 2634 } 2635 2636 list_for_each_entry(dev_data, &pdomain->dev_list, list) { 2637 iommu = get_amd_iommu_from_dev_data(dev_data); 2638 2639 dev_table = get_dev_table(iommu); 2640 pte_root = dev_table[dev_data->devid].data[0]; 2641 2642 pte_root = (enable ? pte_root | DTE_FLAG_HAD : 2643 pte_root & ~DTE_FLAG_HAD); 2644 2645 /* Flush device DTE */ 2646 dev_table[dev_data->devid].data[0] = pte_root; 2647 device_flush_dte(dev_data); 2648 domain_flush = true; 2649 } 2650 2651 /* Flush IOTLB to mark IOPTE dirty on the next translation(s) */ 2652 if (domain_flush) 2653 amd_iommu_domain_flush_all(pdomain); 2654 2655 pdomain->dirty_tracking = enable; 2656 spin_unlock_irqrestore(&pdomain->lock, flags); 2657 2658 return 0; 2659 } 2660 2661 static int amd_iommu_read_and_clear_dirty(struct iommu_domain *domain, 2662 unsigned long iova, size_t size, 2663 unsigned long flags, 2664 struct iommu_dirty_bitmap *dirty) 2665 { 2666 struct protection_domain *pdomain = to_pdomain(domain); 2667 struct io_pgtable_ops *ops = &pdomain->iop.pgtbl.ops; 2668 unsigned long lflags; 2669 2670 if (!ops || !ops->read_and_clear_dirty) 2671 return -EOPNOTSUPP; 2672 2673 spin_lock_irqsave(&pdomain->lock, lflags); 2674 if (!pdomain->dirty_tracking && dirty->bitmap) { 2675 spin_unlock_irqrestore(&pdomain->lock, lflags); 2676 return -EINVAL; 2677 } 2678 spin_unlock_irqrestore(&pdomain->lock, lflags); 2679 2680 return ops->read_and_clear_dirty(ops, iova, size, flags, dirty); 2681 } 2682 2683 static void amd_iommu_get_resv_regions(struct device *dev, 2684 struct list_head *head) 2685 { 2686 struct iommu_resv_region *region; 2687 struct unity_map_entry *entry; 2688 struct amd_iommu *iommu; 2689 struct amd_iommu_pci_seg *pci_seg; 2690 int devid, sbdf; 2691 2692 sbdf = get_device_sbdf_id(dev); 2693 if (sbdf < 0) 2694 return; 2695 2696 devid = PCI_SBDF_TO_DEVID(sbdf); 2697 iommu = get_amd_iommu_from_dev(dev); 2698 pci_seg = iommu->pci_seg; 2699 2700 list_for_each_entry(entry, &pci_seg->unity_map, list) { 2701 int type, prot = 0; 2702 size_t length; 2703 2704 if (devid < entry->devid_start || devid > entry->devid_end) 2705 continue; 2706 2707 type = IOMMU_RESV_DIRECT; 2708 length = entry->address_end - entry->address_start; 2709 if (entry->prot & IOMMU_PROT_IR) 2710 prot |= IOMMU_READ; 2711 if (entry->prot & IOMMU_PROT_IW) 2712 prot |= IOMMU_WRITE; 2713 if (entry->prot & IOMMU_UNITY_MAP_FLAG_EXCL_RANGE) 2714 /* Exclusion range */ 2715 type = IOMMU_RESV_RESERVED; 2716 2717 region = iommu_alloc_resv_region(entry->address_start, 2718 length, prot, type, 2719 GFP_KERNEL); 2720 if (!region) { 2721 dev_err(dev, "Out of memory allocating dm-regions\n"); 2722 return; 2723 } 2724 list_add_tail(®ion->list, head); 2725 } 2726 2727 region = iommu_alloc_resv_region(MSI_RANGE_START, 2728 MSI_RANGE_END - MSI_RANGE_START + 1, 2729 0, IOMMU_RESV_MSI, GFP_KERNEL); 2730 if (!region) 2731 return; 2732 list_add_tail(®ion->list, head); 2733 2734 region = iommu_alloc_resv_region(HT_RANGE_START, 2735 HT_RANGE_END - HT_RANGE_START + 1, 2736 0, IOMMU_RESV_RESERVED, GFP_KERNEL); 2737 if (!region) 2738 return; 2739 list_add_tail(®ion->list, head); 2740 } 2741 2742 static bool amd_iommu_is_attach_deferred(struct device *dev) 2743 { 2744 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2745 2746 return dev_data->defer_attach; 2747 } 2748 2749 static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain) 2750 { 2751 struct protection_domain *dom = to_pdomain(domain); 2752 unsigned long flags; 2753 2754 spin_lock_irqsave(&dom->lock, flags); 2755 amd_iommu_domain_flush_all(dom); 2756 spin_unlock_irqrestore(&dom->lock, flags); 2757 } 2758 2759 static void amd_iommu_iotlb_sync(struct iommu_domain *domain, 2760 struct iommu_iotlb_gather *gather) 2761 { 2762 struct protection_domain *dom = to_pdomain(domain); 2763 unsigned long flags; 2764 2765 spin_lock_irqsave(&dom->lock, flags); 2766 amd_iommu_domain_flush_pages(dom, gather->start, 2767 gather->end - gather->start + 1); 2768 spin_unlock_irqrestore(&dom->lock, flags); 2769 } 2770 2771 static int amd_iommu_def_domain_type(struct device *dev) 2772 { 2773 struct iommu_dev_data *dev_data; 2774 2775 dev_data = dev_iommu_priv_get(dev); 2776 if (!dev_data) 2777 return 0; 2778 2779 /* Always use DMA domain for untrusted device */ 2780 if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted) 2781 return IOMMU_DOMAIN_DMA; 2782 2783 /* 2784 * Do not identity map IOMMUv2 capable devices when: 2785 * - memory encryption is active, because some of those devices 2786 * (AMD GPUs) don't have the encryption bit in their DMA-mask 2787 * and require remapping. 2788 * - SNP is enabled, because it prohibits DTE[Mode]=0. 2789 */ 2790 if (pdev_pasid_supported(dev_data) && 2791 !cc_platform_has(CC_ATTR_MEM_ENCRYPT) && 2792 !amd_iommu_snp_en) { 2793 return IOMMU_DOMAIN_IDENTITY; 2794 } 2795 2796 return 0; 2797 } 2798 2799 static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain) 2800 { 2801 /* IOMMU_PTE_FC is always set */ 2802 return true; 2803 } 2804 2805 static const struct iommu_dirty_ops amd_dirty_ops = { 2806 .set_dirty_tracking = amd_iommu_set_dirty_tracking, 2807 .read_and_clear_dirty = amd_iommu_read_and_clear_dirty, 2808 }; 2809 2810 static int amd_iommu_dev_enable_feature(struct device *dev, 2811 enum iommu_dev_features feat) 2812 { 2813 int ret = 0; 2814 2815 switch (feat) { 2816 case IOMMU_DEV_FEAT_IOPF: 2817 case IOMMU_DEV_FEAT_SVA: 2818 break; 2819 default: 2820 ret = -EINVAL; 2821 break; 2822 } 2823 return ret; 2824 } 2825 2826 static int amd_iommu_dev_disable_feature(struct device *dev, 2827 enum iommu_dev_features feat) 2828 { 2829 int ret = 0; 2830 2831 switch (feat) { 2832 case IOMMU_DEV_FEAT_IOPF: 2833 case IOMMU_DEV_FEAT_SVA: 2834 break; 2835 default: 2836 ret = -EINVAL; 2837 break; 2838 } 2839 return ret; 2840 } 2841 2842 const struct iommu_ops amd_iommu_ops = { 2843 .capable = amd_iommu_capable, 2844 .blocked_domain = &blocked_domain, 2845 .domain_alloc = amd_iommu_domain_alloc, 2846 .domain_alloc_user = amd_iommu_domain_alloc_user, 2847 .domain_alloc_sva = amd_iommu_domain_alloc_sva, 2848 .probe_device = amd_iommu_probe_device, 2849 .release_device = amd_iommu_release_device, 2850 .device_group = amd_iommu_device_group, 2851 .get_resv_regions = amd_iommu_get_resv_regions, 2852 .is_attach_deferred = amd_iommu_is_attach_deferred, 2853 .def_domain_type = amd_iommu_def_domain_type, 2854 .dev_enable_feat = amd_iommu_dev_enable_feature, 2855 .dev_disable_feat = amd_iommu_dev_disable_feature, 2856 .remove_dev_pasid = amd_iommu_remove_dev_pasid, 2857 .page_response = amd_iommu_page_response, 2858 .default_domain_ops = &(const struct iommu_domain_ops) { 2859 .attach_dev = amd_iommu_attach_device, 2860 .map_pages = amd_iommu_map_pages, 2861 .unmap_pages = amd_iommu_unmap_pages, 2862 .iotlb_sync_map = amd_iommu_iotlb_sync_map, 2863 .iova_to_phys = amd_iommu_iova_to_phys, 2864 .flush_iotlb_all = amd_iommu_flush_iotlb_all, 2865 .iotlb_sync = amd_iommu_iotlb_sync, 2866 .free = amd_iommu_domain_free, 2867 .enforce_cache_coherency = amd_iommu_enforce_cache_coherency, 2868 } 2869 }; 2870 2871 #ifdef CONFIG_IRQ_REMAP 2872 2873 /***************************************************************************** 2874 * 2875 * Interrupt Remapping Implementation 2876 * 2877 *****************************************************************************/ 2878 2879 static struct irq_chip amd_ir_chip; 2880 static DEFINE_SPINLOCK(iommu_table_lock); 2881 2882 static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) 2883 { 2884 int ret; 2885 u64 data; 2886 unsigned long flags; 2887 struct iommu_cmd cmd, cmd2; 2888 2889 if (iommu->irtcachedis_enabled) 2890 return; 2891 2892 build_inv_irt(&cmd, devid); 2893 data = atomic64_add_return(1, &iommu->cmd_sem_val); 2894 build_completion_wait(&cmd2, iommu, data); 2895 2896 raw_spin_lock_irqsave(&iommu->lock, flags); 2897 ret = __iommu_queue_command_sync(iommu, &cmd, true); 2898 if (ret) 2899 goto out; 2900 ret = __iommu_queue_command_sync(iommu, &cmd2, false); 2901 if (ret) 2902 goto out; 2903 wait_on_sem(iommu, data); 2904 out: 2905 raw_spin_unlock_irqrestore(&iommu->lock, flags); 2906 } 2907 2908 static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid, 2909 struct irq_remap_table *table) 2910 { 2911 u64 dte; 2912 struct dev_table_entry *dev_table = get_dev_table(iommu); 2913 2914 dte = dev_table[devid].data[2]; 2915 dte &= ~DTE_IRQ_PHYS_ADDR_MASK; 2916 dte |= iommu_virt_to_phys(table->table); 2917 dte |= DTE_IRQ_REMAP_INTCTL; 2918 dte |= DTE_INTTABLEN; 2919 dte |= DTE_IRQ_REMAP_ENABLE; 2920 2921 dev_table[devid].data[2] = dte; 2922 } 2923 2924 static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid) 2925 { 2926 struct irq_remap_table *table; 2927 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 2928 2929 if (WARN_ONCE(!pci_seg->rlookup_table[devid], 2930 "%s: no iommu for devid %x:%x\n", 2931 __func__, pci_seg->id, devid)) 2932 return NULL; 2933 2934 table = pci_seg->irq_lookup_table[devid]; 2935 if (WARN_ONCE(!table, "%s: no table for devid %x:%x\n", 2936 __func__, pci_seg->id, devid)) 2937 return NULL; 2938 2939 return table; 2940 } 2941 2942 static struct irq_remap_table *__alloc_irq_table(void) 2943 { 2944 struct irq_remap_table *table; 2945 2946 table = kzalloc(sizeof(*table), GFP_KERNEL); 2947 if (!table) 2948 return NULL; 2949 2950 table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_KERNEL); 2951 if (!table->table) { 2952 kfree(table); 2953 return NULL; 2954 } 2955 raw_spin_lock_init(&table->lock); 2956 2957 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) 2958 memset(table->table, 0, 2959 MAX_IRQS_PER_TABLE * sizeof(u32)); 2960 else 2961 memset(table->table, 0, 2962 (MAX_IRQS_PER_TABLE * (sizeof(u64) * 2))); 2963 return table; 2964 } 2965 2966 static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid, 2967 struct irq_remap_table *table) 2968 { 2969 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 2970 2971 pci_seg->irq_lookup_table[devid] = table; 2972 set_dte_irq_entry(iommu, devid, table); 2973 iommu_flush_dte(iommu, devid); 2974 } 2975 2976 static int set_remap_table_entry_alias(struct pci_dev *pdev, u16 alias, 2977 void *data) 2978 { 2979 struct irq_remap_table *table = data; 2980 struct amd_iommu_pci_seg *pci_seg; 2981 struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); 2982 2983 if (!iommu) 2984 return -EINVAL; 2985 2986 pci_seg = iommu->pci_seg; 2987 pci_seg->irq_lookup_table[alias] = table; 2988 set_dte_irq_entry(iommu, alias, table); 2989 iommu_flush_dte(pci_seg->rlookup_table[alias], alias); 2990 2991 return 0; 2992 } 2993 2994 static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu, 2995 u16 devid, struct pci_dev *pdev) 2996 { 2997 struct irq_remap_table *table = NULL; 2998 struct irq_remap_table *new_table = NULL; 2999 struct amd_iommu_pci_seg *pci_seg; 3000 unsigned long flags; 3001 u16 alias; 3002 3003 spin_lock_irqsave(&iommu_table_lock, flags); 3004 3005 pci_seg = iommu->pci_seg; 3006 table = pci_seg->irq_lookup_table[devid]; 3007 if (table) 3008 goto out_unlock; 3009 3010 alias = pci_seg->alias_table[devid]; 3011 table = pci_seg->irq_lookup_table[alias]; 3012 if (table) { 3013 set_remap_table_entry(iommu, devid, table); 3014 goto out_wait; 3015 } 3016 spin_unlock_irqrestore(&iommu_table_lock, flags); 3017 3018 /* Nothing there yet, allocate new irq remapping table */ 3019 new_table = __alloc_irq_table(); 3020 if (!new_table) 3021 return NULL; 3022 3023 spin_lock_irqsave(&iommu_table_lock, flags); 3024 3025 table = pci_seg->irq_lookup_table[devid]; 3026 if (table) 3027 goto out_unlock; 3028 3029 table = pci_seg->irq_lookup_table[alias]; 3030 if (table) { 3031 set_remap_table_entry(iommu, devid, table); 3032 goto out_wait; 3033 } 3034 3035 table = new_table; 3036 new_table = NULL; 3037 3038 if (pdev) 3039 pci_for_each_dma_alias(pdev, set_remap_table_entry_alias, 3040 table); 3041 else 3042 set_remap_table_entry(iommu, devid, table); 3043 3044 if (devid != alias) 3045 set_remap_table_entry(iommu, alias, table); 3046 3047 out_wait: 3048 iommu_completion_wait(iommu); 3049 3050 out_unlock: 3051 spin_unlock_irqrestore(&iommu_table_lock, flags); 3052 3053 if (new_table) { 3054 kmem_cache_free(amd_iommu_irq_cache, new_table->table); 3055 kfree(new_table); 3056 } 3057 return table; 3058 } 3059 3060 static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count, 3061 bool align, struct pci_dev *pdev) 3062 { 3063 struct irq_remap_table *table; 3064 int index, c, alignment = 1; 3065 unsigned long flags; 3066 3067 table = alloc_irq_table(iommu, devid, pdev); 3068 if (!table) 3069 return -ENODEV; 3070 3071 if (align) 3072 alignment = roundup_pow_of_two(count); 3073 3074 raw_spin_lock_irqsave(&table->lock, flags); 3075 3076 /* Scan table for free entries */ 3077 for (index = ALIGN(table->min_index, alignment), c = 0; 3078 index < MAX_IRQS_PER_TABLE;) { 3079 if (!iommu->irte_ops->is_allocated(table, index)) { 3080 c += 1; 3081 } else { 3082 c = 0; 3083 index = ALIGN(index + 1, alignment); 3084 continue; 3085 } 3086 3087 if (c == count) { 3088 for (; c != 0; --c) 3089 iommu->irte_ops->set_allocated(table, index - c + 1); 3090 3091 index -= count - 1; 3092 goto out; 3093 } 3094 3095 index++; 3096 } 3097 3098 index = -ENOSPC; 3099 3100 out: 3101 raw_spin_unlock_irqrestore(&table->lock, flags); 3102 3103 return index; 3104 } 3105 3106 static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, 3107 struct irte_ga *irte) 3108 { 3109 struct irq_remap_table *table; 3110 struct irte_ga *entry; 3111 unsigned long flags; 3112 u128 old; 3113 3114 table = get_irq_table(iommu, devid); 3115 if (!table) 3116 return -ENOMEM; 3117 3118 raw_spin_lock_irqsave(&table->lock, flags); 3119 3120 entry = (struct irte_ga *)table->table; 3121 entry = &entry[index]; 3122 3123 /* 3124 * We use cmpxchg16 to atomically update the 128-bit IRTE, 3125 * and it cannot be updated by the hardware or other processors 3126 * behind us, so the return value of cmpxchg16 should be the 3127 * same as the old value. 3128 */ 3129 old = entry->irte; 3130 WARN_ON(!try_cmpxchg128(&entry->irte, &old, irte->irte)); 3131 3132 raw_spin_unlock_irqrestore(&table->lock, flags); 3133 3134 return 0; 3135 } 3136 3137 static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, 3138 struct irte_ga *irte) 3139 { 3140 bool ret; 3141 3142 ret = __modify_irte_ga(iommu, devid, index, irte); 3143 if (ret) 3144 return ret; 3145 3146 iommu_flush_irt_and_complete(iommu, devid); 3147 3148 return 0; 3149 } 3150 3151 static int modify_irte(struct amd_iommu *iommu, 3152 u16 devid, int index, union irte *irte) 3153 { 3154 struct irq_remap_table *table; 3155 unsigned long flags; 3156 3157 table = get_irq_table(iommu, devid); 3158 if (!table) 3159 return -ENOMEM; 3160 3161 raw_spin_lock_irqsave(&table->lock, flags); 3162 table->table[index] = irte->val; 3163 raw_spin_unlock_irqrestore(&table->lock, flags); 3164 3165 iommu_flush_irt_and_complete(iommu, devid); 3166 3167 return 0; 3168 } 3169 3170 static void free_irte(struct amd_iommu *iommu, u16 devid, int index) 3171 { 3172 struct irq_remap_table *table; 3173 unsigned long flags; 3174 3175 table = get_irq_table(iommu, devid); 3176 if (!table) 3177 return; 3178 3179 raw_spin_lock_irqsave(&table->lock, flags); 3180 iommu->irte_ops->clear_allocated(table, index); 3181 raw_spin_unlock_irqrestore(&table->lock, flags); 3182 3183 iommu_flush_irt_and_complete(iommu, devid); 3184 } 3185 3186 static void irte_prepare(void *entry, 3187 u32 delivery_mode, bool dest_mode, 3188 u8 vector, u32 dest_apicid, int devid) 3189 { 3190 union irte *irte = (union irte *) entry; 3191 3192 irte->val = 0; 3193 irte->fields.vector = vector; 3194 irte->fields.int_type = delivery_mode; 3195 irte->fields.destination = dest_apicid; 3196 irte->fields.dm = dest_mode; 3197 irte->fields.valid = 1; 3198 } 3199 3200 static void irte_ga_prepare(void *entry, 3201 u32 delivery_mode, bool dest_mode, 3202 u8 vector, u32 dest_apicid, int devid) 3203 { 3204 struct irte_ga *irte = (struct irte_ga *) entry; 3205 3206 irte->lo.val = 0; 3207 irte->hi.val = 0; 3208 irte->lo.fields_remap.int_type = delivery_mode; 3209 irte->lo.fields_remap.dm = dest_mode; 3210 irte->hi.fields.vector = vector; 3211 irte->lo.fields_remap.destination = APICID_TO_IRTE_DEST_LO(dest_apicid); 3212 irte->hi.fields.destination = APICID_TO_IRTE_DEST_HI(dest_apicid); 3213 irte->lo.fields_remap.valid = 1; 3214 } 3215 3216 static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) 3217 { 3218 union irte *irte = (union irte *) entry; 3219 3220 irte->fields.valid = 1; 3221 modify_irte(iommu, devid, index, irte); 3222 } 3223 3224 static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) 3225 { 3226 struct irte_ga *irte = (struct irte_ga *) entry; 3227 3228 irte->lo.fields_remap.valid = 1; 3229 modify_irte_ga(iommu, devid, index, irte); 3230 } 3231 3232 static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) 3233 { 3234 union irte *irte = (union irte *) entry; 3235 3236 irte->fields.valid = 0; 3237 modify_irte(iommu, devid, index, irte); 3238 } 3239 3240 static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) 3241 { 3242 struct irte_ga *irte = (struct irte_ga *) entry; 3243 3244 irte->lo.fields_remap.valid = 0; 3245 modify_irte_ga(iommu, devid, index, irte); 3246 } 3247 3248 static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index, 3249 u8 vector, u32 dest_apicid) 3250 { 3251 union irte *irte = (union irte *) entry; 3252 3253 irte->fields.vector = vector; 3254 irte->fields.destination = dest_apicid; 3255 modify_irte(iommu, devid, index, irte); 3256 } 3257 3258 static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index, 3259 u8 vector, u32 dest_apicid) 3260 { 3261 struct irte_ga *irte = (struct irte_ga *) entry; 3262 3263 if (!irte->lo.fields_remap.guest_mode) { 3264 irte->hi.fields.vector = vector; 3265 irte->lo.fields_remap.destination = 3266 APICID_TO_IRTE_DEST_LO(dest_apicid); 3267 irte->hi.fields.destination = 3268 APICID_TO_IRTE_DEST_HI(dest_apicid); 3269 modify_irte_ga(iommu, devid, index, irte); 3270 } 3271 } 3272 3273 #define IRTE_ALLOCATED (~1U) 3274 static void irte_set_allocated(struct irq_remap_table *table, int index) 3275 { 3276 table->table[index] = IRTE_ALLOCATED; 3277 } 3278 3279 static void irte_ga_set_allocated(struct irq_remap_table *table, int index) 3280 { 3281 struct irte_ga *ptr = (struct irte_ga *)table->table; 3282 struct irte_ga *irte = &ptr[index]; 3283 3284 memset(&irte->lo.val, 0, sizeof(u64)); 3285 memset(&irte->hi.val, 0, sizeof(u64)); 3286 irte->hi.fields.vector = 0xff; 3287 } 3288 3289 static bool irte_is_allocated(struct irq_remap_table *table, int index) 3290 { 3291 union irte *ptr = (union irte *)table->table; 3292 union irte *irte = &ptr[index]; 3293 3294 return irte->val != 0; 3295 } 3296 3297 static bool irte_ga_is_allocated(struct irq_remap_table *table, int index) 3298 { 3299 struct irte_ga *ptr = (struct irte_ga *)table->table; 3300 struct irte_ga *irte = &ptr[index]; 3301 3302 return irte->hi.fields.vector != 0; 3303 } 3304 3305 static void irte_clear_allocated(struct irq_remap_table *table, int index) 3306 { 3307 table->table[index] = 0; 3308 } 3309 3310 static void irte_ga_clear_allocated(struct irq_remap_table *table, int index) 3311 { 3312 struct irte_ga *ptr = (struct irte_ga *)table->table; 3313 struct irte_ga *irte = &ptr[index]; 3314 3315 memset(&irte->lo.val, 0, sizeof(u64)); 3316 memset(&irte->hi.val, 0, sizeof(u64)); 3317 } 3318 3319 static int get_devid(struct irq_alloc_info *info) 3320 { 3321 switch (info->type) { 3322 case X86_IRQ_ALLOC_TYPE_IOAPIC: 3323 return get_ioapic_devid(info->devid); 3324 case X86_IRQ_ALLOC_TYPE_HPET: 3325 return get_hpet_devid(info->devid); 3326 case X86_IRQ_ALLOC_TYPE_PCI_MSI: 3327 case X86_IRQ_ALLOC_TYPE_PCI_MSIX: 3328 return get_device_sbdf_id(msi_desc_to_dev(info->desc)); 3329 default: 3330 WARN_ON_ONCE(1); 3331 return -1; 3332 } 3333 } 3334 3335 struct irq_remap_ops amd_iommu_irq_ops = { 3336 .prepare = amd_iommu_prepare, 3337 .enable = amd_iommu_enable, 3338 .disable = amd_iommu_disable, 3339 .reenable = amd_iommu_reenable, 3340 .enable_faulting = amd_iommu_enable_faulting, 3341 }; 3342 3343 static void fill_msi_msg(struct msi_msg *msg, u32 index) 3344 { 3345 msg->data = index; 3346 msg->address_lo = 0; 3347 msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW; 3348 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH; 3349 } 3350 3351 static void irq_remapping_prepare_irte(struct amd_ir_data *data, 3352 struct irq_cfg *irq_cfg, 3353 struct irq_alloc_info *info, 3354 int devid, int index, int sub_handle) 3355 { 3356 struct irq_2_irte *irte_info = &data->irq_2_irte; 3357 struct amd_iommu *iommu = data->iommu; 3358 3359 if (!iommu) 3360 return; 3361 3362 data->irq_2_irte.devid = devid; 3363 data->irq_2_irte.index = index + sub_handle; 3364 iommu->irte_ops->prepare(data->entry, APIC_DELIVERY_MODE_FIXED, 3365 apic->dest_mode_logical, irq_cfg->vector, 3366 irq_cfg->dest_apicid, devid); 3367 3368 switch (info->type) { 3369 case X86_IRQ_ALLOC_TYPE_IOAPIC: 3370 case X86_IRQ_ALLOC_TYPE_HPET: 3371 case X86_IRQ_ALLOC_TYPE_PCI_MSI: 3372 case X86_IRQ_ALLOC_TYPE_PCI_MSIX: 3373 fill_msi_msg(&data->msi_entry, irte_info->index); 3374 break; 3375 3376 default: 3377 BUG_ON(1); 3378 break; 3379 } 3380 } 3381 3382 struct amd_irte_ops irte_32_ops = { 3383 .prepare = irte_prepare, 3384 .activate = irte_activate, 3385 .deactivate = irte_deactivate, 3386 .set_affinity = irte_set_affinity, 3387 .set_allocated = irte_set_allocated, 3388 .is_allocated = irte_is_allocated, 3389 .clear_allocated = irte_clear_allocated, 3390 }; 3391 3392 struct amd_irte_ops irte_128_ops = { 3393 .prepare = irte_ga_prepare, 3394 .activate = irte_ga_activate, 3395 .deactivate = irte_ga_deactivate, 3396 .set_affinity = irte_ga_set_affinity, 3397 .set_allocated = irte_ga_set_allocated, 3398 .is_allocated = irte_ga_is_allocated, 3399 .clear_allocated = irte_ga_clear_allocated, 3400 }; 3401 3402 static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, 3403 unsigned int nr_irqs, void *arg) 3404 { 3405 struct irq_alloc_info *info = arg; 3406 struct irq_data *irq_data; 3407 struct amd_ir_data *data = NULL; 3408 struct amd_iommu *iommu; 3409 struct irq_cfg *cfg; 3410 int i, ret, devid, seg, sbdf; 3411 int index; 3412 3413 if (!info) 3414 return -EINVAL; 3415 if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_PCI_MSI) 3416 return -EINVAL; 3417 3418 sbdf = get_devid(info); 3419 if (sbdf < 0) 3420 return -EINVAL; 3421 3422 seg = PCI_SBDF_TO_SEGID(sbdf); 3423 devid = PCI_SBDF_TO_DEVID(sbdf); 3424 iommu = __rlookup_amd_iommu(seg, devid); 3425 if (!iommu) 3426 return -EINVAL; 3427 3428 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 3429 if (ret < 0) 3430 return ret; 3431 3432 if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) { 3433 struct irq_remap_table *table; 3434 3435 table = alloc_irq_table(iommu, devid, NULL); 3436 if (table) { 3437 if (!table->min_index) { 3438 /* 3439 * Keep the first 32 indexes free for IOAPIC 3440 * interrupts. 3441 */ 3442 table->min_index = 32; 3443 for (i = 0; i < 32; ++i) 3444 iommu->irte_ops->set_allocated(table, i); 3445 } 3446 WARN_ON(table->min_index != 32); 3447 index = info->ioapic.pin; 3448 } else { 3449 index = -ENOMEM; 3450 } 3451 } else if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI || 3452 info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX) { 3453 bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI); 3454 3455 index = alloc_irq_index(iommu, devid, nr_irqs, align, 3456 msi_desc_to_pci_dev(info->desc)); 3457 } else { 3458 index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL); 3459 } 3460 3461 if (index < 0) { 3462 pr_warn("Failed to allocate IRTE\n"); 3463 ret = index; 3464 goto out_free_parent; 3465 } 3466 3467 for (i = 0; i < nr_irqs; i++) { 3468 irq_data = irq_domain_get_irq_data(domain, virq + i); 3469 cfg = irq_data ? irqd_cfg(irq_data) : NULL; 3470 if (!cfg) { 3471 ret = -EINVAL; 3472 goto out_free_data; 3473 } 3474 3475 ret = -ENOMEM; 3476 data = kzalloc(sizeof(*data), GFP_KERNEL); 3477 if (!data) 3478 goto out_free_data; 3479 3480 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) 3481 data->entry = kzalloc(sizeof(union irte), GFP_KERNEL); 3482 else 3483 data->entry = kzalloc(sizeof(struct irte_ga), 3484 GFP_KERNEL); 3485 if (!data->entry) { 3486 kfree(data); 3487 goto out_free_data; 3488 } 3489 3490 data->iommu = iommu; 3491 irq_data->hwirq = (devid << 16) + i; 3492 irq_data->chip_data = data; 3493 irq_data->chip = &amd_ir_chip; 3494 irq_remapping_prepare_irte(data, cfg, info, devid, index, i); 3495 irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT); 3496 } 3497 3498 return 0; 3499 3500 out_free_data: 3501 for (i--; i >= 0; i--) { 3502 irq_data = irq_domain_get_irq_data(domain, virq + i); 3503 if (irq_data) 3504 kfree(irq_data->chip_data); 3505 } 3506 for (i = 0; i < nr_irqs; i++) 3507 free_irte(iommu, devid, index + i); 3508 out_free_parent: 3509 irq_domain_free_irqs_common(domain, virq, nr_irqs); 3510 return ret; 3511 } 3512 3513 static void irq_remapping_free(struct irq_domain *domain, unsigned int virq, 3514 unsigned int nr_irqs) 3515 { 3516 struct irq_2_irte *irte_info; 3517 struct irq_data *irq_data; 3518 struct amd_ir_data *data; 3519 int i; 3520 3521 for (i = 0; i < nr_irqs; i++) { 3522 irq_data = irq_domain_get_irq_data(domain, virq + i); 3523 if (irq_data && irq_data->chip_data) { 3524 data = irq_data->chip_data; 3525 irte_info = &data->irq_2_irte; 3526 free_irte(data->iommu, irte_info->devid, irte_info->index); 3527 kfree(data->entry); 3528 kfree(data); 3529 } 3530 } 3531 irq_domain_free_irqs_common(domain, virq, nr_irqs); 3532 } 3533 3534 static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu, 3535 struct amd_ir_data *ir_data, 3536 struct irq_2_irte *irte_info, 3537 struct irq_cfg *cfg); 3538 3539 static int irq_remapping_activate(struct irq_domain *domain, 3540 struct irq_data *irq_data, bool reserve) 3541 { 3542 struct amd_ir_data *data = irq_data->chip_data; 3543 struct irq_2_irte *irte_info = &data->irq_2_irte; 3544 struct amd_iommu *iommu = data->iommu; 3545 struct irq_cfg *cfg = irqd_cfg(irq_data); 3546 3547 if (!iommu) 3548 return 0; 3549 3550 iommu->irte_ops->activate(iommu, data->entry, irte_info->devid, 3551 irte_info->index); 3552 amd_ir_update_irte(irq_data, iommu, data, irte_info, cfg); 3553 return 0; 3554 } 3555 3556 static void irq_remapping_deactivate(struct irq_domain *domain, 3557 struct irq_data *irq_data) 3558 { 3559 struct amd_ir_data *data = irq_data->chip_data; 3560 struct irq_2_irte *irte_info = &data->irq_2_irte; 3561 struct amd_iommu *iommu = data->iommu; 3562 3563 if (iommu) 3564 iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid, 3565 irte_info->index); 3566 } 3567 3568 static int irq_remapping_select(struct irq_domain *d, struct irq_fwspec *fwspec, 3569 enum irq_domain_bus_token bus_token) 3570 { 3571 struct amd_iommu *iommu; 3572 int devid = -1; 3573 3574 if (!amd_iommu_irq_remap) 3575 return 0; 3576 3577 if (x86_fwspec_is_ioapic(fwspec)) 3578 devid = get_ioapic_devid(fwspec->param[0]); 3579 else if (x86_fwspec_is_hpet(fwspec)) 3580 devid = get_hpet_devid(fwspec->param[0]); 3581 3582 if (devid < 0) 3583 return 0; 3584 iommu = __rlookup_amd_iommu((devid >> 16), (devid & 0xffff)); 3585 3586 return iommu && iommu->ir_domain == d; 3587 } 3588 3589 static const struct irq_domain_ops amd_ir_domain_ops = { 3590 .select = irq_remapping_select, 3591 .alloc = irq_remapping_alloc, 3592 .free = irq_remapping_free, 3593 .activate = irq_remapping_activate, 3594 .deactivate = irq_remapping_deactivate, 3595 }; 3596 3597 int amd_iommu_activate_guest_mode(void *data) 3598 { 3599 struct amd_ir_data *ir_data = (struct amd_ir_data *)data; 3600 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; 3601 u64 valid; 3602 3603 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) || !entry) 3604 return 0; 3605 3606 valid = entry->lo.fields_vapic.valid; 3607 3608 entry->lo.val = 0; 3609 entry->hi.val = 0; 3610 3611 entry->lo.fields_vapic.valid = valid; 3612 entry->lo.fields_vapic.guest_mode = 1; 3613 entry->lo.fields_vapic.ga_log_intr = 1; 3614 entry->hi.fields.ga_root_ptr = ir_data->ga_root_ptr; 3615 entry->hi.fields.vector = ir_data->ga_vector; 3616 entry->lo.fields_vapic.ga_tag = ir_data->ga_tag; 3617 3618 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, 3619 ir_data->irq_2_irte.index, entry); 3620 } 3621 EXPORT_SYMBOL(amd_iommu_activate_guest_mode); 3622 3623 int amd_iommu_deactivate_guest_mode(void *data) 3624 { 3625 struct amd_ir_data *ir_data = (struct amd_ir_data *)data; 3626 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; 3627 struct irq_cfg *cfg = ir_data->cfg; 3628 u64 valid; 3629 3630 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) || 3631 !entry || !entry->lo.fields_vapic.guest_mode) 3632 return 0; 3633 3634 valid = entry->lo.fields_remap.valid; 3635 3636 entry->lo.val = 0; 3637 entry->hi.val = 0; 3638 3639 entry->lo.fields_remap.valid = valid; 3640 entry->lo.fields_remap.dm = apic->dest_mode_logical; 3641 entry->lo.fields_remap.int_type = APIC_DELIVERY_MODE_FIXED; 3642 entry->hi.fields.vector = cfg->vector; 3643 entry->lo.fields_remap.destination = 3644 APICID_TO_IRTE_DEST_LO(cfg->dest_apicid); 3645 entry->hi.fields.destination = 3646 APICID_TO_IRTE_DEST_HI(cfg->dest_apicid); 3647 3648 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, 3649 ir_data->irq_2_irte.index, entry); 3650 } 3651 EXPORT_SYMBOL(amd_iommu_deactivate_guest_mode); 3652 3653 static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) 3654 { 3655 int ret; 3656 struct amd_iommu_pi_data *pi_data = vcpu_info; 3657 struct vcpu_data *vcpu_pi_info = pi_data->vcpu_data; 3658 struct amd_ir_data *ir_data = data->chip_data; 3659 struct irq_2_irte *irte_info = &ir_data->irq_2_irte; 3660 struct iommu_dev_data *dev_data; 3661 3662 if (ir_data->iommu == NULL) 3663 return -EINVAL; 3664 3665 dev_data = search_dev_data(ir_data->iommu, irte_info->devid); 3666 3667 /* Note: 3668 * This device has never been set up for guest mode. 3669 * we should not modify the IRTE 3670 */ 3671 if (!dev_data || !dev_data->use_vapic) 3672 return 0; 3673 3674 ir_data->cfg = irqd_cfg(data); 3675 pi_data->ir_data = ir_data; 3676 3677 /* Note: 3678 * SVM tries to set up for VAPIC mode, but we are in 3679 * legacy mode. So, we force legacy mode instead. 3680 */ 3681 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) { 3682 pr_debug("%s: Fall back to using intr legacy remap\n", 3683 __func__); 3684 pi_data->is_guest_mode = false; 3685 } 3686 3687 pi_data->prev_ga_tag = ir_data->cached_ga_tag; 3688 if (pi_data->is_guest_mode) { 3689 ir_data->ga_root_ptr = (pi_data->base >> 12); 3690 ir_data->ga_vector = vcpu_pi_info->vector; 3691 ir_data->ga_tag = pi_data->ga_tag; 3692 ret = amd_iommu_activate_guest_mode(ir_data); 3693 if (!ret) 3694 ir_data->cached_ga_tag = pi_data->ga_tag; 3695 } else { 3696 ret = amd_iommu_deactivate_guest_mode(ir_data); 3697 3698 /* 3699 * This communicates the ga_tag back to the caller 3700 * so that it can do all the necessary clean up. 3701 */ 3702 if (!ret) 3703 ir_data->cached_ga_tag = 0; 3704 } 3705 3706 return ret; 3707 } 3708 3709 3710 static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu, 3711 struct amd_ir_data *ir_data, 3712 struct irq_2_irte *irte_info, 3713 struct irq_cfg *cfg) 3714 { 3715 3716 /* 3717 * Atomically updates the IRTE with the new destination, vector 3718 * and flushes the interrupt entry cache. 3719 */ 3720 iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid, 3721 irte_info->index, cfg->vector, 3722 cfg->dest_apicid); 3723 } 3724 3725 static int amd_ir_set_affinity(struct irq_data *data, 3726 const struct cpumask *mask, bool force) 3727 { 3728 struct amd_ir_data *ir_data = data->chip_data; 3729 struct irq_2_irte *irte_info = &ir_data->irq_2_irte; 3730 struct irq_cfg *cfg = irqd_cfg(data); 3731 struct irq_data *parent = data->parent_data; 3732 struct amd_iommu *iommu = ir_data->iommu; 3733 int ret; 3734 3735 if (!iommu) 3736 return -ENODEV; 3737 3738 ret = parent->chip->irq_set_affinity(parent, mask, force); 3739 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) 3740 return ret; 3741 3742 amd_ir_update_irte(data, iommu, ir_data, irte_info, cfg); 3743 /* 3744 * After this point, all the interrupts will start arriving 3745 * at the new destination. So, time to cleanup the previous 3746 * vector allocation. 3747 */ 3748 vector_schedule_cleanup(cfg); 3749 3750 return IRQ_SET_MASK_OK_DONE; 3751 } 3752 3753 static void ir_compose_msi_msg(struct irq_data *irq_data, struct msi_msg *msg) 3754 { 3755 struct amd_ir_data *ir_data = irq_data->chip_data; 3756 3757 *msg = ir_data->msi_entry; 3758 } 3759 3760 static struct irq_chip amd_ir_chip = { 3761 .name = "AMD-IR", 3762 .irq_ack = apic_ack_irq, 3763 .irq_set_affinity = amd_ir_set_affinity, 3764 .irq_set_vcpu_affinity = amd_ir_set_vcpu_affinity, 3765 .irq_compose_msi_msg = ir_compose_msi_msg, 3766 }; 3767 3768 static const struct msi_parent_ops amdvi_msi_parent_ops = { 3769 .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED | MSI_FLAG_MULTI_PCI_MSI, 3770 .prefix = "IR-", 3771 .init_dev_msi_info = msi_parent_init_dev_msi_info, 3772 }; 3773 3774 int amd_iommu_create_irq_domain(struct amd_iommu *iommu) 3775 { 3776 struct fwnode_handle *fn; 3777 3778 fn = irq_domain_alloc_named_id_fwnode("AMD-IR", iommu->index); 3779 if (!fn) 3780 return -ENOMEM; 3781 iommu->ir_domain = irq_domain_create_hierarchy(arch_get_ir_parent_domain(), 0, 0, 3782 fn, &amd_ir_domain_ops, iommu); 3783 if (!iommu->ir_domain) { 3784 irq_domain_free_fwnode(fn); 3785 return -ENOMEM; 3786 } 3787 3788 irq_domain_update_bus_token(iommu->ir_domain, DOMAIN_BUS_AMDVI); 3789 iommu->ir_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT | 3790 IRQ_DOMAIN_FLAG_ISOLATED_MSI; 3791 iommu->ir_domain->msi_parent_ops = &amdvi_msi_parent_ops; 3792 3793 return 0; 3794 } 3795 3796 int amd_iommu_update_ga(int cpu, bool is_run, void *data) 3797 { 3798 struct amd_ir_data *ir_data = (struct amd_ir_data *)data; 3799 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; 3800 3801 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) || 3802 !entry || !entry->lo.fields_vapic.guest_mode) 3803 return 0; 3804 3805 if (!ir_data->iommu) 3806 return -ENODEV; 3807 3808 if (cpu >= 0) { 3809 entry->lo.fields_vapic.destination = 3810 APICID_TO_IRTE_DEST_LO(cpu); 3811 entry->hi.fields.destination = 3812 APICID_TO_IRTE_DEST_HI(cpu); 3813 } 3814 entry->lo.fields_vapic.is_run = is_run; 3815 3816 return __modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, 3817 ir_data->irq_2_irte.index, entry); 3818 } 3819 EXPORT_SYMBOL(amd_iommu_update_ga); 3820 #endif 3821