1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <jroedel@suse.de> 5 * Leo Duran <leo.duran@amd.com> 6 */ 7 8 #define pr_fmt(fmt) "AMD-Vi: " fmt 9 #define dev_fmt(fmt) pr_fmt(fmt) 10 11 #include <linux/ratelimit.h> 12 #include <linux/pci.h> 13 #include <linux/acpi.h> 14 #include <linux/pci-ats.h> 15 #include <linux/bitmap.h> 16 #include <linux/slab.h> 17 #include <linux/string_choices.h> 18 #include <linux/debugfs.h> 19 #include <linux/scatterlist.h> 20 #include <linux/dma-map-ops.h> 21 #include <linux/dma-direct.h> 22 #include <linux/idr.h> 23 #include <linux/iommu-helper.h> 24 #include <linux/delay.h> 25 #include <linux/amd-iommu.h> 26 #include <linux/notifier.h> 27 #include <linux/export.h> 28 #include <linux/irq.h> 29 #include <linux/irqchip/irq-msi-lib.h> 30 #include <linux/msi.h> 31 #include <linux/irqdomain.h> 32 #include <linux/percpu.h> 33 #include <linux/cc_platform.h> 34 #include <asm/irq_remapping.h> 35 #include <asm/io_apic.h> 36 #include <asm/apic.h> 37 #include <asm/hw_irq.h> 38 #include <asm/proto.h> 39 #include <asm/iommu.h> 40 #include <asm/gart.h> 41 #include <asm/dma.h> 42 #include <uapi/linux/iommufd.h> 43 #include <linux/generic_pt/iommu.h> 44 45 #include "amd_iommu.h" 46 #include "iommufd.h" 47 #include "../irq_remapping.h" 48 #include "../iommu-pages.h" 49 50 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) 51 52 /* Reserved IOVA ranges */ 53 #define MSI_RANGE_START (0xfee00000) 54 #define MSI_RANGE_END (0xfeefffff) 55 #define HT_RANGE_START (0xfd00000000ULL) 56 #define HT_RANGE_END (0xffffffffffULL) 57 58 LIST_HEAD(ioapic_map); 59 LIST_HEAD(hpet_map); 60 LIST_HEAD(acpihid_map); 61 62 const struct iommu_ops amd_iommu_ops; 63 64 int amd_iommu_max_glx_val = -1; 65 66 /* 67 * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap 68 * to know which ones are already in use. 69 */ 70 DEFINE_IDA(pdom_ids); 71 72 static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev, 73 struct iommu_domain *old); 74 75 static void set_dte_entry(struct amd_iommu *iommu, 76 struct iommu_dev_data *dev_data, 77 phys_addr_t top_paddr, unsigned int top_level); 78 79 static int device_flush_dte(struct iommu_dev_data *dev_data); 80 81 static void amd_iommu_change_top(struct pt_iommu *iommu_table, 82 phys_addr_t top_paddr, unsigned int top_level); 83 84 static void iommu_flush_dte_sync(struct amd_iommu *iommu, u16 devid); 85 86 static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid); 87 static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain); 88 static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain, 89 bool enable); 90 91 static void clone_aliases(struct amd_iommu *iommu, struct device *dev); 92 93 static int iommu_completion_wait(struct amd_iommu *iommu); 94 95 /**************************************************************************** 96 * 97 * Helper functions 98 * 99 ****************************************************************************/ 100 101 static __always_inline void amd_iommu_atomic128_set(__int128 *ptr, __int128 val) 102 { 103 /* 104 * Note: 105 * We use arch_cmpxchg128_local() because: 106 * - Need cmpxchg16b instruction mainly for 128-bit store to DTE 107 * (not necessary for cmpxchg since this function is already 108 * protected by a spin_lock for this DTE). 109 * - Neither need LOCK_PREFIX nor try loop because of the spin_lock. 110 */ 111 arch_cmpxchg128_local(ptr, *ptr, val); 112 } 113 114 static void write_dte_upper128(struct dev_table_entry *ptr, struct dev_table_entry *new) 115 { 116 struct dev_table_entry old; 117 118 old.data128[1] = ptr->data128[1]; 119 /* 120 * Preserve DTE_DATA2_INTR_MASK. This needs to be 121 * done here since it requires to be inside 122 * spin_lock(&dev_data->dte_lock) context. 123 */ 124 new->data[2] &= ~DTE_DATA2_INTR_MASK; 125 new->data[2] |= old.data[2] & DTE_DATA2_INTR_MASK; 126 127 amd_iommu_atomic128_set(&ptr->data128[1], new->data128[1]); 128 } 129 130 static void write_dte_lower128(struct dev_table_entry *ptr, struct dev_table_entry *new) 131 { 132 amd_iommu_atomic128_set(&ptr->data128[0], new->data128[0]); 133 } 134 135 /* 136 * Note: 137 * IOMMU reads the entire Device Table entry in a single 256-bit transaction 138 * but the driver is programming DTE using 2 128-bit cmpxchg. So, the driver 139 * need to ensure the following: 140 * - DTE[V|GV] bit is being written last when setting. 141 * - DTE[V|GV] bit is being written first when clearing. 142 * 143 * This function is used only by code, which updates DMA translation part of the DTE. 144 * So, only consider control bits related to DMA when updating the entry. 145 */ 146 static void update_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_data, 147 struct dev_table_entry *new) 148 { 149 unsigned long flags; 150 struct dev_table_entry *dev_table = get_dev_table(iommu); 151 struct dev_table_entry *ptr = &dev_table[dev_data->devid]; 152 153 spin_lock_irqsave(&dev_data->dte_lock, flags); 154 155 if (!(ptr->data[0] & DTE_FLAG_V)) { 156 /* Existing DTE is not valid. */ 157 write_dte_upper128(ptr, new); 158 write_dte_lower128(ptr, new); 159 iommu_flush_dte_sync(iommu, dev_data->devid); 160 } else if (!(new->data[0] & DTE_FLAG_V)) { 161 /* Existing DTE is valid. New DTE is not valid. */ 162 write_dte_lower128(ptr, new); 163 write_dte_upper128(ptr, new); 164 iommu_flush_dte_sync(iommu, dev_data->devid); 165 } else if (!FIELD_GET(DTE_FLAG_GV, ptr->data[0])) { 166 /* 167 * Both DTEs are valid. 168 * Existing DTE has no guest page table. 169 */ 170 write_dte_upper128(ptr, new); 171 write_dte_lower128(ptr, new); 172 iommu_flush_dte_sync(iommu, dev_data->devid); 173 } else if (!FIELD_GET(DTE_FLAG_GV, new->data[0])) { 174 /* 175 * Both DTEs are valid. 176 * Existing DTE has guest page table, 177 * new DTE has no guest page table, 178 */ 179 write_dte_lower128(ptr, new); 180 write_dte_upper128(ptr, new); 181 iommu_flush_dte_sync(iommu, dev_data->devid); 182 } else if (FIELD_GET(DTE_GPT_LEVEL_MASK, ptr->data[2]) != 183 FIELD_GET(DTE_GPT_LEVEL_MASK, new->data[2])) { 184 /* 185 * Both DTEs are valid and have guest page table, 186 * but have different number of levels. So, we need 187 * to upadte both upper and lower 128-bit value, which 188 * require disabling and flushing. 189 */ 190 struct dev_table_entry clear = {}; 191 192 /* First disable DTE */ 193 write_dte_lower128(ptr, &clear); 194 iommu_flush_dte_sync(iommu, dev_data->devid); 195 196 /* Then update DTE */ 197 write_dte_upper128(ptr, new); 198 write_dte_lower128(ptr, new); 199 iommu_flush_dte_sync(iommu, dev_data->devid); 200 } else { 201 /* 202 * Both DTEs are valid and have guest page table, 203 * and same number of levels. We just need to only 204 * update the lower 128-bit. So no need to disable DTE. 205 */ 206 write_dte_lower128(ptr, new); 207 } 208 209 spin_unlock_irqrestore(&dev_data->dte_lock, flags); 210 } 211 212 void amd_iommu_update_dte(struct amd_iommu *iommu, 213 struct iommu_dev_data *dev_data, 214 struct dev_table_entry *new) 215 { 216 update_dte256(iommu, dev_data, new); 217 clone_aliases(iommu, dev_data->dev); 218 device_flush_dte(dev_data); 219 iommu_completion_wait(iommu); 220 } 221 222 static void get_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_data, 223 struct dev_table_entry *dte) 224 { 225 unsigned long flags; 226 struct dev_table_entry *ptr; 227 struct dev_table_entry *dev_table = get_dev_table(iommu); 228 229 ptr = &dev_table[dev_data->devid]; 230 231 spin_lock_irqsave(&dev_data->dte_lock, flags); 232 dte->data128[0] = ptr->data128[0]; 233 dte->data128[1] = ptr->data128[1]; 234 spin_unlock_irqrestore(&dev_data->dte_lock, flags); 235 } 236 237 static inline bool pdom_is_v2_pgtbl_mode(struct protection_domain *pdom) 238 { 239 return (pdom && (pdom->pd_mode == PD_MODE_V2)); 240 } 241 242 static inline bool pdom_is_in_pt_mode(struct protection_domain *pdom) 243 { 244 return (pdom->domain.type == IOMMU_DOMAIN_IDENTITY); 245 } 246 247 /* 248 * We cannot support PASID w/ existing v1 page table in the same domain 249 * since it will be nested. However, existing domain w/ v2 page table 250 * or passthrough mode can be used for PASID. 251 */ 252 static inline bool pdom_is_sva_capable(struct protection_domain *pdom) 253 { 254 return pdom_is_v2_pgtbl_mode(pdom) || pdom_is_in_pt_mode(pdom); 255 } 256 257 static inline int get_acpihid_device_id(struct device *dev, 258 struct acpihid_map_entry **entry) 259 { 260 struct acpi_device *adev = ACPI_COMPANION(dev); 261 struct acpihid_map_entry *p, *p1 = NULL; 262 int hid_count = 0; 263 bool fw_bug; 264 265 if (!adev) 266 return -ENODEV; 267 268 list_for_each_entry(p, &acpihid_map, list) { 269 if (acpi_dev_hid_uid_match(adev, p->hid, 270 p->uid[0] ? p->uid : NULL)) { 271 p1 = p; 272 fw_bug = false; 273 hid_count = 1; 274 break; 275 } 276 277 /* 278 * Count HID matches w/o UID, raise FW_BUG but allow exactly one match 279 */ 280 if (acpi_dev_hid_match(adev, p->hid)) { 281 p1 = p; 282 hid_count++; 283 fw_bug = true; 284 } 285 } 286 287 if (!p1) 288 return -EINVAL; 289 if (fw_bug) 290 dev_err_once(dev, FW_BUG "No ACPI device matched UID, but %d device%s matched HID.\n", 291 hid_count, str_plural(hid_count)); 292 if (hid_count > 1) 293 return -EINVAL; 294 if (entry) 295 *entry = p1; 296 297 return p1->devid; 298 } 299 300 static inline int get_device_sbdf_id(struct device *dev) 301 { 302 int sbdf; 303 304 if (dev_is_pci(dev)) 305 sbdf = get_pci_sbdf_id(to_pci_dev(dev)); 306 else 307 sbdf = get_acpihid_device_id(dev, NULL); 308 309 return sbdf; 310 } 311 312 struct dev_table_entry *get_dev_table(struct amd_iommu *iommu) 313 { 314 struct dev_table_entry *dev_table; 315 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 316 317 BUG_ON(pci_seg == NULL); 318 dev_table = pci_seg->dev_table; 319 BUG_ON(dev_table == NULL); 320 321 return dev_table; 322 } 323 324 static inline u16 get_device_segment(struct device *dev) 325 { 326 u16 seg; 327 328 if (dev_is_pci(dev)) { 329 struct pci_dev *pdev = to_pci_dev(dev); 330 331 seg = pci_domain_nr(pdev->bus); 332 } else { 333 u32 devid = get_acpihid_device_id(dev, NULL); 334 335 seg = PCI_SBDF_TO_SEGID(devid); 336 } 337 338 return seg; 339 } 340 341 /* Writes the specific IOMMU for a device into the PCI segment rlookup table */ 342 void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid) 343 { 344 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 345 346 pci_seg->rlookup_table[devid] = iommu; 347 } 348 349 static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid) 350 { 351 struct amd_iommu_pci_seg *pci_seg; 352 353 for_each_pci_segment(pci_seg) { 354 if (pci_seg->id != seg) 355 continue; 356 /* IVRS may not describe every device on the bus */ 357 if (devid > pci_seg->last_bdf) 358 return NULL; 359 return pci_seg->rlookup_table[devid]; 360 } 361 return NULL; 362 } 363 364 static struct amd_iommu *rlookup_amd_iommu(struct device *dev) 365 { 366 u16 seg = get_device_segment(dev); 367 int devid = get_device_sbdf_id(dev); 368 369 if (devid < 0) 370 return NULL; 371 return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid)); 372 } 373 374 static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid) 375 { 376 struct iommu_dev_data *dev_data; 377 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 378 379 dev_data = kzalloc_obj(*dev_data); 380 if (!dev_data) 381 return NULL; 382 383 mutex_init(&dev_data->mutex); 384 spin_lock_init(&dev_data->dte_lock); 385 dev_data->devid = devid; 386 ratelimit_default_init(&dev_data->rs); 387 388 llist_add(&dev_data->dev_data_list, &pci_seg->dev_data_list); 389 return dev_data; 390 } 391 392 struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid) 393 { 394 struct iommu_dev_data *dev_data; 395 struct llist_node *node; 396 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 397 398 if (llist_empty(&pci_seg->dev_data_list)) 399 return NULL; 400 401 node = pci_seg->dev_data_list.first; 402 llist_for_each_entry(dev_data, node, dev_data_list) { 403 if (dev_data->devid == devid) 404 return dev_data; 405 } 406 407 return NULL; 408 } 409 410 static int clone_alias(struct pci_dev *pdev_origin, u16 alias, void *data) 411 { 412 struct dev_table_entry new; 413 struct amd_iommu *iommu; 414 struct iommu_dev_data *dev_data, *alias_data; 415 struct pci_dev *pdev = data; 416 u16 devid = pci_dev_id(pdev); 417 int ret = 0; 418 419 if (devid == alias) 420 return 0; 421 422 iommu = rlookup_amd_iommu(&pdev->dev); 423 if (!iommu) 424 return 0; 425 426 /* Copy the data from pdev */ 427 dev_data = dev_iommu_priv_get(&pdev->dev); 428 if (!dev_data) { 429 pr_err("%s : Failed to get dev_data for 0x%x\n", __func__, devid); 430 ret = -EINVAL; 431 goto out; 432 } 433 get_dte256(iommu, dev_data, &new); 434 435 /* Setup alias */ 436 alias_data = find_dev_data(iommu, alias); 437 if (!alias_data) { 438 pr_err("%s : Failed to get alias dev_data for 0x%x\n", __func__, alias); 439 ret = -EINVAL; 440 goto out; 441 } 442 update_dte256(iommu, alias_data, &new); 443 444 amd_iommu_set_rlookup_table(iommu, alias); 445 out: 446 return ret; 447 } 448 449 static void clone_aliases(struct amd_iommu *iommu, struct device *dev) 450 { 451 struct pci_dev *pdev; 452 453 if (!dev_is_pci(dev)) 454 return; 455 pdev = to_pci_dev(dev); 456 457 /* 458 * The IVRS alias stored in the alias table may not be 459 * part of the PCI DMA aliases if it's bus differs 460 * from the original device. 461 */ 462 clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], pdev); 463 464 pci_for_each_dma_alias(pdev, clone_alias, pdev); 465 } 466 467 static void setup_aliases(struct amd_iommu *iommu, struct device *dev) 468 { 469 struct pci_dev *pdev = to_pci_dev(dev); 470 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 471 u16 ivrs_alias; 472 473 /* For ACPI HID devices, there are no aliases */ 474 if (!dev_is_pci(dev)) 475 return; 476 477 /* 478 * Add the IVRS alias to the pci aliases if it is on the same 479 * bus. The IVRS table may know about a quirk that we don't. 480 */ 481 ivrs_alias = pci_seg->alias_table[pci_dev_id(pdev)]; 482 if (ivrs_alias != pci_dev_id(pdev) && 483 PCI_BUS_NUM(ivrs_alias) == pdev->bus->number) 484 pci_add_dma_alias(pdev, ivrs_alias & 0xff, 1); 485 486 clone_aliases(iommu, dev); 487 } 488 489 static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid) 490 { 491 struct iommu_dev_data *dev_data; 492 493 dev_data = search_dev_data(iommu, devid); 494 495 if (dev_data == NULL) { 496 dev_data = alloc_dev_data(iommu, devid); 497 if (!dev_data) 498 return NULL; 499 500 if (translation_pre_enabled(iommu)) 501 dev_data->defer_attach = true; 502 } 503 504 return dev_data; 505 } 506 507 /* 508 * Find or create an IOMMU group for a acpihid device. 509 */ 510 static struct iommu_group *acpihid_device_group(struct device *dev) 511 { 512 struct acpihid_map_entry *p, *entry = NULL; 513 int devid; 514 515 devid = get_acpihid_device_id(dev, &entry); 516 if (devid < 0) 517 return ERR_PTR(devid); 518 519 list_for_each_entry(p, &acpihid_map, list) { 520 if ((devid == p->devid) && p->group) 521 entry->group = p->group; 522 } 523 524 if (!entry->group) 525 entry->group = generic_device_group(dev); 526 else 527 iommu_group_ref_get(entry->group); 528 529 return entry->group; 530 } 531 532 static inline bool pdev_pasid_supported(struct iommu_dev_data *dev_data) 533 { 534 return (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP); 535 } 536 537 static u32 pdev_get_caps(struct pci_dev *pdev) 538 { 539 int features; 540 u32 flags = 0; 541 542 if (pci_ats_supported(pdev)) 543 flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP; 544 545 if (pci_pri_supported(pdev)) 546 flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP; 547 548 features = pci_pasid_features(pdev); 549 if (features >= 0) { 550 flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP; 551 552 if (features & PCI_PASID_CAP_EXEC) 553 flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP; 554 555 if (features & PCI_PASID_CAP_PRIV) 556 flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP; 557 } 558 559 return flags; 560 } 561 562 static inline int pdev_enable_cap_ats(struct pci_dev *pdev) 563 { 564 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 565 int ret = -EINVAL; 566 567 if (dev_data->ats_enabled) 568 return 0; 569 570 if (amd_iommu_iotlb_sup && 571 (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP)) { 572 ret = pci_enable_ats(pdev, PAGE_SHIFT); 573 if (!ret) { 574 dev_data->ats_enabled = 1; 575 dev_data->ats_qdep = pci_ats_queue_depth(pdev); 576 } 577 } 578 579 return ret; 580 } 581 582 static inline void pdev_disable_cap_ats(struct pci_dev *pdev) 583 { 584 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 585 586 if (dev_data->ats_enabled) { 587 pci_disable_ats(pdev); 588 dev_data->ats_enabled = 0; 589 } 590 } 591 592 static inline int pdev_enable_cap_pri(struct pci_dev *pdev) 593 { 594 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 595 int ret = -EINVAL; 596 597 if (dev_data->pri_enabled) 598 return 0; 599 600 if (!dev_data->ats_enabled) 601 return 0; 602 603 if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) { 604 /* 605 * First reset the PRI state of the device. 606 * FIXME: Hardcode number of outstanding requests for now 607 */ 608 if (!pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32)) { 609 dev_data->pri_enabled = 1; 610 dev_data->pri_tlp = pci_prg_resp_pasid_required(pdev); 611 612 ret = 0; 613 } 614 } 615 616 return ret; 617 } 618 619 static inline void pdev_disable_cap_pri(struct pci_dev *pdev) 620 { 621 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 622 623 if (dev_data->pri_enabled) { 624 pci_disable_pri(pdev); 625 dev_data->pri_enabled = 0; 626 } 627 } 628 629 static inline int pdev_enable_cap_pasid(struct pci_dev *pdev) 630 { 631 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 632 int ret = -EINVAL; 633 634 if (dev_data->pasid_enabled) 635 return 0; 636 637 if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) { 638 /* Only allow access to user-accessible pages */ 639 ret = pci_enable_pasid(pdev, 0); 640 if (!ret) 641 dev_data->pasid_enabled = 1; 642 } 643 644 return ret; 645 } 646 647 static inline void pdev_disable_cap_pasid(struct pci_dev *pdev) 648 { 649 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 650 651 if (dev_data->pasid_enabled) { 652 pci_disable_pasid(pdev); 653 dev_data->pasid_enabled = 0; 654 } 655 } 656 657 static void pdev_enable_caps(struct pci_dev *pdev) 658 { 659 pdev_enable_cap_pasid(pdev); 660 pdev_enable_cap_ats(pdev); 661 pdev_enable_cap_pri(pdev); 662 } 663 664 static void pdev_disable_caps(struct pci_dev *pdev) 665 { 666 pdev_disable_cap_ats(pdev); 667 pdev_disable_cap_pasid(pdev); 668 pdev_disable_cap_pri(pdev); 669 } 670 671 /* 672 * This function checks if the driver got a valid device from the caller to 673 * avoid dereferencing invalid pointers. 674 */ 675 static bool check_device(struct device *dev) 676 { 677 struct amd_iommu_pci_seg *pci_seg; 678 struct amd_iommu *iommu; 679 int devid, sbdf; 680 681 if (!dev) 682 return false; 683 684 sbdf = get_device_sbdf_id(dev); 685 if (sbdf < 0) 686 return false; 687 devid = PCI_SBDF_TO_DEVID(sbdf); 688 689 iommu = rlookup_amd_iommu(dev); 690 if (!iommu) 691 return false; 692 693 /* Out of our scope? */ 694 pci_seg = iommu->pci_seg; 695 if (devid > pci_seg->last_bdf) 696 return false; 697 698 return true; 699 } 700 701 static int iommu_init_device(struct amd_iommu *iommu, struct device *dev) 702 { 703 struct iommu_dev_data *dev_data; 704 int devid, sbdf; 705 706 if (dev_iommu_priv_get(dev)) 707 return 0; 708 709 sbdf = get_device_sbdf_id(dev); 710 if (sbdf < 0) 711 return sbdf; 712 713 devid = PCI_SBDF_TO_DEVID(sbdf); 714 dev_data = find_dev_data(iommu, devid); 715 if (!dev_data) 716 return -ENOMEM; 717 718 dev_data->dev = dev; 719 720 /* 721 * The dev_iommu_priv_set() needes to be called before setup_aliases. 722 * Otherwise, subsequent call to dev_iommu_priv_get() will fail. 723 */ 724 dev_iommu_priv_set(dev, dev_data); 725 setup_aliases(iommu, dev); 726 727 /* 728 * By default we use passthrough mode for IOMMUv2 capable device. 729 * But if amd_iommu=force_isolation is set (e.g. to debug DMA to 730 * invalid address), we ignore the capability for the device so 731 * it'll be forced to go into translation mode. 732 */ 733 if ((iommu_default_passthrough() || !amd_iommu_force_isolation) && 734 dev_is_pci(dev) && amd_iommu_gt_ppr_supported()) { 735 dev_data->flags = pdev_get_caps(to_pci_dev(dev)); 736 } 737 738 return 0; 739 } 740 741 static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev) 742 { 743 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 744 struct dev_table_entry *dev_table = get_dev_table(iommu); 745 int devid, sbdf; 746 747 sbdf = get_device_sbdf_id(dev); 748 if (sbdf < 0) 749 return; 750 751 devid = PCI_SBDF_TO_DEVID(sbdf); 752 pci_seg->rlookup_table[devid] = NULL; 753 memset(&dev_table[devid], 0, sizeof(struct dev_table_entry)); 754 755 setup_aliases(iommu, dev); 756 } 757 758 759 /**************************************************************************** 760 * 761 * Interrupt handling functions 762 * 763 ****************************************************************************/ 764 765 static void dump_dte_entry(struct amd_iommu *iommu, u16 devid) 766 { 767 int i; 768 struct dev_table_entry dte; 769 struct iommu_dev_data *dev_data = find_dev_data(iommu, devid); 770 771 get_dte256(iommu, dev_data, &dte); 772 773 for (i = 0; i < 4; ++i) 774 pr_err("DTE[%d]: %016llx\n", i, dte.data[i]); 775 } 776 777 static void dump_command(unsigned long phys_addr) 778 { 779 struct iommu_cmd *cmd = iommu_phys_to_virt(phys_addr); 780 int i; 781 782 for (i = 0; i < 4; ++i) 783 pr_err("CMD[%d]: %08x\n", i, cmd->data[i]); 784 } 785 786 static void amd_iommu_report_rmp_hw_error(struct amd_iommu *iommu, volatile u32 *event) 787 { 788 struct iommu_dev_data *dev_data = NULL; 789 int devid, vmg_tag, flags; 790 struct pci_dev *pdev; 791 u64 spa; 792 793 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK; 794 vmg_tag = (event[1]) & 0xFFFF; 795 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; 796 spa = ((u64)event[3] << 32) | (event[2] & 0xFFFFFFF8); 797 798 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), 799 devid & 0xff); 800 if (pdev) 801 dev_data = dev_iommu_priv_get(&pdev->dev); 802 803 if (dev_data) { 804 if (__ratelimit(&dev_data->rs)) { 805 pci_err(pdev, "Event logged [RMP_HW_ERROR vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n", 806 vmg_tag, spa, flags); 807 } 808 } else { 809 pr_err_ratelimited("Event logged [RMP_HW_ERROR device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n", 810 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 811 vmg_tag, spa, flags); 812 } 813 814 if (pdev) 815 pci_dev_put(pdev); 816 } 817 818 static void amd_iommu_report_rmp_fault(struct amd_iommu *iommu, volatile u32 *event) 819 { 820 struct iommu_dev_data *dev_data = NULL; 821 int devid, flags_rmp, vmg_tag, flags; 822 struct pci_dev *pdev; 823 u64 gpa; 824 825 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK; 826 flags_rmp = (event[0] >> EVENT_FLAGS_SHIFT) & 0xFF; 827 vmg_tag = (event[1]) & 0xFFFF; 828 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; 829 gpa = ((u64)event[3] << 32) | event[2]; 830 831 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), 832 devid & 0xff); 833 if (pdev) 834 dev_data = dev_iommu_priv_get(&pdev->dev); 835 836 if (dev_data) { 837 if (__ratelimit(&dev_data->rs)) { 838 pci_err(pdev, "Event logged [RMP_PAGE_FAULT vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n", 839 vmg_tag, gpa, flags_rmp, flags); 840 } 841 } else { 842 pr_err_ratelimited("Event logged [RMP_PAGE_FAULT device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n", 843 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 844 vmg_tag, gpa, flags_rmp, flags); 845 } 846 847 if (pdev) 848 pci_dev_put(pdev); 849 } 850 851 #define IS_IOMMU_MEM_TRANSACTION(flags) \ 852 (((flags) & EVENT_FLAG_I) == 0) 853 854 #define IS_WRITE_REQUEST(flags) \ 855 ((flags) & EVENT_FLAG_RW) 856 857 static void amd_iommu_report_page_fault(struct amd_iommu *iommu, 858 u16 devid, u16 domain_id, 859 u64 address, int flags) 860 { 861 struct iommu_dev_data *dev_data = NULL; 862 struct pci_dev *pdev; 863 864 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), 865 devid & 0xff); 866 if (pdev) 867 dev_data = dev_iommu_priv_get(&pdev->dev); 868 869 if (dev_data) { 870 /* 871 * If this is a DMA fault (for which the I(nterrupt) 872 * bit will be unset), allow report_iommu_fault() to 873 * prevent logging it. 874 */ 875 if (IS_IOMMU_MEM_TRANSACTION(flags)) { 876 /* Device not attached to domain properly */ 877 if (dev_data->domain == NULL) { 878 pr_err_ratelimited("Event logged [Device not attached to domain properly]\n"); 879 pr_err_ratelimited(" device=%04x:%02x:%02x.%x domain=0x%04x\n", 880 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), 881 PCI_FUNC(devid), domain_id); 882 goto out; 883 } 884 885 if (!report_iommu_fault(&dev_data->domain->domain, 886 &pdev->dev, address, 887 IS_WRITE_REQUEST(flags) ? 888 IOMMU_FAULT_WRITE : 889 IOMMU_FAULT_READ)) 890 goto out; 891 } 892 893 if (__ratelimit(&dev_data->rs)) { 894 pci_err(pdev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n", 895 domain_id, address, flags); 896 } 897 } else { 898 pr_err_ratelimited("Event logged [IO_PAGE_FAULT device=%04x:%02x:%02x.%x domain=0x%04x address=0x%llx flags=0x%04x]\n", 899 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 900 domain_id, address, flags); 901 } 902 903 out: 904 if (pdev) 905 pci_dev_put(pdev); 906 } 907 908 static void iommu_print_event(struct amd_iommu *iommu, void *__evt) 909 { 910 struct device *dev = iommu->iommu.dev; 911 int type, devid, flags, tag; 912 volatile u32 *event = __evt; 913 int count = 0; 914 u64 address, ctrl; 915 u32 pasid; 916 917 retry: 918 type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK; 919 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK; 920 pasid = (event[0] & EVENT_DOMID_MASK_HI) | 921 (event[1] & EVENT_DOMID_MASK_LO); 922 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; 923 address = (u64)(((u64)event[3]) << 32) | event[2]; 924 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 925 926 if (type == 0) { 927 /* Did we hit the erratum? */ 928 if (++count == LOOP_TIMEOUT) { 929 pr_err("No event written to event log\n"); 930 return; 931 } 932 udelay(1); 933 goto retry; 934 } 935 936 if (type == EVENT_TYPE_IO_FAULT) { 937 amd_iommu_report_page_fault(iommu, devid, pasid, address, flags); 938 return; 939 } 940 941 switch (type) { 942 case EVENT_TYPE_ILL_DEV: 943 dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", 944 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 945 pasid, address, flags); 946 dev_err(dev, "Control Reg : 0x%llx\n", ctrl); 947 dump_dte_entry(iommu, devid); 948 break; 949 case EVENT_TYPE_DEV_TAB_ERR: 950 dev_err(dev, "Event logged [DEV_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x " 951 "address=0x%llx flags=0x%04x]\n", 952 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 953 address, flags); 954 break; 955 case EVENT_TYPE_PAGE_TAB_ERR: 956 dev_err(dev, "Event logged [PAGE_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x pasid=0x%04x address=0x%llx flags=0x%04x]\n", 957 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 958 pasid, address, flags); 959 break; 960 case EVENT_TYPE_ILL_CMD: 961 dev_err(dev, "Event logged [ILLEGAL_COMMAND_ERROR address=0x%llx]\n", address); 962 dump_command(address); 963 break; 964 case EVENT_TYPE_CMD_HARD_ERR: 965 dev_err(dev, "Event logged [COMMAND_HARDWARE_ERROR address=0x%llx flags=0x%04x]\n", 966 address, flags); 967 break; 968 case EVENT_TYPE_IOTLB_INV_TO: 969 dev_err(dev, "Event logged [IOTLB_INV_TIMEOUT device=%04x:%02x:%02x.%x address=0x%llx]\n", 970 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 971 address); 972 break; 973 case EVENT_TYPE_INV_DEV_REQ: 974 dev_err(dev, "Event logged [INVALID_DEVICE_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", 975 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 976 pasid, address, flags); 977 break; 978 case EVENT_TYPE_RMP_FAULT: 979 amd_iommu_report_rmp_fault(iommu, event); 980 break; 981 case EVENT_TYPE_RMP_HW_ERR: 982 amd_iommu_report_rmp_hw_error(iommu, event); 983 break; 984 case EVENT_TYPE_INV_PPR_REQ: 985 pasid = PPR_PASID(*((u64 *)__evt)); 986 tag = event[1] & 0x03FF; 987 dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x tag=0x%03x]\n", 988 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 989 pasid, address, flags, tag); 990 break; 991 default: 992 dev_err(dev, "Event logged [UNKNOWN event[0]=0x%08x event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n", 993 event[0], event[1], event[2], event[3]); 994 } 995 996 /* 997 * To detect the hardware errata 732 we need to clear the 998 * entry back to zero. This issue does not exist on SNP 999 * enabled system. Also this buffer is not writeable on 1000 * SNP enabled system. 1001 */ 1002 if (!amd_iommu_snp_en) 1003 memset(__evt, 0, 4 * sizeof(u32)); 1004 } 1005 1006 static void iommu_poll_events(struct amd_iommu *iommu) 1007 { 1008 u32 head, tail; 1009 1010 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); 1011 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); 1012 1013 while (head != tail) { 1014 iommu_print_event(iommu, iommu->evt_buf + head); 1015 1016 /* Update head pointer of hardware ring-buffer */ 1017 head = (head + EVTLOG_ENTRY_SIZE) % amd_iommu_evtlog_size; 1018 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); 1019 } 1020 1021 } 1022 1023 #ifdef CONFIG_IRQ_REMAP 1024 static int (*iommu_ga_log_notifier)(u32); 1025 1026 int amd_iommu_register_ga_log_notifier(int (*notifier)(u32)) 1027 { 1028 iommu_ga_log_notifier = notifier; 1029 1030 /* 1031 * Ensure all in-flight IRQ handlers run to completion before returning 1032 * to the caller, e.g. to ensure module code isn't unloaded while it's 1033 * being executed in the IRQ handler. 1034 */ 1035 if (!notifier) 1036 synchronize_rcu(); 1037 1038 return 0; 1039 } 1040 EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier); 1041 1042 static void iommu_poll_ga_log(struct amd_iommu *iommu) 1043 { 1044 u32 head, tail; 1045 1046 if (iommu->ga_log == NULL) 1047 return; 1048 1049 head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET); 1050 tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET); 1051 1052 while (head != tail) { 1053 volatile u64 *raw; 1054 u64 log_entry; 1055 1056 raw = (u64 *)(iommu->ga_log + head); 1057 1058 /* Avoid memcpy function-call overhead */ 1059 log_entry = *raw; 1060 1061 /* Update head pointer of hardware ring-buffer */ 1062 head = (head + GA_ENTRY_SIZE) % GA_LOG_SIZE; 1063 writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); 1064 1065 /* Handle GA entry */ 1066 switch (GA_REQ_TYPE(log_entry)) { 1067 case GA_GUEST_NR: 1068 if (!iommu_ga_log_notifier) 1069 break; 1070 1071 pr_debug("%s: devid=%#x, ga_tag=%#x\n", 1072 __func__, GA_DEVID(log_entry), 1073 GA_TAG(log_entry)); 1074 1075 if (iommu_ga_log_notifier(GA_TAG(log_entry)) != 0) 1076 pr_err("GA log notifier failed.\n"); 1077 break; 1078 default: 1079 break; 1080 } 1081 } 1082 } 1083 1084 static void 1085 amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) 1086 { 1087 if (!irq_remapping_enabled || !dev_is_pci(dev) || 1088 !pci_dev_has_default_msi_parent_domain(to_pci_dev(dev))) 1089 return; 1090 1091 dev_set_msi_domain(dev, iommu->ir_domain); 1092 } 1093 1094 #else /* CONFIG_IRQ_REMAP */ 1095 static inline void 1096 amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) { } 1097 #endif /* !CONFIG_IRQ_REMAP */ 1098 1099 static void amd_iommu_handle_irq(void *data, const char *evt_type, 1100 u32 int_mask, u32 overflow_mask, 1101 void (*int_handler)(struct amd_iommu *), 1102 void (*overflow_handler)(struct amd_iommu *)) 1103 { 1104 struct amd_iommu *iommu = (struct amd_iommu *) data; 1105 u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 1106 u32 mask = int_mask | overflow_mask; 1107 1108 while (status & mask) { 1109 /* Enable interrupt sources again */ 1110 writel(mask, iommu->mmio_base + MMIO_STATUS_OFFSET); 1111 1112 if (int_handler) { 1113 pr_devel("Processing IOMMU (ivhd%d) %s Log\n", 1114 iommu->index, evt_type); 1115 int_handler(iommu); 1116 } 1117 1118 if ((status & overflow_mask) && overflow_handler) 1119 overflow_handler(iommu); 1120 1121 /* 1122 * Hardware bug: ERBT1312 1123 * When re-enabling interrupt (by writing 1 1124 * to clear the bit), the hardware might also try to set 1125 * the interrupt bit in the event status register. 1126 * In this scenario, the bit will be set, and disable 1127 * subsequent interrupts. 1128 * 1129 * Workaround: The IOMMU driver should read back the 1130 * status register and check if the interrupt bits are cleared. 1131 * If not, driver will need to go through the interrupt handler 1132 * again and re-clear the bits 1133 */ 1134 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 1135 } 1136 } 1137 1138 irqreturn_t amd_iommu_int_thread_evtlog(int irq, void *data) 1139 { 1140 amd_iommu_handle_irq(data, "Evt", MMIO_STATUS_EVT_INT_MASK, 1141 MMIO_STATUS_EVT_OVERFLOW_MASK, 1142 iommu_poll_events, amd_iommu_restart_event_logging); 1143 1144 return IRQ_HANDLED; 1145 } 1146 1147 irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data) 1148 { 1149 amd_iommu_handle_irq(data, "PPR", MMIO_STATUS_PPR_INT_MASK, 1150 MMIO_STATUS_PPR_OVERFLOW_MASK, 1151 amd_iommu_poll_ppr_log, amd_iommu_restart_ppr_log); 1152 1153 return IRQ_HANDLED; 1154 } 1155 1156 irqreturn_t amd_iommu_int_thread_galog(int irq, void *data) 1157 { 1158 #ifdef CONFIG_IRQ_REMAP 1159 amd_iommu_handle_irq(data, "GA", MMIO_STATUS_GALOG_INT_MASK, 1160 MMIO_STATUS_GALOG_OVERFLOW_MASK, 1161 iommu_poll_ga_log, amd_iommu_restart_ga_log); 1162 #endif 1163 1164 return IRQ_HANDLED; 1165 } 1166 1167 irqreturn_t amd_iommu_int_thread(int irq, void *data) 1168 { 1169 amd_iommu_int_thread_evtlog(irq, data); 1170 amd_iommu_int_thread_pprlog(irq, data); 1171 amd_iommu_int_thread_galog(irq, data); 1172 1173 return IRQ_HANDLED; 1174 } 1175 1176 /**************************************************************************** 1177 * 1178 * IOMMU command queuing functions 1179 * 1180 ****************************************************************************/ 1181 1182 static void dump_command_buffer(struct amd_iommu *iommu) 1183 { 1184 struct iommu_cmd *cmd; 1185 u32 head, tail; 1186 int i; 1187 1188 head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET); 1189 tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); 1190 1191 pr_err("CMD Buffer head=%llu tail=%llu\n", MMIO_CMD_BUFFER_HEAD(head), 1192 MMIO_CMD_BUFFER_TAIL(tail)); 1193 1194 for (i = 0; i < CMD_BUFFER_ENTRIES; i++) { 1195 cmd = (struct iommu_cmd *)(iommu->cmd_buf + i * sizeof(*cmd)); 1196 pr_err("%3d: %08x %08x %08x %08x\n", i, cmd->data[0], cmd->data[1], cmd->data[2], 1197 cmd->data[3]); 1198 } 1199 } 1200 1201 static int wait_on_sem(struct amd_iommu *iommu, u64 data) 1202 { 1203 int i = 0; 1204 1205 /* 1206 * cmd_sem holds a monotonically non-decreasing completion sequence 1207 * number. 1208 */ 1209 while ((__s64)(READ_ONCE(*iommu->cmd_sem) - data) < 0 && 1210 i < LOOP_TIMEOUT) { 1211 udelay(1); 1212 i += 1; 1213 } 1214 1215 if (i == LOOP_TIMEOUT) { 1216 1217 pr_alert("IOMMU %04x:%02x:%02x.%01x: Completion-Wait loop timed out\n", 1218 iommu->pci_seg->id, PCI_BUS_NUM(iommu->devid), 1219 PCI_SLOT(iommu->devid), PCI_FUNC(iommu->devid)); 1220 1221 if (amd_iommu_dump) 1222 DO_ONCE_LITE(dump_command_buffer, iommu); 1223 1224 return -EIO; 1225 } 1226 1227 return 0; 1228 } 1229 1230 static void copy_cmd_to_buffer(struct amd_iommu *iommu, 1231 struct iommu_cmd *cmd) 1232 { 1233 u8 *target; 1234 u32 tail; 1235 1236 /* Copy command to buffer */ 1237 tail = iommu->cmd_buf_tail; 1238 target = iommu->cmd_buf + tail; 1239 memcpy(target, cmd, sizeof(*cmd)); 1240 1241 tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE; 1242 iommu->cmd_buf_tail = tail; 1243 1244 /* Tell the IOMMU about it */ 1245 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); 1246 } 1247 1248 static void build_completion_wait(struct iommu_cmd *cmd, 1249 struct amd_iommu *iommu, 1250 u64 data) 1251 { 1252 u64 paddr = iommu->cmd_sem_paddr; 1253 1254 memset(cmd, 0, sizeof(*cmd)); 1255 cmd->data[0] = lower_32_bits(paddr) | CMD_COMPL_WAIT_STORE_MASK; 1256 cmd->data[1] = upper_32_bits(paddr); 1257 cmd->data[2] = lower_32_bits(data); 1258 cmd->data[3] = upper_32_bits(data); 1259 CMD_SET_TYPE(cmd, CMD_COMPL_WAIT); 1260 } 1261 1262 static void build_inv_dte(struct iommu_cmd *cmd, u16 devid) 1263 { 1264 memset(cmd, 0, sizeof(*cmd)); 1265 cmd->data[0] = devid; 1266 CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY); 1267 } 1268 1269 /* 1270 * Builds an invalidation address which is suitable for one page or multiple 1271 * pages. Sets the size bit (S) as needed is more than one page is flushed. 1272 */ 1273 static inline u64 build_inv_address(u64 address, u64 last) 1274 { 1275 unsigned int sz_lg2; 1276 1277 address &= GENMASK_U64(63, 12); 1278 sz_lg2 = fls64(address ^ last); 1279 if (sz_lg2 <= 12) 1280 return address; 1281 1282 /* 1283 * Encode sz_lg2 according to Table 14: Example Page Size Encodings 1284 * 1285 * See "Note *": 1286 * Address bits 51:32 can be used to encode page sizes greater 1287 * that 4 Gbytes. 1288 * Which we take to mean that the highest page size has bit 1289 * [51]=0, [50:12]=1 1290 * and that coding happens when sz_lg2 is 52. Fall back to full 1291 * invalidation if the size is too big. 1292 * 1293 */ 1294 if (unlikely(sz_lg2 > 52)) 1295 return CMD_INV_IOMMU_ALL_PAGES_ADDRESS | 1296 CMD_INV_IOMMU_PAGES_SIZE_MASK; 1297 1298 /* 1299 * The sz_lg2 calculation with fls() ensures that: 1300 * address & BIT(sz_lg2 - 1) == 0 1301 * Therefore only the 1's need to be added. 8KB requires no 1's 1302 */ 1303 if (sz_lg2 > 13) 1304 address |= GENMASK_U64(sz_lg2 - 2, 12); 1305 return address | CMD_INV_IOMMU_PAGES_SIZE_MASK; 1306 } 1307 1308 static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, 1309 u64 last, u16 domid, ioasid_t pasid, 1310 u32 flags) 1311 { 1312 u64 inv_address = build_inv_address(address, last); 1313 1314 memset(cmd, 0, sizeof(*cmd)); 1315 1316 cmd->data[1] |= domid; 1317 cmd->data[2] = lower_32_bits(inv_address); 1318 cmd->data[3] = upper_32_bits(inv_address); 1319 cmd->data[2] |= flags; 1320 if (flags & CMD_INV_IOMMU_PAGES_GN_MASK) 1321 cmd->data[0] |= pasid; 1322 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES); 1323 } 1324 1325 static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep, 1326 u64 address, u64 last, 1327 ioasid_t pasid, bool gn) 1328 { 1329 u64 inv_address = build_inv_address(address, last); 1330 1331 memset(cmd, 0, sizeof(*cmd)); 1332 1333 cmd->data[0] = devid; 1334 cmd->data[0] |= (qdep & 0xff) << 24; 1335 cmd->data[1] = devid; 1336 cmd->data[2] = lower_32_bits(inv_address); 1337 cmd->data[3] = upper_32_bits(inv_address); 1338 if (gn) { 1339 cmd->data[0] |= ((pasid >> 8) & 0xff) << 16; 1340 cmd->data[1] |= (pasid & 0xff) << 16; 1341 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK; 1342 } 1343 1344 CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES); 1345 } 1346 1347 static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, u32 pasid, 1348 int status, int tag, u8 gn) 1349 { 1350 memset(cmd, 0, sizeof(*cmd)); 1351 1352 cmd->data[0] = devid; 1353 if (gn) { 1354 cmd->data[1] = pasid; 1355 cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK; 1356 } 1357 cmd->data[3] = tag & 0x1ff; 1358 cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT; 1359 1360 CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR); 1361 } 1362 1363 static void build_inv_all(struct iommu_cmd *cmd) 1364 { 1365 memset(cmd, 0, sizeof(*cmd)); 1366 CMD_SET_TYPE(cmd, CMD_INV_ALL); 1367 } 1368 1369 static void build_inv_irt(struct iommu_cmd *cmd, u16 devid) 1370 { 1371 memset(cmd, 0, sizeof(*cmd)); 1372 cmd->data[0] = devid; 1373 CMD_SET_TYPE(cmd, CMD_INV_IRT); 1374 } 1375 1376 /* 1377 * Writes the command to the IOMMUs command buffer and informs the 1378 * hardware about the new command. 1379 */ 1380 static int __iommu_queue_command_sync(struct amd_iommu *iommu, 1381 struct iommu_cmd *cmd, 1382 bool sync) 1383 { 1384 unsigned int count = 0; 1385 u32 left, next_tail; 1386 1387 next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE; 1388 again: 1389 left = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE; 1390 1391 if (left <= 0x20) { 1392 /* Skip udelay() the first time around */ 1393 if (count++) { 1394 if (count == LOOP_TIMEOUT) { 1395 pr_err("Command buffer timeout\n"); 1396 return -EIO; 1397 } 1398 1399 udelay(1); 1400 } 1401 1402 /* Update head and recheck remaining space */ 1403 iommu->cmd_buf_head = readl(iommu->mmio_base + 1404 MMIO_CMD_HEAD_OFFSET); 1405 1406 goto again; 1407 } 1408 1409 copy_cmd_to_buffer(iommu, cmd); 1410 1411 /* Do we need to make sure all commands are processed? */ 1412 iommu->need_sync = sync; 1413 1414 return 0; 1415 } 1416 1417 static int iommu_queue_command_sync(struct amd_iommu *iommu, 1418 struct iommu_cmd *cmd, 1419 bool sync) 1420 { 1421 unsigned long flags; 1422 int ret; 1423 1424 raw_spin_lock_irqsave(&iommu->lock, flags); 1425 ret = __iommu_queue_command_sync(iommu, cmd, sync); 1426 raw_spin_unlock_irqrestore(&iommu->lock, flags); 1427 1428 return ret; 1429 } 1430 1431 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) 1432 { 1433 return iommu_queue_command_sync(iommu, cmd, true); 1434 } 1435 1436 static u64 get_cmdsem_val(struct amd_iommu *iommu) 1437 { 1438 lockdep_assert_held(&iommu->lock); 1439 return ++iommu->cmd_sem_val; 1440 } 1441 1442 /* 1443 * This function queues a completion wait command into the command 1444 * buffer of an IOMMU 1445 */ 1446 static int iommu_completion_wait(struct amd_iommu *iommu) 1447 { 1448 struct iommu_cmd cmd; 1449 unsigned long flags; 1450 int ret; 1451 u64 data; 1452 1453 raw_spin_lock_irqsave(&iommu->lock, flags); 1454 1455 if (!iommu->need_sync) { 1456 /* 1457 * No command has been queued since the last completion-wait. 1458 * A concurrent CPU may have already queued that CWAIT and 1459 * cleared need_sync; need_sync == false only means a covering 1460 * CWAIT is queued, not that all prior commands have completed. 1461 * Wait for the last allocated sequence number so that any 1462 * command queued before this call (possibly on another CPU) 1463 * is guaranteed to have completed before returning. 1464 */ 1465 data = iommu->cmd_sem_val; 1466 raw_spin_unlock_irqrestore(&iommu->lock, flags); 1467 return wait_on_sem(iommu, data); 1468 } 1469 1470 data = get_cmdsem_val(iommu); 1471 build_completion_wait(&cmd, iommu, data); 1472 1473 ret = __iommu_queue_command_sync(iommu, &cmd, false); 1474 raw_spin_unlock_irqrestore(&iommu->lock, flags); 1475 1476 if (ret) 1477 return ret; 1478 1479 return wait_on_sem(iommu, data); 1480 } 1481 1482 static void domain_flush_complete(struct protection_domain *domain) 1483 { 1484 struct pdom_iommu_info *pdom_iommu_info; 1485 unsigned long i; 1486 1487 lockdep_assert_held(&domain->lock); 1488 1489 /* 1490 * Devices of this domain are behind this IOMMU 1491 * We need to wait for completion of all commands. 1492 */ 1493 xa_for_each(&domain->iommu_array, i, pdom_iommu_info) 1494 iommu_completion_wait(pdom_iommu_info->iommu); 1495 } 1496 1497 static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid) 1498 { 1499 struct iommu_cmd cmd; 1500 1501 build_inv_dte(&cmd, devid); 1502 1503 return iommu_queue_command(iommu, &cmd); 1504 } 1505 1506 static void iommu_flush_dte_sync(struct amd_iommu *iommu, u16 devid) 1507 { 1508 int ret; 1509 1510 ret = iommu_flush_dte(iommu, devid); 1511 if (!ret) 1512 iommu_completion_wait(iommu); 1513 } 1514 1515 static void amd_iommu_flush_dte_all(struct amd_iommu *iommu) 1516 { 1517 u32 devid; 1518 u16 last_bdf = iommu->pci_seg->last_bdf; 1519 1520 for (devid = 0; devid <= last_bdf; ++devid) 1521 iommu_flush_dte(iommu, devid); 1522 1523 iommu_completion_wait(iommu); 1524 } 1525 1526 /* 1527 * This function uses heavy locking and may disable irqs for some time. But 1528 * this is no issue because it is only called during resume. 1529 */ 1530 static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu) 1531 { 1532 u32 dom_id; 1533 u16 last_bdf = iommu->pci_seg->last_bdf; 1534 1535 for (dom_id = 0; dom_id <= last_bdf; ++dom_id) { 1536 struct iommu_cmd cmd; 1537 build_inv_iommu_pages(&cmd, 0, U64_MAX, 1538 dom_id, IOMMU_NO_PASID, 1539 CMD_INV_IOMMU_PAGES_PDE_MASK); 1540 iommu_queue_command(iommu, &cmd); 1541 } 1542 1543 iommu_completion_wait(iommu); 1544 } 1545 1546 static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id) 1547 { 1548 struct iommu_cmd cmd; 1549 1550 build_inv_iommu_pages(&cmd, 0, U64_MAX, 1551 dom_id, IOMMU_NO_PASID, 1552 CMD_INV_IOMMU_PAGES_PDE_MASK); 1553 iommu_queue_command(iommu, &cmd); 1554 1555 iommu_completion_wait(iommu); 1556 } 1557 1558 static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, 1559 u64 address, u64 last, u32 flags) 1560 { 1561 int ret = 0; 1562 struct amd_iommu_viommu *aviommu; 1563 1564 list_for_each_entry(aviommu, &pdom->viommu_list, pdom_list) { 1565 unsigned long i; 1566 struct guest_domain_mapping_info *gdom_info; 1567 struct amd_iommu *iommu = container_of(aviommu->core.iommu_dev, 1568 struct amd_iommu, iommu); 1569 1570 xa_lock(&aviommu->gdomid_array); 1571 xa_for_each(&aviommu->gdomid_array, i, gdom_info) { 1572 struct iommu_cmd cmd; 1573 1574 pr_debug("%s: iommu=%#x, hdom_id=%#x\n", __func__, 1575 iommu->devid, gdom_info->hdom_id); 1576 build_inv_iommu_pages(&cmd, address, last, gdom_info->hdom_id, 1577 IOMMU_NO_PASID, flags); 1578 ret |= iommu_queue_command(iommu, &cmd); 1579 } 1580 xa_unlock(&aviommu->gdomid_array); 1581 } 1582 return ret; 1583 } 1584 1585 static void amd_iommu_flush_all(struct amd_iommu *iommu) 1586 { 1587 struct iommu_cmd cmd; 1588 1589 build_inv_all(&cmd); 1590 1591 iommu_queue_command(iommu, &cmd); 1592 iommu_completion_wait(iommu); 1593 } 1594 1595 static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid) 1596 { 1597 struct iommu_cmd cmd; 1598 1599 build_inv_irt(&cmd, devid); 1600 1601 iommu_queue_command(iommu, &cmd); 1602 } 1603 1604 static void amd_iommu_flush_irt_all(struct amd_iommu *iommu) 1605 { 1606 u32 devid; 1607 u16 last_bdf = iommu->pci_seg->last_bdf; 1608 1609 if (iommu->irtcachedis_enabled) 1610 return; 1611 1612 for (devid = 0; devid <= last_bdf; devid++) 1613 iommu_flush_irt(iommu, devid); 1614 1615 iommu_completion_wait(iommu); 1616 } 1617 1618 void amd_iommu_flush_all_caches(struct amd_iommu *iommu) 1619 { 1620 if (check_feature(FEATURE_IA)) { 1621 amd_iommu_flush_all(iommu); 1622 } else { 1623 amd_iommu_flush_dte_all(iommu); 1624 amd_iommu_flush_irt_all(iommu); 1625 amd_iommu_flush_tlb_all(iommu); 1626 } 1627 } 1628 1629 /* 1630 * Command send function for flushing on-device TLB 1631 */ 1632 static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address, 1633 u64 last, ioasid_t pasid, bool gn) 1634 { 1635 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 1636 struct iommu_cmd cmd; 1637 int qdep = dev_data->ats_qdep; 1638 1639 build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, 1640 last, pasid, gn); 1641 1642 return iommu_queue_command(iommu, &cmd); 1643 } 1644 1645 static int device_flush_dte_alias(struct pci_dev *pdev, u16 alias, void *data) 1646 { 1647 struct amd_iommu *iommu = data; 1648 1649 return iommu_flush_dte(iommu, alias); 1650 } 1651 1652 /* 1653 * Command send function for invalidating a device table entry 1654 */ 1655 static int device_flush_dte(struct iommu_dev_data *dev_data) 1656 { 1657 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 1658 struct pci_dev *pdev = NULL; 1659 struct amd_iommu_pci_seg *pci_seg; 1660 u16 alias; 1661 int ret; 1662 1663 if (dev_is_pci(dev_data->dev)) 1664 pdev = to_pci_dev(dev_data->dev); 1665 1666 if (pdev) 1667 ret = pci_for_each_dma_alias(pdev, 1668 device_flush_dte_alias, iommu); 1669 else 1670 ret = iommu_flush_dte(iommu, dev_data->devid); 1671 if (ret) 1672 return ret; 1673 1674 pci_seg = iommu->pci_seg; 1675 alias = pci_seg->alias_table[dev_data->devid]; 1676 if (alias != dev_data->devid) { 1677 ret = iommu_flush_dte(iommu, alias); 1678 if (ret) 1679 return ret; 1680 } 1681 1682 if (dev_data->ats_enabled) { 1683 /* Invalidate the entire contents of an IOTLB */ 1684 ret = device_flush_iotlb(dev_data, 0, U64_MAX, 1685 IOMMU_NO_PASID, false); 1686 } 1687 1688 return ret; 1689 } 1690 1691 static int domain_flush_pages_v2(struct protection_domain *pdom, 1692 u64 address, u64 last, u32 flags) 1693 { 1694 struct iommu_dev_data *dev_data; 1695 struct iommu_cmd cmd; 1696 int ret = 0; 1697 1698 lockdep_assert_held(&pdom->lock); 1699 list_for_each_entry(dev_data, &pdom->dev_list, list) { 1700 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); 1701 u16 domid = dev_data->gcr3_info.domid; 1702 1703 build_inv_iommu_pages(&cmd, address, last, domid, 1704 IOMMU_NO_PASID, 1705 flags | CMD_INV_IOMMU_PAGES_GN_MASK); 1706 1707 ret |= iommu_queue_command(iommu, &cmd); 1708 } 1709 1710 return ret; 1711 } 1712 1713 static int domain_flush_pages_v1(struct protection_domain *pdom, 1714 u64 address, u64 last, u32 flags) 1715 { 1716 struct pdom_iommu_info *pdom_iommu_info; 1717 struct iommu_cmd cmd; 1718 int ret = 0; 1719 unsigned long i; 1720 1721 lockdep_assert_held(&pdom->lock); 1722 1723 build_inv_iommu_pages(&cmd, address, last, 1724 pdom->id, IOMMU_NO_PASID, flags); 1725 1726 xa_for_each(&pdom->iommu_array, i, pdom_iommu_info) { 1727 /* 1728 * Devices of this domain are behind this IOMMU 1729 * We need a TLB flush 1730 */ 1731 ret |= iommu_queue_command(pdom_iommu_info->iommu, &cmd); 1732 } 1733 1734 /* 1735 * A domain w/ v1 table can be a nest parent, which can have 1736 * multiple nested domains. Each nested domain has 1:1 mapping 1737 * between gDomID and hDomID. Therefore, flush every hDomID 1738 * associated to this nest parent domain. 1739 * 1740 * See drivers/iommu/amd/nested.c: amd_iommu_alloc_domain_nested() 1741 */ 1742 if (!list_empty(&pdom->viommu_list)) 1743 ret |= iommu_flush_pages_v1_hdom_ids(pdom, address, last, flags); 1744 1745 return ret; 1746 } 1747 1748 /* 1749 * TLB invalidation function which is called from the mapping functions. 1750 * It flushes range of PTEs of the domain. 1751 */ 1752 static void __domain_flush_pages(struct protection_domain *domain, 1753 u64 address, u64 last, u32 flags) 1754 { 1755 struct iommu_dev_data *dev_data; 1756 int ret = 0; 1757 ioasid_t pasid = IOMMU_NO_PASID; 1758 bool gn = false; 1759 1760 lockdep_assert_held(&domain->lock); 1761 1762 if (pdom_is_v2_pgtbl_mode(domain)) { 1763 gn = true; 1764 ret = domain_flush_pages_v2(domain, address, last, flags); 1765 } else { 1766 ret = domain_flush_pages_v1(domain, address, last, flags); 1767 } 1768 1769 list_for_each_entry(dev_data, &domain->dev_list, list) { 1770 1771 if (!dev_data->ats_enabled) 1772 continue; 1773 1774 ret |= device_flush_iotlb(dev_data, address, last, pasid, gn); 1775 } 1776 1777 WARN_ON(ret); 1778 } 1779 1780 void amd_iommu_domain_flush_pages(struct protection_domain *domain, 1781 u64 address, u64 last, u32 flags) 1782 { 1783 lockdep_assert_held(&domain->lock); 1784 1785 if (likely(!amd_iommu_np_cache) || 1786 unlikely(address == 0 && last == U64_MAX)) { 1787 __domain_flush_pages(domain, address, last, flags); 1788 1789 /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ 1790 domain_flush_complete(domain); 1791 1792 return; 1793 } 1794 1795 /* 1796 * When NpCache is on, we infer that we run in a VM and use a vIOMMU. 1797 * In such setups it is best to avoid flushes of ranges which are not 1798 * naturally aligned, since it would lead to flushes of unmodified 1799 * PTEs. Such flushes would require the hypervisor to do more work than 1800 * necessary. Therefore, perform repeated flushes of aligned ranges 1801 * until you cover the range. Each iteration flushes the smaller 1802 * between the natural alignment of the address that we flush and the 1803 * greatest naturally aligned region that fits in the range. 1804 */ 1805 while (address <= last) { 1806 unsigned int sz_lg2 = ilog2(last - address + 1); 1807 u64 flush_last; 1808 1809 if (likely(address)) 1810 sz_lg2 = min_t(unsigned int, sz_lg2, __ffs64(address)); 1811 1812 flush_last = address + (1ULL << sz_lg2) - 1; 1813 __domain_flush_pages(domain, address, flush_last, flags); 1814 if (check_add_overflow(flush_last, 1, &address)) 1815 break; 1816 } 1817 1818 /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ 1819 domain_flush_complete(domain); 1820 } 1821 1822 /* Flush the whole IO/TLB for a given protection domain - including PDE */ 1823 static void amd_iommu_domain_flush_all(struct protection_domain *domain) 1824 { 1825 amd_iommu_domain_flush_pages(domain, 0, U64_MAX, 1826 CMD_INV_IOMMU_PAGES_PDE_MASK); 1827 } 1828 1829 void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, 1830 ioasid_t pasid, u64 address, u64 last) 1831 { 1832 struct iommu_cmd cmd; 1833 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); 1834 1835 build_inv_iommu_pages(&cmd, address, last, 1836 dev_data->gcr3_info.domid, pasid, 1837 CMD_INV_IOMMU_PAGES_GN_MASK | 1838 CMD_INV_IOMMU_PAGES_PDE_MASK); 1839 iommu_queue_command(iommu, &cmd); 1840 1841 if (dev_data->ats_enabled) 1842 device_flush_iotlb(dev_data, address, last, pasid, true); 1843 1844 iommu_completion_wait(iommu); 1845 } 1846 1847 static void dev_flush_pasid_all(struct iommu_dev_data *dev_data, 1848 ioasid_t pasid) 1849 { 1850 amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0, U64_MAX); 1851 } 1852 1853 int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag) 1854 { 1855 struct iommu_dev_data *dev_data; 1856 struct amd_iommu *iommu; 1857 struct iommu_cmd cmd; 1858 1859 dev_data = dev_iommu_priv_get(dev); 1860 iommu = get_amd_iommu_from_dev(dev); 1861 1862 build_complete_ppr(&cmd, dev_data->devid, pasid, status, 1863 tag, dev_data->pri_tlp); 1864 1865 return iommu_queue_command(iommu, &cmd); 1866 } 1867 1868 /**************************************************************************** 1869 * 1870 * The next functions belong to the domain allocation. A domain is 1871 * allocated for every IOMMU as the default domain. If device isolation 1872 * is enabled, every device get its own domain. The most important thing 1873 * about domains is the page table mapping the DMA address space they 1874 * contain. 1875 * 1876 ****************************************************************************/ 1877 int amd_iommu_pdom_id_alloc(void) 1878 { 1879 return ida_alloc_range(&pdom_ids, 1, MAX_DOMAIN_ID - 1, GFP_ATOMIC); 1880 } 1881 1882 int amd_iommu_pdom_id_reserve(u16 id, gfp_t gfp) 1883 { 1884 return ida_alloc_range(&pdom_ids, id, id, gfp); 1885 } 1886 1887 void amd_iommu_pdom_id_free(int id) 1888 { 1889 ida_free(&pdom_ids, id); 1890 } 1891 1892 void amd_iommu_pdom_id_destroy(void) 1893 { 1894 ida_destroy(&pdom_ids); 1895 } 1896 1897 static void free_gcr3_tbl_level1(u64 *tbl) 1898 { 1899 u64 *ptr; 1900 int i; 1901 1902 for (i = 0; i < 512; ++i) { 1903 if (!(tbl[i] & GCR3_VALID)) 1904 continue; 1905 1906 ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK); 1907 1908 iommu_free_pages(ptr); 1909 } 1910 } 1911 1912 static void free_gcr3_tbl_level2(u64 *tbl) 1913 { 1914 u64 *ptr; 1915 int i; 1916 1917 for (i = 0; i < 512; ++i) { 1918 if (!(tbl[i] & GCR3_VALID)) 1919 continue; 1920 1921 ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK); 1922 1923 free_gcr3_tbl_level1(ptr); 1924 } 1925 } 1926 1927 static void free_gcr3_table(struct gcr3_tbl_info *gcr3_info) 1928 { 1929 if (gcr3_info->glx == 2) 1930 free_gcr3_tbl_level2(gcr3_info->gcr3_tbl); 1931 else if (gcr3_info->glx == 1) 1932 free_gcr3_tbl_level1(gcr3_info->gcr3_tbl); 1933 else 1934 WARN_ON_ONCE(gcr3_info->glx != 0); 1935 1936 gcr3_info->glx = 0; 1937 1938 /* Free per device domain ID */ 1939 amd_iommu_pdom_id_free(gcr3_info->domid); 1940 1941 iommu_free_pages(gcr3_info->gcr3_tbl); 1942 gcr3_info->gcr3_tbl = NULL; 1943 } 1944 1945 /* 1946 * Number of GCR3 table levels required. Level must be 4-Kbyte 1947 * page and can contain up to 512 entries. 1948 */ 1949 static int get_gcr3_levels(int pasids) 1950 { 1951 int levels; 1952 1953 if (pasids == -1) 1954 return amd_iommu_max_glx_val; 1955 1956 levels = get_count_order(pasids); 1957 1958 return levels ? (DIV_ROUND_UP(levels, 9) - 1) : levels; 1959 } 1960 1961 static int setup_gcr3_table(struct gcr3_tbl_info *gcr3_info, 1962 struct amd_iommu *iommu, int pasids) 1963 { 1964 int levels = get_gcr3_levels(pasids); 1965 int nid = iommu ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; 1966 int domid; 1967 1968 if (levels > amd_iommu_max_glx_val) 1969 return -EINVAL; 1970 1971 if (gcr3_info->gcr3_tbl) 1972 return -EBUSY; 1973 1974 /* Allocate per device domain ID */ 1975 domid = amd_iommu_pdom_id_alloc(); 1976 if (domid <= 0) 1977 return -ENOSPC; 1978 gcr3_info->domid = domid; 1979 1980 gcr3_info->gcr3_tbl = iommu_alloc_pages_node_sz(nid, GFP_ATOMIC, SZ_4K); 1981 if (gcr3_info->gcr3_tbl == NULL) { 1982 amd_iommu_pdom_id_free(domid); 1983 return -ENOMEM; 1984 } 1985 1986 gcr3_info->glx = levels; 1987 1988 return 0; 1989 } 1990 1991 static u64 *__get_gcr3_pte(struct gcr3_tbl_info *gcr3_info, 1992 ioasid_t pasid, bool alloc) 1993 { 1994 int index; 1995 u64 *pte; 1996 u64 *root = gcr3_info->gcr3_tbl; 1997 int level = gcr3_info->glx; 1998 1999 while (true) { 2000 2001 index = (pasid >> (9 * level)) & 0x1ff; 2002 pte = &root[index]; 2003 2004 if (level == 0) 2005 break; 2006 2007 if (!(*pte & GCR3_VALID)) { 2008 if (!alloc) 2009 return NULL; 2010 2011 root = (void *)get_zeroed_page(GFP_ATOMIC); 2012 if (root == NULL) 2013 return NULL; 2014 2015 *pte = iommu_virt_to_phys(root) | GCR3_VALID; 2016 } 2017 2018 root = iommu_phys_to_virt(*pte & PAGE_MASK); 2019 2020 level -= 1; 2021 } 2022 2023 return pte; 2024 } 2025 2026 static int update_gcr3(struct iommu_dev_data *dev_data, 2027 ioasid_t pasid, unsigned long gcr3, bool set) 2028 { 2029 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 2030 u64 *pte; 2031 2032 pte = __get_gcr3_pte(gcr3_info, pasid, true); 2033 if (pte == NULL) 2034 return -ENOMEM; 2035 2036 if (set) 2037 *pte = (gcr3 & PAGE_MASK) | GCR3_VALID; 2038 else 2039 *pte = 0; 2040 2041 dev_flush_pasid_all(dev_data, pasid); 2042 return 0; 2043 } 2044 2045 int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid, 2046 unsigned long gcr3) 2047 { 2048 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 2049 int ret; 2050 2051 iommu_group_mutex_assert(dev_data->dev); 2052 2053 ret = update_gcr3(dev_data, pasid, gcr3, true); 2054 if (ret) 2055 return ret; 2056 2057 gcr3_info->pasid_cnt++; 2058 return ret; 2059 } 2060 2061 int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid) 2062 { 2063 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 2064 int ret; 2065 2066 iommu_group_mutex_assert(dev_data->dev); 2067 2068 ret = update_gcr3(dev_data, pasid, 0, false); 2069 if (ret) 2070 return ret; 2071 2072 gcr3_info->pasid_cnt--; 2073 return ret; 2074 } 2075 2076 /* 2077 * Note: 2078 * The old value for GCR3 table and GPT have been cleared from caller. 2079 */ 2080 static void set_dte_gcr3_table(struct iommu_dev_data *dev_data, 2081 struct dev_table_entry *new) 2082 { 2083 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 2084 u64 gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl); 2085 2086 new->data[0] |= DTE_FLAG_TV | 2087 (dev_data->ppr ? DTE_FLAG_PPR : 0) | 2088 (pdom_is_v2_pgtbl_mode(dev_data->domain) ? DTE_FLAG_GIOV : 0) | 2089 DTE_FLAG_GV | 2090 FIELD_PREP(DTE_GLX, gcr3_info->glx) | 2091 FIELD_PREP(DTE_GCR3_14_12, gcr3 >> 12) | 2092 DTE_FLAG_IR | DTE_FLAG_IW; 2093 2094 new->data[1] |= FIELD_PREP(DTE_DOMID_MASK, dev_data->gcr3_info.domid) | 2095 FIELD_PREP(DTE_GCR3_30_15, gcr3 >> 15) | 2096 (dev_data->ats_enabled ? DTE_FLAG_IOTLB : 0) | 2097 FIELD_PREP(DTE_GCR3_51_31, gcr3 >> 31); 2098 2099 /* Guest page table can only support 4 and 5 levels */ 2100 if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) 2101 new->data[2] |= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_5_LEVEL); 2102 else 2103 new->data[2] |= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_4_LEVEL); 2104 } 2105 2106 void amd_iommu_set_dte_v1(struct iommu_dev_data *dev_data, 2107 struct protection_domain *domain, u16 domid, 2108 struct pt_iommu_amdv1_hw_info *pt_info, 2109 struct dev_table_entry *new) 2110 { 2111 u64 host_pt_root = __sme_set(pt_info->host_pt_root); 2112 2113 /* Note Dirty tracking is used for v1 table only for now */ 2114 new->data[0] |= DTE_FLAG_TV | 2115 FIELD_PREP(DTE_MODE_MASK, pt_info->mode) | 2116 (domain->dirty_tracking ? DTE_FLAG_HAD : 0) | 2117 FIELD_PREP(DTE_HOST_TRP, host_pt_root >> 12) | 2118 DTE_FLAG_IR | DTE_FLAG_IW; 2119 2120 new->data[1] |= FIELD_PREP(DTE_DOMID_MASK, domid) | 2121 (dev_data->ats_enabled ? DTE_FLAG_IOTLB : 0); 2122 } 2123 2124 static void set_dte_v1(struct iommu_dev_data *dev_data, 2125 struct protection_domain *domain, u16 domid, 2126 phys_addr_t top_paddr, unsigned int top_level, 2127 struct dev_table_entry *new) 2128 { 2129 struct pt_iommu_amdv1_hw_info pt_info; 2130 2131 /* 2132 * When updating the IO pagetable, the new top and level 2133 * are provided as parameters. For other operations i.e. 2134 * device attach, retrieve the current pagetable info 2135 * via the IOMMU PT API. 2136 */ 2137 if (top_paddr) { 2138 pt_info.host_pt_root = top_paddr; 2139 pt_info.mode = top_level + 1; 2140 } else { 2141 WARN_ON(top_paddr || top_level); 2142 pt_iommu_amdv1_hw_info(&domain->amdv1, &pt_info); 2143 } 2144 2145 amd_iommu_set_dte_v1(dev_data, domain, domid, &pt_info, new); 2146 } 2147 2148 static void set_dte_passthrough(struct iommu_dev_data *dev_data, 2149 struct protection_domain *domain, 2150 struct dev_table_entry *new) 2151 { 2152 new->data[0] |= DTE_FLAG_TV | DTE_FLAG_IR | DTE_FLAG_IW; 2153 2154 new->data[1] |= FIELD_PREP(DTE_DOMID_MASK, domain->id) | 2155 (dev_data->ats_enabled ? DTE_FLAG_IOTLB : 0); 2156 2157 } 2158 2159 static void set_dte_entry(struct amd_iommu *iommu, 2160 struct iommu_dev_data *dev_data, 2161 phys_addr_t top_paddr, unsigned int top_level) 2162 { 2163 u32 old_domid; 2164 struct dev_table_entry new = {}; 2165 struct protection_domain *domain = dev_data->domain; 2166 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 2167 struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid]; 2168 2169 amd_iommu_make_clear_dte(dev_data, &new); 2170 2171 old_domid = READ_ONCE(dte->data[1]) & DTE_DOMID_MASK; 2172 if (gcr3_info->gcr3_tbl) 2173 set_dte_gcr3_table(dev_data, &new); 2174 else if (domain->domain.type == IOMMU_DOMAIN_IDENTITY) 2175 set_dte_passthrough(dev_data, domain, &new); 2176 else if ((domain->domain.type & __IOMMU_DOMAIN_PAGING) && 2177 domain->pd_mode == PD_MODE_V1) 2178 set_dte_v1(dev_data, domain, domain->id, top_paddr, top_level, &new); 2179 else 2180 WARN_ON(true); 2181 2182 amd_iommu_update_dte(iommu, dev_data, &new); 2183 2184 /* 2185 * A kdump kernel might be replacing a domain ID that was copied from 2186 * the previous kernel--if so, it needs to flush the translation cache 2187 * entries for the old domain ID that is being overwritten 2188 */ 2189 if (old_domid) { 2190 amd_iommu_flush_tlb_domid(iommu, old_domid); 2191 } 2192 } 2193 2194 /* 2195 * Clear DMA-remap related flags to block all DMA (blockeded domain) 2196 */ 2197 static void clear_dte_entry(struct amd_iommu *iommu, struct iommu_dev_data *dev_data) 2198 { 2199 struct dev_table_entry new = {}; 2200 2201 amd_iommu_make_clear_dte(dev_data, &new); 2202 amd_iommu_update_dte(iommu, dev_data, &new); 2203 } 2204 2205 /* Update and flush DTE for the given device */ 2206 static void dev_update_dte(struct iommu_dev_data *dev_data, bool set) 2207 { 2208 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); 2209 2210 if (set) 2211 set_dte_entry(iommu, dev_data, 0, 0); 2212 else 2213 clear_dte_entry(iommu, dev_data); 2214 } 2215 2216 /* 2217 * If domain is SVA capable then initialize GCR3 table. Also if domain is 2218 * in v2 page table mode then update GCR3[0]. 2219 */ 2220 static int init_gcr3_table(struct iommu_dev_data *dev_data, 2221 struct protection_domain *pdom) 2222 { 2223 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 2224 int max_pasids = dev_data->max_pasids; 2225 struct pt_iommu_x86_64_hw_info pt_info; 2226 int ret = 0; 2227 2228 /* 2229 * If domain is in pt mode then setup GCR3 table only if device 2230 * is PASID capable 2231 */ 2232 if (pdom_is_in_pt_mode(pdom) && !pdev_pasid_supported(dev_data)) 2233 return ret; 2234 2235 /* 2236 * By default, setup GCR3 table to support MAX PASIDs 2237 * supported by the device/IOMMU. 2238 */ 2239 ret = setup_gcr3_table(&dev_data->gcr3_info, iommu, 2240 max_pasids > 0 ? max_pasids : 1); 2241 if (ret) 2242 return ret; 2243 2244 /* Setup GCR3[0] only if domain is setup with v2 page table mode */ 2245 if (!pdom_is_v2_pgtbl_mode(pdom)) 2246 return ret; 2247 2248 pt_iommu_x86_64_hw_info(&pdom->amdv2, &pt_info); 2249 ret = update_gcr3(dev_data, 0, __sme_set(pt_info.gcr3_pt), true); 2250 if (ret) 2251 free_gcr3_table(&dev_data->gcr3_info); 2252 2253 return ret; 2254 } 2255 2256 static void destroy_gcr3_table(struct iommu_dev_data *dev_data, 2257 struct protection_domain *pdom) 2258 { 2259 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 2260 2261 if (pdom_is_v2_pgtbl_mode(pdom)) 2262 update_gcr3(dev_data, 0, 0, false); 2263 2264 if (gcr3_info->gcr3_tbl == NULL) 2265 return; 2266 2267 free_gcr3_table(gcr3_info); 2268 } 2269 2270 static int pdom_attach_iommu(struct amd_iommu *iommu, 2271 struct protection_domain *pdom) 2272 { 2273 struct pdom_iommu_info *pdom_iommu_info, *curr; 2274 unsigned long flags; 2275 int ret = 0; 2276 2277 spin_lock_irqsave(&pdom->lock, flags); 2278 2279 pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index); 2280 if (pdom_iommu_info) { 2281 pdom_iommu_info->refcnt++; 2282 goto out_unlock; 2283 } 2284 2285 pdom_iommu_info = kzalloc_obj(*pdom_iommu_info, GFP_ATOMIC); 2286 if (!pdom_iommu_info) { 2287 ret = -ENOMEM; 2288 goto out_unlock; 2289 } 2290 2291 pdom_iommu_info->iommu = iommu; 2292 pdom_iommu_info->refcnt = 1; 2293 2294 curr = xa_cmpxchg(&pdom->iommu_array, iommu->index, 2295 NULL, pdom_iommu_info, GFP_ATOMIC); 2296 if (curr) { 2297 kfree(pdom_iommu_info); 2298 ret = -ENOSPC; 2299 goto out_unlock; 2300 } 2301 2302 out_unlock: 2303 spin_unlock_irqrestore(&pdom->lock, flags); 2304 return ret; 2305 } 2306 2307 static void pdom_detach_iommu(struct amd_iommu *iommu, 2308 struct protection_domain *pdom) 2309 { 2310 struct pdom_iommu_info *pdom_iommu_info; 2311 unsigned long flags; 2312 2313 spin_lock_irqsave(&pdom->lock, flags); 2314 2315 pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index); 2316 if (!pdom_iommu_info) { 2317 spin_unlock_irqrestore(&pdom->lock, flags); 2318 return; 2319 } 2320 2321 pdom_iommu_info->refcnt--; 2322 if (pdom_iommu_info->refcnt == 0) { 2323 xa_erase(&pdom->iommu_array, iommu->index); 2324 kfree(pdom_iommu_info); 2325 } 2326 2327 spin_unlock_irqrestore(&pdom->lock, flags); 2328 } 2329 2330 /* 2331 * If a device is not yet associated with a domain, this function makes the 2332 * device visible in the domain 2333 */ 2334 static int attach_device(struct device *dev, 2335 struct protection_domain *domain) 2336 { 2337 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2338 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 2339 struct pci_dev *pdev; 2340 unsigned long flags; 2341 int ret = 0; 2342 2343 mutex_lock(&dev_data->mutex); 2344 2345 if (dev_data->domain != NULL) { 2346 ret = -EBUSY; 2347 goto out; 2348 } 2349 2350 /* Do reference counting */ 2351 ret = pdom_attach_iommu(iommu, domain); 2352 if (ret) 2353 goto out; 2354 2355 /* Setup GCR3 table */ 2356 if (pdom_is_sva_capable(domain)) { 2357 ret = init_gcr3_table(dev_data, domain); 2358 if (ret) { 2359 pdom_detach_iommu(iommu, domain); 2360 goto out; 2361 } 2362 } 2363 2364 pdev = dev_is_pci(dev_data->dev) ? to_pci_dev(dev_data->dev) : NULL; 2365 if (pdev && pdom_is_sva_capable(domain)) { 2366 pdev_enable_caps(pdev); 2367 2368 /* 2369 * Device can continue to function even if IOPF 2370 * enablement failed. Hence in error path just 2371 * disable device PRI support. 2372 */ 2373 if (amd_iommu_iopf_add_device(iommu, dev_data)) 2374 pdev_disable_cap_pri(pdev); 2375 } else if (pdev) { 2376 pdev_enable_cap_ats(pdev); 2377 } 2378 2379 /* Update data structures */ 2380 dev_data->domain = domain; 2381 spin_lock_irqsave(&domain->lock, flags); 2382 list_add(&dev_data->list, &domain->dev_list); 2383 spin_unlock_irqrestore(&domain->lock, flags); 2384 2385 /* Update device table */ 2386 dev_update_dte(dev_data, true); 2387 2388 out: 2389 mutex_unlock(&dev_data->mutex); 2390 2391 return ret; 2392 } 2393 2394 /* 2395 * Removes a device from a protection domain (with devtable_lock held) 2396 */ 2397 static void detach_device(struct device *dev) 2398 { 2399 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2400 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 2401 struct protection_domain *domain = dev_data->domain; 2402 unsigned long flags; 2403 2404 mutex_lock(&dev_data->mutex); 2405 2406 /* 2407 * First check if the device is still attached. It might already 2408 * be detached from its domain because the generic 2409 * iommu_detach_group code detached it and we try again here in 2410 * our alias handling. 2411 */ 2412 if (WARN_ON(!dev_data->domain)) 2413 goto out; 2414 2415 /* Remove IOPF handler */ 2416 if (dev_data->ppr) { 2417 iopf_queue_flush_dev(dev); 2418 amd_iommu_iopf_remove_device(iommu, dev_data); 2419 } 2420 2421 if (dev_is_pci(dev)) 2422 pdev_disable_caps(to_pci_dev(dev)); 2423 2424 /* Clear DTE and flush the entry */ 2425 dev_update_dte(dev_data, false); 2426 2427 /* Flush IOTLB and wait for the flushes to finish */ 2428 spin_lock_irqsave(&domain->lock, flags); 2429 amd_iommu_domain_flush_all(domain); 2430 list_del(&dev_data->list); 2431 spin_unlock_irqrestore(&domain->lock, flags); 2432 2433 /* Clear GCR3 table */ 2434 if (pdom_is_sva_capable(domain)) 2435 destroy_gcr3_table(dev_data, domain); 2436 2437 /* Update data structures */ 2438 dev_data->domain = NULL; 2439 2440 /* decrease reference counters - needs to happen after the flushes */ 2441 pdom_detach_iommu(iommu, domain); 2442 2443 out: 2444 mutex_unlock(&dev_data->mutex); 2445 } 2446 2447 static struct iommu_device *amd_iommu_probe_device(struct device *dev) 2448 { 2449 struct iommu_device *iommu_dev; 2450 struct amd_iommu *iommu; 2451 struct iommu_dev_data *dev_data; 2452 int ret; 2453 2454 if (!check_device(dev)) 2455 return ERR_PTR(-ENODEV); 2456 2457 iommu = rlookup_amd_iommu(dev); 2458 if (!iommu) 2459 return ERR_PTR(-ENODEV); 2460 2461 /* Not registered yet? */ 2462 if (!iommu->iommu.ops) 2463 return ERR_PTR(-ENODEV); 2464 2465 if (dev_iommu_priv_get(dev)) 2466 return &iommu->iommu; 2467 2468 ret = iommu_init_device(iommu, dev); 2469 if (ret) { 2470 dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); 2471 iommu_dev = ERR_PTR(ret); 2472 iommu_ignore_device(iommu, dev); 2473 goto out_err; 2474 } 2475 2476 amd_iommu_set_pci_msi_domain(dev, iommu); 2477 iommu_dev = &iommu->iommu; 2478 2479 /* 2480 * If IOMMU and device supports PASID then it will contain max 2481 * supported PASIDs, else it will be zero. 2482 */ 2483 dev_data = dev_iommu_priv_get(dev); 2484 if (amd_iommu_pasid_supported() && dev_is_pci(dev) && 2485 pdev_pasid_supported(dev_data)) { 2486 dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids, 2487 pci_max_pasids(to_pci_dev(dev))); 2488 } 2489 2490 if (amd_iommu_pgtable == PD_MODE_NONE) { 2491 pr_warn_once("%s: DMA translation not supported by iommu.\n", 2492 __func__); 2493 iommu_dev = ERR_PTR(-ENODEV); 2494 goto out_err; 2495 } 2496 2497 iommu_completion_wait(iommu); 2498 2499 if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2)) 2500 dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K; 2501 else 2502 dev_data->max_irqs = MAX_IRQS_PER_TABLE_512; 2503 2504 if (dev_is_pci(dev)) 2505 pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT); 2506 2507 out_err: 2508 return iommu_dev; 2509 } 2510 2511 static void amd_iommu_release_device(struct device *dev) 2512 { 2513 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2514 2515 WARN_ON(dev_data->domain); 2516 2517 /* 2518 * We keep dev_data around for unplugged devices and reuse it when the 2519 * device is re-plugged - not doing so would introduce a ton of races. 2520 */ 2521 } 2522 2523 static struct iommu_group *amd_iommu_device_group(struct device *dev) 2524 { 2525 if (dev_is_pci(dev)) 2526 return pci_device_group(dev); 2527 2528 return acpihid_device_group(dev); 2529 } 2530 2531 /***************************************************************************** 2532 * 2533 * The following functions belong to the exported interface of AMD IOMMU 2534 * 2535 * This interface allows access to lower level functions of the IOMMU 2536 * like protection domain handling and assignement of devices to domains 2537 * which is not possible with the dma_ops interface. 2538 * 2539 *****************************************************************************/ 2540 2541 static void protection_domain_init(struct protection_domain *domain) 2542 { 2543 spin_lock_init(&domain->lock); 2544 INIT_LIST_HEAD(&domain->dev_list); 2545 INIT_LIST_HEAD(&domain->dev_data_list); 2546 INIT_LIST_HEAD(&domain->viommu_list); 2547 xa_init(&domain->iommu_array); 2548 } 2549 2550 struct protection_domain *protection_domain_alloc(void) 2551 { 2552 struct protection_domain *domain; 2553 int domid; 2554 2555 domain = kzalloc_obj(*domain); 2556 if (!domain) 2557 return NULL; 2558 2559 domid = amd_iommu_pdom_id_alloc(); 2560 if (domid <= 0) { 2561 kfree(domain); 2562 return NULL; 2563 } 2564 domain->id = domid; 2565 2566 protection_domain_init(domain); 2567 2568 return domain; 2569 } 2570 2571 static bool amd_iommu_hd_support(struct amd_iommu *iommu) 2572 { 2573 if (amd_iommu_hatdis) 2574 return false; 2575 2576 return iommu && (iommu->features & FEATURE_HDSUP); 2577 } 2578 2579 static spinlock_t *amd_iommu_get_top_lock(struct pt_iommu *iommupt) 2580 { 2581 struct protection_domain *pdom = 2582 container_of(iommupt, struct protection_domain, iommu); 2583 2584 return &pdom->lock; 2585 } 2586 2587 /* 2588 * Update all HW references to the domain with a new pgtable configuration. 2589 */ 2590 static void amd_iommu_change_top(struct pt_iommu *iommu_table, 2591 phys_addr_t top_paddr, unsigned int top_level) 2592 { 2593 struct protection_domain *pdom = 2594 container_of(iommu_table, struct protection_domain, iommu); 2595 struct iommu_dev_data *dev_data; 2596 2597 lockdep_assert_held(&pdom->lock); 2598 2599 /* Update the DTE for all devices attached to this domain */ 2600 list_for_each_entry(dev_data, &pdom->dev_list, list) { 2601 struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev); 2602 2603 /* Update the HW references with the new level and top ptr */ 2604 set_dte_entry(iommu, dev_data, top_paddr, top_level); 2605 clone_aliases(iommu, dev_data->dev); 2606 } 2607 2608 list_for_each_entry(dev_data, &pdom->dev_list, list) 2609 device_flush_dte(dev_data); 2610 2611 domain_flush_complete(pdom); 2612 } 2613 2614 /* 2615 * amd_iommu_iotlb_sync_map() is used to generate flushes for non-present to 2616 * present (ie mapping) operations. It is a NOP if the IOMMU doesn't have non 2617 * present caching (like hypervisor shadowing). 2618 */ 2619 static int amd_iommu_iotlb_sync_map(struct iommu_domain *dom, 2620 unsigned long iova, size_t size) 2621 { 2622 struct protection_domain *domain = to_pdomain(dom); 2623 unsigned long flags; 2624 2625 if (likely(!amd_iommu_np_cache)) 2626 return 0; 2627 2628 spin_lock_irqsave(&domain->lock, flags); 2629 amd_iommu_domain_flush_pages(domain, iova, iova + size - 1, 2630 CMD_INV_IOMMU_PAGES_PDE_MASK); 2631 spin_unlock_irqrestore(&domain->lock, flags); 2632 return 0; 2633 } 2634 2635 static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain) 2636 { 2637 struct protection_domain *dom = to_pdomain(domain); 2638 unsigned long flags; 2639 2640 spin_lock_irqsave(&dom->lock, flags); 2641 amd_iommu_domain_flush_all(dom); 2642 spin_unlock_irqrestore(&dom->lock, flags); 2643 } 2644 2645 static void amd_iommu_iotlb_sync(struct iommu_domain *domain, 2646 struct iommu_iotlb_gather *gather) 2647 { 2648 struct protection_domain *dom = to_pdomain(domain); 2649 unsigned long flags; 2650 2651 spin_lock_irqsave(&dom->lock, flags); 2652 amd_iommu_domain_flush_pages(dom, gather->start, gather->end, 2653 iommu_pages_list_empty(&gather->freelist) ? 2654 0 : CMD_INV_IOMMU_PAGES_PDE_MASK); 2655 spin_unlock_irqrestore(&dom->lock, flags); 2656 iommu_put_pages_list(&gather->freelist); 2657 } 2658 2659 static const struct pt_iommu_driver_ops amd_hw_driver_ops_v1 = { 2660 .get_top_lock = amd_iommu_get_top_lock, 2661 .change_top = amd_iommu_change_top, 2662 }; 2663 2664 static const struct iommu_domain_ops amdv1_ops = { 2665 IOMMU_PT_DOMAIN_OPS(amdv1), 2666 .iotlb_sync_map = amd_iommu_iotlb_sync_map, 2667 .flush_iotlb_all = amd_iommu_flush_iotlb_all, 2668 .iotlb_sync = amd_iommu_iotlb_sync, 2669 .attach_dev = amd_iommu_attach_device, 2670 .free = amd_iommu_domain_free, 2671 .enforce_cache_coherency = amd_iommu_enforce_cache_coherency, 2672 }; 2673 2674 static const struct iommu_dirty_ops amdv1_dirty_ops = { 2675 IOMMU_PT_DIRTY_OPS(amdv1), 2676 .set_dirty_tracking = amd_iommu_set_dirty_tracking, 2677 }; 2678 2679 static struct iommu_domain *amd_iommu_domain_alloc_paging_v1(struct device *dev, 2680 u32 flags) 2681 { 2682 struct pt_iommu_amdv1_cfg cfg = {}; 2683 struct protection_domain *domain; 2684 int ret; 2685 2686 if (amd_iommu_hatdis) 2687 return ERR_PTR(-EOPNOTSUPP); 2688 2689 domain = protection_domain_alloc(); 2690 if (!domain) 2691 return ERR_PTR(-ENOMEM); 2692 2693 domain->pd_mode = PD_MODE_V1; 2694 domain->iommu.driver_ops = &amd_hw_driver_ops_v1; 2695 domain->iommu.nid = dev_to_node(dev); 2696 if (flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) 2697 domain->domain.dirty_ops = &amdv1_dirty_ops; 2698 2699 /* 2700 * Someday FORCE_COHERENCE should be set by 2701 * amd_iommu_enforce_cache_coherency() like VT-d does. 2702 */ 2703 cfg.common.features = BIT(PT_FEAT_DYNAMIC_TOP) | 2704 BIT(PT_FEAT_AMDV1_ENCRYPT_TABLES) | 2705 BIT(PT_FEAT_AMDV1_FORCE_COHERENCE); 2706 2707 /* 2708 * AMD's IOMMU can flush as many pages as necessary in a single flush. 2709 * Unless we run in a virtual machine, which can be inferred according 2710 * to whether "non-present cache" is on, it is probably best to prefer 2711 * (potentially) too extensive TLB flushing (i.e., more misses) over 2712 * multiple TLB flushes (i.e., more flushes). For virtual machines the 2713 * hypervisor needs to synchronize the host IOMMU PTEs with those of 2714 * the guest, and the trade-off is different: unnecessary TLB flushes 2715 * should be avoided. 2716 */ 2717 if (amd_iommu_np_cache) 2718 cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE_NO_GAPS); 2719 else 2720 cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE); 2721 2722 cfg.common.hw_max_vasz_lg2 = amd_iommu_hpt_vasize; 2723 cfg.common.hw_max_oasz_lg2 = 52; 2724 cfg.starting_level = 2; 2725 domain->domain.ops = &amdv1_ops; 2726 2727 ret = pt_iommu_amdv1_init(&domain->amdv1, &cfg, GFP_KERNEL); 2728 if (ret) { 2729 amd_iommu_domain_free(&domain->domain); 2730 return ERR_PTR(ret); 2731 } 2732 2733 /* 2734 * Narrow the supported page sizes to those selected by the kernel 2735 * command line. 2736 */ 2737 domain->domain.pgsize_bitmap &= amd_iommu_pgsize_bitmap; 2738 return &domain->domain; 2739 } 2740 2741 static const struct iommu_domain_ops amdv2_ops = { 2742 IOMMU_PT_DOMAIN_OPS(x86_64), 2743 .iotlb_sync_map = amd_iommu_iotlb_sync_map, 2744 .flush_iotlb_all = amd_iommu_flush_iotlb_all, 2745 .iotlb_sync = amd_iommu_iotlb_sync, 2746 .attach_dev = amd_iommu_attach_device, 2747 .free = amd_iommu_domain_free, 2748 /* 2749 * Note the AMDv2 page table format does not support a Force Coherency 2750 * bit, so enforce_cache_coherency should not be set. However VFIO is 2751 * not prepared to handle a case where some domains will support 2752 * enforcement and others do not. VFIO and iommufd will have to be fixed 2753 * before it can fully use the V2 page table. See the comment in 2754 * iommufd_hwpt_paging_alloc(). For now leave things as they have 2755 * historically been and lie about enforce_cache_coherencey. 2756 */ 2757 .enforce_cache_coherency = amd_iommu_enforce_cache_coherency, 2758 }; 2759 2760 static struct iommu_domain *amd_iommu_domain_alloc_paging_v2(struct device *dev, 2761 u32 flags) 2762 { 2763 struct pt_iommu_x86_64_cfg cfg = {}; 2764 struct protection_domain *domain; 2765 int ret; 2766 2767 if (!amd_iommu_v2_pgtbl_supported()) 2768 return ERR_PTR(-EOPNOTSUPP); 2769 2770 domain = protection_domain_alloc(); 2771 if (!domain) 2772 return ERR_PTR(-ENOMEM); 2773 2774 domain->pd_mode = PD_MODE_V2; 2775 domain->iommu.nid = dev_to_node(dev); 2776 2777 cfg.common.features = BIT(PT_FEAT_X86_64_AMD_ENCRYPT_TABLES); 2778 if (amd_iommu_np_cache) 2779 cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE_NO_GAPS); 2780 else 2781 cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE); 2782 2783 /* 2784 * The v2 table behaves differently if it is attached to PASID 0 vs a 2785 * non-zero PASID. On PASID 0 it has no sign extension and the full 2786 * 57/48 bits decode the lower addresses. Otherwise it behaves like a 2787 * normal sign extended x86 page table. Since we want the domain to work 2788 * in both modes the top bit is removed and PT_FEAT_SIGN_EXTEND is not 2789 * set which creates a table that is compatible in both modes. 2790 */ 2791 if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) { 2792 cfg.common.hw_max_vasz_lg2 = 56; 2793 cfg.top_level = 4; 2794 } else { 2795 cfg.common.hw_max_vasz_lg2 = 47; 2796 cfg.top_level = 3; 2797 } 2798 cfg.common.hw_max_oasz_lg2 = 52; 2799 domain->domain.ops = &amdv2_ops; 2800 2801 ret = pt_iommu_x86_64_init(&domain->amdv2, &cfg, GFP_KERNEL); 2802 if (ret) { 2803 amd_iommu_domain_free(&domain->domain); 2804 return ERR_PTR(ret); 2805 } 2806 return &domain->domain; 2807 } 2808 2809 static inline bool is_nest_parent_supported(u32 flags) 2810 { 2811 /* Only allow nest parent when these features are supported */ 2812 return check_feature(FEATURE_GT) && 2813 check_feature(FEATURE_GIOSUP) && 2814 check_feature2(FEATURE_GCR3TRPMODE); 2815 } 2816 2817 static struct iommu_domain * 2818 amd_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags, 2819 const struct iommu_user_data *user_data) 2820 2821 { 2822 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); 2823 const u32 supported_flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING | 2824 IOMMU_HWPT_ALLOC_PASID | 2825 IOMMU_HWPT_ALLOC_NEST_PARENT; 2826 2827 if ((flags & ~supported_flags) || user_data) 2828 return ERR_PTR(-EOPNOTSUPP); 2829 2830 switch (flags & supported_flags) { 2831 case IOMMU_HWPT_ALLOC_DIRTY_TRACKING: 2832 case IOMMU_HWPT_ALLOC_NEST_PARENT: 2833 case IOMMU_HWPT_ALLOC_DIRTY_TRACKING | IOMMU_HWPT_ALLOC_NEST_PARENT: 2834 /* 2835 * Allocate domain with v1 page table for dirty tracking 2836 * and/or Nest parent. 2837 */ 2838 if ((flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) && 2839 !amd_iommu_hd_support(iommu)) 2840 break; 2841 2842 if ((flags & IOMMU_HWPT_ALLOC_NEST_PARENT) && 2843 !is_nest_parent_supported(flags)) 2844 break; 2845 2846 return amd_iommu_domain_alloc_paging_v1(dev, flags); 2847 case IOMMU_HWPT_ALLOC_PASID: 2848 /* Allocate domain with v2 page table if IOMMU supports PASID. */ 2849 if (!amd_iommu_pasid_supported()) 2850 break; 2851 return amd_iommu_domain_alloc_paging_v2(dev, flags); 2852 case 0: { 2853 struct iommu_domain *ret; 2854 2855 /* If nothing specific is required use the kernel commandline default */ 2856 if (amd_iommu_pgtable == PD_MODE_V1) { 2857 ret = amd_iommu_domain_alloc_paging_v1(dev, flags); 2858 if (ret != ERR_PTR(-EOPNOTSUPP)) 2859 return ret; 2860 return amd_iommu_domain_alloc_paging_v2(dev, flags); 2861 } 2862 ret = amd_iommu_domain_alloc_paging_v2(dev, flags); 2863 if (ret != ERR_PTR(-EOPNOTSUPP)) 2864 return ret; 2865 return amd_iommu_domain_alloc_paging_v1(dev, flags); 2866 } 2867 default: 2868 break; 2869 } 2870 return ERR_PTR(-EOPNOTSUPP); 2871 } 2872 2873 void amd_iommu_domain_free(struct iommu_domain *dom) 2874 { 2875 struct protection_domain *domain = to_pdomain(dom); 2876 2877 WARN_ON(!list_empty(&domain->dev_list)); 2878 pt_iommu_deinit(&domain->iommu); 2879 amd_iommu_pdom_id_free(domain->id); 2880 kfree(domain); 2881 } 2882 2883 static int blocked_domain_attach_device(struct iommu_domain *domain, 2884 struct device *dev, 2885 struct iommu_domain *old) 2886 { 2887 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2888 2889 if (dev_data->domain) 2890 detach_device(dev); 2891 2892 /* Clear DTE and flush the entry */ 2893 mutex_lock(&dev_data->mutex); 2894 dev_update_dte(dev_data, false); 2895 mutex_unlock(&dev_data->mutex); 2896 2897 return 0; 2898 } 2899 2900 static int blocked_domain_set_dev_pasid(struct iommu_domain *domain, 2901 struct device *dev, ioasid_t pasid, 2902 struct iommu_domain *old) 2903 { 2904 amd_iommu_remove_dev_pasid(dev, pasid, old); 2905 return 0; 2906 } 2907 2908 static struct iommu_domain blocked_domain = { 2909 .type = IOMMU_DOMAIN_BLOCKED, 2910 .ops = &(const struct iommu_domain_ops) { 2911 .attach_dev = blocked_domain_attach_device, 2912 .set_dev_pasid = blocked_domain_set_dev_pasid, 2913 } 2914 }; 2915 2916 static struct protection_domain identity_domain; 2917 2918 static int amd_iommu_identity_attach(struct iommu_domain *dom, struct device *dev, 2919 struct iommu_domain *old) 2920 { 2921 /* 2922 * Don't allow attaching a device to the identity domain if SNP is 2923 * enabled. 2924 */ 2925 if (amd_iommu_snp_en) 2926 return -EINVAL; 2927 2928 return amd_iommu_attach_device(dom, dev, old); 2929 } 2930 2931 static const struct iommu_domain_ops identity_domain_ops = { 2932 .attach_dev = amd_iommu_identity_attach, 2933 }; 2934 2935 void amd_iommu_init_identity_domain(void) 2936 { 2937 struct iommu_domain *domain = &identity_domain.domain; 2938 2939 domain->type = IOMMU_DOMAIN_IDENTITY; 2940 domain->ops = &identity_domain_ops; 2941 domain->owner = &amd_iommu_ops; 2942 2943 identity_domain.id = amd_iommu_pdom_id_alloc(); 2944 2945 protection_domain_init(&identity_domain); 2946 } 2947 2948 static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev, 2949 struct iommu_domain *old) 2950 { 2951 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2952 struct protection_domain *domain = to_pdomain(dom); 2953 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); 2954 int ret; 2955 2956 /* 2957 * Skip attach device to domain if new domain is same as 2958 * devices current domain 2959 */ 2960 if (dev_data->domain == domain) 2961 return 0; 2962 2963 dev_data->defer_attach = false; 2964 2965 /* 2966 * Restrict to devices with compatible IOMMU hardware support 2967 * when enforcement of dirty tracking is enabled. 2968 */ 2969 if (dom->dirty_ops && !amd_iommu_hd_support(iommu)) 2970 return -EINVAL; 2971 2972 if (dev_data->domain) 2973 detach_device(dev); 2974 2975 ret = attach_device(dev, domain); 2976 2977 #ifdef CONFIG_IRQ_REMAP 2978 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) { 2979 if (dom->type == IOMMU_DOMAIN_UNMANAGED) 2980 dev_data->use_vapic = 1; 2981 else 2982 dev_data->use_vapic = 0; 2983 } 2984 #endif 2985 2986 return ret; 2987 } 2988 2989 static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) 2990 { 2991 switch (cap) { 2992 case IOMMU_CAP_CACHE_COHERENCY: 2993 return true; 2994 case IOMMU_CAP_NOEXEC: 2995 return false; 2996 case IOMMU_CAP_PRE_BOOT_PROTECTION: 2997 return amdr_ivrs_remap_support; 2998 case IOMMU_CAP_ENFORCE_CACHE_COHERENCY: 2999 return true; 3000 case IOMMU_CAP_DIRTY_TRACKING: { 3001 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); 3002 3003 return amd_iommu_hd_support(iommu); 3004 } 3005 case IOMMU_CAP_PCI_ATS_SUPPORTED: { 3006 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 3007 3008 return amd_iommu_iotlb_sup && 3009 (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP); 3010 } 3011 default: 3012 break; 3013 } 3014 3015 return false; 3016 } 3017 3018 static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain, 3019 bool enable) 3020 { 3021 struct protection_domain *pdomain = to_pdomain(domain); 3022 struct dev_table_entry *dte; 3023 struct iommu_dev_data *dev_data; 3024 bool domain_flush = false; 3025 struct amd_iommu *iommu; 3026 unsigned long flags; 3027 u64 new; 3028 3029 spin_lock_irqsave(&pdomain->lock, flags); 3030 if (!(pdomain->dirty_tracking ^ enable)) { 3031 spin_unlock_irqrestore(&pdomain->lock, flags); 3032 return 0; 3033 } 3034 3035 list_for_each_entry(dev_data, &pdomain->dev_list, list) { 3036 spin_lock(&dev_data->dte_lock); 3037 iommu = get_amd_iommu_from_dev_data(dev_data); 3038 dte = &get_dev_table(iommu)[dev_data->devid]; 3039 new = dte->data[0]; 3040 new = (enable ? new | DTE_FLAG_HAD : new & ~DTE_FLAG_HAD); 3041 dte->data[0] = new; 3042 spin_unlock(&dev_data->dte_lock); 3043 3044 /* Flush device DTE */ 3045 device_flush_dte(dev_data); 3046 domain_flush = true; 3047 } 3048 3049 /* Flush IOTLB to mark IOPTE dirty on the next translation(s) */ 3050 if (domain_flush) 3051 amd_iommu_domain_flush_all(pdomain); 3052 3053 pdomain->dirty_tracking = enable; 3054 spin_unlock_irqrestore(&pdomain->lock, flags); 3055 3056 return 0; 3057 } 3058 3059 static void amd_iommu_get_resv_regions(struct device *dev, 3060 struct list_head *head) 3061 { 3062 struct iommu_resv_region *region; 3063 struct unity_map_entry *entry; 3064 struct amd_iommu *iommu; 3065 struct amd_iommu_pci_seg *pci_seg; 3066 int devid, sbdf; 3067 3068 sbdf = get_device_sbdf_id(dev); 3069 if (sbdf < 0) 3070 return; 3071 3072 devid = PCI_SBDF_TO_DEVID(sbdf); 3073 iommu = get_amd_iommu_from_dev(dev); 3074 pci_seg = iommu->pci_seg; 3075 3076 list_for_each_entry(entry, &pci_seg->unity_map, list) { 3077 int type, prot = 0; 3078 size_t length; 3079 3080 if (devid < entry->devid_start || devid > entry->devid_end) 3081 continue; 3082 3083 type = IOMMU_RESV_DIRECT; 3084 length = entry->address_end - entry->address_start; 3085 if (entry->prot & IOMMU_PROT_IR) 3086 prot |= IOMMU_READ; 3087 if (entry->prot & IOMMU_PROT_IW) 3088 prot |= IOMMU_WRITE; 3089 3090 region = iommu_alloc_resv_region(entry->address_start, 3091 length, prot, type, 3092 GFP_KERNEL); 3093 if (!region) { 3094 dev_err(dev, "Out of memory allocating dm-regions\n"); 3095 return; 3096 } 3097 list_add_tail(®ion->list, head); 3098 } 3099 3100 region = iommu_alloc_resv_region(MSI_RANGE_START, 3101 MSI_RANGE_END - MSI_RANGE_START + 1, 3102 0, IOMMU_RESV_MSI, GFP_KERNEL); 3103 if (!region) 3104 return; 3105 list_add_tail(®ion->list, head); 3106 3107 if (amd_iommu_ht_range_ignore()) 3108 return; 3109 3110 region = iommu_alloc_resv_region(HT_RANGE_START, 3111 HT_RANGE_END - HT_RANGE_START + 1, 3112 0, IOMMU_RESV_RESERVED, GFP_KERNEL); 3113 if (!region) 3114 return; 3115 list_add_tail(®ion->list, head); 3116 } 3117 3118 static bool amd_iommu_is_attach_deferred(struct device *dev) 3119 { 3120 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 3121 3122 return dev_data->defer_attach; 3123 } 3124 3125 static int amd_iommu_def_domain_type(struct device *dev) 3126 { 3127 struct iommu_dev_data *dev_data; 3128 3129 dev_data = dev_iommu_priv_get(dev); 3130 if (!dev_data) 3131 return 0; 3132 3133 /* Always use DMA domain for untrusted device */ 3134 if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted) 3135 return IOMMU_DOMAIN_DMA; 3136 3137 /* 3138 * Do not identity map IOMMUv2 capable devices when: 3139 * - memory encryption is active, because some of those devices 3140 * (AMD GPUs) don't have the encryption bit in their DMA-mask 3141 * and require remapping. 3142 * - SNP is enabled, because it prohibits DTE[Mode]=0. 3143 */ 3144 if (pdev_pasid_supported(dev_data) && 3145 !cc_platform_has(CC_ATTR_MEM_ENCRYPT) && 3146 !amd_iommu_snp_en) { 3147 return IOMMU_DOMAIN_IDENTITY; 3148 } 3149 3150 return 0; 3151 } 3152 3153 static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain) 3154 { 3155 /* IOMMU_PTE_FC is always set */ 3156 return true; 3157 } 3158 3159 const struct iommu_ops amd_iommu_ops = { 3160 .capable = amd_iommu_capable, 3161 .hw_info = amd_iommufd_hw_info, 3162 .blocked_domain = &blocked_domain, 3163 .release_domain = &blocked_domain, 3164 .identity_domain = &identity_domain.domain, 3165 .domain_alloc_paging_flags = amd_iommu_domain_alloc_paging_flags, 3166 .domain_alloc_sva = amd_iommu_domain_alloc_sva, 3167 .probe_device = amd_iommu_probe_device, 3168 .release_device = amd_iommu_release_device, 3169 .device_group = amd_iommu_device_group, 3170 .get_resv_regions = amd_iommu_get_resv_regions, 3171 .is_attach_deferred = amd_iommu_is_attach_deferred, 3172 .def_domain_type = amd_iommu_def_domain_type, 3173 .page_response = amd_iommu_page_response, 3174 .get_viommu_size = amd_iommufd_get_viommu_size, 3175 .viommu_init = amd_iommufd_viommu_init, 3176 }; 3177 3178 #ifdef CONFIG_IRQ_REMAP 3179 3180 /***************************************************************************** 3181 * 3182 * Interrupt Remapping Implementation 3183 * 3184 *****************************************************************************/ 3185 3186 static struct irq_chip amd_ir_chip; 3187 static DEFINE_SPINLOCK(iommu_table_lock); 3188 3189 static int iommu_flush_dev_irt(struct pci_dev *unused, u16 devid, void *data) 3190 { 3191 int ret; 3192 struct iommu_cmd cmd; 3193 struct amd_iommu *iommu = data; 3194 3195 build_inv_irt(&cmd, devid); 3196 ret = __iommu_queue_command_sync(iommu, &cmd, true); 3197 return ret; 3198 } 3199 3200 static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) 3201 { 3202 int ret; 3203 u64 data; 3204 unsigned long flags; 3205 struct iommu_cmd cmd; 3206 struct pci_dev *pdev = NULL; 3207 struct iommu_dev_data *dev_data = search_dev_data(iommu, devid); 3208 3209 if (iommu->irtcachedis_enabled) 3210 return; 3211 3212 if (dev_data && dev_data->dev && dev_is_pci(dev_data->dev)) 3213 pdev = to_pci_dev(dev_data->dev); 3214 3215 raw_spin_lock_irqsave(&iommu->lock, flags); 3216 data = get_cmdsem_val(iommu); 3217 build_completion_wait(&cmd, iommu, data); 3218 3219 if (pdev) 3220 ret = pci_for_each_dma_alias(pdev, iommu_flush_dev_irt, iommu); 3221 else 3222 ret = iommu_flush_dev_irt(NULL, devid, iommu); 3223 if (ret) 3224 goto out_err; 3225 3226 ret = __iommu_queue_command_sync(iommu, &cmd, false); 3227 if (ret) 3228 goto out_err; 3229 raw_spin_unlock_irqrestore(&iommu->lock, flags); 3230 3231 wait_on_sem(iommu, data); 3232 return; 3233 3234 out_err: 3235 raw_spin_unlock_irqrestore(&iommu->lock, flags); 3236 } 3237 3238 static inline u8 iommu_get_int_tablen(struct iommu_dev_data *dev_data) 3239 { 3240 if (dev_data && dev_data->max_irqs == MAX_IRQS_PER_TABLE_2K) 3241 return DTE_INTTABLEN_2K; 3242 return DTE_INTTABLEN_512; 3243 } 3244 3245 static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid, 3246 struct irq_remap_table *table) 3247 { 3248 u64 new; 3249 struct dev_table_entry *dte = &get_dev_table(iommu)[devid]; 3250 struct iommu_dev_data *dev_data = search_dev_data(iommu, devid); 3251 3252 if (dev_data) 3253 spin_lock(&dev_data->dte_lock); 3254 3255 new = READ_ONCE(dte->data[2]); 3256 new &= ~DTE_IRQ_PHYS_ADDR_MASK; 3257 new |= iommu_virt_to_phys(table->table); 3258 new |= DTE_IRQ_REMAP_INTCTL; 3259 new |= iommu_get_int_tablen(dev_data); 3260 new |= DTE_IRQ_REMAP_ENABLE; 3261 WRITE_ONCE(dte->data[2], new); 3262 3263 if (dev_data) 3264 spin_unlock(&dev_data->dte_lock); 3265 } 3266 3267 static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid) 3268 { 3269 struct irq_remap_table *table; 3270 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 3271 3272 if (WARN_ONCE(!pci_seg->rlookup_table[devid], 3273 "%s: no iommu for devid %x:%x\n", 3274 __func__, pci_seg->id, devid)) 3275 return NULL; 3276 3277 table = pci_seg->irq_lookup_table[devid]; 3278 if (WARN_ONCE(!table, "%s: no table for devid %x:%x\n", 3279 __func__, pci_seg->id, devid)) 3280 return NULL; 3281 3282 return table; 3283 } 3284 3285 static struct irq_remap_table *__alloc_irq_table(int nid, size_t size) 3286 { 3287 struct irq_remap_table *table; 3288 3289 table = kzalloc_obj(*table); 3290 if (!table) 3291 return NULL; 3292 3293 table->table = iommu_alloc_pages_node_sz( 3294 nid, GFP_KERNEL, max(DTE_INTTAB_ALIGNMENT, size)); 3295 if (!table->table) { 3296 kfree(table); 3297 return NULL; 3298 } 3299 raw_spin_lock_init(&table->lock); 3300 3301 return table; 3302 } 3303 3304 static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid, 3305 struct irq_remap_table *table) 3306 { 3307 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 3308 3309 pci_seg->irq_lookup_table[devid] = table; 3310 set_dte_irq_entry(iommu, devid, table); 3311 iommu_flush_dte(iommu, devid); 3312 } 3313 3314 static int set_remap_table_entry_alias(struct pci_dev *pdev, u16 alias, 3315 void *data) 3316 { 3317 struct irq_remap_table *table = data; 3318 struct amd_iommu_pci_seg *pci_seg; 3319 struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); 3320 3321 if (!iommu) 3322 return -EINVAL; 3323 3324 pci_seg = iommu->pci_seg; 3325 pci_seg->irq_lookup_table[alias] = table; 3326 set_dte_irq_entry(iommu, alias, table); 3327 iommu_flush_dte(pci_seg->rlookup_table[alias], alias); 3328 3329 return 0; 3330 } 3331 3332 static inline size_t get_irq_table_size(unsigned int max_irqs) 3333 { 3334 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) 3335 return max_irqs * sizeof(u32); 3336 3337 return max_irqs * (sizeof(u64) * 2); 3338 } 3339 3340 static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu, 3341 u16 devid, struct pci_dev *pdev, 3342 unsigned int max_irqs) 3343 { 3344 struct irq_remap_table *table = NULL; 3345 struct irq_remap_table *new_table = NULL; 3346 struct amd_iommu_pci_seg *pci_seg; 3347 unsigned long flags; 3348 int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; 3349 u16 alias; 3350 3351 spin_lock_irqsave(&iommu_table_lock, flags); 3352 3353 pci_seg = iommu->pci_seg; 3354 table = pci_seg->irq_lookup_table[devid]; 3355 if (table) 3356 goto out_unlock; 3357 3358 alias = pci_seg->alias_table[devid]; 3359 table = pci_seg->irq_lookup_table[alias]; 3360 if (table) { 3361 set_remap_table_entry(iommu, devid, table); 3362 goto out_wait; 3363 } 3364 spin_unlock_irqrestore(&iommu_table_lock, flags); 3365 3366 /* Nothing there yet, allocate new irq remapping table */ 3367 new_table = __alloc_irq_table(nid, get_irq_table_size(max_irqs)); 3368 if (!new_table) 3369 return NULL; 3370 3371 spin_lock_irqsave(&iommu_table_lock, flags); 3372 3373 table = pci_seg->irq_lookup_table[devid]; 3374 if (table) 3375 goto out_unlock; 3376 3377 table = pci_seg->irq_lookup_table[alias]; 3378 if (table) { 3379 set_remap_table_entry(iommu, devid, table); 3380 goto out_wait; 3381 } 3382 3383 table = new_table; 3384 new_table = NULL; 3385 3386 if (pdev) 3387 pci_for_each_dma_alias(pdev, set_remap_table_entry_alias, 3388 table); 3389 else 3390 set_remap_table_entry(iommu, devid, table); 3391 3392 if (devid != alias) 3393 set_remap_table_entry(iommu, alias, table); 3394 3395 out_wait: 3396 iommu_completion_wait(iommu); 3397 3398 out_unlock: 3399 spin_unlock_irqrestore(&iommu_table_lock, flags); 3400 3401 if (new_table) { 3402 iommu_free_pages(new_table->table); 3403 kfree(new_table); 3404 } 3405 return table; 3406 } 3407 3408 static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count, 3409 bool align, struct pci_dev *pdev, 3410 unsigned long max_irqs) 3411 { 3412 struct irq_remap_table *table; 3413 int index, c, alignment = 1; 3414 unsigned long flags; 3415 3416 table = alloc_irq_table(iommu, devid, pdev, max_irqs); 3417 if (!table) 3418 return -ENODEV; 3419 3420 if (align) 3421 alignment = roundup_pow_of_two(count); 3422 3423 raw_spin_lock_irqsave(&table->lock, flags); 3424 3425 /* Scan table for free entries */ 3426 for (index = ALIGN(table->min_index, alignment), c = 0; 3427 index < max_irqs;) { 3428 if (!iommu->irte_ops->is_allocated(table, index)) { 3429 c += 1; 3430 } else { 3431 c = 0; 3432 index = ALIGN(index + 1, alignment); 3433 continue; 3434 } 3435 3436 if (c == count) { 3437 for (; c != 0; --c) 3438 iommu->irte_ops->set_allocated(table, index - c + 1); 3439 3440 index -= count - 1; 3441 goto out; 3442 } 3443 3444 index++; 3445 } 3446 3447 index = -ENOSPC; 3448 3449 out: 3450 raw_spin_unlock_irqrestore(&table->lock, flags); 3451 3452 return index; 3453 } 3454 3455 static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, 3456 struct irte_ga *irte) 3457 { 3458 struct irq_remap_table *table; 3459 struct irte_ga *entry; 3460 unsigned long flags; 3461 u128 old; 3462 3463 table = get_irq_table(iommu, devid); 3464 if (!table) 3465 return -ENOMEM; 3466 3467 raw_spin_lock_irqsave(&table->lock, flags); 3468 3469 entry = (struct irte_ga *)table->table; 3470 entry = &entry[index]; 3471 3472 /* 3473 * We use cmpxchg16 to atomically update the 128-bit IRTE, 3474 * and it cannot be updated by the hardware or other processors 3475 * behind us, so the return value of cmpxchg16 should be the 3476 * same as the old value. 3477 */ 3478 old = entry->irte; 3479 WARN_ON(!try_cmpxchg128(&entry->irte, &old, irte->irte)); 3480 3481 raw_spin_unlock_irqrestore(&table->lock, flags); 3482 3483 return 0; 3484 } 3485 3486 static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, 3487 struct irte_ga *irte) 3488 { 3489 int ret; 3490 3491 ret = __modify_irte_ga(iommu, devid, index, irte); 3492 if (ret) 3493 return ret; 3494 3495 iommu_flush_irt_and_complete(iommu, devid); 3496 3497 return 0; 3498 } 3499 3500 static int modify_irte(struct amd_iommu *iommu, 3501 u16 devid, int index, union irte *irte) 3502 { 3503 struct irq_remap_table *table; 3504 unsigned long flags; 3505 3506 table = get_irq_table(iommu, devid); 3507 if (!table) 3508 return -ENOMEM; 3509 3510 raw_spin_lock_irqsave(&table->lock, flags); 3511 table->table[index] = irte->val; 3512 raw_spin_unlock_irqrestore(&table->lock, flags); 3513 3514 iommu_flush_irt_and_complete(iommu, devid); 3515 3516 return 0; 3517 } 3518 3519 static void free_irte(struct amd_iommu *iommu, u16 devid, int index) 3520 { 3521 struct irq_remap_table *table; 3522 unsigned long flags; 3523 3524 table = get_irq_table(iommu, devid); 3525 if (!table) 3526 return; 3527 3528 raw_spin_lock_irqsave(&table->lock, flags); 3529 iommu->irte_ops->clear_allocated(table, index); 3530 raw_spin_unlock_irqrestore(&table->lock, flags); 3531 3532 iommu_flush_irt_and_complete(iommu, devid); 3533 } 3534 3535 static void irte_prepare(void *entry, 3536 u32 delivery_mode, bool dest_mode, 3537 u8 vector, u32 dest_apicid, int devid) 3538 { 3539 union irte *irte = (union irte *) entry; 3540 3541 irte->val = 0; 3542 irte->fields.vector = vector; 3543 irte->fields.int_type = delivery_mode; 3544 irte->fields.destination = dest_apicid; 3545 irte->fields.dm = dest_mode; 3546 irte->fields.valid = 1; 3547 } 3548 3549 static void irte_ga_prepare(void *entry, 3550 u32 delivery_mode, bool dest_mode, 3551 u8 vector, u32 dest_apicid, int devid) 3552 { 3553 struct irte_ga *irte = (struct irte_ga *) entry; 3554 3555 irte->lo.val = 0; 3556 irte->hi.val = 0; 3557 irte->lo.fields_remap.int_type = delivery_mode; 3558 irte->lo.fields_remap.dm = dest_mode; 3559 irte->hi.fields.vector = vector; 3560 irte->lo.fields_remap.destination = APICID_TO_IRTE_DEST_LO(dest_apicid); 3561 irte->hi.fields.destination = APICID_TO_IRTE_DEST_HI(dest_apicid); 3562 irte->lo.fields_remap.valid = 1; 3563 } 3564 3565 static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) 3566 { 3567 union irte *irte = (union irte *) entry; 3568 3569 irte->fields.valid = 1; 3570 modify_irte(iommu, devid, index, irte); 3571 } 3572 3573 static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) 3574 { 3575 struct irte_ga *irte = (struct irte_ga *) entry; 3576 3577 irte->lo.fields_remap.valid = 1; 3578 modify_irte_ga(iommu, devid, index, irte); 3579 } 3580 3581 static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) 3582 { 3583 union irte *irte = (union irte *) entry; 3584 3585 irte->fields.valid = 0; 3586 modify_irte(iommu, devid, index, irte); 3587 } 3588 3589 static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) 3590 { 3591 struct irte_ga *irte = (struct irte_ga *) entry; 3592 3593 irte->lo.fields_remap.valid = 0; 3594 modify_irte_ga(iommu, devid, index, irte); 3595 } 3596 3597 static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index, 3598 u8 vector, u32 dest_apicid) 3599 { 3600 union irte *irte = (union irte *) entry; 3601 3602 irte->fields.vector = vector; 3603 irte->fields.destination = dest_apicid; 3604 modify_irte(iommu, devid, index, irte); 3605 } 3606 3607 static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index, 3608 u8 vector, u32 dest_apicid) 3609 { 3610 struct irte_ga *irte = (struct irte_ga *) entry; 3611 3612 if (!irte->lo.fields_remap.guest_mode) { 3613 irte->hi.fields.vector = vector; 3614 irte->lo.fields_remap.destination = 3615 APICID_TO_IRTE_DEST_LO(dest_apicid); 3616 irte->hi.fields.destination = 3617 APICID_TO_IRTE_DEST_HI(dest_apicid); 3618 modify_irte_ga(iommu, devid, index, irte); 3619 } 3620 } 3621 3622 #define IRTE_ALLOCATED (~1U) 3623 static void irte_set_allocated(struct irq_remap_table *table, int index) 3624 { 3625 table->table[index] = IRTE_ALLOCATED; 3626 } 3627 3628 static void irte_ga_set_allocated(struct irq_remap_table *table, int index) 3629 { 3630 struct irte_ga *ptr = (struct irte_ga *)table->table; 3631 struct irte_ga *irte = &ptr[index]; 3632 3633 memset(&irte->lo.val, 0, sizeof(u64)); 3634 memset(&irte->hi.val, 0, sizeof(u64)); 3635 irte->hi.fields.vector = 0xff; 3636 } 3637 3638 static bool irte_is_allocated(struct irq_remap_table *table, int index) 3639 { 3640 union irte *ptr = (union irte *)table->table; 3641 union irte *irte = &ptr[index]; 3642 3643 return irte->val != 0; 3644 } 3645 3646 static bool irte_ga_is_allocated(struct irq_remap_table *table, int index) 3647 { 3648 struct irte_ga *ptr = (struct irte_ga *)table->table; 3649 struct irte_ga *irte = &ptr[index]; 3650 3651 return irte->hi.fields.vector != 0; 3652 } 3653 3654 static void irte_clear_allocated(struct irq_remap_table *table, int index) 3655 { 3656 table->table[index] = 0; 3657 } 3658 3659 static void irte_ga_clear_allocated(struct irq_remap_table *table, int index) 3660 { 3661 struct irte_ga *ptr = (struct irte_ga *)table->table; 3662 struct irte_ga *irte = &ptr[index]; 3663 3664 memset(&irte->lo.val, 0, sizeof(u64)); 3665 memset(&irte->hi.val, 0, sizeof(u64)); 3666 } 3667 3668 static int get_devid(struct irq_alloc_info *info) 3669 { 3670 switch (info->type) { 3671 case X86_IRQ_ALLOC_TYPE_IOAPIC: 3672 return get_ioapic_devid(info->devid); 3673 case X86_IRQ_ALLOC_TYPE_HPET: 3674 return get_hpet_devid(info->devid); 3675 case X86_IRQ_ALLOC_TYPE_PCI_MSI: 3676 case X86_IRQ_ALLOC_TYPE_PCI_MSIX: 3677 return get_device_sbdf_id(msi_desc_to_dev(info->desc)); 3678 default: 3679 WARN_ON_ONCE(1); 3680 return -1; 3681 } 3682 } 3683 3684 struct irq_remap_ops amd_iommu_irq_ops = { 3685 .prepare = amd_iommu_prepare, 3686 .enable = amd_iommu_enable, 3687 .disable = amd_iommu_disable, 3688 .reenable = amd_iommu_reenable, 3689 .enable_faulting = amd_iommu_enable_faulting, 3690 }; 3691 3692 static void fill_msi_msg(struct msi_msg *msg, u32 index) 3693 { 3694 msg->data = index; 3695 msg->address_lo = 0; 3696 msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW; 3697 /* 3698 * The struct msi_msg.dest_mode_logical is used to set the DM bit 3699 * in MSI Message Address Register. For device w/ 2K int-remap support, 3700 * this is bit must be set to 1 regardless of the actual destination 3701 * mode, which is signified by the IRTE[DM]. 3702 */ 3703 if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2)) 3704 msg->arch_addr_lo.dest_mode_logical = true; 3705 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH; 3706 } 3707 3708 static void irq_remapping_prepare_irte(struct amd_ir_data *data, 3709 struct irq_cfg *irq_cfg, 3710 struct irq_alloc_info *info, 3711 int devid, int index, int sub_handle) 3712 { 3713 struct irq_2_irte *irte_info = &data->irq_2_irte; 3714 struct amd_iommu *iommu = data->iommu; 3715 3716 if (!iommu) 3717 return; 3718 3719 data->irq_2_irte.devid = devid; 3720 data->irq_2_irte.index = index + sub_handle; 3721 iommu->irte_ops->prepare(data->entry, APIC_DELIVERY_MODE_FIXED, 3722 apic->dest_mode_logical, irq_cfg->vector, 3723 irq_cfg->dest_apicid, devid); 3724 3725 switch (info->type) { 3726 case X86_IRQ_ALLOC_TYPE_IOAPIC: 3727 case X86_IRQ_ALLOC_TYPE_HPET: 3728 case X86_IRQ_ALLOC_TYPE_PCI_MSI: 3729 case X86_IRQ_ALLOC_TYPE_PCI_MSIX: 3730 fill_msi_msg(&data->msi_entry, irte_info->index); 3731 break; 3732 3733 default: 3734 BUG_ON(1); 3735 break; 3736 } 3737 } 3738 3739 struct amd_irte_ops irte_32_ops = { 3740 .prepare = irte_prepare, 3741 .activate = irte_activate, 3742 .deactivate = irte_deactivate, 3743 .set_affinity = irte_set_affinity, 3744 .set_allocated = irte_set_allocated, 3745 .is_allocated = irte_is_allocated, 3746 .clear_allocated = irte_clear_allocated, 3747 }; 3748 3749 struct amd_irte_ops irte_128_ops = { 3750 .prepare = irte_ga_prepare, 3751 .activate = irte_ga_activate, 3752 .deactivate = irte_ga_deactivate, 3753 .set_affinity = irte_ga_set_affinity, 3754 .set_allocated = irte_ga_set_allocated, 3755 .is_allocated = irte_ga_is_allocated, 3756 .clear_allocated = irte_ga_clear_allocated, 3757 }; 3758 3759 static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, 3760 unsigned int nr_irqs, void *arg) 3761 { 3762 struct irq_alloc_info *info = arg; 3763 struct irq_data *irq_data; 3764 struct amd_ir_data *data = NULL; 3765 struct amd_iommu *iommu; 3766 struct irq_cfg *cfg; 3767 struct iommu_dev_data *dev_data; 3768 unsigned long max_irqs; 3769 int i, ret, devid, seg, sbdf; 3770 int index; 3771 3772 if (!info) 3773 return -EINVAL; 3774 if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_PCI_MSI) 3775 return -EINVAL; 3776 3777 sbdf = get_devid(info); 3778 if (sbdf < 0) 3779 return -EINVAL; 3780 3781 seg = PCI_SBDF_TO_SEGID(sbdf); 3782 devid = PCI_SBDF_TO_DEVID(sbdf); 3783 iommu = __rlookup_amd_iommu(seg, devid); 3784 if (!iommu) 3785 return -EINVAL; 3786 3787 dev_data = search_dev_data(iommu, devid); 3788 max_irqs = dev_data ? dev_data->max_irqs : MAX_IRQS_PER_TABLE_512; 3789 3790 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 3791 if (ret < 0) 3792 return ret; 3793 3794 if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) { 3795 struct irq_remap_table *table; 3796 3797 table = alloc_irq_table(iommu, devid, NULL, max_irqs); 3798 if (table) { 3799 if (!table->min_index) { 3800 /* 3801 * Keep the first 32 indexes free for IOAPIC 3802 * interrupts. 3803 */ 3804 table->min_index = 32; 3805 for (i = 0; i < 32; ++i) 3806 iommu->irte_ops->set_allocated(table, i); 3807 } 3808 WARN_ON(table->min_index != 32); 3809 index = info->ioapic.pin; 3810 } else { 3811 index = -ENOMEM; 3812 } 3813 } else if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI || 3814 info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX) { 3815 bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI); 3816 3817 index = alloc_irq_index(iommu, devid, nr_irqs, align, 3818 msi_desc_to_pci_dev(info->desc), 3819 max_irqs); 3820 } else { 3821 index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL, 3822 max_irqs); 3823 } 3824 3825 if (index < 0) { 3826 pr_warn("Failed to allocate IRTE\n"); 3827 ret = index; 3828 goto out_free_parent; 3829 } 3830 3831 for (i = 0; i < nr_irqs; i++) { 3832 irq_data = irq_domain_get_irq_data(domain, virq + i); 3833 cfg = irq_data ? irqd_cfg(irq_data) : NULL; 3834 if (!cfg) { 3835 ret = -EINVAL; 3836 goto out_free_data; 3837 } 3838 3839 ret = -ENOMEM; 3840 data = kzalloc_obj(*data); 3841 if (!data) 3842 goto out_free_data; 3843 3844 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) 3845 data->entry = kzalloc_obj(union irte); 3846 else 3847 data->entry = kzalloc_obj(struct irte_ga); 3848 if (!data->entry) { 3849 kfree(data); 3850 goto out_free_data; 3851 } 3852 3853 data->iommu = iommu; 3854 irq_data->hwirq = (devid << 16) + i; 3855 irq_data->chip_data = data; 3856 irq_data->chip = &amd_ir_chip; 3857 irq_remapping_prepare_irte(data, cfg, info, devid, index, i); 3858 } 3859 3860 return 0; 3861 3862 out_free_data: 3863 for (i--; i >= 0; i--) { 3864 irq_data = irq_domain_get_irq_data(domain, virq + i); 3865 if (irq_data) 3866 kfree(irq_data->chip_data); 3867 } 3868 for (i = 0; i < nr_irqs; i++) 3869 free_irte(iommu, devid, index + i); 3870 out_free_parent: 3871 irq_domain_free_irqs_common(domain, virq, nr_irqs); 3872 return ret; 3873 } 3874 3875 static void irq_remapping_free(struct irq_domain *domain, unsigned int virq, 3876 unsigned int nr_irqs) 3877 { 3878 struct irq_2_irte *irte_info; 3879 struct irq_data *irq_data; 3880 struct amd_ir_data *data; 3881 int i; 3882 3883 for (i = 0; i < nr_irqs; i++) { 3884 irq_data = irq_domain_get_irq_data(domain, virq + i); 3885 if (irq_data && irq_data->chip_data) { 3886 data = irq_data->chip_data; 3887 irte_info = &data->irq_2_irte; 3888 free_irte(data->iommu, irte_info->devid, irte_info->index); 3889 kfree(data->entry); 3890 kfree(data); 3891 } 3892 } 3893 irq_domain_free_irqs_common(domain, virq, nr_irqs); 3894 } 3895 3896 static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu, 3897 struct amd_ir_data *ir_data, 3898 struct irq_2_irte *irte_info, 3899 struct irq_cfg *cfg); 3900 3901 static int irq_remapping_activate(struct irq_domain *domain, 3902 struct irq_data *irq_data, bool reserve) 3903 { 3904 struct amd_ir_data *data = irq_data->chip_data; 3905 struct irq_2_irte *irte_info = &data->irq_2_irte; 3906 struct amd_iommu *iommu = data->iommu; 3907 struct irq_cfg *cfg = irqd_cfg(irq_data); 3908 3909 if (!iommu) 3910 return 0; 3911 3912 iommu->irte_ops->activate(iommu, data->entry, irte_info->devid, 3913 irte_info->index); 3914 amd_ir_update_irte(irq_data, iommu, data, irte_info, cfg); 3915 return 0; 3916 } 3917 3918 static void irq_remapping_deactivate(struct irq_domain *domain, 3919 struct irq_data *irq_data) 3920 { 3921 struct amd_ir_data *data = irq_data->chip_data; 3922 struct irq_2_irte *irte_info = &data->irq_2_irte; 3923 struct amd_iommu *iommu = data->iommu; 3924 3925 if (iommu) 3926 iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid, 3927 irte_info->index); 3928 } 3929 3930 static int irq_remapping_select(struct irq_domain *d, struct irq_fwspec *fwspec, 3931 enum irq_domain_bus_token bus_token) 3932 { 3933 struct amd_iommu *iommu; 3934 int devid = -1; 3935 3936 if (!amd_iommu_irq_remap) 3937 return 0; 3938 3939 if (x86_fwspec_is_ioapic(fwspec)) 3940 devid = get_ioapic_devid(fwspec->param[0]); 3941 else if (x86_fwspec_is_hpet(fwspec)) 3942 devid = get_hpet_devid(fwspec->param[0]); 3943 3944 if (devid < 0) 3945 return 0; 3946 iommu = __rlookup_amd_iommu((devid >> 16), (devid & 0xffff)); 3947 3948 return iommu && iommu->ir_domain == d; 3949 } 3950 3951 static const struct irq_domain_ops amd_ir_domain_ops = { 3952 .select = irq_remapping_select, 3953 .alloc = irq_remapping_alloc, 3954 .free = irq_remapping_free, 3955 .activate = irq_remapping_activate, 3956 .deactivate = irq_remapping_deactivate, 3957 }; 3958 3959 static void __amd_iommu_update_ga(struct irte_ga *entry, int cpu, 3960 bool ga_log_intr) 3961 { 3962 if (cpu >= 0) { 3963 entry->lo.fields_vapic.destination = 3964 APICID_TO_IRTE_DEST_LO(cpu); 3965 entry->hi.fields.destination = 3966 APICID_TO_IRTE_DEST_HI(cpu); 3967 entry->lo.fields_vapic.is_run = true; 3968 entry->lo.fields_vapic.ga_log_intr = false; 3969 } else { 3970 entry->lo.fields_vapic.is_run = false; 3971 entry->lo.fields_vapic.ga_log_intr = ga_log_intr; 3972 } 3973 } 3974 3975 /* 3976 * Update the pCPU information for an IRTE that is configured to post IRQs to 3977 * a vCPU, without issuing an IOMMU invalidation for the IRTE. 3978 * 3979 * If the vCPU is associated with a pCPU (@cpu >= 0), configure the Destination 3980 * with the pCPU's APIC ID, set IsRun, and clear GALogIntr. If the vCPU isn't 3981 * associated with a pCPU (@cpu < 0), clear IsRun and set/clear GALogIntr based 3982 * on input from the caller (e.g. KVM only requests GALogIntr when the vCPU is 3983 * blocking and requires a notification wake event). I.e. treat vCPUs that are 3984 * associated with a pCPU as running. This API is intended to be used when a 3985 * vCPU is scheduled in/out (or stops running for any reason), to do a fast 3986 * update of IsRun, GALogIntr, and (conditionally) Destination. 3987 * 3988 * Per the IOMMU spec, the Destination, IsRun, and GATag fields are not cached 3989 * and thus don't require an invalidation to ensure the IOMMU consumes fresh 3990 * information. 3991 */ 3992 int amd_iommu_update_ga(void *data, int cpu, bool ga_log_intr) 3993 { 3994 struct amd_ir_data *ir_data = (struct amd_ir_data *)data; 3995 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; 3996 3997 if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))) 3998 return -EINVAL; 3999 4000 if (!entry || !entry->lo.fields_vapic.guest_mode) 4001 return 0; 4002 4003 if (!ir_data->iommu) 4004 return -ENODEV; 4005 4006 __amd_iommu_update_ga(entry, cpu, ga_log_intr); 4007 4008 return __modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, 4009 ir_data->irq_2_irte.index, entry); 4010 } 4011 EXPORT_SYMBOL(amd_iommu_update_ga); 4012 4013 int amd_iommu_activate_guest_mode(void *data, int cpu, bool ga_log_intr) 4014 { 4015 struct amd_ir_data *ir_data = (struct amd_ir_data *)data; 4016 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; 4017 u64 valid; 4018 4019 if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))) 4020 return -EINVAL; 4021 4022 if (!entry) 4023 return 0; 4024 4025 valid = entry->lo.fields_vapic.valid; 4026 4027 entry->lo.val = 0; 4028 entry->hi.val = 0; 4029 4030 entry->lo.fields_vapic.valid = valid; 4031 entry->lo.fields_vapic.guest_mode = 1; 4032 entry->hi.fields.ga_root_ptr = ir_data->ga_root_ptr; 4033 entry->hi.fields.vector = ir_data->ga_vector; 4034 entry->lo.fields_vapic.ga_tag = ir_data->ga_tag; 4035 4036 __amd_iommu_update_ga(entry, cpu, ga_log_intr); 4037 4038 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, 4039 ir_data->irq_2_irte.index, entry); 4040 } 4041 EXPORT_SYMBOL(amd_iommu_activate_guest_mode); 4042 4043 int amd_iommu_deactivate_guest_mode(void *data) 4044 { 4045 struct amd_ir_data *ir_data = (struct amd_ir_data *)data; 4046 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; 4047 struct irq_cfg *cfg = ir_data->cfg; 4048 u64 valid; 4049 4050 if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))) 4051 return -EINVAL; 4052 4053 if (!entry || !entry->lo.fields_vapic.guest_mode) 4054 return 0; 4055 4056 valid = entry->lo.fields_remap.valid; 4057 4058 entry->lo.val = 0; 4059 entry->hi.val = 0; 4060 4061 entry->lo.fields_remap.valid = valid; 4062 entry->lo.fields_remap.dm = apic->dest_mode_logical; 4063 entry->lo.fields_remap.int_type = APIC_DELIVERY_MODE_FIXED; 4064 entry->hi.fields.vector = cfg->vector; 4065 entry->lo.fields_remap.destination = 4066 APICID_TO_IRTE_DEST_LO(cfg->dest_apicid); 4067 entry->hi.fields.destination = 4068 APICID_TO_IRTE_DEST_HI(cfg->dest_apicid); 4069 4070 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, 4071 ir_data->irq_2_irte.index, entry); 4072 } 4073 EXPORT_SYMBOL(amd_iommu_deactivate_guest_mode); 4074 4075 static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *info) 4076 { 4077 int ret; 4078 struct amd_iommu_pi_data *pi_data = info; 4079 struct amd_ir_data *ir_data = data->chip_data; 4080 struct irq_2_irte *irte_info = &ir_data->irq_2_irte; 4081 struct iommu_dev_data *dev_data; 4082 4083 if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))) 4084 return -EINVAL; 4085 4086 if (ir_data->iommu == NULL) 4087 return -EINVAL; 4088 4089 dev_data = search_dev_data(ir_data->iommu, irte_info->devid); 4090 4091 /* Note: 4092 * This device has never been set up for guest mode. 4093 * we should not modify the IRTE 4094 */ 4095 if (!dev_data || !dev_data->use_vapic) 4096 return -EINVAL; 4097 4098 ir_data->cfg = irqd_cfg(data); 4099 4100 if (pi_data) { 4101 pi_data->ir_data = ir_data; 4102 4103 ir_data->ga_root_ptr = (pi_data->vapic_addr >> 12); 4104 ir_data->ga_vector = pi_data->vector; 4105 ir_data->ga_tag = pi_data->ga_tag; 4106 if (pi_data->is_guest_mode) 4107 ret = amd_iommu_activate_guest_mode(ir_data, pi_data->cpu, 4108 pi_data->ga_log_intr); 4109 else 4110 ret = amd_iommu_deactivate_guest_mode(ir_data); 4111 } else { 4112 ret = amd_iommu_deactivate_guest_mode(ir_data); 4113 } 4114 4115 return ret; 4116 } 4117 4118 4119 static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu, 4120 struct amd_ir_data *ir_data, 4121 struct irq_2_irte *irte_info, 4122 struct irq_cfg *cfg) 4123 { 4124 4125 /* 4126 * Atomically updates the IRTE with the new destination, vector 4127 * and flushes the interrupt entry cache. 4128 */ 4129 iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid, 4130 irte_info->index, cfg->vector, 4131 cfg->dest_apicid); 4132 } 4133 4134 static int amd_ir_set_affinity(struct irq_data *data, 4135 const struct cpumask *mask, bool force) 4136 { 4137 struct amd_ir_data *ir_data = data->chip_data; 4138 struct irq_2_irte *irte_info = &ir_data->irq_2_irte; 4139 struct irq_cfg *cfg = irqd_cfg(data); 4140 struct irq_data *parent = data->parent_data; 4141 struct amd_iommu *iommu = ir_data->iommu; 4142 int ret; 4143 4144 if (!iommu) 4145 return -ENODEV; 4146 4147 ret = parent->chip->irq_set_affinity(parent, mask, force); 4148 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) 4149 return ret; 4150 4151 amd_ir_update_irte(data, iommu, ir_data, irte_info, cfg); 4152 /* 4153 * After this point, all the interrupts will start arriving 4154 * at the new destination. So, time to cleanup the previous 4155 * vector allocation. 4156 */ 4157 vector_schedule_cleanup(cfg); 4158 4159 return IRQ_SET_MASK_OK_DONE; 4160 } 4161 4162 static void ir_compose_msi_msg(struct irq_data *irq_data, struct msi_msg *msg) 4163 { 4164 struct amd_ir_data *ir_data = irq_data->chip_data; 4165 4166 *msg = ir_data->msi_entry; 4167 } 4168 4169 static struct irq_chip amd_ir_chip = { 4170 .name = "AMD-IR", 4171 .irq_ack = apic_ack_irq, 4172 .irq_set_affinity = amd_ir_set_affinity, 4173 .irq_set_vcpu_affinity = amd_ir_set_vcpu_affinity, 4174 .irq_compose_msi_msg = ir_compose_msi_msg, 4175 }; 4176 4177 static const struct msi_parent_ops amdvi_msi_parent_ops = { 4178 .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED | MSI_FLAG_MULTI_PCI_MSI, 4179 .bus_select_token = DOMAIN_BUS_AMDVI, 4180 .bus_select_mask = MATCH_PCI_MSI, 4181 .prefix = "IR-", 4182 .init_dev_msi_info = msi_parent_init_dev_msi_info, 4183 }; 4184 4185 int amd_iommu_create_irq_domain(struct amd_iommu *iommu) 4186 { 4187 struct irq_domain_info info = { 4188 .fwnode = irq_domain_alloc_named_id_fwnode("AMD-IR", iommu->index), 4189 .ops = &amd_ir_domain_ops, 4190 .domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI, 4191 .host_data = iommu, 4192 .parent = arch_get_ir_parent_domain(), 4193 }; 4194 4195 if (!info.fwnode) 4196 return -ENOMEM; 4197 4198 iommu->ir_domain = msi_create_parent_irq_domain(&info, &amdvi_msi_parent_ops); 4199 if (!iommu->ir_domain) { 4200 irq_domain_free_fwnode(info.fwnode); 4201 return -ENOMEM; 4202 } 4203 return 0; 4204 } 4205 #endif 4206 4207 MODULE_IMPORT_NS("GENERIC_PT_IOMMU"); 4208