1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Volume Management Device driver 4 * Copyright (c) 2015, Intel Corporation. 5 */ 6 7 #include <linux/device.h> 8 #include <linux/interrupt.h> 9 #include <linux/irq.h> 10 #include <linux/irqchip/irq-msi-lib.h> 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/msi.h> 14 #include <linux/pci.h> 15 #include <linux/pci-acpi.h> 16 #include <linux/pci-ecam.h> 17 #include <linux/srcu.h> 18 #include <linux/rculist.h> 19 #include <linux/rcupdate.h> 20 21 #include <xen/xen.h> 22 23 #include <asm/irqdomain.h> 24 25 #define VMD_CFGBAR 0 26 #define VMD_MEMBAR1 2 27 #define VMD_MEMBAR2 4 28 29 #define PCI_REG_VMCAP 0x40 30 #define BUS_RESTRICT_CAP(vmcap) (vmcap & 0x1) 31 #define PCI_REG_VMCONFIG 0x44 32 #define BUS_RESTRICT_CFG(vmcfg) ((vmcfg >> 8) & 0x3) 33 #define VMCONFIG_MSI_REMAP 0x2 34 #define PCI_REG_VMLOCK 0x70 35 #define MB2_SHADOW_EN(vmlock) (vmlock & 0x2) 36 37 #define MB2_SHADOW_OFFSET 0x2000 38 #define MB2_SHADOW_SIZE 16 39 40 enum vmd_features { 41 /* 42 * Device may contain registers which hint the physical location of the 43 * membars, in order to allow proper address translation during 44 * resource assignment to enable guest virtualization 45 */ 46 VMD_FEAT_HAS_MEMBAR_SHADOW = (1 << 0), 47 48 /* 49 * Device may provide root port configuration information which limits 50 * bus numbering 51 */ 52 VMD_FEAT_HAS_BUS_RESTRICTIONS = (1 << 1), 53 54 /* 55 * Device contains physical location shadow registers in 56 * vendor-specific capability space 57 */ 58 VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP = (1 << 2), 59 60 /* 61 * Device may use MSI-X vector 0 for software triggering and will not 62 * be used for MSI remapping 63 */ 64 VMD_FEAT_OFFSET_FIRST_VECTOR = (1 << 3), 65 66 /* 67 * Device can bypass remapping MSI-X transactions into its MSI-X table, 68 * avoiding the requirement of a VMD MSI domain for child device 69 * interrupt handling. 70 */ 71 VMD_FEAT_CAN_BYPASS_MSI_REMAP = (1 << 4), 72 73 /* 74 * Enable ASPM on the PCIE root ports and set the default LTR of the 75 * storage devices on platforms where these values are not configured by 76 * BIOS. This is needed for laptops, which require these settings for 77 * proper power management of the SoC. 78 */ 79 VMD_FEAT_BIOS_PM_QUIRK = (1 << 5), 80 }; 81 82 #define VMD_BIOS_PM_QUIRK_LTR 0x1003 /* 3145728 ns */ 83 84 #define VMD_FEATS_CLIENT (VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP | \ 85 VMD_FEAT_HAS_BUS_RESTRICTIONS | \ 86 VMD_FEAT_OFFSET_FIRST_VECTOR | \ 87 VMD_FEAT_BIOS_PM_QUIRK) 88 89 static DEFINE_IDA(vmd_instance_ida); 90 91 /* 92 * Lock for manipulating VMD IRQ lists. 93 */ 94 static DEFINE_RAW_SPINLOCK(list_lock); 95 96 /** 97 * struct vmd_irq - private data to map driver IRQ to the VMD shared vector 98 * @node: list item for parent traversal. 99 * @irq: back pointer to parent. 100 * @enabled: true if driver enabled IRQ 101 * @virq: the virtual IRQ value provided to the requesting driver. 102 * 103 * Every MSI/MSI-X IRQ requested for a device in a VMD domain will be mapped to 104 * a VMD IRQ using this structure. 105 */ 106 struct vmd_irq { 107 struct list_head node; 108 struct vmd_irq_list *irq; 109 bool enabled; 110 unsigned int virq; 111 }; 112 113 /** 114 * struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector 115 * @irq_list: the list of irq's the VMD one demuxes to. 116 * @srcu: SRCU struct for local synchronization. 117 * @count: number of child IRQs assigned to this vector; used to track 118 * sharing. 119 * @virq: The underlying VMD Linux interrupt number 120 */ 121 struct vmd_irq_list { 122 struct list_head irq_list; 123 struct srcu_struct srcu; 124 unsigned int count; 125 unsigned int virq; 126 }; 127 128 struct vmd_dev { 129 struct pci_dev *dev; 130 131 raw_spinlock_t cfg_lock; 132 void __iomem *cfgbar; 133 134 int msix_count; 135 struct vmd_irq_list *irqs; 136 137 struct pci_sysdata sysdata; 138 struct resource resources[3]; 139 struct irq_domain *irq_domain; 140 struct pci_bus *bus; 141 u8 busn_start; 142 u8 first_vec; 143 char *name; 144 int instance; 145 }; 146 147 static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus) 148 { 149 return container_of(bus->sysdata, struct vmd_dev, sysdata); 150 } 151 152 static inline unsigned int index_from_irqs(struct vmd_dev *vmd, 153 struct vmd_irq_list *irqs) 154 { 155 return irqs - vmd->irqs; 156 } 157 158 /* 159 * Drivers managing a device in a VMD domain allocate their own IRQs as before, 160 * but the MSI entry for the hardware it's driving will be programmed with a 161 * destination ID for the VMD MSI-X table. The VMD muxes interrupts in its 162 * domain into one of its own, and the VMD driver de-muxes these for the 163 * handlers sharing that VMD IRQ. The vmd irq_domain provides the operations 164 * and irq_chip to set this up. 165 */ 166 static void vmd_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) 167 { 168 struct vmd_irq *vmdirq = data->chip_data; 169 struct vmd_irq_list *irq = vmdirq->irq; 170 struct vmd_dev *vmd = irq_data_get_irq_handler_data(data); 171 172 memset(msg, 0, sizeof(*msg)); 173 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH; 174 msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW; 175 msg->arch_addr_lo.destid_0_7 = index_from_irqs(vmd, irq); 176 } 177 178 static void vmd_irq_enable(struct irq_data *data) 179 { 180 struct vmd_irq *vmdirq = data->chip_data; 181 182 scoped_guard(raw_spinlock_irqsave, &list_lock) { 183 WARN_ON(vmdirq->enabled); 184 list_add_tail_rcu(&vmdirq->node, &vmdirq->irq->irq_list); 185 vmdirq->enabled = true; 186 } 187 } 188 189 static void vmd_pci_msi_enable(struct irq_data *data) 190 { 191 vmd_irq_enable(data->parent_data); 192 data->chip->irq_unmask(data); 193 } 194 195 static unsigned int vmd_pci_msi_startup(struct irq_data *data) 196 { 197 vmd_pci_msi_enable(data); 198 return 0; 199 } 200 201 static void vmd_irq_disable(struct irq_data *data) 202 { 203 struct vmd_irq *vmdirq = data->chip_data; 204 205 scoped_guard(raw_spinlock_irqsave, &list_lock) { 206 if (vmdirq->enabled) { 207 list_del_rcu(&vmdirq->node); 208 vmdirq->enabled = false; 209 } 210 } 211 } 212 213 static void vmd_pci_msi_disable(struct irq_data *data) 214 { 215 data->chip->irq_mask(data); 216 vmd_irq_disable(data->parent_data); 217 } 218 219 static void vmd_pci_msi_shutdown(struct irq_data *data) 220 { 221 vmd_pci_msi_disable(data); 222 } 223 224 static struct irq_chip vmd_msi_controller = { 225 .name = "VMD-MSI", 226 .irq_compose_msi_msg = vmd_compose_msi_msg, 227 }; 228 229 /* 230 * XXX: We can be even smarter selecting the best IRQ once we solve the 231 * affinity problem. 232 */ 233 static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *desc) 234 { 235 int i, best; 236 237 if (vmd->msix_count == 1 + vmd->first_vec) 238 return &vmd->irqs[vmd->first_vec]; 239 240 /* 241 * White list for fast-interrupt handlers. All others will share the 242 * "slow" interrupt vector. 243 */ 244 switch (msi_desc_to_pci_dev(desc)->class) { 245 case PCI_CLASS_STORAGE_EXPRESS: 246 break; 247 default: 248 return &vmd->irqs[vmd->first_vec]; 249 } 250 251 scoped_guard(raw_spinlock_irq, &list_lock) { 252 best = vmd->first_vec + 1; 253 for (i = best; i < vmd->msix_count; i++) 254 if (vmd->irqs[i].count < vmd->irqs[best].count) 255 best = i; 256 vmd->irqs[best].count++; 257 } 258 259 return &vmd->irqs[best]; 260 } 261 262 static void vmd_msi_free(struct irq_domain *domain, unsigned int virq, 263 unsigned int nr_irqs); 264 265 static int vmd_msi_alloc(struct irq_domain *domain, unsigned int virq, 266 unsigned int nr_irqs, void *arg) 267 { 268 struct msi_desc *desc = ((msi_alloc_info_t *)arg)->desc; 269 struct vmd_dev *vmd = domain->host_data; 270 struct vmd_irq *vmdirq; 271 272 for (int i = 0; i < nr_irqs; ++i) { 273 vmdirq = kzalloc(sizeof(*vmdirq), GFP_KERNEL); 274 if (!vmdirq) { 275 vmd_msi_free(domain, virq, i); 276 return -ENOMEM; 277 } 278 279 INIT_LIST_HEAD(&vmdirq->node); 280 vmdirq->irq = vmd_next_irq(vmd, desc); 281 vmdirq->virq = virq + i; 282 283 irq_domain_set_info(domain, virq + i, vmdirq->irq->virq, 284 &vmd_msi_controller, vmdirq, 285 handle_untracked_irq, vmd, NULL); 286 } 287 288 return 0; 289 } 290 291 static void vmd_msi_free(struct irq_domain *domain, unsigned int virq, 292 unsigned int nr_irqs) 293 { 294 struct irq_data *irq_data; 295 struct vmd_irq *vmdirq; 296 297 for (int i = 0; i < nr_irqs; ++i) { 298 irq_data = irq_domain_get_irq_data(domain, virq + i); 299 vmdirq = irq_data->chip_data; 300 301 synchronize_srcu(&vmdirq->irq->srcu); 302 303 /* XXX: Potential optimization to rebalance */ 304 scoped_guard(raw_spinlock_irq, &list_lock) 305 vmdirq->irq->count--; 306 307 kfree(vmdirq); 308 } 309 } 310 311 static const struct irq_domain_ops vmd_msi_domain_ops = { 312 .alloc = vmd_msi_alloc, 313 .free = vmd_msi_free, 314 }; 315 316 static bool vmd_init_dev_msi_info(struct device *dev, struct irq_domain *domain, 317 struct irq_domain *real_parent, 318 struct msi_domain_info *info) 319 { 320 if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info)) 321 return false; 322 323 info->chip->irq_startup = vmd_pci_msi_startup; 324 info->chip->irq_shutdown = vmd_pci_msi_shutdown; 325 info->chip->irq_enable = vmd_pci_msi_enable; 326 info->chip->irq_disable = vmd_pci_msi_disable; 327 return true; 328 } 329 330 #define VMD_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | MSI_FLAG_PCI_MSIX) 331 #define VMD_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_NO_AFFINITY) 332 333 static const struct msi_parent_ops vmd_msi_parent_ops = { 334 .supported_flags = VMD_MSI_FLAGS_SUPPORTED, 335 .required_flags = VMD_MSI_FLAGS_REQUIRED, 336 .bus_select_token = DOMAIN_BUS_VMD_MSI, 337 .bus_select_mask = MATCH_PCI_MSI, 338 .prefix = "VMD-", 339 .init_dev_msi_info = vmd_init_dev_msi_info, 340 }; 341 342 static int vmd_create_irq_domain(struct vmd_dev *vmd) 343 { 344 struct irq_domain_info info = { 345 .size = vmd->msix_count, 346 .ops = &vmd_msi_domain_ops, 347 .host_data = vmd, 348 }; 349 350 info.fwnode = irq_domain_alloc_named_id_fwnode("VMD-MSI", 351 vmd->sysdata.domain); 352 if (!info.fwnode) 353 return -ENODEV; 354 355 vmd->irq_domain = msi_create_parent_irq_domain(&info, 356 &vmd_msi_parent_ops); 357 if (!vmd->irq_domain) { 358 irq_domain_free_fwnode(info.fwnode); 359 return -ENODEV; 360 } 361 362 return 0; 363 } 364 365 static void vmd_set_msi_remapping(struct vmd_dev *vmd, bool enable) 366 { 367 u16 reg; 368 369 pci_read_config_word(vmd->dev, PCI_REG_VMCONFIG, ®); 370 reg = enable ? (reg & ~VMCONFIG_MSI_REMAP) : 371 (reg | VMCONFIG_MSI_REMAP); 372 pci_write_config_word(vmd->dev, PCI_REG_VMCONFIG, reg); 373 } 374 375 static void vmd_remove_irq_domain(struct vmd_dev *vmd) 376 { 377 /* 378 * Some production BIOS won't enable remapping between soft reboots. 379 * Ensure remapping is restored before unloading the driver. 380 */ 381 if (!vmd->msix_count) 382 vmd_set_msi_remapping(vmd, true); 383 384 if (vmd->irq_domain) { 385 struct fwnode_handle *fn = vmd->irq_domain->fwnode; 386 387 irq_domain_remove(vmd->irq_domain); 388 irq_domain_free_fwnode(fn); 389 } 390 } 391 392 static void __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus, 393 unsigned int devfn, int reg, int len) 394 { 395 unsigned int busnr_ecam = bus->number - vmd->busn_start; 396 u32 offset = PCIE_ECAM_OFFSET(busnr_ecam, devfn, reg); 397 398 if (offset + len >= resource_size(&vmd->dev->resource[VMD_CFGBAR])) 399 return NULL; 400 401 return vmd->cfgbar + offset; 402 } 403 404 /* 405 * CPU may deadlock if config space is not serialized on some versions of this 406 * hardware, so all config space access is done under a spinlock. 407 */ 408 static int vmd_pci_read(struct pci_bus *bus, unsigned int devfn, int reg, 409 int len, u32 *value) 410 { 411 struct vmd_dev *vmd = vmd_from_bus(bus); 412 void __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len); 413 414 if (!addr) 415 return -EFAULT; 416 417 guard(raw_spinlock_irqsave)(&vmd->cfg_lock); 418 switch (len) { 419 case 1: 420 *value = readb(addr); 421 return 0; 422 case 2: 423 *value = readw(addr); 424 return 0; 425 case 4: 426 *value = readl(addr); 427 return 0; 428 default: 429 return -EINVAL; 430 } 431 } 432 433 /* 434 * VMD h/w converts non-posted config writes to posted memory writes. The 435 * read-back in this function forces the completion so it returns only after 436 * the config space was written, as expected. 437 */ 438 static int vmd_pci_write(struct pci_bus *bus, unsigned int devfn, int reg, 439 int len, u32 value) 440 { 441 struct vmd_dev *vmd = vmd_from_bus(bus); 442 void __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len); 443 444 if (!addr) 445 return -EFAULT; 446 447 guard(raw_spinlock_irqsave)(&vmd->cfg_lock); 448 switch (len) { 449 case 1: 450 writeb(value, addr); 451 readb(addr); 452 return 0; 453 case 2: 454 writew(value, addr); 455 readw(addr); 456 return 0; 457 case 4: 458 writel(value, addr); 459 readl(addr); 460 return 0; 461 default: 462 return -EINVAL; 463 } 464 } 465 466 static struct pci_ops vmd_ops = { 467 .read = vmd_pci_read, 468 .write = vmd_pci_write, 469 }; 470 471 #ifdef CONFIG_ACPI 472 static struct acpi_device *vmd_acpi_find_companion(struct pci_dev *pci_dev) 473 { 474 struct pci_host_bridge *bridge; 475 u32 busnr, addr; 476 477 if (pci_dev->bus->ops != &vmd_ops) 478 return NULL; 479 480 bridge = pci_find_host_bridge(pci_dev->bus); 481 busnr = pci_dev->bus->number - bridge->bus->number; 482 /* 483 * The address computation below is only applicable to relative bus 484 * numbers below 32. 485 */ 486 if (busnr > 31) 487 return NULL; 488 489 addr = (busnr << 24) | ((u32)pci_dev->devfn << 16) | 0x8000FFFFU; 490 491 dev_dbg(&pci_dev->dev, "Looking for ACPI companion (address 0x%x)\n", 492 addr); 493 494 return acpi_find_child_device(ACPI_COMPANION(bridge->dev.parent), addr, 495 false); 496 } 497 498 static bool hook_installed; 499 500 static void vmd_acpi_begin(void) 501 { 502 if (pci_acpi_set_companion_lookup_hook(vmd_acpi_find_companion)) 503 return; 504 505 hook_installed = true; 506 } 507 508 static void vmd_acpi_end(void) 509 { 510 if (!hook_installed) 511 return; 512 513 pci_acpi_clear_companion_lookup_hook(); 514 hook_installed = false; 515 } 516 #else 517 static inline void vmd_acpi_begin(void) { } 518 static inline void vmd_acpi_end(void) { } 519 #endif /* CONFIG_ACPI */ 520 521 static void vmd_domain_reset(struct vmd_dev *vmd) 522 { 523 u16 bus, max_buses = resource_size(&vmd->resources[0]); 524 u8 dev, functions, fn, hdr_type; 525 char __iomem *base; 526 527 for (bus = 0; bus < max_buses; bus++) { 528 for (dev = 0; dev < 32; dev++) { 529 base = vmd->cfgbar + PCIE_ECAM_OFFSET(bus, 530 PCI_DEVFN(dev, 0), 0); 531 532 hdr_type = readb(base + PCI_HEADER_TYPE); 533 534 functions = (hdr_type & PCI_HEADER_TYPE_MFD) ? 8 : 1; 535 for (fn = 0; fn < functions; fn++) { 536 base = vmd->cfgbar + PCIE_ECAM_OFFSET(bus, 537 PCI_DEVFN(dev, fn), 0); 538 539 hdr_type = readb(base + PCI_HEADER_TYPE) & 540 PCI_HEADER_TYPE_MASK; 541 542 if (hdr_type != PCI_HEADER_TYPE_BRIDGE || 543 (readw(base + PCI_CLASS_DEVICE) != 544 PCI_CLASS_BRIDGE_PCI)) 545 continue; 546 547 /* 548 * Temporarily disable the I/O range before updating 549 * PCI_IO_BASE. 550 */ 551 writel(0x0000ffff, base + PCI_IO_BASE_UPPER16); 552 /* Update lower 16 bits of I/O base/limit */ 553 writew(0x00f0, base + PCI_IO_BASE); 554 /* Update upper 16 bits of I/O base/limit */ 555 writel(0, base + PCI_IO_BASE_UPPER16); 556 557 /* MMIO Base/Limit */ 558 writel(0x0000fff0, base + PCI_MEMORY_BASE); 559 560 /* Prefetchable MMIO Base/Limit */ 561 writel(0, base + PCI_PREF_LIMIT_UPPER32); 562 writel(0x0000fff0, base + PCI_PREF_MEMORY_BASE); 563 writel(0xffffffff, base + PCI_PREF_BASE_UPPER32); 564 } 565 } 566 } 567 } 568 569 static void vmd_attach_resources(struct vmd_dev *vmd) 570 { 571 vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1]; 572 vmd->dev->resource[VMD_MEMBAR2].child = &vmd->resources[2]; 573 } 574 575 static void vmd_detach_resources(struct vmd_dev *vmd) 576 { 577 vmd->dev->resource[VMD_MEMBAR1].child = NULL; 578 vmd->dev->resource[VMD_MEMBAR2].child = NULL; 579 } 580 581 static int vmd_get_phys_offsets(struct vmd_dev *vmd, bool native_hint, 582 resource_size_t *offset1, 583 resource_size_t *offset2) 584 { 585 struct pci_dev *dev = vmd->dev; 586 u64 phys1, phys2; 587 588 if (native_hint) { 589 u32 vmlock; 590 int ret; 591 592 ret = pci_read_config_dword(dev, PCI_REG_VMLOCK, &vmlock); 593 if (ret || PCI_POSSIBLE_ERROR(vmlock)) 594 return -ENODEV; 595 596 if (MB2_SHADOW_EN(vmlock)) { 597 void __iomem *membar2; 598 599 membar2 = pci_iomap(dev, VMD_MEMBAR2, 0); 600 if (!membar2) 601 return -ENOMEM; 602 phys1 = readq(membar2 + MB2_SHADOW_OFFSET); 603 phys2 = readq(membar2 + MB2_SHADOW_OFFSET + 8); 604 pci_iounmap(dev, membar2); 605 } else 606 return 0; 607 } else { 608 /* Hypervisor-Emulated Vendor-Specific Capability */ 609 int pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); 610 u32 reg, regu; 611 612 pci_read_config_dword(dev, pos + 4, ®); 613 614 /* "SHDW" */ 615 if (pos && reg == 0x53484457) { 616 pci_read_config_dword(dev, pos + 8, ®); 617 pci_read_config_dword(dev, pos + 12, ®u); 618 phys1 = (u64) regu << 32 | reg; 619 620 pci_read_config_dword(dev, pos + 16, ®); 621 pci_read_config_dword(dev, pos + 20, ®u); 622 phys2 = (u64) regu << 32 | reg; 623 } else 624 return 0; 625 } 626 627 *offset1 = dev->resource[VMD_MEMBAR1].start - 628 (phys1 & PCI_BASE_ADDRESS_MEM_MASK); 629 *offset2 = dev->resource[VMD_MEMBAR2].start - 630 (phys2 & PCI_BASE_ADDRESS_MEM_MASK); 631 632 return 0; 633 } 634 635 static int vmd_get_bus_number_start(struct vmd_dev *vmd) 636 { 637 struct pci_dev *dev = vmd->dev; 638 u16 reg; 639 640 pci_read_config_word(dev, PCI_REG_VMCAP, ®); 641 if (BUS_RESTRICT_CAP(reg)) { 642 pci_read_config_word(dev, PCI_REG_VMCONFIG, ®); 643 644 switch (BUS_RESTRICT_CFG(reg)) { 645 case 0: 646 vmd->busn_start = 0; 647 break; 648 case 1: 649 vmd->busn_start = 128; 650 break; 651 case 2: 652 vmd->busn_start = 224; 653 break; 654 default: 655 pci_err(dev, "Unknown Bus Offset Setting (%d)\n", 656 BUS_RESTRICT_CFG(reg)); 657 return -ENODEV; 658 } 659 } 660 661 return 0; 662 } 663 664 static irqreturn_t vmd_irq(int irq, void *data) 665 { 666 struct vmd_irq_list *irqs = data; 667 struct vmd_irq *vmdirq; 668 int idx; 669 670 idx = srcu_read_lock(&irqs->srcu); 671 list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node) 672 generic_handle_irq(vmdirq->virq); 673 srcu_read_unlock(&irqs->srcu, idx); 674 675 return IRQ_HANDLED; 676 } 677 678 static int vmd_alloc_irqs(struct vmd_dev *vmd) 679 { 680 struct pci_dev *dev = vmd->dev; 681 int i, err; 682 683 vmd->msix_count = pci_msix_vec_count(dev); 684 if (vmd->msix_count < 0) 685 return -ENODEV; 686 687 vmd->msix_count = pci_alloc_irq_vectors(dev, vmd->first_vec + 1, 688 vmd->msix_count, PCI_IRQ_MSIX); 689 if (vmd->msix_count < 0) 690 return vmd->msix_count; 691 692 vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs), 693 GFP_KERNEL); 694 if (!vmd->irqs) 695 return -ENOMEM; 696 697 for (i = 0; i < vmd->msix_count; i++) { 698 err = init_srcu_struct(&vmd->irqs[i].srcu); 699 if (err) 700 return err; 701 702 INIT_LIST_HEAD(&vmd->irqs[i].irq_list); 703 vmd->irqs[i].virq = pci_irq_vector(dev, i); 704 err = devm_request_irq(&dev->dev, vmd->irqs[i].virq, 705 vmd_irq, IRQF_NO_THREAD, 706 vmd->name, &vmd->irqs[i]); 707 if (err) 708 return err; 709 } 710 711 return 0; 712 } 713 714 /* 715 * Since VMD is an aperture to regular PCIe root ports, only allow it to 716 * control features that the OS is allowed to control on the physical PCI bus. 717 */ 718 static void vmd_copy_host_bridge_flags(struct pci_host_bridge *root_bridge, 719 struct pci_host_bridge *vmd_bridge) 720 { 721 vmd_bridge->native_pcie_hotplug = root_bridge->native_pcie_hotplug; 722 vmd_bridge->native_shpc_hotplug = root_bridge->native_shpc_hotplug; 723 vmd_bridge->native_aer = root_bridge->native_aer; 724 vmd_bridge->native_pme = root_bridge->native_pme; 725 vmd_bridge->native_ltr = root_bridge->native_ltr; 726 vmd_bridge->native_dpc = root_bridge->native_dpc; 727 } 728 729 /* 730 * Enable ASPM and LTR settings on devices that aren't configured by BIOS. 731 */ 732 static int vmd_pm_enable_quirk(struct pci_dev *pdev, void *userdata) 733 { 734 unsigned long features = *(unsigned long *)userdata; 735 u16 ltr = VMD_BIOS_PM_QUIRK_LTR; 736 u32 ltr_reg; 737 int pos; 738 739 if (!(features & VMD_FEAT_BIOS_PM_QUIRK)) 740 return 0; 741 742 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_LTR); 743 if (!pos) 744 goto out_state_change; 745 746 /* 747 * Skip if the max snoop LTR is non-zero, indicating BIOS has set it 748 * so the LTR quirk is not needed. 749 */ 750 pci_read_config_dword(pdev, pos + PCI_LTR_MAX_SNOOP_LAT, <r_reg); 751 if (!!(ltr_reg & (PCI_LTR_VALUE_MASK | PCI_LTR_SCALE_MASK))) 752 goto out_state_change; 753 754 /* 755 * Set the default values to the maximum required by the platform to 756 * allow the deepest power management savings. Write as a DWORD where 757 * the lower word is the max snoop latency and the upper word is the 758 * max non-snoop latency. 759 */ 760 ltr_reg = (ltr << 16) | ltr; 761 pci_write_config_dword(pdev, pos + PCI_LTR_MAX_SNOOP_LAT, ltr_reg); 762 pci_info(pdev, "VMD: Default LTR value set by driver\n"); 763 764 out_state_change: 765 /* 766 * Ensure devices are in D0 before enabling PCI-PM L1 PM Substates, per 767 * PCIe r6.0, sec 5.5.4. 768 */ 769 pci_set_power_state_locked(pdev, PCI_D0); 770 pci_enable_link_state_locked(pdev, PCIE_LINK_STATE_ALL); 771 return 0; 772 } 773 774 static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) 775 { 776 struct pci_sysdata *sd = &vmd->sysdata; 777 struct resource *res; 778 u32 upper_bits; 779 unsigned long flags; 780 LIST_HEAD(resources); 781 resource_size_t offset[2] = {0}; 782 resource_size_t membar2_offset = 0x2000; 783 struct pci_bus *child; 784 struct pci_dev *dev; 785 int ret; 786 787 /* 788 * Shadow registers may exist in certain VMD device ids which allow 789 * guests to correctly assign host physical addresses to the root ports 790 * and child devices. These registers will either return the host value 791 * or 0, depending on an enable bit in the VMD device. 792 */ 793 if (features & VMD_FEAT_HAS_MEMBAR_SHADOW) { 794 membar2_offset = MB2_SHADOW_OFFSET + MB2_SHADOW_SIZE; 795 ret = vmd_get_phys_offsets(vmd, true, &offset[0], &offset[1]); 796 if (ret) 797 return ret; 798 } else if (features & VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP) { 799 ret = vmd_get_phys_offsets(vmd, false, &offset[0], &offset[1]); 800 if (ret) 801 return ret; 802 } 803 804 /* 805 * Certain VMD devices may have a root port configuration option which 806 * limits the bus range to between 0-127, 128-255, or 224-255 807 */ 808 if (features & VMD_FEAT_HAS_BUS_RESTRICTIONS) { 809 ret = vmd_get_bus_number_start(vmd); 810 if (ret) 811 return ret; 812 } 813 814 res = &vmd->dev->resource[VMD_CFGBAR]; 815 vmd->resources[0] = (struct resource) { 816 .name = "VMD CFGBAR", 817 .start = vmd->busn_start, 818 .end = vmd->busn_start + (resource_size(res) >> 20) - 1, 819 .flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED, 820 }; 821 822 /* 823 * If the window is below 4GB, clear IORESOURCE_MEM_64 so we can 824 * put 32-bit resources in the window. 825 * 826 * There's no hardware reason why a 64-bit window *couldn't* 827 * contain a 32-bit resource, but pbus_size_mem() computes the 828 * bridge window size assuming a 64-bit window will contain no 829 * 32-bit resources. __pci_assign_resource() enforces that 830 * artificial restriction to make sure everything will fit. 831 * 832 * The only way we could use a 64-bit non-prefetchable MEMBAR is 833 * if its address is <4GB so that we can convert it to a 32-bit 834 * resource. To be visible to the host OS, all VMD endpoints must 835 * be initially configured by platform BIOS, which includes setting 836 * up these resources. We can assume the device is configured 837 * according to the platform needs. 838 */ 839 res = &vmd->dev->resource[VMD_MEMBAR1]; 840 upper_bits = upper_32_bits(res->end); 841 flags = res->flags & ~IORESOURCE_SIZEALIGN; 842 if (!upper_bits) 843 flags &= ~IORESOURCE_MEM_64; 844 vmd->resources[1] = (struct resource) { 845 .name = "VMD MEMBAR1", 846 .start = res->start, 847 .end = res->end, 848 .flags = flags, 849 .parent = res, 850 }; 851 852 res = &vmd->dev->resource[VMD_MEMBAR2]; 853 upper_bits = upper_32_bits(res->end); 854 flags = res->flags & ~IORESOURCE_SIZEALIGN; 855 if (!upper_bits) 856 flags &= ~IORESOURCE_MEM_64; 857 vmd->resources[2] = (struct resource) { 858 .name = "VMD MEMBAR2", 859 .start = res->start + membar2_offset, 860 .end = res->end, 861 .flags = flags, 862 .parent = res, 863 }; 864 865 /* 866 * Currently MSI remapping must be enabled in guest passthrough mode 867 * due to some missing interrupt remapping plumbing. This is probably 868 * acceptable because the guest is usually CPU-limited and MSI 869 * remapping doesn't become a performance bottleneck. 870 */ 871 if (!(features & VMD_FEAT_CAN_BYPASS_MSI_REMAP) || 872 offset[0] || offset[1]) { 873 ret = vmd_alloc_irqs(vmd); 874 if (ret) 875 return ret; 876 877 vmd_set_msi_remapping(vmd, true); 878 879 ret = vmd_create_irq_domain(vmd); 880 if (ret) 881 return ret; 882 } else { 883 vmd_set_msi_remapping(vmd, false); 884 } 885 886 pci_add_resource(&resources, &vmd->resources[0]); 887 pci_add_resource_offset(&resources, &vmd->resources[1], offset[0]); 888 pci_add_resource_offset(&resources, &vmd->resources[2], offset[1]); 889 890 sd->vmd_dev = vmd->dev; 891 892 /* 893 * Emulated domains start at 0x10000 to not clash with ACPI _SEG 894 * domains. Per ACPI r6.0, sec 6.5.6, _SEG returns an integer, of 895 * which the lower 16 bits are the PCI Segment Group (domain) number. 896 * Other bits are currently reserved. 897 */ 898 sd->domain = pci_bus_find_emul_domain_nr(0, 0x10000, INT_MAX); 899 if (sd->domain < 0) 900 return sd->domain; 901 902 sd->node = pcibus_to_node(vmd->dev->bus); 903 904 vmd->bus = pci_create_root_bus(&vmd->dev->dev, vmd->busn_start, 905 &vmd_ops, sd, &resources); 906 if (!vmd->bus) { 907 pci_bus_release_emul_domain_nr(sd->domain); 908 pci_free_resource_list(&resources); 909 vmd_remove_irq_domain(vmd); 910 return -ENODEV; 911 } 912 913 vmd_copy_host_bridge_flags(pci_find_host_bridge(vmd->dev->bus), 914 to_pci_host_bridge(vmd->bus->bridge)); 915 916 vmd_attach_resources(vmd); 917 if (vmd->irq_domain) 918 dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain); 919 else 920 dev_set_msi_domain(&vmd->bus->dev, 921 dev_get_msi_domain(&vmd->dev->dev)); 922 923 WARN(sysfs_create_link(&vmd->dev->dev.kobj, &vmd->bus->dev.kobj, 924 "domain"), "Can't create symlink to domain\n"); 925 926 vmd_acpi_begin(); 927 928 pci_scan_child_bus(vmd->bus); 929 vmd_domain_reset(vmd); 930 931 /* When Intel VMD is enabled, the OS does not discover the Root Ports 932 * owned by Intel VMD within the MMCFG space. pci_reset_bus() applies 933 * a reset to the parent of the PCI device supplied as argument. This 934 * is why we pass a child device, so the reset can be triggered at 935 * the Intel bridge level and propagated to all the children in the 936 * hierarchy. 937 */ 938 list_for_each_entry(child, &vmd->bus->children, node) { 939 if (!list_empty(&child->devices)) { 940 dev = list_first_entry(&child->devices, 941 struct pci_dev, bus_list); 942 ret = pci_reset_bus(dev); 943 if (ret) 944 pci_warn(dev, "can't reset device: %d\n", ret); 945 946 break; 947 } 948 } 949 950 pci_assign_unassigned_bus_resources(vmd->bus); 951 952 pci_walk_bus(vmd->bus, vmd_pm_enable_quirk, &features); 953 954 /* 955 * VMD root buses are virtual and don't return true on pci_is_pcie() 956 * and will fail pcie_bus_configure_settings() early. It can instead be 957 * run on each of the real root ports. 958 */ 959 list_for_each_entry(child, &vmd->bus->children, node) 960 pcie_bus_configure_settings(child); 961 962 pci_bus_add_devices(vmd->bus); 963 964 vmd_acpi_end(); 965 return 0; 966 } 967 968 static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id) 969 { 970 unsigned long features = (unsigned long) id->driver_data; 971 struct vmd_dev *vmd; 972 int err; 973 974 if (xen_domain()) { 975 /* 976 * Xen doesn't have knowledge about devices in the VMD bus 977 * because the config space of devices behind the VMD bridge is 978 * not known to Xen, and hence Xen cannot discover or configure 979 * them in any way. 980 * 981 * Bypass of MSI remapping won't work in that case as direct 982 * write by Linux to the MSI entries won't result in functional 983 * interrupts, as Xen is the entity that manages the host 984 * interrupt controller and must configure interrupts. However 985 * multiplexing of interrupts by the VMD bridge will work under 986 * Xen, so force the usage of that mode which must always be 987 * supported by VMD bridges. 988 */ 989 features &= ~VMD_FEAT_CAN_BYPASS_MSI_REMAP; 990 } 991 992 if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20)) 993 return -ENOMEM; 994 995 vmd = devm_kzalloc(&dev->dev, sizeof(*vmd), GFP_KERNEL); 996 if (!vmd) 997 return -ENOMEM; 998 999 vmd->dev = dev; 1000 vmd->sysdata.domain = PCI_DOMAIN_NR_NOT_SET; 1001 vmd->instance = ida_alloc(&vmd_instance_ida, GFP_KERNEL); 1002 if (vmd->instance < 0) 1003 return vmd->instance; 1004 1005 vmd->name = devm_kasprintf(&dev->dev, GFP_KERNEL, "vmd%d", 1006 vmd->instance); 1007 if (!vmd->name) { 1008 err = -ENOMEM; 1009 goto out_release_instance; 1010 } 1011 1012 err = pcim_enable_device(dev); 1013 if (err < 0) 1014 goto out_release_instance; 1015 1016 vmd->cfgbar = pcim_iomap(dev, VMD_CFGBAR, 0); 1017 if (!vmd->cfgbar) { 1018 err = -ENOMEM; 1019 goto out_release_instance; 1020 } 1021 1022 pci_set_master(dev); 1023 if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) && 1024 dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32))) { 1025 err = -ENODEV; 1026 goto out_release_instance; 1027 } 1028 1029 if (features & VMD_FEAT_OFFSET_FIRST_VECTOR) 1030 vmd->first_vec = 1; 1031 1032 raw_spin_lock_init(&vmd->cfg_lock); 1033 pci_set_drvdata(dev, vmd); 1034 err = vmd_enable_domain(vmd, features); 1035 if (err) 1036 goto out_release_instance; 1037 1038 dev_info(&vmd->dev->dev, "Bound to PCI domain %04x\n", 1039 vmd->sysdata.domain); 1040 return 0; 1041 1042 out_release_instance: 1043 ida_free(&vmd_instance_ida, vmd->instance); 1044 return err; 1045 } 1046 1047 static void vmd_cleanup_srcu(struct vmd_dev *vmd) 1048 { 1049 int i; 1050 1051 for (i = 0; i < vmd->msix_count; i++) 1052 cleanup_srcu_struct(&vmd->irqs[i].srcu); 1053 } 1054 1055 static void vmd_remove(struct pci_dev *dev) 1056 { 1057 struct vmd_dev *vmd = pci_get_drvdata(dev); 1058 1059 pci_stop_root_bus(vmd->bus); 1060 sysfs_remove_link(&vmd->dev->dev.kobj, "domain"); 1061 pci_remove_root_bus(vmd->bus); 1062 vmd_cleanup_srcu(vmd); 1063 vmd_detach_resources(vmd); 1064 vmd_remove_irq_domain(vmd); 1065 ida_free(&vmd_instance_ida, vmd->instance); 1066 pci_bus_release_emul_domain_nr(vmd->sysdata.domain); 1067 } 1068 1069 static void vmd_shutdown(struct pci_dev *dev) 1070 { 1071 struct vmd_dev *vmd = pci_get_drvdata(dev); 1072 1073 vmd_remove_irq_domain(vmd); 1074 } 1075 1076 #ifdef CONFIG_PM_SLEEP 1077 static int vmd_suspend(struct device *dev) 1078 { 1079 struct pci_dev *pdev = to_pci_dev(dev); 1080 struct vmd_dev *vmd = pci_get_drvdata(pdev); 1081 int i; 1082 1083 for (i = 0; i < vmd->msix_count; i++) 1084 devm_free_irq(dev, vmd->irqs[i].virq, &vmd->irqs[i]); 1085 1086 return 0; 1087 } 1088 1089 static int vmd_resume(struct device *dev) 1090 { 1091 struct pci_dev *pdev = to_pci_dev(dev); 1092 struct vmd_dev *vmd = pci_get_drvdata(pdev); 1093 int err, i; 1094 1095 vmd_set_msi_remapping(vmd, !!vmd->irq_domain); 1096 1097 for (i = 0; i < vmd->msix_count; i++) { 1098 err = devm_request_irq(dev, vmd->irqs[i].virq, 1099 vmd_irq, IRQF_NO_THREAD, 1100 vmd->name, &vmd->irqs[i]); 1101 if (err) 1102 return err; 1103 } 1104 1105 return 0; 1106 } 1107 #endif 1108 static SIMPLE_DEV_PM_OPS(vmd_dev_pm_ops, vmd_suspend, vmd_resume); 1109 1110 static const struct pci_device_id vmd_ids[] = { 1111 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_201D), 1112 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP,}, 1113 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_28C0), 1114 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW | 1115 VMD_FEAT_HAS_BUS_RESTRICTIONS | 1116 VMD_FEAT_CAN_BYPASS_MSI_REMAP,}, 1117 {PCI_VDEVICE(INTEL, 0x467f), 1118 .driver_data = VMD_FEATS_CLIENT,}, 1119 {PCI_VDEVICE(INTEL, 0x4c3d), 1120 .driver_data = VMD_FEATS_CLIENT,}, 1121 {PCI_VDEVICE(INTEL, 0xa77f), 1122 .driver_data = VMD_FEATS_CLIENT,}, 1123 {PCI_VDEVICE(INTEL, 0x7d0b), 1124 .driver_data = VMD_FEATS_CLIENT,}, 1125 {PCI_VDEVICE(INTEL, 0xad0b), 1126 .driver_data = VMD_FEATS_CLIENT,}, 1127 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_9A0B), 1128 .driver_data = VMD_FEATS_CLIENT,}, 1129 {PCI_VDEVICE(INTEL, 0xb60b), 1130 .driver_data = VMD_FEATS_CLIENT,}, 1131 {PCI_VDEVICE(INTEL, 0xb06f), 1132 .driver_data = VMD_FEATS_CLIENT,}, 1133 {PCI_VDEVICE(INTEL, 0xb07f), 1134 .driver_data = VMD_FEATS_CLIENT,}, 1135 {0,} 1136 }; 1137 MODULE_DEVICE_TABLE(pci, vmd_ids); 1138 1139 static struct pci_driver vmd_drv = { 1140 .name = "vmd", 1141 .id_table = vmd_ids, 1142 .probe = vmd_probe, 1143 .remove = vmd_remove, 1144 .shutdown = vmd_shutdown, 1145 .driver = { 1146 .pm = &vmd_dev_pm_ops, 1147 }, 1148 }; 1149 module_pci_driver(vmd_drv); 1150 1151 MODULE_AUTHOR("Intel Corporation"); 1152 MODULE_DESCRIPTION("Volume Management Device driver"); 1153 MODULE_LICENSE("GPL v2"); 1154 MODULE_VERSION("0.6"); 1155