Lines Matching +full:msi +full:- +full:specifier
1 // SPDX-License-Identifier: GPL-2.0
8 * This driver acts as a paravirtual front-end for PCI Express root buses.
9 * When a PCI Express function (either an entire device or an SR-IOV
13 * VM within Hyper-V, there may seem to be no PCI bus at all in the VM
18 * to the VM using this front-end will appear at "device 0", the domain will
24 * MSI or MSI-X) associated with the functions on the bus. As interrupts are
28 * vector. This driver does not support level-triggered (line-based)
32 * The rest of this driver mostly maps PCI concepts onto underlying Hyper-V
34 * by Hyper-V is mapped into a single page of memory space, and the
37 * the PCI back-end driver in Hyper-V.
43 #include <linux/pci-ecam.h>
47 #include <linux/msi.h>
71 #define CPU_AFFINITY_ALL -1ULL
74 * Supported protocol versions in the order of probing - highest go
86 #define CFG_PAGE_SIZE (PCI_CONFIG_MMIO_LENGTH - CFG_PAGE_OFFSET)
153 * Function numbers are 8-bits wide on Express, as interpreted through ARI,
156 * the Hyper-V parent partition.
209 * occupied by this Message-Signaled
210 * Interrupt. For "MSI", as first defined
212 * 32. For "MSI-X," as first defined in PCI
213 * 3.0, this must be 1, as each MSI-X table
227 * struct hv_msi_desc2 - 1.2 version of hv_msi_desc
233 * occupied by this Message-Signaled
234 * Interrupt. For "MSI", as first defined
236 * 32. For "MSI-X," as first defined in PCI
237 * 3.0, this must be 1, as each MSI-X table
251 * struct hv_msi_desc3 - 1.3 version of hv_msi_desc
271 * a message-signaled interrupt, either MSI
272 * or MSI-X.
598 return cfg->vector; in hv_msi_get_int_vector()
604 * hv_arch_irq_unmask() - "Unmask" the IRQ by setting its current
608 * Build new a destination for the MSI and make a hypercall to
630 pbus = pdev->bus; in hv_arch_irq_unmask()
631 hbus = container_of(pbus->sysdata, struct hv_pcibus_device, sysdata); in hv_arch_irq_unmask()
632 int_desc = data->chip_data; in hv_arch_irq_unmask()
634 dev_warn(&hbus->hdev->device, "%s() can not unmask irq %u\n", in hv_arch_irq_unmask()
635 __func__, data->irq); in hv_arch_irq_unmask()
643 params->partition_id = HV_PARTITION_ID_SELF; in hv_arch_irq_unmask()
644 params->int_entry.source = HV_INTERRUPT_SOURCE_MSI; in hv_arch_irq_unmask()
645 params->int_entry.msi_entry.address.as_uint32 = int_desc->address & 0xffffffff; in hv_arch_irq_unmask()
646 params->int_entry.msi_entry.data.as_uint32 = int_desc->data; in hv_arch_irq_unmask()
647 params->device_id = (hbus->hdev->dev_instance.b[5] << 24) | in hv_arch_irq_unmask()
648 (hbus->hdev->dev_instance.b[4] << 16) | in hv_arch_irq_unmask()
649 (hbus->hdev->dev_instance.b[7] << 8) | in hv_arch_irq_unmask()
650 (hbus->hdev->dev_instance.b[6] & 0xf8) | in hv_arch_irq_unmask()
651 PCI_FUNC(pdev->devfn); in hv_arch_irq_unmask()
652 params->int_target.vector = hv_msi_get_int_vector(data); in hv_arch_irq_unmask()
654 if (hbus->protocol_version >= PCI_PROTOCOL_VERSION_1_2) { in hv_arch_irq_unmask()
662 params->int_target.flags |= in hv_arch_irq_unmask()
671 nr_bank = cpumask_to_vpset(¶ms->int_target.vp_set, tmp); in hv_arch_irq_unmask()
680 * var-sized hypercall, var-size starts after vp_mask (thus in hv_arch_irq_unmask()
687 params->int_target.vp_mask |= in hv_arch_irq_unmask()
703 * refer to cpu_disable_common() -> fixup_irqs() -> in hv_arch_irq_unmask()
704 * irq_migrate_all_off_this_cpu() -> migrate_one_irq(). in hv_arch_irq_unmask()
709 * into the irqdata data structure in migrate_one_irq() -> in hv_arch_irq_unmask()
714 if (!hv_result_success(res) && hbus->state != hv_pcibus_removing) in hv_arch_irq_unmask()
715 dev_err(&hbus->hdev->device, in hv_arch_irq_unmask()
722 * starting with a power of two to satisfy power of 2 multi-MSI requirement.
725 #define HV_PCI_MSI_SPI_NR (1020 - HV_PCI_MSI_SPI_START)
736 /* Hyper-V vPCI MSI GIC IRQ domain */
739 /* Hyper-V PCI MSI IRQ chip */
741 .name = "MSI",
750 return irqd->parent_data->hwirq; in hv_msi_get_int_vector()
764 struct hv_pci_chip_data *chip_data = domain->host_data; in hv_pci_vec_irq_free()
766 int first = d->hwirq - HV_PCI_MSI_SPI_START; in hv_pci_vec_irq_free()
769 mutex_lock(&chip_data->map_lock); in hv_pci_vec_irq_free()
770 bitmap_release_region(chip_data->spi_map, in hv_pci_vec_irq_free()
773 mutex_unlock(&chip_data->map_lock); in hv_pci_vec_irq_free()
794 struct hv_pci_chip_data *chip_data = domain->host_data; in hv_pci_vec_alloc_device_irq()
798 mutex_lock(&chip_data->map_lock); in hv_pci_vec_alloc_device_irq()
799 index = bitmap_find_free_region(chip_data->spi_map, in hv_pci_vec_alloc_device_irq()
802 mutex_unlock(&chip_data->map_lock); in hv_pci_vec_alloc_device_irq()
804 return -ENOSPC; in hv_pci_vec_alloc_device_irq()
819 fwspec.fwnode = domain->parent->fwnode; in hv_pci_vec_irq_gic_domain_alloc()
829 * Since the interrupt specifier is not coming from ACPI or DT, the in hv_pci_vec_irq_gic_domain_alloc()
833 d = irq_domain_get_irq_data(domain->parent, virq); in hv_pci_vec_irq_gic_domain_alloc()
835 return d->chip->irq_set_type(d, IRQ_TYPE_EDGE_RISING); in hv_pci_vec_irq_gic_domain_alloc()
861 domain->host_data); in hv_pci_vec_irq_domain_alloc()
870 * composing MSI from the hypervisor. GIC will eventually set the right
894 int ret = -ENOMEM; in hv_pci_irqchip_init()
900 mutex_init(&chip_data->map_lock); in hv_pci_irqchip_init()
915 pr_err("Failed to create Hyper-V arm64 vPCI MSI IRQ domain\n"); in hv_pci_irqchip_init()
936 * registers which Hyper-V already supports, so no hypercall needed.
942 * hv_pci_generic_compl() - Invoked for a completion packet
956 comp_pkt->completion_status = resp->status; in hv_pci_generic_compl()
957 complete(&comp_pkt->host_event); in hv_pci_generic_compl()
965 refcount_inc(&hpdev->refs); in get_pcichild()
970 if (refcount_dec_and_test(&hpdev->refs)) in put_pcichild()
982 if (hdev->channel->rescind) { in wait_for_response()
983 dev_warn_once(&hdev->device, "The device is gone.\n"); in wait_for_response()
984 return -ENODEV; in wait_for_response()
995 * devfn_to_wslot() - Convert from Linux PCI slot to Windows
1014 * wslot_to_devfn() - Convert from Windows PCI slot to Linux
1037 * to use the per-cpu input argument page. Use it for in hv_pci_read_mmio()
1042 in->gpa = gpa; in hv_pci_read_mmio()
1043 in->size = size; in hv_pci_read_mmio()
1049 *val = *(u8 *)(out->data); in hv_pci_read_mmio()
1052 *val = *(u16 *)(out->data); in hv_pci_read_mmio()
1055 *val = *(u32 *)(out->data); in hv_pci_read_mmio()
1070 * to use the per-cpu input argument memory. in hv_pci_write_mmio()
1073 in->gpa = gpa; in hv_pci_write_mmio()
1074 in->size = size; in hv_pci_write_mmio()
1077 *(u8 *)(in->data) = val; in hv_pci_write_mmio()
1080 *(u16 *)(in->data) = val; in hv_pci_write_mmio()
1083 *(u32 *)(in->data) = val; in hv_pci_write_mmio()
1095 * of pages in memory-mapped I/O space. Writing to the first page chooses
1102 * _hv_pcifront_read_config() - Internal PCI config read
1111 struct hv_pcibus_device *hbus = hpdev->hbus; in _hv_pcifront_read_config()
1112 struct device *dev = &hbus->hdev->device; in _hv_pcifront_read_config()
1120 memcpy(val, ((u8 *)&hpdev->desc.v_id) + where, size); in _hv_pcifront_read_config()
1123 memcpy(val, ((u8 *)&hpdev->desc.rev) + where - in _hv_pcifront_read_config()
1127 memcpy(val, (u8 *)&hpdev->desc.subsystem_id + where - in _hv_pcifront_read_config()
1136 * Interrupt Line and Interrupt PIN are hard-wired to zero in _hv_pcifront_read_config()
1137 * because this front-end only supports message-signaled in _hv_pcifront_read_config()
1143 spin_lock_irqsave(&hbus->config_lock, flags); in _hv_pcifront_read_config()
1144 if (hbus->use_calls) { in _hv_pcifront_read_config()
1145 phys_addr_t addr = hbus->mem_config->start + offset; in _hv_pcifront_read_config()
1147 hv_pci_write_mmio(dev, hbus->mem_config->start, 4, in _hv_pcifront_read_config()
1148 hpdev->desc.win_slot.slot); in _hv_pcifront_read_config()
1151 void __iomem *addr = hbus->cfg_addr + offset; in _hv_pcifront_read_config()
1154 writel(hpdev->desc.win_slot.slot, hbus->cfg_addr); in _hv_pcifront_read_config()
1175 spin_unlock_irqrestore(&hbus->config_lock, flags); in _hv_pcifront_read_config()
1183 struct hv_pcibus_device *hbus = hpdev->hbus; in hv_pcifront_get_vendor_id()
1184 struct device *dev = &hbus->hdev->device; in hv_pcifront_get_vendor_id()
1189 spin_lock_irqsave(&hbus->config_lock, flags); in hv_pcifront_get_vendor_id()
1191 if (hbus->use_calls) { in hv_pcifront_get_vendor_id()
1192 phys_addr_t addr = hbus->mem_config->start + in hv_pcifront_get_vendor_id()
1195 hv_pci_write_mmio(dev, hbus->mem_config->start, 4, in hv_pcifront_get_vendor_id()
1196 hpdev->desc.win_slot.slot); in hv_pcifront_get_vendor_id()
1200 void __iomem *addr = hbus->cfg_addr + CFG_PAGE_OFFSET + in hv_pcifront_get_vendor_id()
1203 writel(hpdev->desc.win_slot.slot, hbus->cfg_addr); in hv_pcifront_get_vendor_id()
1214 spin_unlock_irqrestore(&hbus->config_lock, flags); in hv_pcifront_get_vendor_id()
1220 * _hv_pcifront_write_config() - Internal PCI config write
1229 struct hv_pcibus_device *hbus = hpdev->hbus; in _hv_pcifront_write_config()
1230 struct device *dev = &hbus->hdev->device; in _hv_pcifront_write_config()
1236 /* SSIDs and ROM BARs are read-only */ in _hv_pcifront_write_config()
1238 spin_lock_irqsave(&hbus->config_lock, flags); in _hv_pcifront_write_config()
1240 if (hbus->use_calls) { in _hv_pcifront_write_config()
1241 phys_addr_t addr = hbus->mem_config->start + offset; in _hv_pcifront_write_config()
1243 hv_pci_write_mmio(dev, hbus->mem_config->start, 4, in _hv_pcifront_write_config()
1244 hpdev->desc.win_slot.slot); in _hv_pcifront_write_config()
1247 void __iomem *addr = hbus->cfg_addr + offset; in _hv_pcifront_write_config()
1250 writel(hpdev->desc.win_slot.slot, hbus->cfg_addr); in _hv_pcifront_write_config()
1271 spin_unlock_irqrestore(&hbus->config_lock, flags); in _hv_pcifront_write_config()
1278 * hv_pcifront_read_config() - Read configuration space
1292 container_of(bus->sysdata, struct hv_pcibus_device, sysdata); in hv_pcifront_read_config()
1306 * hv_pcifront_write_config() - Write configuration space
1320 container_of(bus->sysdata, struct hv_pcibus_device, sysdata); in hv_pcifront_write_config()
1342 * Hyper-V SR-IOV provides a backchannel mechanism in software for
1345 * but instead of doing reads and writes in 32-bit chunks through a very slow
1348 * Nearly every SR-IOV device contains just such a communications channel in
1372 * hv_pci_read_config_compl() - Invoked when a response packet
1388 comp->comp_pkt.completion_status = -1; in hv_pci_read_config_compl()
1392 data_len = resp_packet_size - hdr_len; in hv_pci_read_config_compl()
1393 if (data_len > 0 && read_resp->status == 0) { in hv_pci_read_config_compl()
1394 comp->bytes_returned = min(comp->len, data_len); in hv_pci_read_config_compl()
1395 memcpy(comp->buf, read_resp->bytes, comp->bytes_returned); in hv_pci_read_config_compl()
1397 comp->bytes_returned = 0; in hv_pci_read_config_compl()
1400 comp->comp_pkt.completion_status = read_resp->status; in hv_pci_read_config_compl()
1402 complete(&comp->comp_pkt.host_event); in hv_pci_read_config_compl()
1406 * hv_read_config_block() - Sends a read config block request to
1407 * the back-end driver running in the Hyper-V parent partition.
1412 * @bytes_returned: Size which came back from the back-end driver.
1414 * Return: 0 on success, -errno on failure
1421 container_of(pdev->bus->sysdata, struct hv_pcibus_device, in hv_read_config_block()
1432 return -EINVAL; in hv_read_config_block()
1442 read_blk->message_type.type = PCI_READ_BLOCK; in hv_read_config_block()
1443 read_blk->wslot.slot = devfn_to_wslot(pdev->devfn); in hv_read_config_block()
1444 read_blk->block_id = block_id; in hv_read_config_block()
1445 read_blk->bytes_requested = len; in hv_read_config_block()
1447 ret = vmbus_sendpacket(hbus->hdev->channel, read_blk, in hv_read_config_block()
1454 ret = wait_for_response(hbus->hdev, &comp_pkt.comp_pkt.host_event); in hv_read_config_block()
1460 dev_err(&hbus->hdev->device, in hv_read_config_block()
1464 return -EIO; in hv_read_config_block()
1472 * hv_pci_write_config_compl() - Invoked when a response packet for a write
1483 comp_pkt->completion_status = resp->status; in hv_pci_write_config_compl()
1484 complete(&comp_pkt->host_event); in hv_pci_write_config_compl()
1488 * hv_write_config_block() - Sends a write config block request to the
1489 * back-end driver running in the Hyper-V parent partition.
1495 * Return: 0 on success, -errno on failure
1501 container_of(pdev->bus->sysdata, struct hv_pcibus_device, in hv_write_config_block()
1514 return -EINVAL; in hv_write_config_block()
1522 write_blk->message_type.type = PCI_WRITE_BLOCK; in hv_write_config_block()
1523 write_blk->wslot.slot = devfn_to_wslot(pdev->devfn); in hv_write_config_block()
1524 write_blk->block_id = block_id; in hv_write_config_block()
1525 write_blk->byte_count = len; in hv_write_config_block()
1526 memcpy(write_blk->bytes, buf, len); in hv_write_config_block()
1533 * specified in write_blk->byte_count. in hv_write_config_block()
1537 ret = vmbus_sendpacket(hbus->hdev->channel, write_blk, pkt_size, in hv_write_config_block()
1543 ret = wait_for_response(hbus->hdev, &comp_pkt.host_event); in hv_write_config_block()
1548 dev_err(&hbus->hdev->device, in hv_write_config_block()
1551 return -EIO; in hv_write_config_block()
1558 * hv_register_block_invalidate() - Invoked when a config block invalidation
1559 * arrives from the back-end driver.
1564 * Return: 0 on success, -errno on failure
1571 container_of(pdev->bus->sysdata, struct hv_pcibus_device, in hv_register_block_invalidate()
1575 hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn)); in hv_register_block_invalidate()
1577 return -ENODEV; in hv_register_block_invalidate()
1579 hpdev->block_invalidate = block_invalidate; in hv_register_block_invalidate()
1580 hpdev->invalidate_context = context; in hv_register_block_invalidate()
1597 if (!int_desc->vector_count) { in hv_int_desc_free()
1603 int_pkt->message_type.type = in hv_int_desc_free()
1605 int_pkt->wslot.slot = hpdev->desc.win_slot.slot; in hv_int_desc_free()
1606 int_pkt->int_desc = *int_desc; in hv_int_desc_free()
1607 vmbus_sendpacket(hpdev->hbus->hdev->channel, int_pkt, sizeof(*int_pkt), in hv_int_desc_free()
1613 * hv_msi_free() - Free the MSI.
1615 * @info: Extra MSI-related context
1618 * The Hyper-V parent partition and hypervisor are tracking the
1631 struct msi_desc *msi = irq_data_get_msi_desc(irq_data); in hv_msi_free() local
1633 pdev = msi_desc_to_pci_dev(msi); in hv_msi_free()
1634 hbus = info->data; in hv_msi_free()
1639 irq_data->chip_data = NULL; in hv_msi_free()
1640 hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn)); in hv_msi_free()
1653 if (data->parent_data->chip->irq_mask) in hv_irq_mask()
1661 if (data->parent_data->chip->irq_unmask) in hv_irq_unmask()
1679 comp_pkt->comp_pkt.completion_status = -1; in hv_pci_compose_compl()
1682 comp_pkt->comp_pkt.completion_status = resp->status; in hv_pci_compose_compl()
1683 comp_pkt->int_desc = int_resp->int_desc; in hv_pci_compose_compl()
1685 complete(&comp_pkt->comp_pkt.host_event); in hv_pci_compose_compl()
1692 int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE; in hv_compose_msi_req_v1()
1693 int_pkt->wslot.slot = slot; in hv_compose_msi_req_v1()
1694 int_pkt->int_desc.vector = vector; in hv_compose_msi_req_v1()
1695 int_pkt->int_desc.vector_count = vector_count; in hv_compose_msi_req_v1()
1696 int_pkt->int_desc.delivery_mode = DELIVERY_MODE; in hv_compose_msi_req_v1()
1699 * Create MSI w/ dummy vCPU set, overwritten by subsequent retarget in in hv_compose_msi_req_v1()
1702 int_pkt->int_desc.cpu_mask = CPU_AFFINITY_ALL; in hv_compose_msi_req_v1()
1710 * interrupted is specified later in hv_irq_unmask() and communicated to Hyper-V
1712 * not irrelevant because Hyper-V chooses the physical CPU to handle the
1714 * hv_compose_msi_msg(). Hyper-V's choice of pCPU is not visible to the guest,
1716 * performance bottleneck. So we spread out the dummy vCPUs to influence Hyper-V
1719 * For the single-MSI and MSI-X cases, it's OK for hv_compose_msi_req_get_cpu()
1721 * hv_compose_msi_msg() contains the "real" vCPU, causing Hyper-V to choose a
1722 * new pCPU for the interrupt. But for the multi-MSI case, the second call to
1724 * original dummy vCPU is used. This dummy vCPU must be round-robin'ed so that
1725 * the pCPUs are spread out. All interrupts for a multi-MSI device end up using
1729 * With Hyper-V in Nov 2022, the HVCALL_RETARGET_INTERRUPT hypercall does *not*
1730 * cause Hyper-V to reselect the pCPU based on the specified vCPU. Such an
1732 * dummy vCPU selection won't matter, and interrupts for the same multi-MSI
1737 * Create MSI w/ dummy vCPU set targeting just one vCPU, overwritten
1746 * Make sure the dummy vCPU values for multi-MSI don't all point to vCPU0.
1752 /* -1 means starting with CPU 0 */ in hv_compose_multi_msi_req_get_cpu()
1753 static int cpu_next = -1; in hv_compose_multi_msi_req_get_cpu()
1773 int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE2; in hv_compose_msi_req_v2()
1774 int_pkt->wslot.slot = slot; in hv_compose_msi_req_v2()
1775 int_pkt->int_desc.vector = vector; in hv_compose_msi_req_v2()
1776 int_pkt->int_desc.vector_count = vector_count; in hv_compose_msi_req_v2()
1777 int_pkt->int_desc.delivery_mode = DELIVERY_MODE; in hv_compose_msi_req_v2()
1778 int_pkt->int_desc.processor_array[0] = in hv_compose_msi_req_v2()
1780 int_pkt->int_desc.processor_count = 1; in hv_compose_msi_req_v2()
1789 int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE3; in hv_compose_msi_req_v3()
1790 int_pkt->wslot.slot = slot; in hv_compose_msi_req_v3()
1791 int_pkt->int_desc.vector = vector; in hv_compose_msi_req_v3()
1792 int_pkt->int_desc.reserved = 0; in hv_compose_msi_req_v3()
1793 int_pkt->int_desc.vector_count = vector_count; in hv_compose_msi_req_v3()
1794 int_pkt->int_desc.delivery_mode = DELIVERY_MODE; in hv_compose_msi_req_v3()
1795 int_pkt->int_desc.processor_array[0] = in hv_compose_msi_req_v3()
1797 int_pkt->int_desc.processor_count = 1; in hv_compose_msi_req_v3()
1803 * hv_compose_msi_msg() - Supplies a valid MSI address/data
1804 * @data: Everything about this MSI
1845 multi_msi = !msi_desc->pci.msi_attrib.is_msix && in hv_compose_msi_msg()
1846 msi_desc->nvec_used > 1; in hv_compose_msi_msg()
1849 if (data->chip_data && multi_msi) { in hv_compose_msi_msg()
1850 int_desc = data->chip_data; in hv_compose_msi_msg()
1851 msg->address_hi = int_desc->address >> 32; in hv_compose_msi_msg()
1852 msg->address_lo = int_desc->address & 0xffffffff; in hv_compose_msi_msg()
1853 msg->data = int_desc->data; in hv_compose_msi_msg()
1859 pbus = pdev->bus; in hv_compose_msi_msg()
1860 hbus = container_of(pbus->sysdata, struct hv_pcibus_device, sysdata); in hv_compose_msi_msg()
1861 channel = hbus->hdev->channel; in hv_compose_msi_msg()
1862 hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn)); in hv_compose_msi_msg()
1867 if (data->chip_data && !multi_msi) { in hv_compose_msi_msg()
1868 int_desc = data->chip_data; in hv_compose_msi_msg()
1869 data->chip_data = NULL; in hv_compose_msi_msg()
1879 * If this is not the first MSI of Multi MSI, we already have in hv_compose_msi_msg()
1882 if (msi_desc->irq != data->irq) { in hv_compose_msi_msg()
1883 data->chip_data = int_desc; in hv_compose_msi_msg()
1884 int_desc->address = msi_desc->msg.address_lo | in hv_compose_msi_msg()
1885 (u64)msi_desc->msg.address_hi << 32; in hv_compose_msi_msg()
1886 int_desc->data = msi_desc->msg.data + in hv_compose_msi_msg()
1887 (data->irq - msi_desc->irq); in hv_compose_msi_msg()
1888 msg->address_hi = msi_desc->msg.address_hi; in hv_compose_msi_msg()
1889 msg->address_lo = msi_desc->msg.address_lo; in hv_compose_msi_msg()
1890 msg->data = int_desc->data; in hv_compose_msi_msg()
1897 * to be aligned with the count, and also not zero. Multi-msi in hv_compose_msi_msg()
1901 vector_count = msi_desc->nvec_used; in hv_compose_msi_msg()
1919 switch (hbus->protocol_version) { in hv_compose_msi_msg()
1922 hpdev->desc.win_slot.slot, in hv_compose_msi_msg()
1931 hpdev->desc.win_slot.slot, in hv_compose_msi_msg()
1939 hpdev->desc.win_slot.slot, in hv_compose_msi_msg()
1949 dev_err(&hbus->hdev->device, in hv_compose_msi_msg()
1954 ret = vmbus_sendpacket_getid(hpdev->hbus->hdev->channel, &ctxt.int_pkts, in hv_compose_msi_msg()
1959 dev_err(&hbus->hdev->device, in hv_compose_msi_msg()
1969 tasklet_disable_in_atomic(&channel->callback_event); in hv_compose_msi_msg()
1980 dev_err_once(&hbus->hdev->device, in hv_compose_msi_msg()
1992 spin_lock_irqsave(&channel->sched_lock, flags); in hv_compose_msi_msg()
1993 if (unlikely(channel->onchannel_callback == NULL)) { in hv_compose_msi_msg()
1994 spin_unlock_irqrestore(&channel->sched_lock, flags); in hv_compose_msi_msg()
1998 spin_unlock_irqrestore(&channel->sched_lock, flags); in hv_compose_msi_msg()
2003 tasklet_enable(&channel->callback_event); in hv_compose_msi_msg()
2006 dev_err(&hbus->hdev->device, in hv_compose_msi_msg()
2018 data->chip_data = int_desc; in hv_compose_msi_msg()
2021 msg->address_hi = comp.int_desc.address >> 32; in hv_compose_msi_msg()
2022 msg->address_lo = comp.int_desc.address & 0xffffffff; in hv_compose_msi_msg()
2023 msg->data = comp.int_desc.data; in hv_compose_msi_msg()
2029 tasklet_enable(&channel->callback_event); in hv_compose_msi_msg()
2034 * been 're-used', i.e., already removed and (re-)mapped.) in hv_compose_msi_msg()
2044 msg->address_hi = 0; in hv_compose_msi_msg()
2045 msg->address_lo = 0; in hv_compose_msi_msg()
2046 msg->data = 0; in hv_compose_msi_msg()
2051 .name = "Hyper-V PCIe MSI",
2069 * hv_pcie_init_irq_domain() - Initialize IRQ domain
2074 * devices only support MSI and MSI-X, not line-based interrupts
2075 * or simulations of line-based interrupts through PCIe's
2076 * fabric-layer messages. Because interrupts are remapped, we
2077 * can support multi-message MSI here.
2083 hbus->msi_info.chip = &hv_msi_irq_chip; in hv_pcie_init_irq_domain()
2084 hbus->msi_info.ops = &hv_msi_ops; in hv_pcie_init_irq_domain()
2085 hbus->msi_info.flags = (MSI_FLAG_USE_DEF_DOM_OPS | in hv_pcie_init_irq_domain()
2088 hbus->msi_info.handler = FLOW_HANDLER; in hv_pcie_init_irq_domain()
2089 hbus->msi_info.handler_name = FLOW_NAME; in hv_pcie_init_irq_domain()
2090 hbus->msi_info.data = hbus; in hv_pcie_init_irq_domain()
2091 hbus->irq_domain = pci_msi_create_irq_domain(hbus->fwnode, in hv_pcie_init_irq_domain()
2092 &hbus->msi_info, in hv_pcie_init_irq_domain()
2094 if (!hbus->irq_domain) { in hv_pcie_init_irq_domain()
2095 dev_err(&hbus->hdev->device, in hv_pcie_init_irq_domain()
2096 "Failed to build an MSI IRQ domain\n"); in hv_pcie_init_irq_domain()
2097 return -ENODEV; in hv_pcie_init_irq_domain()
2100 dev_set_msi_domain(&hbus->bridge->dev, hbus->irq_domain); in hv_pcie_init_irq_domain()
2106 * get_bar_size() - Get the address space consumed by a BAR
2107 * @bar_val: Value that a BAR returned after -1 was written
2126 * survey_child_resources() - Total all MMIO requirements
2139 event = xchg(&hbus->survey_event, NULL); in survey_child_resources()
2144 if (hbus->low_mmio_space || hbus->high_mmio_space) { in survey_child_resources()
2149 spin_lock_irqsave(&hbus->device_list_lock, flags); in survey_child_resources()
2156 list_for_each_entry(hpdev, &hbus->children, list_entry) { in survey_child_resources()
2158 if (hpdev->probed_bar[i] & PCI_BASE_ADDRESS_SPACE_IO) in survey_child_resources()
2159 dev_err(&hbus->hdev->device, in survey_child_resources()
2162 if (hpdev->probed_bar[i] != 0) { in survey_child_resources()
2168 bar_val = hpdev->probed_bar[i]; in survey_child_resources()
2171 ((u64)hpdev->probed_bar[++i] << 32); in survey_child_resources()
2178 hbus->high_mmio_space += bar_size; in survey_child_resources()
2180 hbus->low_mmio_space += bar_size; in survey_child_resources()
2185 spin_unlock_irqrestore(&hbus->device_list_lock, flags); in survey_child_resources()
2190 * prepopulate_bars() - Fill in BARs with defaults
2198 * the memory-mapped I/O space will be aligned on the largest
2215 if (hbus->low_mmio_space) { in prepopulate_bars()
2216 low_size = 1ULL << (63 - __builtin_clzll(hbus->low_mmio_space)); in prepopulate_bars()
2217 low_base = hbus->low_mmio_res->start; in prepopulate_bars()
2220 if (hbus->high_mmio_space) { in prepopulate_bars()
2222 (63 - __builtin_clzll(hbus->high_mmio_space)); in prepopulate_bars()
2223 high_base = hbus->high_mmio_res->start; in prepopulate_bars()
2226 spin_lock_irqsave(&hbus->device_list_lock, flags); in prepopulate_bars()
2234 * If the memory enable bit is already set, Hyper-V silently ignores in prepopulate_bars()
2239 list_for_each_entry(hpdev, &hbus->children, list_entry) { in prepopulate_bars()
2247 list_for_each_entry(hpdev, &hbus->children, list_entry) { in prepopulate_bars()
2249 bar_val = hpdev->probed_bar[i]; in prepopulate_bars()
2255 ((u64)hpdev->probed_bar[i + 1] in prepopulate_bars()
2289 * to be pre-set. Actually here we intentionally in prepopulate_bars()
2291 * in the core PCI driver doesn't cause Hyper-V in prepopulate_bars()
2305 spin_unlock_irqrestore(&hbus->device_list_lock, flags); in prepopulate_bars()
2314 * same ordered workqueue. Therefore hbus->children list will not change
2323 list_for_each_entry(hpdev, &hbus->children, list_entry) { in hv_pci_assign_slots()
2324 if (hpdev->pci_slot) in hv_pci_assign_slots()
2327 slot_nr = PCI_SLOT(wslot_to_devfn(hpdev->desc.win_slot.slot)); in hv_pci_assign_slots()
2328 snprintf(name, SLOT_NAME_SIZE, "%u", hpdev->desc.ser); in hv_pci_assign_slots()
2329 hpdev->pci_slot = pci_create_slot(hbus->bridge->bus, slot_nr, in hv_pci_assign_slots()
2331 if (IS_ERR(hpdev->pci_slot)) { in hv_pci_assign_slots()
2333 hpdev->pci_slot = NULL; in hv_pci_assign_slots()
2345 list_for_each_entry(hpdev, &hbus->children, list_entry) { in hv_pci_remove_slots()
2346 if (!hpdev->pci_slot) in hv_pci_remove_slots()
2348 pci_destroy_slot(hpdev->pci_slot); in hv_pci_remove_slots()
2349 hpdev->pci_slot = NULL; in hv_pci_remove_slots()
2359 struct pci_bus *bus = hbus->bridge->bus; in hv_pci_assign_numa_node()
2362 list_for_each_entry(dev, &bus->devices, bus_list) { in hv_pci_assign_numa_node()
2363 hv_dev = get_pcichild_wslot(hbus, devfn_to_wslot(dev->devfn)); in hv_pci_assign_numa_node()
2367 if (hv_dev->desc.flags & HV_PCI_DEVICE_FLAG_NUMA_AFFINITY && in hv_pci_assign_numa_node()
2368 hv_dev->desc.virtual_numa_node < num_possible_nodes()) in hv_pci_assign_numa_node()
2375 set_dev_node(&dev->dev, in hv_pci_assign_numa_node()
2377 hv_dev->desc.virtual_numa_node)); in hv_pci_assign_numa_node()
2384 * create_root_hv_pci_bus() - Expose a new root PCI bus
2387 * Return: 0 on success, -errno on failure
2392 struct pci_host_bridge *bridge = hbus->bridge; in create_root_hv_pci_bus()
2394 bridge->dev.parent = &hbus->hdev->device; in create_root_hv_pci_bus()
2395 bridge->sysdata = &hbus->sysdata; in create_root_hv_pci_bus()
2396 bridge->ops = &hv_pcifront_ops; in create_root_hv_pci_bus()
2404 pci_bus_assign_resources(bridge->bus); in create_root_hv_pci_bus()
2406 pci_bus_add_devices(bridge->bus); in create_root_hv_pci_bus()
2408 hbus->state = hv_pcibus_installed; in create_root_hv_pci_bus()
2418 * q_resource_requirements() - Query Resource Requirements
2435 status = (resp_packet_size < sizeof(*q_res_req)) ? -1 : resp->status; in q_resource_requirements()
2437 dev_err(&completion->hpdev->hbus->hdev->device, in q_resource_requirements()
2442 completion->hpdev->probed_bar[i] = in q_resource_requirements()
2443 q_res_req->probed_bar[i]; in q_resource_requirements()
2447 complete(&completion->host_event); in q_resource_requirements()
2451 * new_pcichild_device() - Create a new child device
2478 hpdev->hbus = hbus; in new_pcichild_device()
2486 res_req->message_type.type = PCI_QUERY_RESOURCE_REQUIREMENTS; in new_pcichild_device()
2487 res_req->wslot.slot = desc->win_slot.slot; in new_pcichild_device()
2489 ret = vmbus_sendpacket(hbus->hdev->channel, res_req, in new_pcichild_device()
2497 if (wait_for_response(hbus->hdev, &comp_pkt.host_event)) in new_pcichild_device()
2500 hpdev->desc = *desc; in new_pcichild_device()
2501 refcount_set(&hpdev->refs, 1); in new_pcichild_device()
2503 spin_lock_irqsave(&hbus->device_list_lock, flags); in new_pcichild_device()
2505 list_add_tail(&hpdev->list_entry, &hbus->children); in new_pcichild_device()
2506 spin_unlock_irqrestore(&hbus->device_list_lock, flags); in new_pcichild_device()
2515 * get_pcichild_wslot() - Find device from slot
2533 spin_lock_irqsave(&hbus->device_list_lock, flags); in get_pcichild_wslot()
2534 list_for_each_entry(iter, &hbus->children, list_entry) { in get_pcichild_wslot()
2535 if (iter->desc.win_slot.slot == wslot) { in get_pcichild_wslot()
2541 spin_unlock_irqrestore(&hbus->device_list_lock, flags); in get_pcichild_wslot()
2547 * pci_devices_present_work() - Handle new list of child devices
2552 * debug the interaction between Hyper-V and Linux. This
2567 * because both are pushed to the ordered workqueue hbus->wq.
2582 hbus = dr_wrk->bus; in pci_devices_present_work()
2588 spin_lock_irqsave(&hbus->device_list_lock, flags); in pci_devices_present_work()
2589 while (!list_empty(&hbus->dr_list)) { in pci_devices_present_work()
2590 dr = list_first_entry(&hbus->dr_list, struct hv_dr_state, in pci_devices_present_work()
2592 list_del(&dr->list_entry); in pci_devices_present_work()
2595 if (!list_empty(&hbus->dr_list)) { in pci_devices_present_work()
2600 spin_unlock_irqrestore(&hbus->device_list_lock, flags); in pci_devices_present_work()
2605 mutex_lock(&hbus->state_lock); in pci_devices_present_work()
2608 spin_lock_irqsave(&hbus->device_list_lock, flags); in pci_devices_present_work()
2609 list_for_each_entry(hpdev, &hbus->children, list_entry) { in pci_devices_present_work()
2610 hpdev->reported_missing = true; in pci_devices_present_work()
2612 spin_unlock_irqrestore(&hbus->device_list_lock, flags); in pci_devices_present_work()
2615 for (child_no = 0; child_no < dr->device_count; child_no++) { in pci_devices_present_work()
2617 new_desc = &dr->func[child_no]; in pci_devices_present_work()
2619 spin_lock_irqsave(&hbus->device_list_lock, flags); in pci_devices_present_work()
2620 list_for_each_entry(hpdev, &hbus->children, list_entry) { in pci_devices_present_work()
2621 if ((hpdev->desc.win_slot.slot == new_desc->win_slot.slot) && in pci_devices_present_work()
2622 (hpdev->desc.v_id == new_desc->v_id) && in pci_devices_present_work()
2623 (hpdev->desc.d_id == new_desc->d_id) && in pci_devices_present_work()
2624 (hpdev->desc.ser == new_desc->ser)) { in pci_devices_present_work()
2625 hpdev->reported_missing = false; in pci_devices_present_work()
2629 spin_unlock_irqrestore(&hbus->device_list_lock, flags); in pci_devices_present_work()
2634 dev_err(&hbus->hdev->device, in pci_devices_present_work()
2640 spin_lock_irqsave(&hbus->device_list_lock, flags); in pci_devices_present_work()
2643 list_for_each_entry(hpdev, &hbus->children, list_entry) { in pci_devices_present_work()
2644 if (hpdev->reported_missing) { in pci_devices_present_work()
2647 list_move_tail(&hpdev->list_entry, &removed); in pci_devices_present_work()
2652 spin_unlock_irqrestore(&hbus->device_list_lock, flags); in pci_devices_present_work()
2658 list_del(&hpdev->list_entry); in pci_devices_present_work()
2660 if (hpdev->pci_slot) in pci_devices_present_work()
2661 pci_destroy_slot(hpdev->pci_slot); in pci_devices_present_work()
2666 switch (hbus->state) { in pci_devices_present_work()
2673 pci_scan_child_bus(hbus->bridge->bus); in pci_devices_present_work()
2688 mutex_unlock(&hbus->state_lock); in pci_devices_present_work()
2694 * hv_pci_start_relations_work() - Queue work to start device discovery
2698 * Return: 0 on success, -errno on failure
2707 if (hbus->state == hv_pcibus_removing) { in hv_pci_start_relations_work()
2708 dev_info(&hbus->hdev->device, in hv_pci_start_relations_work()
2710 return -ENOENT; in hv_pci_start_relations_work()
2715 return -ENOMEM; in hv_pci_start_relations_work()
2717 INIT_WORK(&dr_wrk->wrk, pci_devices_present_work); in hv_pci_start_relations_work()
2718 dr_wrk->bus = hbus; in hv_pci_start_relations_work()
2720 spin_lock_irqsave(&hbus->device_list_lock, flags); in hv_pci_start_relations_work()
2726 pending_dr = !list_empty(&hbus->dr_list); in hv_pci_start_relations_work()
2727 list_add_tail(&dr->list_entry, &hbus->dr_list); in hv_pci_start_relations_work()
2728 spin_unlock_irqrestore(&hbus->device_list_lock, flags); in hv_pci_start_relations_work()
2733 queue_work(hbus->wq, &dr_wrk->wrk); in hv_pci_start_relations_work()
2739 * hv_pci_devices_present() - Handle list of new children
2753 dr = kzalloc(struct_size(dr, func, relations->device_count), in hv_pci_devices_present()
2758 dr->device_count = relations->device_count; in hv_pci_devices_present()
2759 for (i = 0; i < dr->device_count; i++) { in hv_pci_devices_present()
2760 dr->func[i].v_id = relations->func[i].v_id; in hv_pci_devices_present()
2761 dr->func[i].d_id = relations->func[i].d_id; in hv_pci_devices_present()
2762 dr->func[i].rev = relations->func[i].rev; in hv_pci_devices_present()
2763 dr->func[i].prog_intf = relations->func[i].prog_intf; in hv_pci_devices_present()
2764 dr->func[i].subclass = relations->func[i].subclass; in hv_pci_devices_present()
2765 dr->func[i].base_class = relations->func[i].base_class; in hv_pci_devices_present()
2766 dr->func[i].subsystem_id = relations->func[i].subsystem_id; in hv_pci_devices_present()
2767 dr->func[i].win_slot = relations->func[i].win_slot; in hv_pci_devices_present()
2768 dr->func[i].ser = relations->func[i].ser; in hv_pci_devices_present()
2776 * hv_pci_devices_present2() - Handle list of new children
2788 dr = kzalloc(struct_size(dr, func, relations->device_count), in hv_pci_devices_present2()
2793 dr->device_count = relations->device_count; in hv_pci_devices_present2()
2794 for (i = 0; i < dr->device_count; i++) { in hv_pci_devices_present2()
2795 dr->func[i].v_id = relations->func[i].v_id; in hv_pci_devices_present2()
2796 dr->func[i].d_id = relations->func[i].d_id; in hv_pci_devices_present2()
2797 dr->func[i].rev = relations->func[i].rev; in hv_pci_devices_present2()
2798 dr->func[i].prog_intf = relations->func[i].prog_intf; in hv_pci_devices_present2()
2799 dr->func[i].subclass = relations->func[i].subclass; in hv_pci_devices_present2()
2800 dr->func[i].base_class = relations->func[i].base_class; in hv_pci_devices_present2()
2801 dr->func[i].subsystem_id = relations->func[i].subsystem_id; in hv_pci_devices_present2()
2802 dr->func[i].win_slot = relations->func[i].win_slot; in hv_pci_devices_present2()
2803 dr->func[i].ser = relations->func[i].ser; in hv_pci_devices_present2()
2804 dr->func[i].flags = relations->func[i].flags; in hv_pci_devices_present2()
2805 dr->func[i].virtual_numa_node = in hv_pci_devices_present2()
2806 relations->func[i].virtual_numa_node; in hv_pci_devices_present2()
2814 * hv_eject_device_work() - Asynchronously handles ejection
2836 hbus = hpdev->hbus; in hv_eject_device_work()
2838 mutex_lock(&hbus->state_lock); in hv_eject_device_work()
2843 * must be done without constructs like pci_domain_nr(hbus->bridge->bus) in hv_eject_device_work()
2844 * because hbus->bridge->bus may not exist yet. in hv_eject_device_work()
2846 wslot = wslot_to_devfn(hpdev->desc.win_slot.slot); in hv_eject_device_work()
2847 pdev = pci_get_domain_bus_and_slot(hbus->bridge->domain_nr, 0, wslot); in hv_eject_device_work()
2855 spin_lock_irqsave(&hbus->device_list_lock, flags); in hv_eject_device_work()
2856 list_del(&hpdev->list_entry); in hv_eject_device_work()
2857 spin_unlock_irqrestore(&hbus->device_list_lock, flags); in hv_eject_device_work()
2859 if (hpdev->pci_slot) in hv_eject_device_work()
2860 pci_destroy_slot(hpdev->pci_slot); in hv_eject_device_work()
2864 ejct_pkt->message_type.type = PCI_EJECTION_COMPLETE; in hv_eject_device_work()
2865 ejct_pkt->wslot.slot = hpdev->desc.win_slot.slot; in hv_eject_device_work()
2866 vmbus_sendpacket(hbus->hdev->channel, ejct_pkt, in hv_eject_device_work()
2877 mutex_unlock(&hbus->state_lock); in hv_eject_device_work()
2881 * hv_pci_eject_device() - Handles device ejection
2885 * just schedules work so that we don't re-enter the packet
2890 struct hv_pcibus_device *hbus = hpdev->hbus; in hv_pci_eject_device()
2891 struct hv_device *hdev = hbus->hdev; in hv_pci_eject_device()
2893 if (hbus->state == hv_pcibus_removing) { in hv_pci_eject_device()
2894 dev_info(&hdev->device, "PCI VMBus EJECT: ignored\n"); in hv_pci_eject_device()
2899 INIT_WORK(&hpdev->wrk, hv_eject_device_work); in hv_pci_eject_device()
2900 queue_work(hbus->wq, &hpdev->wrk); in hv_pci_eject_device()
2904 * hv_pci_onchannelcallback() - Handles incoming packets
2915 struct vmbus_channel *chan = hbus->hdev->channel; in hv_pci_onchannelcallback()
2939 if (ret == -ENOBUFS) { in hv_pci_onchannelcallback()
2961 switch (desc->type) { in hv_pci_onchannelcallback()
2969 dev_err(&hbus->hdev->device, in hv_pci_onchannelcallback()
2977 * Call ->completion_func() within the critical section to make in hv_pci_onchannelcallback()
2983 comp_packet->completion_func(comp_packet->compl_ctxt, in hv_pci_onchannelcallback()
2992 switch (new_message->message_type.type) { in hv_pci_onchannelcallback()
2999 bus_rel->device_count)) { in hv_pci_onchannelcallback()
3000 dev_err(&hbus->hdev->device, in hv_pci_onchannelcallback()
3014 bus_rel2->device_count)) { in hv_pci_onchannelcallback()
3015 dev_err(&hbus->hdev->device, in hv_pci_onchannelcallback()
3027 dev_err(&hbus->hdev->device, in hv_pci_onchannelcallback()
3032 dev_message->wslot.slot); in hv_pci_onchannelcallback()
3043 dev_err(&hbus->hdev->device, in hv_pci_onchannelcallback()
3048 inval->wslot.slot); in hv_pci_onchannelcallback()
3050 if (hpdev->block_invalidate) { in hv_pci_onchannelcallback()
3051 hpdev->block_invalidate( in hv_pci_onchannelcallback()
3052 hpdev->invalidate_context, in hv_pci_onchannelcallback()
3053 inval->block_mask); in hv_pci_onchannelcallback()
3060 dev_warn(&hbus->hdev->device, in hv_pci_onchannelcallback()
3062 new_message->message_type.type); in hv_pci_onchannelcallback()
3068 dev_err(&hbus->hdev->device, in hv_pci_onchannelcallback()
3070 desc->type, req_id, bytes_recvd); in hv_pci_onchannelcallback()
3079 * hv_pci_protocol_negotiation() - Set up protocol
3082 * the order of probing - highest go first.
3116 return -ENOMEM; in hv_pci_protocol_negotiation()
3119 pkt->completion_func = hv_pci_generic_compl; in hv_pci_protocol_negotiation()
3120 pkt->compl_ctxt = &comp_pkt; in hv_pci_protocol_negotiation()
3121 version_req = (struct pci_version_request *)&pkt->message; in hv_pci_protocol_negotiation()
3122 version_req->message_type.type = PCI_QUERY_PROTOCOL_VERSION; in hv_pci_protocol_negotiation()
3125 version_req->protocol_version = version[i]; in hv_pci_protocol_negotiation()
3126 ret = vmbus_sendpacket(hdev->channel, version_req, in hv_pci_protocol_negotiation()
3134 dev_err(&hdev->device, in hv_pci_protocol_negotiation()
3135 "PCI Pass-through VSP failed to request version: %d", in hv_pci_protocol_negotiation()
3141 hbus->protocol_version = version[i]; in hv_pci_protocol_negotiation()
3142 dev_info(&hdev->device, in hv_pci_protocol_negotiation()
3144 hbus->protocol_version); in hv_pci_protocol_negotiation()
3149 dev_err(&hdev->device, in hv_pci_protocol_negotiation()
3150 "PCI Pass-through VSP failed version request: %#x", in hv_pci_protocol_negotiation()
3152 ret = -EPROTO; in hv_pci_protocol_negotiation()
3159 dev_err(&hdev->device, in hv_pci_protocol_negotiation()
3160 "PCI pass-through VSP failed to find supported version"); in hv_pci_protocol_negotiation()
3161 ret = -EPROTO; in hv_pci_protocol_negotiation()
3169 * hv_pci_free_bridge_windows() - Release memory regions for the
3180 if (hbus->low_mmio_space && hbus->low_mmio_res) { in hv_pci_free_bridge_windows()
3181 hbus->low_mmio_res->flags |= IORESOURCE_BUSY; in hv_pci_free_bridge_windows()
3182 vmbus_free_mmio(hbus->low_mmio_res->start, in hv_pci_free_bridge_windows()
3183 resource_size(hbus->low_mmio_res)); in hv_pci_free_bridge_windows()
3186 if (hbus->high_mmio_space && hbus->high_mmio_res) { in hv_pci_free_bridge_windows()
3187 hbus->high_mmio_res->flags |= IORESOURCE_BUSY; in hv_pci_free_bridge_windows()
3188 vmbus_free_mmio(hbus->high_mmio_res->start, in hv_pci_free_bridge_windows()
3189 resource_size(hbus->high_mmio_res)); in hv_pci_free_bridge_windows()
3194 * hv_pci_allocate_bridge_windows() - Allocate memory regions
3216 * Return: 0 on success, -errno on failure
3223 if (hbus->low_mmio_space) { in hv_pci_allocate_bridge_windows()
3224 align = 1ULL << (63 - __builtin_clzll(hbus->low_mmio_space)); in hv_pci_allocate_bridge_windows()
3225 ret = vmbus_allocate_mmio(&hbus->low_mmio_res, hbus->hdev, 0, in hv_pci_allocate_bridge_windows()
3227 hbus->low_mmio_space, in hv_pci_allocate_bridge_windows()
3230 dev_err(&hbus->hdev->device, in hv_pci_allocate_bridge_windows()
3232 hbus->low_mmio_space); in hv_pci_allocate_bridge_windows()
3237 hbus->low_mmio_res->flags |= IORESOURCE_WINDOW; in hv_pci_allocate_bridge_windows()
3238 hbus->low_mmio_res->flags &= ~IORESOURCE_BUSY; in hv_pci_allocate_bridge_windows()
3239 pci_add_resource(&hbus->bridge->windows, hbus->low_mmio_res); in hv_pci_allocate_bridge_windows()
3242 if (hbus->high_mmio_space) { in hv_pci_allocate_bridge_windows()
3243 align = 1ULL << (63 - __builtin_clzll(hbus->high_mmio_space)); in hv_pci_allocate_bridge_windows()
3244 ret = vmbus_allocate_mmio(&hbus->high_mmio_res, hbus->hdev, in hv_pci_allocate_bridge_windows()
3245 0x100000000, -1, in hv_pci_allocate_bridge_windows()
3246 hbus->high_mmio_space, align, in hv_pci_allocate_bridge_windows()
3249 dev_err(&hbus->hdev->device, in hv_pci_allocate_bridge_windows()
3251 hbus->high_mmio_space); in hv_pci_allocate_bridge_windows()
3256 hbus->high_mmio_res->flags |= IORESOURCE_WINDOW; in hv_pci_allocate_bridge_windows()
3257 hbus->high_mmio_res->flags &= ~IORESOURCE_BUSY; in hv_pci_allocate_bridge_windows()
3258 pci_add_resource(&hbus->bridge->windows, hbus->high_mmio_res); in hv_pci_allocate_bridge_windows()
3264 if (hbus->low_mmio_res) { in hv_pci_allocate_bridge_windows()
3265 vmbus_free_mmio(hbus->low_mmio_res->start, in hv_pci_allocate_bridge_windows()
3266 resource_size(hbus->low_mmio_res)); in hv_pci_allocate_bridge_windows()
3273 * hv_allocate_config_window() - Find MMIO space for PCI Config
3276 * This function claims memory-mapped I/O space for accessing
3279 * Return: 0 on success, -errno on failure
3289 ret = vmbus_allocate_mmio(&hbus->mem_config, hbus->hdev, 0, -1, in hv_allocate_config_window()
3302 hbus->mem_config->flags |= IORESOURCE_BUSY; in hv_allocate_config_window()
3309 vmbus_free_mmio(hbus->mem_config->start, PCI_CONFIG_MMIO_LENGTH); in hv_free_config_window()
3315 * hv_pci_enter_d0() - Bring the "bus" into the D0 power state
3318 * Return: 0 on success, -errno on failure
3332 * powered-on state. This includes telling the host which region in hv_pci_enter_d0()
3333 * of memory-mapped I/O space has been chosen for configuration space in hv_pci_enter_d0()
3338 return -ENOMEM; in hv_pci_enter_d0()
3341 pkt->completion_func = hv_pci_generic_compl; in hv_pci_enter_d0()
3342 pkt->compl_ctxt = &comp_pkt; in hv_pci_enter_d0()
3343 d0_entry = (struct pci_bus_d0_entry *)&pkt->message; in hv_pci_enter_d0()
3344 d0_entry->message_type.type = PCI_BUS_D0ENTRY; in hv_pci_enter_d0()
3345 d0_entry->mmio_base = hbus->mem_config->start; in hv_pci_enter_d0()
3347 ret = vmbus_sendpacket(hdev->channel, d0_entry, sizeof(*d0_entry), in hv_pci_enter_d0()
3366 dev_err(&hdev->device, "Retrying D0 Entry\n"); in hv_pci_enter_d0()
3376 hbus->wslot_res_allocated = 255; in hv_pci_enter_d0()
3384 dev_err(&hdev->device, in hv_pci_enter_d0()
3389 dev_err(&hdev->device, in hv_pci_enter_d0()
3390 "PCI Pass-through VSP failed D0 Entry with status %x\n", in hv_pci_enter_d0()
3392 ret = -EPROTO; in hv_pci_enter_d0()
3404 * hv_pci_query_relations() - Ask host to send list of child
3408 * Return: 0 on success, -errno on failure
3419 if (cmpxchg(&hbus->survey_event, NULL, &comp)) in hv_pci_query_relations()
3420 return -ENOTEMPTY; in hv_pci_query_relations()
3425 ret = vmbus_sendpacket(hdev->channel, &message, sizeof(message), in hv_pci_query_relations()
3432 * vmbus_sendpacket() or wait_for_response() returns -ENODEV but we in hv_pci_query_relations()
3434 * channel callback already scheduled a work to hbus->wq, which can be in hv_pci_query_relations()
3435 * running pci_devices_present_work() -> survey_child_resources() -> in hv_pci_query_relations()
3436 * complete(&hbus->survey_event), even after hv_pci_query_relations() in hv_pci_query_relations()
3439 * raw_spin_lock_irqsave(). Flush hbus->wq before we exit from in hv_pci_query_relations()
3441 * -ENODEV, there can't be any more work item scheduled to hbus->wq in hv_pci_query_relations()
3442 * after the flush_workqueue(): see vmbus_onoffer_rescind() -> in hv_pci_query_relations()
3443 * vmbus_reset_channel_cb(), vmbus_rescind_cleanup() -> in hv_pci_query_relations()
3444 * channel->rescind = true. in hv_pci_query_relations()
3446 flush_workqueue(hbus->wq); in hv_pci_query_relations()
3452 * hv_send_resources_allocated() - Report local resource choices
3466 * Return: 0 on success, -errno on failure
3480 size_res = (hbus->protocol_version < PCI_PROTOCOL_VERSION_1_2) in hv_send_resources_allocated()
3485 return -ENOMEM; in hv_send_resources_allocated()
3496 pkt->completion_func = hv_pci_generic_compl; in hv_send_resources_allocated()
3497 pkt->compl_ctxt = &comp_pkt; in hv_send_resources_allocated()
3499 if (hbus->protocol_version < PCI_PROTOCOL_VERSION_1_2) { in hv_send_resources_allocated()
3501 (struct pci_resources_assigned *)&pkt->message; in hv_send_resources_allocated()
3502 res_assigned->message_type.type = in hv_send_resources_allocated()
3504 res_assigned->wslot.slot = hpdev->desc.win_slot.slot; in hv_send_resources_allocated()
3507 (struct pci_resources_assigned2 *)&pkt->message; in hv_send_resources_allocated()
3508 res_assigned2->message_type.type = in hv_send_resources_allocated()
3510 res_assigned2->wslot.slot = hpdev->desc.win_slot.slot; in hv_send_resources_allocated()
3514 ret = vmbus_sendpacket(hdev->channel, &pkt->message, in hv_send_resources_allocated()
3524 ret = -EPROTO; in hv_send_resources_allocated()
3525 dev_err(&hdev->device, in hv_send_resources_allocated()
3531 hbus->wslot_res_allocated = wslot; in hv_send_resources_allocated()
3539 * hv_send_resources_released() - Report local resources
3543 * Return: 0 on success, -errno on failure
3553 for (wslot = hbus->wslot_res_allocated; wslot >= 0; wslot--) { in hv_send_resources_released()
3560 pkt.wslot.slot = hpdev->desc.win_slot.slot; in hv_send_resources_released()
3564 ret = vmbus_sendpacket(hdev->channel, &pkt, sizeof(pkt), 0, in hv_send_resources_released()
3569 hbus->wslot_res_allocated = wslot - 1; in hv_send_resources_released()
3572 hbus->wslot_res_allocated = -1; in hv_send_resources_released()
3587 * hv_get_dom_num() - Get a valid PCI domain number
3611 * hv_put_dom_num() - Mark the PCI domain number as free
3620 * hv_pci_probe() - New VMBus channel probe, for a root PCI bus
3624 * Return: 0 on success, -errno on failure
3635 bridge = devm_pci_alloc_host_bridge(&hdev->device, 0); in hv_pci_probe()
3637 return -ENOMEM; in hv_pci_probe()
3641 return -ENOMEM; in hv_pci_probe()
3643 hbus->bridge = bridge; in hv_pci_probe()
3644 mutex_init(&hbus->state_lock); in hv_pci_probe()
3645 hbus->state = hv_pcibus_init; in hv_pci_probe()
3646 hbus->wslot_res_allocated = -1; in hv_pci_probe()
3654 * Note that, since this code only runs in a Hyper-V VM, Hyper-V in hv_pci_probe()
3661 dom_req = hdev->dev_instance.b[5] << 8 | hdev->dev_instance.b[4]; in hv_pci_probe()
3665 dev_err(&hdev->device, in hv_pci_probe()
3667 ret = -EINVAL; in hv_pci_probe()
3672 dev_info(&hdev->device, in hv_pci_probe()
3676 hbus->bridge->domain_nr = dom; in hv_pci_probe()
3678 hbus->sysdata.domain = dom; in hv_pci_probe()
3679 hbus->use_calls = !!(ms_hyperv.hints & HV_X64_USE_MMIO_HYPERCALLS); in hv_pci_probe()
3688 hbus->sysdata.parent = hdev->device.parent; in hv_pci_probe()
3689 hbus->use_calls = false; in hv_pci_probe()
3692 hbus->hdev = hdev; in hv_pci_probe()
3693 INIT_LIST_HEAD(&hbus->children); in hv_pci_probe()
3694 INIT_LIST_HEAD(&hbus->dr_list); in hv_pci_probe()
3695 spin_lock_init(&hbus->config_lock); in hv_pci_probe()
3696 spin_lock_init(&hbus->device_list_lock); in hv_pci_probe()
3697 hbus->wq = alloc_ordered_workqueue("hv_pci_%x", 0, in hv_pci_probe()
3698 hbus->bridge->domain_nr); in hv_pci_probe()
3699 if (!hbus->wq) { in hv_pci_probe()
3700 ret = -ENOMEM; in hv_pci_probe()
3704 hdev->channel->next_request_id_callback = vmbus_next_request_id; in hv_pci_probe()
3705 hdev->channel->request_addr_callback = vmbus_request_addr; in hv_pci_probe()
3706 hdev->channel->rqstor_size = HV_PCI_RQSTOR_SIZE; in hv_pci_probe()
3708 ret = vmbus_open(hdev->channel, pci_ring_size, pci_ring_size, NULL, 0, in hv_pci_probe()
3724 hbus->cfg_addr = ioremap(hbus->mem_config->start, in hv_pci_probe()
3726 if (!hbus->cfg_addr) { in hv_pci_probe()
3727 dev_err(&hdev->device, in hv_pci_probe()
3729 ret = -ENOMEM; in hv_pci_probe()
3733 name = kasprintf(GFP_KERNEL, "%pUL", &hdev->dev_instance); in hv_pci_probe()
3735 ret = -ENOMEM; in hv_pci_probe()
3739 hbus->fwnode = irq_domain_alloc_named_fwnode(name); in hv_pci_probe()
3741 if (!hbus->fwnode) { in hv_pci_probe()
3742 ret = -ENOMEM; in hv_pci_probe()
3754 mutex_lock(&hbus->state_lock); in hv_pci_probe()
3770 hbus->state = hv_pcibus_probed; in hv_pci_probe()
3776 mutex_unlock(&hbus->state_lock); in hv_pci_probe()
3784 mutex_unlock(&hbus->state_lock); in hv_pci_probe()
3786 irq_domain_remove(hbus->irq_domain); in hv_pci_probe()
3788 irq_domain_free_fwnode(hbus->fwnode); in hv_pci_probe()
3790 iounmap(hbus->cfg_addr); in hv_pci_probe()
3794 vmbus_close(hdev->channel); in hv_pci_probe()
3796 destroy_workqueue(hbus->wq); in hv_pci_probe()
3798 hv_put_dom_num(hbus->bridge->domain_nr); in hv_pci_probe()
3807 struct vmbus_channel *chan = hdev->channel; in hv_pci_bus_exit()
3820 * access the per-channel ringbuffer any longer. in hv_pci_bus_exit()
3822 if (chan->rescind) in hv_pci_bus_exit()
3830 spin_lock_irqsave(&hbus->device_list_lock, flags); in hv_pci_bus_exit()
3831 list_for_each_entry_safe(hpdev, tmp, &hbus->children, list_entry) in hv_pci_bus_exit()
3832 list_move_tail(&hpdev->list_entry, &removed); in hv_pci_bus_exit()
3833 spin_unlock_irqrestore(&hbus->device_list_lock, flags); in hv_pci_bus_exit()
3837 list_del(&hpdev->list_entry); in hv_pci_bus_exit()
3838 if (hpdev->pci_slot) in hv_pci_bus_exit()
3839 pci_destroy_slot(hpdev->pci_slot); in hv_pci_bus_exit()
3848 dev_err(&hdev->device, in hv_pci_bus_exit()
3877 return -ETIMEDOUT; in hv_pci_bus_exit()
3884 * hv_pci_remove() - Remove routine for this VMBus channel
3892 if (hbus->state == hv_pcibus_installed) { in hv_pci_remove()
3893 tasklet_disable(&hdev->channel->callback_event); in hv_pci_remove()
3894 hbus->state = hv_pcibus_removing; in hv_pci_remove()
3895 tasklet_enable(&hdev->channel->callback_event); in hv_pci_remove()
3896 destroy_workqueue(hbus->wq); in hv_pci_remove()
3897 hbus->wq = NULL; in hv_pci_remove()
3900 * on hbus-wq. We can't race with hv_pci_devices_present() in hv_pci_remove()
3906 pci_stop_root_bus(hbus->bridge->bus); in hv_pci_remove()
3908 pci_remove_root_bus(hbus->bridge->bus); in hv_pci_remove()
3914 vmbus_close(hdev->channel); in hv_pci_remove()
3916 iounmap(hbus->cfg_addr); in hv_pci_remove()
3919 irq_domain_remove(hbus->irq_domain); in hv_pci_remove()
3920 irq_domain_free_fwnode(hbus->fwnode); in hv_pci_remove()
3922 hv_put_dom_num(hbus->bridge->domain_nr); in hv_pci_remove()
3939 * items onto hbus->wq in hv_pci_devices_present() and in hv_pci_suspend()
3942 * the work item handler pci_devices_present_work() -> in hv_pci_suspend()
3946 * callback tasklet, sets hbus->state to hv_pcibus_removing, and in hv_pci_suspend()
3947 * re-enables the tasklet. This way, when hv_pci_suspend() proceeds, in hv_pci_suspend()
3949 * hbus->wq and safely closes the vmbus channel. in hv_pci_suspend()
3951 tasklet_disable(&hdev->channel->callback_event); in hv_pci_suspend()
3954 old_state = hbus->state; in hv_pci_suspend()
3955 if (hbus->state == hv_pcibus_installed) in hv_pci_suspend()
3956 hbus->state = hv_pcibus_removing; in hv_pci_suspend()
3958 tasklet_enable(&hdev->channel->callback_event); in hv_pci_suspend()
3961 return -EINVAL; in hv_pci_suspend()
3963 flush_workqueue(hbus->wq); in hv_pci_suspend()
3969 vmbus_close(hdev->channel); in hv_pci_suspend()
3980 if (!pdev->msi_enabled && !pdev->msix_enabled) in hv_pci_restore_msi_msg()
3983 msi_lock_descs(&pdev->dev); in hv_pci_restore_msi_msg()
3984 msi_for_each_desc(entry, &pdev->dev, MSI_DESC_ASSOCIATED) { in hv_pci_restore_msi_msg()
3985 irq_data = irq_get_irq_data(entry->irq); in hv_pci_restore_msi_msg()
3987 ret = -EINVAL; in hv_pci_restore_msi_msg()
3991 hv_compose_msi_msg(irq_data, &entry->msg); in hv_pci_restore_msi_msg()
3993 msi_unlock_descs(&pdev->dev); in hv_pci_restore_msi_msg()
3999 * Upon resume, pci_restore_msi_state() -> ... -> __pci_write_msi_msg()
4000 * directly writes the MSI/MSI-X registers via MMIO, but since Hyper-V
4002 * must be used to ask Hyper-V to re-create the IOMMU Interrupt Remapping
4007 pci_walk_bus(hbus->bridge->bus, hv_pci_restore_msi_msg, NULL); in hv_pci_restore_msi_state()
4016 hbus->state = hv_pcibus_init; in hv_pci_resume()
4018 hdev->channel->next_request_id_callback = vmbus_next_request_id; in hv_pci_resume()
4019 hdev->channel->request_addr_callback = vmbus_request_addr; in hv_pci_resume()
4020 hdev->channel->rqstor_size = HV_PCI_RQSTOR_SIZE; in hv_pci_resume()
4022 ret = vmbus_open(hdev->channel, pci_ring_size, pci_ring_size, NULL, 0, in hv_pci_resume()
4028 version[0] = hbus->protocol_version; in hv_pci_resume()
4037 mutex_lock(&hbus->state_lock); in hv_pci_resume()
4051 hbus->state = hv_pcibus_installed; in hv_pci_resume()
4052 mutex_unlock(&hbus->state_lock); in hv_pci_resume()
4056 mutex_unlock(&hbus->state_lock); in hv_pci_resume()
4058 vmbus_close(hdev->channel); in hv_pci_resume()
4063 /* PCI Pass-through Class ID */
4064 /* 44C4F61D-4444-4400-9D52-802E27EDE19F */
4094 return -ENODEV; in init_hv_pci_drv()
4114 MODULE_DESCRIPTION("Hyper-V PCI");