1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 4 * Author: Alex Williamson <alex.williamson@redhat.com> 5 * 6 * Derived from original vfio: 7 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 8 * Author: Tom Lyon, pugs@cisco.com 9 */ 10 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 13 #include <linux/device.h> 14 #include <linux/eventfd.h> 15 #include <linux/file.h> 16 #include <linux/interrupt.h> 17 #include <linux/iommu.h> 18 #include <linux/module.h> 19 #include <linux/mutex.h> 20 #include <linux/notifier.h> 21 #include <linux/pci.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/slab.h> 24 #include <linux/types.h> 25 #include <linux/uaccess.h> 26 #include <linux/vfio.h> 27 #include <linux/vgaarb.h> 28 #include <linux/nospec.h> 29 #include <linux/sched/mm.h> 30 31 #include "vfio_pci_private.h" 32 33 #define DRIVER_VERSION "0.2" 34 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 35 #define DRIVER_DESC "VFIO PCI - User Level meta-driver" 36 37 static char ids[1024] __initdata; 38 module_param_string(ids, ids, sizeof(ids), 0); 39 MODULE_PARM_DESC(ids, "Initial PCI IDs to add to the vfio driver, format is \"vendor:device[:subvendor[:subdevice[:class[:class_mask]]]]\" and multiple comma separated entries can be specified"); 40 41 static bool nointxmask; 42 module_param_named(nointxmask, nointxmask, bool, S_IRUGO | S_IWUSR); 43 MODULE_PARM_DESC(nointxmask, 44 "Disable support for PCI 2.3 style INTx masking. If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag."); 45 46 #ifdef CONFIG_VFIO_PCI_VGA 47 static bool disable_vga; 48 module_param(disable_vga, bool, S_IRUGO); 49 MODULE_PARM_DESC(disable_vga, "Disable VGA resource access through vfio-pci"); 50 #endif 51 52 static bool disable_idle_d3; 53 module_param(disable_idle_d3, bool, S_IRUGO | S_IWUSR); 54 MODULE_PARM_DESC(disable_idle_d3, 55 "Disable using the PCI D3 low power state for idle, unused devices"); 56 57 static bool enable_sriov; 58 #ifdef CONFIG_PCI_IOV 59 module_param(enable_sriov, bool, 0644); 60 MODULE_PARM_DESC(enable_sriov, "Enable support for SR-IOV configuration. Enabling SR-IOV on a PF typically requires support of the userspace PF driver, enabling VFs without such support may result in non-functional VFs or PF."); 61 #endif 62 63 static inline bool vfio_vga_disabled(void) 64 { 65 #ifdef CONFIG_VFIO_PCI_VGA 66 return disable_vga; 67 #else 68 return true; 69 #endif 70 } 71 72 /* 73 * Our VGA arbiter participation is limited since we don't know anything 74 * about the device itself. However, if the device is the only VGA device 75 * downstream of a bridge and VFIO VGA support is disabled, then we can 76 * safely return legacy VGA IO and memory as not decoded since the user 77 * has no way to get to it and routing can be disabled externally at the 78 * bridge. 79 */ 80 static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga) 81 { 82 struct vfio_pci_device *vdev = opaque; 83 struct pci_dev *tmp = NULL, *pdev = vdev->pdev; 84 unsigned char max_busnr; 85 unsigned int decodes; 86 87 if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus)) 88 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | 89 VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM; 90 91 max_busnr = pci_bus_max_busnr(pdev->bus); 92 decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 93 94 while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) { 95 if (tmp == pdev || 96 pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) || 97 pci_is_root_bus(tmp->bus)) 98 continue; 99 100 if (tmp->bus->number >= pdev->bus->number && 101 tmp->bus->number <= max_busnr) { 102 pci_dev_put(tmp); 103 decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM; 104 break; 105 } 106 } 107 108 return decodes; 109 } 110 111 static inline bool vfio_pci_is_vga(struct pci_dev *pdev) 112 { 113 return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA; 114 } 115 116 static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev) 117 { 118 struct resource *res; 119 int i; 120 struct vfio_pci_dummy_resource *dummy_res; 121 122 INIT_LIST_HEAD(&vdev->dummy_resources_list); 123 124 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 125 int bar = i + PCI_STD_RESOURCES; 126 127 res = &vdev->pdev->resource[bar]; 128 129 if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP)) 130 goto no_mmap; 131 132 if (!(res->flags & IORESOURCE_MEM)) 133 goto no_mmap; 134 135 /* 136 * The PCI core shouldn't set up a resource with a 137 * type but zero size. But there may be bugs that 138 * cause us to do that. 139 */ 140 if (!resource_size(res)) 141 goto no_mmap; 142 143 if (resource_size(res) >= PAGE_SIZE) { 144 vdev->bar_mmap_supported[bar] = true; 145 continue; 146 } 147 148 if (!(res->start & ~PAGE_MASK)) { 149 /* 150 * Add a dummy resource to reserve the remainder 151 * of the exclusive page in case that hot-add 152 * device's bar is assigned into it. 153 */ 154 dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL); 155 if (dummy_res == NULL) 156 goto no_mmap; 157 158 dummy_res->resource.name = "vfio sub-page reserved"; 159 dummy_res->resource.start = res->end + 1; 160 dummy_res->resource.end = res->start + PAGE_SIZE - 1; 161 dummy_res->resource.flags = res->flags; 162 if (request_resource(res->parent, 163 &dummy_res->resource)) { 164 kfree(dummy_res); 165 goto no_mmap; 166 } 167 dummy_res->index = bar; 168 list_add(&dummy_res->res_next, 169 &vdev->dummy_resources_list); 170 vdev->bar_mmap_supported[bar] = true; 171 continue; 172 } 173 /* 174 * Here we don't handle the case when the BAR is not page 175 * aligned because we can't expect the BAR will be 176 * assigned into the same location in a page in guest 177 * when we passthrough the BAR. And it's hard to access 178 * this BAR in userspace because we have no way to get 179 * the BAR's location in a page. 180 */ 181 no_mmap: 182 vdev->bar_mmap_supported[bar] = false; 183 } 184 } 185 186 static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev); 187 static void vfio_pci_disable(struct vfio_pci_device *vdev); 188 static int vfio_pci_try_zap_and_vma_lock_cb(struct pci_dev *pdev, void *data); 189 190 /* 191 * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND 192 * _and_ the ability detect when the device is asserting INTx via PCI_STATUS. 193 * If a device implements the former but not the latter we would typically 194 * expect broken_intx_masking be set and require an exclusive interrupt. 195 * However since we do have control of the device's ability to assert INTx, 196 * we can instead pretend that the device does not implement INTx, virtualizing 197 * the pin register to report zero and maintaining DisINTx set on the host. 198 */ 199 static bool vfio_pci_nointx(struct pci_dev *pdev) 200 { 201 switch (pdev->vendor) { 202 case PCI_VENDOR_ID_INTEL: 203 switch (pdev->device) { 204 /* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */ 205 case 0x1572: 206 case 0x1574: 207 case 0x1580 ... 0x1581: 208 case 0x1583 ... 0x158b: 209 case 0x37d0 ... 0x37d2: 210 return true; 211 default: 212 return false; 213 } 214 } 215 216 return false; 217 } 218 219 static void vfio_pci_probe_power_state(struct vfio_pci_device *vdev) 220 { 221 struct pci_dev *pdev = vdev->pdev; 222 u16 pmcsr; 223 224 if (!pdev->pm_cap) 225 return; 226 227 pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr); 228 229 vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET); 230 } 231 232 /* 233 * pci_set_power_state() wrapper handling devices which perform a soft reset on 234 * D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev, 235 * restore when returned to D0. Saved separately from pci_saved_state for use 236 * by PM capability emulation and separately from pci_dev internal saved state 237 * to avoid it being overwritten and consumed around other resets. 238 */ 239 int vfio_pci_set_power_state(struct vfio_pci_device *vdev, pci_power_t state) 240 { 241 struct pci_dev *pdev = vdev->pdev; 242 bool needs_restore = false, needs_save = false; 243 int ret; 244 245 if (vdev->needs_pm_restore) { 246 if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) { 247 pci_save_state(pdev); 248 needs_save = true; 249 } 250 251 if (pdev->current_state >= PCI_D3hot && state <= PCI_D0) 252 needs_restore = true; 253 } 254 255 ret = pci_set_power_state(pdev, state); 256 257 if (!ret) { 258 /* D3 might be unsupported via quirk, skip unless in D3 */ 259 if (needs_save && pdev->current_state >= PCI_D3hot) { 260 vdev->pm_save = pci_store_saved_state(pdev); 261 } else if (needs_restore) { 262 pci_load_and_free_saved_state(pdev, &vdev->pm_save); 263 pci_restore_state(pdev); 264 } 265 } 266 267 return ret; 268 } 269 270 static int vfio_pci_enable(struct vfio_pci_device *vdev) 271 { 272 struct pci_dev *pdev = vdev->pdev; 273 int ret; 274 u16 cmd; 275 u8 msix_pos; 276 277 vfio_pci_set_power_state(vdev, PCI_D0); 278 279 /* Don't allow our initial saved state to include busmaster */ 280 pci_clear_master(pdev); 281 282 ret = pci_enable_device(pdev); 283 if (ret) 284 return ret; 285 286 /* If reset fails because of the device lock, fail this path entirely */ 287 ret = pci_try_reset_function(pdev); 288 if (ret == -EAGAIN) { 289 pci_disable_device(pdev); 290 return ret; 291 } 292 293 vdev->reset_works = !ret; 294 pci_save_state(pdev); 295 vdev->pci_saved_state = pci_store_saved_state(pdev); 296 if (!vdev->pci_saved_state) 297 pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__); 298 299 if (likely(!nointxmask)) { 300 if (vfio_pci_nointx(pdev)) { 301 pci_info(pdev, "Masking broken INTx support\n"); 302 vdev->nointx = true; 303 pci_intx(pdev, 0); 304 } else 305 vdev->pci_2_3 = pci_intx_mask_supported(pdev); 306 } 307 308 pci_read_config_word(pdev, PCI_COMMAND, &cmd); 309 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) { 310 cmd &= ~PCI_COMMAND_INTX_DISABLE; 311 pci_write_config_word(pdev, PCI_COMMAND, cmd); 312 } 313 314 ret = vfio_config_init(vdev); 315 if (ret) { 316 kfree(vdev->pci_saved_state); 317 vdev->pci_saved_state = NULL; 318 pci_disable_device(pdev); 319 return ret; 320 } 321 322 msix_pos = pdev->msix_cap; 323 if (msix_pos) { 324 u16 flags; 325 u32 table; 326 327 pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags); 328 pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table); 329 330 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR; 331 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET; 332 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16; 333 } else 334 vdev->msix_bar = 0xFF; 335 336 if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev)) 337 vdev->has_vga = true; 338 339 340 if (vfio_pci_is_vga(pdev) && 341 pdev->vendor == PCI_VENDOR_ID_INTEL && 342 IS_ENABLED(CONFIG_VFIO_PCI_IGD)) { 343 ret = vfio_pci_igd_init(vdev); 344 if (ret) { 345 pci_warn(pdev, "Failed to setup Intel IGD regions\n"); 346 goto disable_exit; 347 } 348 } 349 350 if (pdev->vendor == PCI_VENDOR_ID_NVIDIA && 351 IS_ENABLED(CONFIG_VFIO_PCI_NVLINK2)) { 352 ret = vfio_pci_nvdia_v100_nvlink2_init(vdev); 353 if (ret && ret != -ENODEV) { 354 pci_warn(pdev, "Failed to setup NVIDIA NV2 RAM region\n"); 355 goto disable_exit; 356 } 357 } 358 359 if (pdev->vendor == PCI_VENDOR_ID_IBM && 360 IS_ENABLED(CONFIG_VFIO_PCI_NVLINK2)) { 361 ret = vfio_pci_ibm_npu2_init(vdev); 362 if (ret && ret != -ENODEV) { 363 pci_warn(pdev, "Failed to setup NVIDIA NV2 ATSD region\n"); 364 goto disable_exit; 365 } 366 } 367 368 vfio_pci_probe_mmaps(vdev); 369 370 return 0; 371 372 disable_exit: 373 vfio_pci_disable(vdev); 374 return ret; 375 } 376 377 static void vfio_pci_disable(struct vfio_pci_device *vdev) 378 { 379 struct pci_dev *pdev = vdev->pdev; 380 struct vfio_pci_dummy_resource *dummy_res, *tmp; 381 struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp; 382 int i, bar; 383 384 /* Stop the device from further DMA */ 385 pci_clear_master(pdev); 386 387 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE | 388 VFIO_IRQ_SET_ACTION_TRIGGER, 389 vdev->irq_type, 0, 0, NULL); 390 391 /* Device closed, don't need mutex here */ 392 list_for_each_entry_safe(ioeventfd, ioeventfd_tmp, 393 &vdev->ioeventfds_list, next) { 394 vfio_virqfd_disable(&ioeventfd->virqfd); 395 list_del(&ioeventfd->next); 396 kfree(ioeventfd); 397 } 398 vdev->ioeventfds_nr = 0; 399 400 vdev->virq_disabled = false; 401 402 for (i = 0; i < vdev->num_regions; i++) 403 vdev->region[i].ops->release(vdev, &vdev->region[i]); 404 405 vdev->num_regions = 0; 406 kfree(vdev->region); 407 vdev->region = NULL; /* don't krealloc a freed pointer */ 408 409 vfio_config_free(vdev); 410 411 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 412 bar = i + PCI_STD_RESOURCES; 413 if (!vdev->barmap[bar]) 414 continue; 415 pci_iounmap(pdev, vdev->barmap[bar]); 416 pci_release_selected_regions(pdev, 1 << bar); 417 vdev->barmap[bar] = NULL; 418 } 419 420 list_for_each_entry_safe(dummy_res, tmp, 421 &vdev->dummy_resources_list, res_next) { 422 list_del(&dummy_res->res_next); 423 release_resource(&dummy_res->resource); 424 kfree(dummy_res); 425 } 426 427 vdev->needs_reset = true; 428 429 /* 430 * If we have saved state, restore it. If we can reset the device, 431 * even better. Resetting with current state seems better than 432 * nothing, but saving and restoring current state without reset 433 * is just busy work. 434 */ 435 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) { 436 pci_info(pdev, "%s: Couldn't reload saved state\n", __func__); 437 438 if (!vdev->reset_works) 439 goto out; 440 441 pci_save_state(pdev); 442 } 443 444 /* 445 * Disable INTx and MSI, presumably to avoid spurious interrupts 446 * during reset. Stolen from pci_reset_function() 447 */ 448 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 449 450 /* 451 * Try to get the locks ourselves to prevent a deadlock. The 452 * success of this is dependent on being able to lock the device, 453 * which is not always possible. 454 * We can not use the "try" reset interface here, which will 455 * overwrite the previously restored configuration information. 456 */ 457 if (vdev->reset_works && pci_cfg_access_trylock(pdev)) { 458 if (device_trylock(&pdev->dev)) { 459 if (!__pci_reset_function_locked(pdev)) 460 vdev->needs_reset = false; 461 device_unlock(&pdev->dev); 462 } 463 pci_cfg_access_unlock(pdev); 464 } 465 466 pci_restore_state(pdev); 467 out: 468 pci_disable_device(pdev); 469 470 vfio_pci_try_bus_reset(vdev); 471 472 if (!disable_idle_d3) 473 vfio_pci_set_power_state(vdev, PCI_D3hot); 474 } 475 476 static struct pci_driver vfio_pci_driver; 477 478 static struct vfio_pci_device *get_pf_vdev(struct vfio_pci_device *vdev, 479 struct vfio_device **pf_dev) 480 { 481 struct pci_dev *physfn = pci_physfn(vdev->pdev); 482 483 if (!vdev->pdev->is_virtfn) 484 return NULL; 485 486 *pf_dev = vfio_device_get_from_dev(&physfn->dev); 487 if (!*pf_dev) 488 return NULL; 489 490 if (pci_dev_driver(physfn) != &vfio_pci_driver) { 491 vfio_device_put(*pf_dev); 492 return NULL; 493 } 494 495 return vfio_device_data(*pf_dev); 496 } 497 498 static void vfio_pci_vf_token_user_add(struct vfio_pci_device *vdev, int val) 499 { 500 struct vfio_device *pf_dev; 501 struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev, &pf_dev); 502 503 if (!pf_vdev) 504 return; 505 506 mutex_lock(&pf_vdev->vf_token->lock); 507 pf_vdev->vf_token->users += val; 508 WARN_ON(pf_vdev->vf_token->users < 0); 509 mutex_unlock(&pf_vdev->vf_token->lock); 510 511 vfio_device_put(pf_dev); 512 } 513 514 static void vfio_pci_release(void *device_data) 515 { 516 struct vfio_pci_device *vdev = device_data; 517 518 mutex_lock(&vdev->reflck->lock); 519 520 if (!(--vdev->refcnt)) { 521 vfio_pci_vf_token_user_add(vdev, -1); 522 vfio_spapr_pci_eeh_release(vdev->pdev); 523 vfio_pci_disable(vdev); 524 mutex_lock(&vdev->igate); 525 if (vdev->err_trigger) { 526 eventfd_ctx_put(vdev->err_trigger); 527 vdev->err_trigger = NULL; 528 } 529 mutex_unlock(&vdev->igate); 530 531 mutex_lock(&vdev->igate); 532 if (vdev->req_trigger) { 533 eventfd_ctx_put(vdev->req_trigger); 534 vdev->req_trigger = NULL; 535 } 536 mutex_unlock(&vdev->igate); 537 } 538 539 mutex_unlock(&vdev->reflck->lock); 540 541 module_put(THIS_MODULE); 542 } 543 544 static int vfio_pci_open(void *device_data) 545 { 546 struct vfio_pci_device *vdev = device_data; 547 int ret = 0; 548 549 if (!try_module_get(THIS_MODULE)) 550 return -ENODEV; 551 552 mutex_lock(&vdev->reflck->lock); 553 554 if (!vdev->refcnt) { 555 ret = vfio_pci_enable(vdev); 556 if (ret) 557 goto error; 558 559 vfio_spapr_pci_eeh_open(vdev->pdev); 560 vfio_pci_vf_token_user_add(vdev, 1); 561 } 562 vdev->refcnt++; 563 error: 564 mutex_unlock(&vdev->reflck->lock); 565 if (ret) 566 module_put(THIS_MODULE); 567 return ret; 568 } 569 570 static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type) 571 { 572 if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) { 573 u8 pin; 574 575 if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || 576 vdev->nointx || vdev->pdev->is_virtfn) 577 return 0; 578 579 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin); 580 581 return pin ? 1 : 0; 582 } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) { 583 u8 pos; 584 u16 flags; 585 586 pos = vdev->pdev->msi_cap; 587 if (pos) { 588 pci_read_config_word(vdev->pdev, 589 pos + PCI_MSI_FLAGS, &flags); 590 return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1); 591 } 592 } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) { 593 u8 pos; 594 u16 flags; 595 596 pos = vdev->pdev->msix_cap; 597 if (pos) { 598 pci_read_config_word(vdev->pdev, 599 pos + PCI_MSIX_FLAGS, &flags); 600 601 return (flags & PCI_MSIX_FLAGS_QSIZE) + 1; 602 } 603 } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) { 604 if (pci_is_pcie(vdev->pdev)) 605 return 1; 606 } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) { 607 return 1; 608 } 609 610 return 0; 611 } 612 613 static int vfio_pci_count_devs(struct pci_dev *pdev, void *data) 614 { 615 (*(int *)data)++; 616 return 0; 617 } 618 619 struct vfio_pci_fill_info { 620 int max; 621 int cur; 622 struct vfio_pci_dependent_device *devices; 623 }; 624 625 static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data) 626 { 627 struct vfio_pci_fill_info *fill = data; 628 struct iommu_group *iommu_group; 629 630 if (fill->cur == fill->max) 631 return -EAGAIN; /* Something changed, try again */ 632 633 iommu_group = iommu_group_get(&pdev->dev); 634 if (!iommu_group) 635 return -EPERM; /* Cannot reset non-isolated devices */ 636 637 fill->devices[fill->cur].group_id = iommu_group_id(iommu_group); 638 fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus); 639 fill->devices[fill->cur].bus = pdev->bus->number; 640 fill->devices[fill->cur].devfn = pdev->devfn; 641 fill->cur++; 642 iommu_group_put(iommu_group); 643 return 0; 644 } 645 646 struct vfio_pci_group_entry { 647 struct vfio_group *group; 648 int id; 649 }; 650 651 struct vfio_pci_group_info { 652 int count; 653 struct vfio_pci_group_entry *groups; 654 }; 655 656 static int vfio_pci_validate_devs(struct pci_dev *pdev, void *data) 657 { 658 struct vfio_pci_group_info *info = data; 659 struct iommu_group *group; 660 int id, i; 661 662 group = iommu_group_get(&pdev->dev); 663 if (!group) 664 return -EPERM; 665 666 id = iommu_group_id(group); 667 668 for (i = 0; i < info->count; i++) 669 if (info->groups[i].id == id) 670 break; 671 672 iommu_group_put(group); 673 674 return (i == info->count) ? -EINVAL : 0; 675 } 676 677 static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot) 678 { 679 for (; pdev; pdev = pdev->bus->self) 680 if (pdev->bus == slot->bus) 681 return (pdev->slot == slot); 682 return false; 683 } 684 685 struct vfio_pci_walk_info { 686 int (*fn)(struct pci_dev *, void *data); 687 void *data; 688 struct pci_dev *pdev; 689 bool slot; 690 int ret; 691 }; 692 693 static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data) 694 { 695 struct vfio_pci_walk_info *walk = data; 696 697 if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot)) 698 walk->ret = walk->fn(pdev, walk->data); 699 700 return walk->ret; 701 } 702 703 static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev, 704 int (*fn)(struct pci_dev *, 705 void *data), void *data, 706 bool slot) 707 { 708 struct vfio_pci_walk_info walk = { 709 .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0, 710 }; 711 712 pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk); 713 714 return walk.ret; 715 } 716 717 static int msix_mmappable_cap(struct vfio_pci_device *vdev, 718 struct vfio_info_cap *caps) 719 { 720 struct vfio_info_cap_header header = { 721 .id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE, 722 .version = 1 723 }; 724 725 return vfio_info_add_capability(caps, &header, sizeof(header)); 726 } 727 728 int vfio_pci_register_dev_region(struct vfio_pci_device *vdev, 729 unsigned int type, unsigned int subtype, 730 const struct vfio_pci_regops *ops, 731 size_t size, u32 flags, void *data) 732 { 733 struct vfio_pci_region *region; 734 735 region = krealloc(vdev->region, 736 (vdev->num_regions + 1) * sizeof(*region), 737 GFP_KERNEL); 738 if (!region) 739 return -ENOMEM; 740 741 vdev->region = region; 742 vdev->region[vdev->num_regions].type = type; 743 vdev->region[vdev->num_regions].subtype = subtype; 744 vdev->region[vdev->num_regions].ops = ops; 745 vdev->region[vdev->num_regions].size = size; 746 vdev->region[vdev->num_regions].flags = flags; 747 vdev->region[vdev->num_regions].data = data; 748 749 vdev->num_regions++; 750 751 return 0; 752 } 753 754 struct vfio_devices { 755 struct vfio_device **devices; 756 int cur_index; 757 int max_index; 758 }; 759 760 static long vfio_pci_ioctl(void *device_data, 761 unsigned int cmd, unsigned long arg) 762 { 763 struct vfio_pci_device *vdev = device_data; 764 unsigned long minsz; 765 766 if (cmd == VFIO_DEVICE_GET_INFO) { 767 struct vfio_device_info info; 768 769 minsz = offsetofend(struct vfio_device_info, num_irqs); 770 771 if (copy_from_user(&info, (void __user *)arg, minsz)) 772 return -EFAULT; 773 774 if (info.argsz < minsz) 775 return -EINVAL; 776 777 info.flags = VFIO_DEVICE_FLAGS_PCI; 778 779 if (vdev->reset_works) 780 info.flags |= VFIO_DEVICE_FLAGS_RESET; 781 782 info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; 783 info.num_irqs = VFIO_PCI_NUM_IRQS; 784 785 return copy_to_user((void __user *)arg, &info, minsz) ? 786 -EFAULT : 0; 787 788 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 789 struct pci_dev *pdev = vdev->pdev; 790 struct vfio_region_info info; 791 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 792 int i, ret; 793 794 minsz = offsetofend(struct vfio_region_info, offset); 795 796 if (copy_from_user(&info, (void __user *)arg, minsz)) 797 return -EFAULT; 798 799 if (info.argsz < minsz) 800 return -EINVAL; 801 802 switch (info.index) { 803 case VFIO_PCI_CONFIG_REGION_INDEX: 804 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 805 info.size = pdev->cfg_size; 806 info.flags = VFIO_REGION_INFO_FLAG_READ | 807 VFIO_REGION_INFO_FLAG_WRITE; 808 break; 809 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 810 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 811 info.size = pci_resource_len(pdev, info.index); 812 if (!info.size) { 813 info.flags = 0; 814 break; 815 } 816 817 info.flags = VFIO_REGION_INFO_FLAG_READ | 818 VFIO_REGION_INFO_FLAG_WRITE; 819 if (vdev->bar_mmap_supported[info.index]) { 820 info.flags |= VFIO_REGION_INFO_FLAG_MMAP; 821 if (info.index == vdev->msix_bar) { 822 ret = msix_mmappable_cap(vdev, &caps); 823 if (ret) 824 return ret; 825 } 826 } 827 828 break; 829 case VFIO_PCI_ROM_REGION_INDEX: 830 { 831 void __iomem *io; 832 size_t size; 833 u16 cmd; 834 835 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 836 info.flags = 0; 837 838 /* Report the BAR size, not the ROM size */ 839 info.size = pci_resource_len(pdev, info.index); 840 if (!info.size) { 841 /* Shadow ROMs appear as PCI option ROMs */ 842 if (pdev->resource[PCI_ROM_RESOURCE].flags & 843 IORESOURCE_ROM_SHADOW) 844 info.size = 0x20000; 845 else 846 break; 847 } 848 849 /* 850 * Is it really there? Enable memory decode for 851 * implicit access in pci_map_rom(). 852 */ 853 cmd = vfio_pci_memory_lock_and_enable(vdev); 854 io = pci_map_rom(pdev, &size); 855 if (io) { 856 info.flags = VFIO_REGION_INFO_FLAG_READ; 857 pci_unmap_rom(pdev, io); 858 } else { 859 info.size = 0; 860 } 861 vfio_pci_memory_unlock_and_restore(vdev, cmd); 862 863 break; 864 } 865 case VFIO_PCI_VGA_REGION_INDEX: 866 if (!vdev->has_vga) 867 return -EINVAL; 868 869 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 870 info.size = 0xc0000; 871 info.flags = VFIO_REGION_INFO_FLAG_READ | 872 VFIO_REGION_INFO_FLAG_WRITE; 873 874 break; 875 default: 876 { 877 struct vfio_region_info_cap_type cap_type = { 878 .header.id = VFIO_REGION_INFO_CAP_TYPE, 879 .header.version = 1 }; 880 881 if (info.index >= 882 VFIO_PCI_NUM_REGIONS + vdev->num_regions) 883 return -EINVAL; 884 info.index = array_index_nospec(info.index, 885 VFIO_PCI_NUM_REGIONS + 886 vdev->num_regions); 887 888 i = info.index - VFIO_PCI_NUM_REGIONS; 889 890 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 891 info.size = vdev->region[i].size; 892 info.flags = vdev->region[i].flags; 893 894 cap_type.type = vdev->region[i].type; 895 cap_type.subtype = vdev->region[i].subtype; 896 897 ret = vfio_info_add_capability(&caps, &cap_type.header, 898 sizeof(cap_type)); 899 if (ret) 900 return ret; 901 902 if (vdev->region[i].ops->add_capability) { 903 ret = vdev->region[i].ops->add_capability(vdev, 904 &vdev->region[i], &caps); 905 if (ret) 906 return ret; 907 } 908 } 909 } 910 911 if (caps.size) { 912 info.flags |= VFIO_REGION_INFO_FLAG_CAPS; 913 if (info.argsz < sizeof(info) + caps.size) { 914 info.argsz = sizeof(info) + caps.size; 915 info.cap_offset = 0; 916 } else { 917 vfio_info_cap_shift(&caps, sizeof(info)); 918 if (copy_to_user((void __user *)arg + 919 sizeof(info), caps.buf, 920 caps.size)) { 921 kfree(caps.buf); 922 return -EFAULT; 923 } 924 info.cap_offset = sizeof(info); 925 } 926 927 kfree(caps.buf); 928 } 929 930 return copy_to_user((void __user *)arg, &info, minsz) ? 931 -EFAULT : 0; 932 933 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 934 struct vfio_irq_info info; 935 936 minsz = offsetofend(struct vfio_irq_info, count); 937 938 if (copy_from_user(&info, (void __user *)arg, minsz)) 939 return -EFAULT; 940 941 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 942 return -EINVAL; 943 944 switch (info.index) { 945 case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX: 946 case VFIO_PCI_REQ_IRQ_INDEX: 947 break; 948 case VFIO_PCI_ERR_IRQ_INDEX: 949 if (pci_is_pcie(vdev->pdev)) 950 break; 951 /* fall through */ 952 default: 953 return -EINVAL; 954 } 955 956 info.flags = VFIO_IRQ_INFO_EVENTFD; 957 958 info.count = vfio_pci_get_irq_count(vdev, info.index); 959 960 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 961 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 962 VFIO_IRQ_INFO_AUTOMASKED); 963 else 964 info.flags |= VFIO_IRQ_INFO_NORESIZE; 965 966 return copy_to_user((void __user *)arg, &info, minsz) ? 967 -EFAULT : 0; 968 969 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 970 struct vfio_irq_set hdr; 971 u8 *data = NULL; 972 int max, ret = 0; 973 size_t data_size = 0; 974 975 minsz = offsetofend(struct vfio_irq_set, count); 976 977 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 978 return -EFAULT; 979 980 max = vfio_pci_get_irq_count(vdev, hdr.index); 981 982 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, 983 VFIO_PCI_NUM_IRQS, &data_size); 984 if (ret) 985 return ret; 986 987 if (data_size) { 988 data = memdup_user((void __user *)(arg + minsz), 989 data_size); 990 if (IS_ERR(data)) 991 return PTR_ERR(data); 992 } 993 994 mutex_lock(&vdev->igate); 995 996 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, 997 hdr.start, hdr.count, data); 998 999 mutex_unlock(&vdev->igate); 1000 kfree(data); 1001 1002 return ret; 1003 1004 } else if (cmd == VFIO_DEVICE_RESET) { 1005 int ret; 1006 1007 if (!vdev->reset_works) 1008 return -EINVAL; 1009 1010 vfio_pci_zap_and_down_write_memory_lock(vdev); 1011 ret = pci_try_reset_function(vdev->pdev); 1012 up_write(&vdev->memory_lock); 1013 1014 return ret; 1015 1016 } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) { 1017 struct vfio_pci_hot_reset_info hdr; 1018 struct vfio_pci_fill_info fill = { 0 }; 1019 struct vfio_pci_dependent_device *devices = NULL; 1020 bool slot = false; 1021 int ret = 0; 1022 1023 minsz = offsetofend(struct vfio_pci_hot_reset_info, count); 1024 1025 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1026 return -EFAULT; 1027 1028 if (hdr.argsz < minsz) 1029 return -EINVAL; 1030 1031 hdr.flags = 0; 1032 1033 /* Can we do a slot or bus reset or neither? */ 1034 if (!pci_probe_reset_slot(vdev->pdev->slot)) 1035 slot = true; 1036 else if (pci_probe_reset_bus(vdev->pdev->bus)) 1037 return -ENODEV; 1038 1039 /* How many devices are affected? */ 1040 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 1041 vfio_pci_count_devs, 1042 &fill.max, slot); 1043 if (ret) 1044 return ret; 1045 1046 WARN_ON(!fill.max); /* Should always be at least one */ 1047 1048 /* 1049 * If there's enough space, fill it now, otherwise return 1050 * -ENOSPC and the number of devices affected. 1051 */ 1052 if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) { 1053 ret = -ENOSPC; 1054 hdr.count = fill.max; 1055 goto reset_info_exit; 1056 } 1057 1058 devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL); 1059 if (!devices) 1060 return -ENOMEM; 1061 1062 fill.devices = devices; 1063 1064 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 1065 vfio_pci_fill_devs, 1066 &fill, slot); 1067 1068 /* 1069 * If a device was removed between counting and filling, 1070 * we may come up short of fill.max. If a device was 1071 * added, we'll have a return of -EAGAIN above. 1072 */ 1073 if (!ret) 1074 hdr.count = fill.cur; 1075 1076 reset_info_exit: 1077 if (copy_to_user((void __user *)arg, &hdr, minsz)) 1078 ret = -EFAULT; 1079 1080 if (!ret) { 1081 if (copy_to_user((void __user *)(arg + minsz), devices, 1082 hdr.count * sizeof(*devices))) 1083 ret = -EFAULT; 1084 } 1085 1086 kfree(devices); 1087 return ret; 1088 1089 } else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) { 1090 struct vfio_pci_hot_reset hdr; 1091 int32_t *group_fds; 1092 struct vfio_pci_group_entry *groups; 1093 struct vfio_pci_group_info info; 1094 struct vfio_devices devs = { .cur_index = 0 }; 1095 bool slot = false; 1096 int i, group_idx, mem_idx = 0, count = 0, ret = 0; 1097 1098 minsz = offsetofend(struct vfio_pci_hot_reset, count); 1099 1100 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1101 return -EFAULT; 1102 1103 if (hdr.argsz < minsz || hdr.flags) 1104 return -EINVAL; 1105 1106 /* Can we do a slot or bus reset or neither? */ 1107 if (!pci_probe_reset_slot(vdev->pdev->slot)) 1108 slot = true; 1109 else if (pci_probe_reset_bus(vdev->pdev->bus)) 1110 return -ENODEV; 1111 1112 /* 1113 * We can't let userspace give us an arbitrarily large 1114 * buffer to copy, so verify how many we think there 1115 * could be. Note groups can have multiple devices so 1116 * one group per device is the max. 1117 */ 1118 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 1119 vfio_pci_count_devs, 1120 &count, slot); 1121 if (ret) 1122 return ret; 1123 1124 /* Somewhere between 1 and count is OK */ 1125 if (!hdr.count || hdr.count > count) 1126 return -EINVAL; 1127 1128 group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL); 1129 groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL); 1130 if (!group_fds || !groups) { 1131 kfree(group_fds); 1132 kfree(groups); 1133 return -ENOMEM; 1134 } 1135 1136 if (copy_from_user(group_fds, (void __user *)(arg + minsz), 1137 hdr.count * sizeof(*group_fds))) { 1138 kfree(group_fds); 1139 kfree(groups); 1140 return -EFAULT; 1141 } 1142 1143 /* 1144 * For each group_fd, get the group through the vfio external 1145 * user interface and store the group and iommu ID. This 1146 * ensures the group is held across the reset. 1147 */ 1148 for (group_idx = 0; group_idx < hdr.count; group_idx++) { 1149 struct vfio_group *group; 1150 struct fd f = fdget(group_fds[group_idx]); 1151 if (!f.file) { 1152 ret = -EBADF; 1153 break; 1154 } 1155 1156 group = vfio_group_get_external_user(f.file); 1157 fdput(f); 1158 if (IS_ERR(group)) { 1159 ret = PTR_ERR(group); 1160 break; 1161 } 1162 1163 groups[group_idx].group = group; 1164 groups[group_idx].id = 1165 vfio_external_user_iommu_id(group); 1166 } 1167 1168 kfree(group_fds); 1169 1170 /* release reference to groups on error */ 1171 if (ret) 1172 goto hot_reset_release; 1173 1174 info.count = hdr.count; 1175 info.groups = groups; 1176 1177 /* 1178 * Test whether all the affected devices are contained 1179 * by the set of groups provided by the user. 1180 */ 1181 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 1182 vfio_pci_validate_devs, 1183 &info, slot); 1184 if (ret) 1185 goto hot_reset_release; 1186 1187 devs.max_index = count; 1188 devs.devices = kcalloc(count, sizeof(struct vfio_device *), 1189 GFP_KERNEL); 1190 if (!devs.devices) { 1191 ret = -ENOMEM; 1192 goto hot_reset_release; 1193 } 1194 1195 /* 1196 * We need to get memory_lock for each device, but devices 1197 * can share mmap_lock, therefore we need to zap and hold 1198 * the vma_lock for each device, and only then get each 1199 * memory_lock. 1200 */ 1201 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 1202 vfio_pci_try_zap_and_vma_lock_cb, 1203 &devs, slot); 1204 if (ret) 1205 goto hot_reset_release; 1206 1207 for (; mem_idx < devs.cur_index; mem_idx++) { 1208 struct vfio_pci_device *tmp; 1209 1210 tmp = vfio_device_data(devs.devices[mem_idx]); 1211 1212 ret = down_write_trylock(&tmp->memory_lock); 1213 if (!ret) { 1214 ret = -EBUSY; 1215 goto hot_reset_release; 1216 } 1217 mutex_unlock(&tmp->vma_lock); 1218 } 1219 1220 /* User has access, do the reset */ 1221 ret = pci_reset_bus(vdev->pdev); 1222 1223 hot_reset_release: 1224 for (i = 0; i < devs.cur_index; i++) { 1225 struct vfio_device *device; 1226 struct vfio_pci_device *tmp; 1227 1228 device = devs.devices[i]; 1229 tmp = vfio_device_data(device); 1230 1231 if (i < mem_idx) 1232 up_write(&tmp->memory_lock); 1233 else 1234 mutex_unlock(&tmp->vma_lock); 1235 vfio_device_put(device); 1236 } 1237 kfree(devs.devices); 1238 1239 for (group_idx--; group_idx >= 0; group_idx--) 1240 vfio_group_put_external_user(groups[group_idx].group); 1241 1242 kfree(groups); 1243 return ret; 1244 } else if (cmd == VFIO_DEVICE_IOEVENTFD) { 1245 struct vfio_device_ioeventfd ioeventfd; 1246 int count; 1247 1248 minsz = offsetofend(struct vfio_device_ioeventfd, fd); 1249 1250 if (copy_from_user(&ioeventfd, (void __user *)arg, minsz)) 1251 return -EFAULT; 1252 1253 if (ioeventfd.argsz < minsz) 1254 return -EINVAL; 1255 1256 if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK) 1257 return -EINVAL; 1258 1259 count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK; 1260 1261 if (hweight8(count) != 1 || ioeventfd.fd < -1) 1262 return -EINVAL; 1263 1264 return vfio_pci_ioeventfd(vdev, ioeventfd.offset, 1265 ioeventfd.data, count, ioeventfd.fd); 1266 } else if (cmd == VFIO_DEVICE_FEATURE) { 1267 struct vfio_device_feature feature; 1268 uuid_t uuid; 1269 1270 minsz = offsetofend(struct vfio_device_feature, flags); 1271 1272 if (copy_from_user(&feature, (void __user *)arg, minsz)) 1273 return -EFAULT; 1274 1275 if (feature.argsz < minsz) 1276 return -EINVAL; 1277 1278 /* Check unknown flags */ 1279 if (feature.flags & ~(VFIO_DEVICE_FEATURE_MASK | 1280 VFIO_DEVICE_FEATURE_SET | 1281 VFIO_DEVICE_FEATURE_GET | 1282 VFIO_DEVICE_FEATURE_PROBE)) 1283 return -EINVAL; 1284 1285 /* GET & SET are mutually exclusive except with PROBE */ 1286 if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) && 1287 (feature.flags & VFIO_DEVICE_FEATURE_SET) && 1288 (feature.flags & VFIO_DEVICE_FEATURE_GET)) 1289 return -EINVAL; 1290 1291 switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) { 1292 case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN: 1293 if (!vdev->vf_token) 1294 return -ENOTTY; 1295 1296 /* 1297 * We do not support GET of the VF Token UUID as this 1298 * could expose the token of the previous device user. 1299 */ 1300 if (feature.flags & VFIO_DEVICE_FEATURE_GET) 1301 return -EINVAL; 1302 1303 if (feature.flags & VFIO_DEVICE_FEATURE_PROBE) 1304 return 0; 1305 1306 /* Don't SET unless told to do so */ 1307 if (!(feature.flags & VFIO_DEVICE_FEATURE_SET)) 1308 return -EINVAL; 1309 1310 if (feature.argsz < minsz + sizeof(uuid)) 1311 return -EINVAL; 1312 1313 if (copy_from_user(&uuid, (void __user *)(arg + minsz), 1314 sizeof(uuid))) 1315 return -EFAULT; 1316 1317 mutex_lock(&vdev->vf_token->lock); 1318 uuid_copy(&vdev->vf_token->uuid, &uuid); 1319 mutex_unlock(&vdev->vf_token->lock); 1320 1321 return 0; 1322 default: 1323 return -ENOTTY; 1324 } 1325 } 1326 1327 return -ENOTTY; 1328 } 1329 1330 static ssize_t vfio_pci_rw(void *device_data, char __user *buf, 1331 size_t count, loff_t *ppos, bool iswrite) 1332 { 1333 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 1334 struct vfio_pci_device *vdev = device_data; 1335 1336 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) 1337 return -EINVAL; 1338 1339 switch (index) { 1340 case VFIO_PCI_CONFIG_REGION_INDEX: 1341 return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite); 1342 1343 case VFIO_PCI_ROM_REGION_INDEX: 1344 if (iswrite) 1345 return -EINVAL; 1346 return vfio_pci_bar_rw(vdev, buf, count, ppos, false); 1347 1348 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1349 return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite); 1350 1351 case VFIO_PCI_VGA_REGION_INDEX: 1352 return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite); 1353 default: 1354 index -= VFIO_PCI_NUM_REGIONS; 1355 return vdev->region[index].ops->rw(vdev, buf, 1356 count, ppos, iswrite); 1357 } 1358 1359 return -EINVAL; 1360 } 1361 1362 static ssize_t vfio_pci_read(void *device_data, char __user *buf, 1363 size_t count, loff_t *ppos) 1364 { 1365 if (!count) 1366 return 0; 1367 1368 return vfio_pci_rw(device_data, buf, count, ppos, false); 1369 } 1370 1371 static ssize_t vfio_pci_write(void *device_data, const char __user *buf, 1372 size_t count, loff_t *ppos) 1373 { 1374 if (!count) 1375 return 0; 1376 1377 return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true); 1378 } 1379 1380 /* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */ 1381 static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try) 1382 { 1383 struct vfio_pci_mmap_vma *mmap_vma, *tmp; 1384 1385 /* 1386 * Lock ordering: 1387 * vma_lock is nested under mmap_lock for vm_ops callback paths. 1388 * The memory_lock semaphore is used by both code paths calling 1389 * into this function to zap vmas and the vm_ops.fault callback 1390 * to protect the memory enable state of the device. 1391 * 1392 * When zapping vmas we need to maintain the mmap_lock => vma_lock 1393 * ordering, which requires using vma_lock to walk vma_list to 1394 * acquire an mm, then dropping vma_lock to get the mmap_lock and 1395 * reacquiring vma_lock. This logic is derived from similar 1396 * requirements in uverbs_user_mmap_disassociate(). 1397 * 1398 * mmap_lock must always be the top-level lock when it is taken. 1399 * Therefore we can only hold the memory_lock write lock when 1400 * vma_list is empty, as we'd need to take mmap_lock to clear 1401 * entries. vma_list can only be guaranteed empty when holding 1402 * vma_lock, thus memory_lock is nested under vma_lock. 1403 * 1404 * This enables the vm_ops.fault callback to acquire vma_lock, 1405 * followed by memory_lock read lock, while already holding 1406 * mmap_lock without risk of deadlock. 1407 */ 1408 while (1) { 1409 struct mm_struct *mm = NULL; 1410 1411 if (try) { 1412 if (!mutex_trylock(&vdev->vma_lock)) 1413 return 0; 1414 } else { 1415 mutex_lock(&vdev->vma_lock); 1416 } 1417 while (!list_empty(&vdev->vma_list)) { 1418 mmap_vma = list_first_entry(&vdev->vma_list, 1419 struct vfio_pci_mmap_vma, 1420 vma_next); 1421 mm = mmap_vma->vma->vm_mm; 1422 if (mmget_not_zero(mm)) 1423 break; 1424 1425 list_del(&mmap_vma->vma_next); 1426 kfree(mmap_vma); 1427 mm = NULL; 1428 } 1429 if (!mm) 1430 return 1; 1431 mutex_unlock(&vdev->vma_lock); 1432 1433 if (try) { 1434 if (!mmap_read_trylock(mm)) { 1435 mmput(mm); 1436 return 0; 1437 } 1438 } else { 1439 mmap_read_lock(mm); 1440 } 1441 if (mmget_still_valid(mm)) { 1442 if (try) { 1443 if (!mutex_trylock(&vdev->vma_lock)) { 1444 mmap_read_unlock(mm); 1445 mmput(mm); 1446 return 0; 1447 } 1448 } else { 1449 mutex_lock(&vdev->vma_lock); 1450 } 1451 list_for_each_entry_safe(mmap_vma, tmp, 1452 &vdev->vma_list, vma_next) { 1453 struct vm_area_struct *vma = mmap_vma->vma; 1454 1455 if (vma->vm_mm != mm) 1456 continue; 1457 1458 list_del(&mmap_vma->vma_next); 1459 kfree(mmap_vma); 1460 1461 zap_vma_ptes(vma, vma->vm_start, 1462 vma->vm_end - vma->vm_start); 1463 } 1464 mutex_unlock(&vdev->vma_lock); 1465 } 1466 mmap_read_unlock(mm); 1467 mmput(mm); 1468 } 1469 } 1470 1471 void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device *vdev) 1472 { 1473 vfio_pci_zap_and_vma_lock(vdev, false); 1474 down_write(&vdev->memory_lock); 1475 mutex_unlock(&vdev->vma_lock); 1476 } 1477 1478 u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_device *vdev) 1479 { 1480 u16 cmd; 1481 1482 down_write(&vdev->memory_lock); 1483 pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd); 1484 if (!(cmd & PCI_COMMAND_MEMORY)) 1485 pci_write_config_word(vdev->pdev, PCI_COMMAND, 1486 cmd | PCI_COMMAND_MEMORY); 1487 1488 return cmd; 1489 } 1490 1491 void vfio_pci_memory_unlock_and_restore(struct vfio_pci_device *vdev, u16 cmd) 1492 { 1493 pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd); 1494 up_write(&vdev->memory_lock); 1495 } 1496 1497 /* Caller holds vma_lock */ 1498 static int __vfio_pci_add_vma(struct vfio_pci_device *vdev, 1499 struct vm_area_struct *vma) 1500 { 1501 struct vfio_pci_mmap_vma *mmap_vma; 1502 1503 mmap_vma = kmalloc(sizeof(*mmap_vma), GFP_KERNEL); 1504 if (!mmap_vma) 1505 return -ENOMEM; 1506 1507 mmap_vma->vma = vma; 1508 list_add(&mmap_vma->vma_next, &vdev->vma_list); 1509 1510 return 0; 1511 } 1512 1513 /* 1514 * Zap mmaps on open so that we can fault them in on access and therefore 1515 * our vma_list only tracks mappings accessed since last zap. 1516 */ 1517 static void vfio_pci_mmap_open(struct vm_area_struct *vma) 1518 { 1519 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); 1520 } 1521 1522 static void vfio_pci_mmap_close(struct vm_area_struct *vma) 1523 { 1524 struct vfio_pci_device *vdev = vma->vm_private_data; 1525 struct vfio_pci_mmap_vma *mmap_vma; 1526 1527 mutex_lock(&vdev->vma_lock); 1528 list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) { 1529 if (mmap_vma->vma == vma) { 1530 list_del(&mmap_vma->vma_next); 1531 kfree(mmap_vma); 1532 break; 1533 } 1534 } 1535 mutex_unlock(&vdev->vma_lock); 1536 } 1537 1538 static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf) 1539 { 1540 struct vm_area_struct *vma = vmf->vma; 1541 struct vfio_pci_device *vdev = vma->vm_private_data; 1542 vm_fault_t ret = VM_FAULT_NOPAGE; 1543 1544 mutex_lock(&vdev->vma_lock); 1545 down_read(&vdev->memory_lock); 1546 1547 if (!__vfio_pci_memory_enabled(vdev)) { 1548 ret = VM_FAULT_SIGBUS; 1549 mutex_unlock(&vdev->vma_lock); 1550 goto up_out; 1551 } 1552 1553 if (__vfio_pci_add_vma(vdev, vma)) { 1554 ret = VM_FAULT_OOM; 1555 mutex_unlock(&vdev->vma_lock); 1556 goto up_out; 1557 } 1558 1559 mutex_unlock(&vdev->vma_lock); 1560 1561 if (remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 1562 vma->vm_end - vma->vm_start, vma->vm_page_prot)) 1563 ret = VM_FAULT_SIGBUS; 1564 1565 up_out: 1566 up_read(&vdev->memory_lock); 1567 return ret; 1568 } 1569 1570 static const struct vm_operations_struct vfio_pci_mmap_ops = { 1571 .open = vfio_pci_mmap_open, 1572 .close = vfio_pci_mmap_close, 1573 .fault = vfio_pci_mmap_fault, 1574 }; 1575 1576 static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma) 1577 { 1578 struct vfio_pci_device *vdev = device_data; 1579 struct pci_dev *pdev = vdev->pdev; 1580 unsigned int index; 1581 u64 phys_len, req_len, pgoff, req_start; 1582 int ret; 1583 1584 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 1585 1586 if (vma->vm_end < vma->vm_start) 1587 return -EINVAL; 1588 if ((vma->vm_flags & VM_SHARED) == 0) 1589 return -EINVAL; 1590 if (index >= VFIO_PCI_NUM_REGIONS) { 1591 int regnum = index - VFIO_PCI_NUM_REGIONS; 1592 struct vfio_pci_region *region = vdev->region + regnum; 1593 1594 if (region && region->ops && region->ops->mmap && 1595 (region->flags & VFIO_REGION_INFO_FLAG_MMAP)) 1596 return region->ops->mmap(vdev, region, vma); 1597 return -EINVAL; 1598 } 1599 if (index >= VFIO_PCI_ROM_REGION_INDEX) 1600 return -EINVAL; 1601 if (!vdev->bar_mmap_supported[index]) 1602 return -EINVAL; 1603 1604 phys_len = PAGE_ALIGN(pci_resource_len(pdev, index)); 1605 req_len = vma->vm_end - vma->vm_start; 1606 pgoff = vma->vm_pgoff & 1607 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 1608 req_start = pgoff << PAGE_SHIFT; 1609 1610 if (req_start + req_len > phys_len) 1611 return -EINVAL; 1612 1613 /* 1614 * Even though we don't make use of the barmap for the mmap, 1615 * we need to request the region and the barmap tracks that. 1616 */ 1617 if (!vdev->barmap[index]) { 1618 ret = pci_request_selected_regions(pdev, 1619 1 << index, "vfio-pci"); 1620 if (ret) 1621 return ret; 1622 1623 vdev->barmap[index] = pci_iomap(pdev, index, 0); 1624 if (!vdev->barmap[index]) { 1625 pci_release_selected_regions(pdev, 1 << index); 1626 return -ENOMEM; 1627 } 1628 } 1629 1630 vma->vm_private_data = vdev; 1631 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 1632 vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff; 1633 1634 /* 1635 * See remap_pfn_range(), called from vfio_pci_fault() but we can't 1636 * change vm_flags within the fault handler. Set them now. 1637 */ 1638 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 1639 vma->vm_ops = &vfio_pci_mmap_ops; 1640 1641 return 0; 1642 } 1643 1644 static void vfio_pci_request(void *device_data, unsigned int count) 1645 { 1646 struct vfio_pci_device *vdev = device_data; 1647 struct pci_dev *pdev = vdev->pdev; 1648 1649 mutex_lock(&vdev->igate); 1650 1651 if (vdev->req_trigger) { 1652 if (!(count % 10)) 1653 pci_notice_ratelimited(pdev, 1654 "Relaying device request to user (#%u)\n", 1655 count); 1656 eventfd_signal(vdev->req_trigger, 1); 1657 } else if (count == 0) { 1658 pci_warn(pdev, 1659 "No device request channel registered, blocked until released by user\n"); 1660 } 1661 1662 mutex_unlock(&vdev->igate); 1663 } 1664 1665 static int vfio_pci_validate_vf_token(struct vfio_pci_device *vdev, 1666 bool vf_token, uuid_t *uuid) 1667 { 1668 /* 1669 * There's always some degree of trust or collaboration between SR-IOV 1670 * PF and VFs, even if just that the PF hosts the SR-IOV capability and 1671 * can disrupt VFs with a reset, but often the PF has more explicit 1672 * access to deny service to the VF or access data passed through the 1673 * VF. We therefore require an opt-in via a shared VF token (UUID) to 1674 * represent this trust. This both prevents that a VF driver might 1675 * assume the PF driver is a trusted, in-kernel driver, and also that 1676 * a PF driver might be replaced with a rogue driver, unknown to in-use 1677 * VF drivers. 1678 * 1679 * Therefore when presented with a VF, if the PF is a vfio device and 1680 * it is bound to the vfio-pci driver, the user needs to provide a VF 1681 * token to access the device, in the form of appending a vf_token to 1682 * the device name, for example: 1683 * 1684 * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3" 1685 * 1686 * When presented with a PF which has VFs in use, the user must also 1687 * provide the current VF token to prove collaboration with existing 1688 * VF users. If VFs are not in use, the VF token provided for the PF 1689 * device will act to set the VF token. 1690 * 1691 * If the VF token is provided but unused, an error is generated. 1692 */ 1693 if (!vdev->pdev->is_virtfn && !vdev->vf_token && !vf_token) 1694 return 0; /* No VF token provided or required */ 1695 1696 if (vdev->pdev->is_virtfn) { 1697 struct vfio_device *pf_dev; 1698 struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev, &pf_dev); 1699 bool match; 1700 1701 if (!pf_vdev) { 1702 if (!vf_token) 1703 return 0; /* PF is not vfio-pci, no VF token */ 1704 1705 pci_info_ratelimited(vdev->pdev, 1706 "VF token incorrectly provided, PF not bound to vfio-pci\n"); 1707 return -EINVAL; 1708 } 1709 1710 if (!vf_token) { 1711 vfio_device_put(pf_dev); 1712 pci_info_ratelimited(vdev->pdev, 1713 "VF token required to access device\n"); 1714 return -EACCES; 1715 } 1716 1717 mutex_lock(&pf_vdev->vf_token->lock); 1718 match = uuid_equal(uuid, &pf_vdev->vf_token->uuid); 1719 mutex_unlock(&pf_vdev->vf_token->lock); 1720 1721 vfio_device_put(pf_dev); 1722 1723 if (!match) { 1724 pci_info_ratelimited(vdev->pdev, 1725 "Incorrect VF token provided for device\n"); 1726 return -EACCES; 1727 } 1728 } else if (vdev->vf_token) { 1729 mutex_lock(&vdev->vf_token->lock); 1730 if (vdev->vf_token->users) { 1731 if (!vf_token) { 1732 mutex_unlock(&vdev->vf_token->lock); 1733 pci_info_ratelimited(vdev->pdev, 1734 "VF token required to access device\n"); 1735 return -EACCES; 1736 } 1737 1738 if (!uuid_equal(uuid, &vdev->vf_token->uuid)) { 1739 mutex_unlock(&vdev->vf_token->lock); 1740 pci_info_ratelimited(vdev->pdev, 1741 "Incorrect VF token provided for device\n"); 1742 return -EACCES; 1743 } 1744 } else if (vf_token) { 1745 uuid_copy(&vdev->vf_token->uuid, uuid); 1746 } 1747 1748 mutex_unlock(&vdev->vf_token->lock); 1749 } else if (vf_token) { 1750 pci_info_ratelimited(vdev->pdev, 1751 "VF token incorrectly provided, not a PF or VF\n"); 1752 return -EINVAL; 1753 } 1754 1755 return 0; 1756 } 1757 1758 #define VF_TOKEN_ARG "vf_token=" 1759 1760 static int vfio_pci_match(void *device_data, char *buf) 1761 { 1762 struct vfio_pci_device *vdev = device_data; 1763 bool vf_token = false; 1764 uuid_t uuid; 1765 int ret; 1766 1767 if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev)))) 1768 return 0; /* No match */ 1769 1770 if (strlen(buf) > strlen(pci_name(vdev->pdev))) { 1771 buf += strlen(pci_name(vdev->pdev)); 1772 1773 if (*buf != ' ') 1774 return 0; /* No match: non-whitespace after name */ 1775 1776 while (*buf) { 1777 if (*buf == ' ') { 1778 buf++; 1779 continue; 1780 } 1781 1782 if (!vf_token && !strncmp(buf, VF_TOKEN_ARG, 1783 strlen(VF_TOKEN_ARG))) { 1784 buf += strlen(VF_TOKEN_ARG); 1785 1786 if (strlen(buf) < UUID_STRING_LEN) 1787 return -EINVAL; 1788 1789 ret = uuid_parse(buf, &uuid); 1790 if (ret) 1791 return ret; 1792 1793 vf_token = true; 1794 buf += UUID_STRING_LEN; 1795 } else { 1796 /* Unknown/duplicate option */ 1797 return -EINVAL; 1798 } 1799 } 1800 } 1801 1802 ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid); 1803 if (ret) 1804 return ret; 1805 1806 return 1; /* Match */ 1807 } 1808 1809 static const struct vfio_device_ops vfio_pci_ops = { 1810 .name = "vfio-pci", 1811 .open = vfio_pci_open, 1812 .release = vfio_pci_release, 1813 .ioctl = vfio_pci_ioctl, 1814 .read = vfio_pci_read, 1815 .write = vfio_pci_write, 1816 .mmap = vfio_pci_mmap, 1817 .request = vfio_pci_request, 1818 .match = vfio_pci_match, 1819 }; 1820 1821 static int vfio_pci_reflck_attach(struct vfio_pci_device *vdev); 1822 static void vfio_pci_reflck_put(struct vfio_pci_reflck *reflck); 1823 static struct pci_driver vfio_pci_driver; 1824 1825 static int vfio_pci_bus_notifier(struct notifier_block *nb, 1826 unsigned long action, void *data) 1827 { 1828 struct vfio_pci_device *vdev = container_of(nb, 1829 struct vfio_pci_device, nb); 1830 struct device *dev = data; 1831 struct pci_dev *pdev = to_pci_dev(dev); 1832 struct pci_dev *physfn = pci_physfn(pdev); 1833 1834 if (action == BUS_NOTIFY_ADD_DEVICE && 1835 pdev->is_virtfn && physfn == vdev->pdev) { 1836 pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n", 1837 pci_name(pdev)); 1838 pdev->driver_override = kasprintf(GFP_KERNEL, "%s", 1839 vfio_pci_ops.name); 1840 } else if (action == BUS_NOTIFY_BOUND_DRIVER && 1841 pdev->is_virtfn && physfn == vdev->pdev) { 1842 struct pci_driver *drv = pci_dev_driver(pdev); 1843 1844 if (drv && drv != &vfio_pci_driver) 1845 pci_warn(vdev->pdev, 1846 "VF %s bound to driver %s while PF bound to vfio-pci\n", 1847 pci_name(pdev), drv->name); 1848 } 1849 1850 return 0; 1851 } 1852 1853 static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1854 { 1855 struct vfio_pci_device *vdev; 1856 struct iommu_group *group; 1857 int ret; 1858 1859 if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL) 1860 return -EINVAL; 1861 1862 /* 1863 * Prevent binding to PFs with VFs enabled, the VFs might be in use 1864 * by the host or other users. We cannot capture the VFs if they 1865 * already exist, nor can we track VF users. Disabling SR-IOV here 1866 * would initiate removing the VFs, which would unbind the driver, 1867 * which is prone to blocking if that VF is also in use by vfio-pci. 1868 * Just reject these PFs and let the user sort it out. 1869 */ 1870 if (pci_num_vf(pdev)) { 1871 pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n"); 1872 return -EBUSY; 1873 } 1874 1875 group = vfio_iommu_group_get(&pdev->dev); 1876 if (!group) 1877 return -EINVAL; 1878 1879 vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); 1880 if (!vdev) { 1881 ret = -ENOMEM; 1882 goto out_group_put; 1883 } 1884 1885 vdev->pdev = pdev; 1886 vdev->irq_type = VFIO_PCI_NUM_IRQS; 1887 mutex_init(&vdev->igate); 1888 spin_lock_init(&vdev->irqlock); 1889 mutex_init(&vdev->ioeventfds_lock); 1890 INIT_LIST_HEAD(&vdev->ioeventfds_list); 1891 mutex_init(&vdev->vma_lock); 1892 INIT_LIST_HEAD(&vdev->vma_list); 1893 init_rwsem(&vdev->memory_lock); 1894 1895 ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev); 1896 if (ret) 1897 goto out_free; 1898 1899 ret = vfio_pci_reflck_attach(vdev); 1900 if (ret) 1901 goto out_del_group_dev; 1902 1903 if (pdev->is_physfn) { 1904 vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL); 1905 if (!vdev->vf_token) { 1906 ret = -ENOMEM; 1907 goto out_reflck; 1908 } 1909 1910 mutex_init(&vdev->vf_token->lock); 1911 uuid_gen(&vdev->vf_token->uuid); 1912 1913 vdev->nb.notifier_call = vfio_pci_bus_notifier; 1914 ret = bus_register_notifier(&pci_bus_type, &vdev->nb); 1915 if (ret) 1916 goto out_vf_token; 1917 } 1918 1919 if (vfio_pci_is_vga(pdev)) { 1920 vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode); 1921 vga_set_legacy_decoding(pdev, 1922 vfio_pci_set_vga_decode(vdev, false)); 1923 } 1924 1925 vfio_pci_probe_power_state(vdev); 1926 1927 if (!disable_idle_d3) { 1928 /* 1929 * pci-core sets the device power state to an unknown value at 1930 * bootup and after being removed from a driver. The only 1931 * transition it allows from this unknown state is to D0, which 1932 * typically happens when a driver calls pci_enable_device(). 1933 * We're not ready to enable the device yet, but we do want to 1934 * be able to get to D3. Therefore first do a D0 transition 1935 * before going to D3. 1936 */ 1937 vfio_pci_set_power_state(vdev, PCI_D0); 1938 vfio_pci_set_power_state(vdev, PCI_D3hot); 1939 } 1940 1941 return ret; 1942 1943 out_vf_token: 1944 kfree(vdev->vf_token); 1945 out_reflck: 1946 vfio_pci_reflck_put(vdev->reflck); 1947 out_del_group_dev: 1948 vfio_del_group_dev(&pdev->dev); 1949 out_free: 1950 kfree(vdev); 1951 out_group_put: 1952 vfio_iommu_group_put(group, &pdev->dev); 1953 return ret; 1954 } 1955 1956 static void vfio_pci_remove(struct pci_dev *pdev) 1957 { 1958 struct vfio_pci_device *vdev; 1959 1960 pci_disable_sriov(pdev); 1961 1962 vdev = vfio_del_group_dev(&pdev->dev); 1963 if (!vdev) 1964 return; 1965 1966 if (vdev->vf_token) { 1967 WARN_ON(vdev->vf_token->users); 1968 mutex_destroy(&vdev->vf_token->lock); 1969 kfree(vdev->vf_token); 1970 } 1971 1972 if (vdev->nb.notifier_call) 1973 bus_unregister_notifier(&pci_bus_type, &vdev->nb); 1974 1975 vfio_pci_reflck_put(vdev->reflck); 1976 1977 vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev); 1978 kfree(vdev->region); 1979 mutex_destroy(&vdev->ioeventfds_lock); 1980 1981 if (!disable_idle_d3) 1982 vfio_pci_set_power_state(vdev, PCI_D0); 1983 1984 kfree(vdev->pm_save); 1985 kfree(vdev); 1986 1987 if (vfio_pci_is_vga(pdev)) { 1988 vga_client_register(pdev, NULL, NULL, NULL); 1989 vga_set_legacy_decoding(pdev, 1990 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | 1991 VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM); 1992 } 1993 } 1994 1995 static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev, 1996 pci_channel_state_t state) 1997 { 1998 struct vfio_pci_device *vdev; 1999 struct vfio_device *device; 2000 2001 device = vfio_device_get_from_dev(&pdev->dev); 2002 if (device == NULL) 2003 return PCI_ERS_RESULT_DISCONNECT; 2004 2005 vdev = vfio_device_data(device); 2006 if (vdev == NULL) { 2007 vfio_device_put(device); 2008 return PCI_ERS_RESULT_DISCONNECT; 2009 } 2010 2011 mutex_lock(&vdev->igate); 2012 2013 if (vdev->err_trigger) 2014 eventfd_signal(vdev->err_trigger, 1); 2015 2016 mutex_unlock(&vdev->igate); 2017 2018 vfio_device_put(device); 2019 2020 return PCI_ERS_RESULT_CAN_RECOVER; 2021 } 2022 2023 static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn) 2024 { 2025 struct vfio_pci_device *vdev; 2026 struct vfio_device *device; 2027 int ret = 0; 2028 2029 might_sleep(); 2030 2031 if (!enable_sriov) 2032 return -ENOENT; 2033 2034 device = vfio_device_get_from_dev(&pdev->dev); 2035 if (!device) 2036 return -ENODEV; 2037 2038 vdev = vfio_device_data(device); 2039 if (!vdev) { 2040 vfio_device_put(device); 2041 return -ENODEV; 2042 } 2043 2044 if (nr_virtfn == 0) 2045 pci_disable_sriov(pdev); 2046 else 2047 ret = pci_enable_sriov(pdev, nr_virtfn); 2048 2049 vfio_device_put(device); 2050 2051 return ret < 0 ? ret : nr_virtfn; 2052 } 2053 2054 static const struct pci_error_handlers vfio_err_handlers = { 2055 .error_detected = vfio_pci_aer_err_detected, 2056 }; 2057 2058 static struct pci_driver vfio_pci_driver = { 2059 .name = "vfio-pci", 2060 .id_table = NULL, /* only dynamic ids */ 2061 .probe = vfio_pci_probe, 2062 .remove = vfio_pci_remove, 2063 .sriov_configure = vfio_pci_sriov_configure, 2064 .err_handler = &vfio_err_handlers, 2065 }; 2066 2067 static DEFINE_MUTEX(reflck_lock); 2068 2069 static struct vfio_pci_reflck *vfio_pci_reflck_alloc(void) 2070 { 2071 struct vfio_pci_reflck *reflck; 2072 2073 reflck = kzalloc(sizeof(*reflck), GFP_KERNEL); 2074 if (!reflck) 2075 return ERR_PTR(-ENOMEM); 2076 2077 kref_init(&reflck->kref); 2078 mutex_init(&reflck->lock); 2079 2080 return reflck; 2081 } 2082 2083 static void vfio_pci_reflck_get(struct vfio_pci_reflck *reflck) 2084 { 2085 kref_get(&reflck->kref); 2086 } 2087 2088 static int vfio_pci_reflck_find(struct pci_dev *pdev, void *data) 2089 { 2090 struct vfio_pci_reflck **preflck = data; 2091 struct vfio_device *device; 2092 struct vfio_pci_device *vdev; 2093 2094 device = vfio_device_get_from_dev(&pdev->dev); 2095 if (!device) 2096 return 0; 2097 2098 if (pci_dev_driver(pdev) != &vfio_pci_driver) { 2099 vfio_device_put(device); 2100 return 0; 2101 } 2102 2103 vdev = vfio_device_data(device); 2104 2105 if (vdev->reflck) { 2106 vfio_pci_reflck_get(vdev->reflck); 2107 *preflck = vdev->reflck; 2108 vfio_device_put(device); 2109 return 1; 2110 } 2111 2112 vfio_device_put(device); 2113 return 0; 2114 } 2115 2116 static int vfio_pci_reflck_attach(struct vfio_pci_device *vdev) 2117 { 2118 bool slot = !pci_probe_reset_slot(vdev->pdev->slot); 2119 2120 mutex_lock(&reflck_lock); 2121 2122 if (pci_is_root_bus(vdev->pdev->bus) || 2123 vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_reflck_find, 2124 &vdev->reflck, slot) <= 0) 2125 vdev->reflck = vfio_pci_reflck_alloc(); 2126 2127 mutex_unlock(&reflck_lock); 2128 2129 return PTR_ERR_OR_ZERO(vdev->reflck); 2130 } 2131 2132 static void vfio_pci_reflck_release(struct kref *kref) 2133 { 2134 struct vfio_pci_reflck *reflck = container_of(kref, 2135 struct vfio_pci_reflck, 2136 kref); 2137 2138 kfree(reflck); 2139 mutex_unlock(&reflck_lock); 2140 } 2141 2142 static void vfio_pci_reflck_put(struct vfio_pci_reflck *reflck) 2143 { 2144 kref_put_mutex(&reflck->kref, vfio_pci_reflck_release, &reflck_lock); 2145 } 2146 2147 static int vfio_pci_get_unused_devs(struct pci_dev *pdev, void *data) 2148 { 2149 struct vfio_devices *devs = data; 2150 struct vfio_device *device; 2151 struct vfio_pci_device *vdev; 2152 2153 if (devs->cur_index == devs->max_index) 2154 return -ENOSPC; 2155 2156 device = vfio_device_get_from_dev(&pdev->dev); 2157 if (!device) 2158 return -EINVAL; 2159 2160 if (pci_dev_driver(pdev) != &vfio_pci_driver) { 2161 vfio_device_put(device); 2162 return -EBUSY; 2163 } 2164 2165 vdev = vfio_device_data(device); 2166 2167 /* Fault if the device is not unused */ 2168 if (vdev->refcnt) { 2169 vfio_device_put(device); 2170 return -EBUSY; 2171 } 2172 2173 devs->devices[devs->cur_index++] = device; 2174 return 0; 2175 } 2176 2177 static int vfio_pci_try_zap_and_vma_lock_cb(struct pci_dev *pdev, void *data) 2178 { 2179 struct vfio_devices *devs = data; 2180 struct vfio_device *device; 2181 struct vfio_pci_device *vdev; 2182 2183 if (devs->cur_index == devs->max_index) 2184 return -ENOSPC; 2185 2186 device = vfio_device_get_from_dev(&pdev->dev); 2187 if (!device) 2188 return -EINVAL; 2189 2190 if (pci_dev_driver(pdev) != &vfio_pci_driver) { 2191 vfio_device_put(device); 2192 return -EBUSY; 2193 } 2194 2195 vdev = vfio_device_data(device); 2196 2197 /* 2198 * Locking multiple devices is prone to deadlock, runaway and 2199 * unwind if we hit contention. 2200 */ 2201 if (!vfio_pci_zap_and_vma_lock(vdev, true)) { 2202 vfio_device_put(device); 2203 return -EBUSY; 2204 } 2205 2206 devs->devices[devs->cur_index++] = device; 2207 return 0; 2208 } 2209 2210 /* 2211 * If a bus or slot reset is available for the provided device and: 2212 * - All of the devices affected by that bus or slot reset are unused 2213 * (!refcnt) 2214 * - At least one of the affected devices is marked dirty via 2215 * needs_reset (such as by lack of FLR support) 2216 * Then attempt to perform that bus or slot reset. Callers are required 2217 * to hold vdev->reflck->lock, protecting the bus/slot reset group from 2218 * concurrent opens. A vfio_device reference is acquired for each device 2219 * to prevent unbinds during the reset operation. 2220 * 2221 * NB: vfio-core considers a group to be viable even if some devices are 2222 * bound to drivers like pci-stub or pcieport. Here we require all devices 2223 * to be bound to vfio_pci since that's the only way we can be sure they 2224 * stay put. 2225 */ 2226 static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev) 2227 { 2228 struct vfio_devices devs = { .cur_index = 0 }; 2229 int i = 0, ret = -EINVAL; 2230 bool slot = false; 2231 struct vfio_pci_device *tmp; 2232 2233 if (!pci_probe_reset_slot(vdev->pdev->slot)) 2234 slot = true; 2235 else if (pci_probe_reset_bus(vdev->pdev->bus)) 2236 return; 2237 2238 if (vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs, 2239 &i, slot) || !i) 2240 return; 2241 2242 devs.max_index = i; 2243 devs.devices = kcalloc(i, sizeof(struct vfio_device *), GFP_KERNEL); 2244 if (!devs.devices) 2245 return; 2246 2247 if (vfio_pci_for_each_slot_or_bus(vdev->pdev, 2248 vfio_pci_get_unused_devs, 2249 &devs, slot)) 2250 goto put_devs; 2251 2252 /* Does at least one need a reset? */ 2253 for (i = 0; i < devs.cur_index; i++) { 2254 tmp = vfio_device_data(devs.devices[i]); 2255 if (tmp->needs_reset) { 2256 ret = pci_reset_bus(vdev->pdev); 2257 break; 2258 } 2259 } 2260 2261 put_devs: 2262 for (i = 0; i < devs.cur_index; i++) { 2263 tmp = vfio_device_data(devs.devices[i]); 2264 2265 /* 2266 * If reset was successful, affected devices no longer need 2267 * a reset and we should return all the collateral devices 2268 * to low power. If not successful, we either didn't reset 2269 * the bus or timed out waiting for it, so let's not touch 2270 * the power state. 2271 */ 2272 if (!ret) { 2273 tmp->needs_reset = false; 2274 2275 if (tmp != vdev && !disable_idle_d3) 2276 vfio_pci_set_power_state(tmp, PCI_D3hot); 2277 } 2278 2279 vfio_device_put(devs.devices[i]); 2280 } 2281 2282 kfree(devs.devices); 2283 } 2284 2285 static void __exit vfio_pci_cleanup(void) 2286 { 2287 pci_unregister_driver(&vfio_pci_driver); 2288 vfio_pci_uninit_perm_bits(); 2289 } 2290 2291 static void __init vfio_pci_fill_ids(void) 2292 { 2293 char *p, *id; 2294 int rc; 2295 2296 /* no ids passed actually */ 2297 if (ids[0] == '\0') 2298 return; 2299 2300 /* add ids specified in the module parameter */ 2301 p = ids; 2302 while ((id = strsep(&p, ","))) { 2303 unsigned int vendor, device, subvendor = PCI_ANY_ID, 2304 subdevice = PCI_ANY_ID, class = 0, class_mask = 0; 2305 int fields; 2306 2307 if (!strlen(id)) 2308 continue; 2309 2310 fields = sscanf(id, "%x:%x:%x:%x:%x:%x", 2311 &vendor, &device, &subvendor, &subdevice, 2312 &class, &class_mask); 2313 2314 if (fields < 2) { 2315 pr_warn("invalid id string \"%s\"\n", id); 2316 continue; 2317 } 2318 2319 rc = pci_add_dynid(&vfio_pci_driver, vendor, device, 2320 subvendor, subdevice, class, class_mask, 0); 2321 if (rc) 2322 pr_warn("failed to add dynamic id [%04x:%04x[%04x:%04x]] class %#08x/%08x (%d)\n", 2323 vendor, device, subvendor, subdevice, 2324 class, class_mask, rc); 2325 else 2326 pr_info("add [%04x:%04x[%04x:%04x]] class %#08x/%08x\n", 2327 vendor, device, subvendor, subdevice, 2328 class, class_mask); 2329 } 2330 } 2331 2332 static int __init vfio_pci_init(void) 2333 { 2334 int ret; 2335 2336 /* Allocate shared config space permision data used by all devices */ 2337 ret = vfio_pci_init_perm_bits(); 2338 if (ret) 2339 return ret; 2340 2341 /* Register and scan for devices */ 2342 ret = pci_register_driver(&vfio_pci_driver); 2343 if (ret) 2344 goto out_driver; 2345 2346 vfio_pci_fill_ids(); 2347 2348 return 0; 2349 2350 out_driver: 2351 vfio_pci_uninit_perm_bits(); 2352 return ret; 2353 } 2354 2355 module_init(vfio_pci_init); 2356 module_exit(vfio_pci_cleanup); 2357 2358 MODULE_VERSION(DRIVER_VERSION); 2359 MODULE_LICENSE("GPL v2"); 2360 MODULE_AUTHOR(DRIVER_AUTHOR); 2361 MODULE_DESCRIPTION(DRIVER_DESC); 2362