1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 4 * Author: Alex Williamson <alex.williamson@redhat.com> 5 * 6 * Derived from original vfio: 7 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 8 * Author: Tom Lyon, pugs@cisco.com 9 */ 10 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 13 #include <linux/aperture.h> 14 #include <linux/device.h> 15 #include <linux/eventfd.h> 16 #include <linux/file.h> 17 #include <linux/interrupt.h> 18 #include <linux/iommu.h> 19 #include <linux/module.h> 20 #include <linux/mutex.h> 21 #include <linux/notifier.h> 22 #include <linux/pci.h> 23 #include <linux/pm_runtime.h> 24 #include <linux/slab.h> 25 #include <linux/types.h> 26 #include <linux/uaccess.h> 27 #include <linux/vgaarb.h> 28 #include <linux/nospec.h> 29 #include <linux/sched/mm.h> 30 #include <linux/iommufd.h> 31 #if IS_ENABLED(CONFIG_EEH) 32 #include <asm/eeh.h> 33 #endif 34 35 #include "vfio_pci_priv.h" 36 37 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 38 #define DRIVER_DESC "core driver for VFIO based PCI devices" 39 40 static bool nointxmask; 41 static bool disable_vga; 42 static bool disable_idle_d3; 43 44 /* List of PF's that vfio_pci_core_sriov_configure() has been called on */ 45 static DEFINE_MUTEX(vfio_pci_sriov_pfs_mutex); 46 static LIST_HEAD(vfio_pci_sriov_pfs); 47 48 struct vfio_pci_dummy_resource { 49 struct resource resource; 50 int index; 51 struct list_head res_next; 52 }; 53 54 struct vfio_pci_vf_token { 55 struct mutex lock; 56 uuid_t uuid; 57 int users; 58 }; 59 60 struct vfio_pci_mmap_vma { 61 struct vm_area_struct *vma; 62 struct list_head vma_next; 63 }; 64 65 static inline bool vfio_vga_disabled(void) 66 { 67 #ifdef CONFIG_VFIO_PCI_VGA 68 return disable_vga; 69 #else 70 return true; 71 #endif 72 } 73 74 /* 75 * Our VGA arbiter participation is limited since we don't know anything 76 * about the device itself. However, if the device is the only VGA device 77 * downstream of a bridge and VFIO VGA support is disabled, then we can 78 * safely return legacy VGA IO and memory as not decoded since the user 79 * has no way to get to it and routing can be disabled externally at the 80 * bridge. 81 */ 82 static unsigned int vfio_pci_set_decode(struct pci_dev *pdev, bool single_vga) 83 { 84 struct pci_dev *tmp = NULL; 85 unsigned char max_busnr; 86 unsigned int decodes; 87 88 if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus)) 89 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | 90 VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM; 91 92 max_busnr = pci_bus_max_busnr(pdev->bus); 93 decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 94 95 while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) { 96 if (tmp == pdev || 97 pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) || 98 pci_is_root_bus(tmp->bus)) 99 continue; 100 101 if (tmp->bus->number >= pdev->bus->number && 102 tmp->bus->number <= max_busnr) { 103 pci_dev_put(tmp); 104 decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM; 105 break; 106 } 107 } 108 109 return decodes; 110 } 111 112 static void vfio_pci_probe_mmaps(struct vfio_pci_core_device *vdev) 113 { 114 struct resource *res; 115 int i; 116 struct vfio_pci_dummy_resource *dummy_res; 117 118 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 119 int bar = i + PCI_STD_RESOURCES; 120 121 res = &vdev->pdev->resource[bar]; 122 123 if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP)) 124 goto no_mmap; 125 126 if (!(res->flags & IORESOURCE_MEM)) 127 goto no_mmap; 128 129 /* 130 * The PCI core shouldn't set up a resource with a 131 * type but zero size. But there may be bugs that 132 * cause us to do that. 133 */ 134 if (!resource_size(res)) 135 goto no_mmap; 136 137 if (resource_size(res) >= PAGE_SIZE) { 138 vdev->bar_mmap_supported[bar] = true; 139 continue; 140 } 141 142 if (!(res->start & ~PAGE_MASK)) { 143 /* 144 * Add a dummy resource to reserve the remainder 145 * of the exclusive page in case that hot-add 146 * device's bar is assigned into it. 147 */ 148 dummy_res = 149 kzalloc(sizeof(*dummy_res), GFP_KERNEL_ACCOUNT); 150 if (dummy_res == NULL) 151 goto no_mmap; 152 153 dummy_res->resource.name = "vfio sub-page reserved"; 154 dummy_res->resource.start = res->end + 1; 155 dummy_res->resource.end = res->start + PAGE_SIZE - 1; 156 dummy_res->resource.flags = res->flags; 157 if (request_resource(res->parent, 158 &dummy_res->resource)) { 159 kfree(dummy_res); 160 goto no_mmap; 161 } 162 dummy_res->index = bar; 163 list_add(&dummy_res->res_next, 164 &vdev->dummy_resources_list); 165 vdev->bar_mmap_supported[bar] = true; 166 continue; 167 } 168 /* 169 * Here we don't handle the case when the BAR is not page 170 * aligned because we can't expect the BAR will be 171 * assigned into the same location in a page in guest 172 * when we passthrough the BAR. And it's hard to access 173 * this BAR in userspace because we have no way to get 174 * the BAR's location in a page. 175 */ 176 no_mmap: 177 vdev->bar_mmap_supported[bar] = false; 178 } 179 } 180 181 struct vfio_pci_group_info; 182 static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set); 183 static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set, 184 struct vfio_pci_group_info *groups, 185 struct iommufd_ctx *iommufd_ctx); 186 187 /* 188 * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND 189 * _and_ the ability detect when the device is asserting INTx via PCI_STATUS. 190 * If a device implements the former but not the latter we would typically 191 * expect broken_intx_masking be set and require an exclusive interrupt. 192 * However since we do have control of the device's ability to assert INTx, 193 * we can instead pretend that the device does not implement INTx, virtualizing 194 * the pin register to report zero and maintaining DisINTx set on the host. 195 */ 196 static bool vfio_pci_nointx(struct pci_dev *pdev) 197 { 198 switch (pdev->vendor) { 199 case PCI_VENDOR_ID_INTEL: 200 switch (pdev->device) { 201 /* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */ 202 case 0x1572: 203 case 0x1574: 204 case 0x1580 ... 0x1581: 205 case 0x1583 ... 0x158b: 206 case 0x37d0 ... 0x37d2: 207 /* X550 */ 208 case 0x1563: 209 return true; 210 default: 211 return false; 212 } 213 } 214 215 return false; 216 } 217 218 static void vfio_pci_probe_power_state(struct vfio_pci_core_device *vdev) 219 { 220 struct pci_dev *pdev = vdev->pdev; 221 u16 pmcsr; 222 223 if (!pdev->pm_cap) 224 return; 225 226 pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr); 227 228 vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET); 229 } 230 231 /* 232 * pci_set_power_state() wrapper handling devices which perform a soft reset on 233 * D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev, 234 * restore when returned to D0. Saved separately from pci_saved_state for use 235 * by PM capability emulation and separately from pci_dev internal saved state 236 * to avoid it being overwritten and consumed around other resets. 237 */ 238 int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t state) 239 { 240 struct pci_dev *pdev = vdev->pdev; 241 bool needs_restore = false, needs_save = false; 242 int ret; 243 244 /* Prevent changing power state for PFs with VFs enabled */ 245 if (pci_num_vf(pdev) && state > PCI_D0) 246 return -EBUSY; 247 248 if (vdev->needs_pm_restore) { 249 if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) { 250 pci_save_state(pdev); 251 needs_save = true; 252 } 253 254 if (pdev->current_state >= PCI_D3hot && state <= PCI_D0) 255 needs_restore = true; 256 } 257 258 ret = pci_set_power_state(pdev, state); 259 260 if (!ret) { 261 /* D3 might be unsupported via quirk, skip unless in D3 */ 262 if (needs_save && pdev->current_state >= PCI_D3hot) { 263 /* 264 * The current PCI state will be saved locally in 265 * 'pm_save' during the D3hot transition. When the 266 * device state is changed to D0 again with the current 267 * function, then pci_store_saved_state() will restore 268 * the state and will free the memory pointed by 269 * 'pm_save'. There are few cases where the PCI power 270 * state can be changed to D0 without the involvement 271 * of the driver. For these cases, free the earlier 272 * allocated memory first before overwriting 'pm_save' 273 * to prevent the memory leak. 274 */ 275 kfree(vdev->pm_save); 276 vdev->pm_save = pci_store_saved_state(pdev); 277 } else if (needs_restore) { 278 pci_load_and_free_saved_state(pdev, &vdev->pm_save); 279 pci_restore_state(pdev); 280 } 281 } 282 283 return ret; 284 } 285 286 static int vfio_pci_runtime_pm_entry(struct vfio_pci_core_device *vdev, 287 struct eventfd_ctx *efdctx) 288 { 289 /* 290 * The vdev power related flags are protected with 'memory_lock' 291 * semaphore. 292 */ 293 vfio_pci_zap_and_down_write_memory_lock(vdev); 294 if (vdev->pm_runtime_engaged) { 295 up_write(&vdev->memory_lock); 296 return -EINVAL; 297 } 298 299 vdev->pm_runtime_engaged = true; 300 vdev->pm_wake_eventfd_ctx = efdctx; 301 pm_runtime_put_noidle(&vdev->pdev->dev); 302 up_write(&vdev->memory_lock); 303 304 return 0; 305 } 306 307 static int vfio_pci_core_pm_entry(struct vfio_device *device, u32 flags, 308 void __user *arg, size_t argsz) 309 { 310 struct vfio_pci_core_device *vdev = 311 container_of(device, struct vfio_pci_core_device, vdev); 312 int ret; 313 314 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0); 315 if (ret != 1) 316 return ret; 317 318 /* 319 * Inside vfio_pci_runtime_pm_entry(), only the runtime PM usage count 320 * will be decremented. The pm_runtime_put() will be invoked again 321 * while returning from the ioctl and then the device can go into 322 * runtime suspended state. 323 */ 324 return vfio_pci_runtime_pm_entry(vdev, NULL); 325 } 326 327 static int vfio_pci_core_pm_entry_with_wakeup( 328 struct vfio_device *device, u32 flags, 329 struct vfio_device_low_power_entry_with_wakeup __user *arg, 330 size_t argsz) 331 { 332 struct vfio_pci_core_device *vdev = 333 container_of(device, struct vfio_pci_core_device, vdev); 334 struct vfio_device_low_power_entry_with_wakeup entry; 335 struct eventfd_ctx *efdctx; 336 int ret; 337 338 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 339 sizeof(entry)); 340 if (ret != 1) 341 return ret; 342 343 if (copy_from_user(&entry, arg, sizeof(entry))) 344 return -EFAULT; 345 346 if (entry.wakeup_eventfd < 0) 347 return -EINVAL; 348 349 efdctx = eventfd_ctx_fdget(entry.wakeup_eventfd); 350 if (IS_ERR(efdctx)) 351 return PTR_ERR(efdctx); 352 353 ret = vfio_pci_runtime_pm_entry(vdev, efdctx); 354 if (ret) 355 eventfd_ctx_put(efdctx); 356 357 return ret; 358 } 359 360 static void __vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev) 361 { 362 if (vdev->pm_runtime_engaged) { 363 vdev->pm_runtime_engaged = false; 364 pm_runtime_get_noresume(&vdev->pdev->dev); 365 366 if (vdev->pm_wake_eventfd_ctx) { 367 eventfd_ctx_put(vdev->pm_wake_eventfd_ctx); 368 vdev->pm_wake_eventfd_ctx = NULL; 369 } 370 } 371 } 372 373 static void vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev) 374 { 375 /* 376 * The vdev power related flags are protected with 'memory_lock' 377 * semaphore. 378 */ 379 down_write(&vdev->memory_lock); 380 __vfio_pci_runtime_pm_exit(vdev); 381 up_write(&vdev->memory_lock); 382 } 383 384 static int vfio_pci_core_pm_exit(struct vfio_device *device, u32 flags, 385 void __user *arg, size_t argsz) 386 { 387 struct vfio_pci_core_device *vdev = 388 container_of(device, struct vfio_pci_core_device, vdev); 389 int ret; 390 391 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0); 392 if (ret != 1) 393 return ret; 394 395 /* 396 * The device is always in the active state here due to pm wrappers 397 * around ioctls. If the device had entered a low power state and 398 * pm_wake_eventfd_ctx is valid, vfio_pci_core_runtime_resume() has 399 * already signaled the eventfd and exited low power mode itself. 400 * pm_runtime_engaged protects the redundant call here. 401 */ 402 vfio_pci_runtime_pm_exit(vdev); 403 return 0; 404 } 405 406 #ifdef CONFIG_PM 407 static int vfio_pci_core_runtime_suspend(struct device *dev) 408 { 409 struct vfio_pci_core_device *vdev = dev_get_drvdata(dev); 410 411 down_write(&vdev->memory_lock); 412 /* 413 * The user can move the device into D3hot state before invoking 414 * power management IOCTL. Move the device into D0 state here and then 415 * the pci-driver core runtime PM suspend function will move the device 416 * into the low power state. Also, for the devices which have 417 * NoSoftRst-, it will help in restoring the original state 418 * (saved locally in 'vdev->pm_save'). 419 */ 420 vfio_pci_set_power_state(vdev, PCI_D0); 421 up_write(&vdev->memory_lock); 422 423 /* 424 * If INTx is enabled, then mask INTx before going into the runtime 425 * suspended state and unmask the same in the runtime resume. 426 * If INTx has already been masked by the user, then 427 * vfio_pci_intx_mask() will return false and in that case, INTx 428 * should not be unmasked in the runtime resume. 429 */ 430 vdev->pm_intx_masked = ((vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) && 431 vfio_pci_intx_mask(vdev)); 432 433 return 0; 434 } 435 436 static int vfio_pci_core_runtime_resume(struct device *dev) 437 { 438 struct vfio_pci_core_device *vdev = dev_get_drvdata(dev); 439 440 /* 441 * Resume with a pm_wake_eventfd_ctx signals the eventfd and exit 442 * low power mode. 443 */ 444 down_write(&vdev->memory_lock); 445 if (vdev->pm_wake_eventfd_ctx) { 446 eventfd_signal(vdev->pm_wake_eventfd_ctx); 447 __vfio_pci_runtime_pm_exit(vdev); 448 } 449 up_write(&vdev->memory_lock); 450 451 if (vdev->pm_intx_masked) 452 vfio_pci_intx_unmask(vdev); 453 454 return 0; 455 } 456 #endif /* CONFIG_PM */ 457 458 /* 459 * The pci-driver core runtime PM routines always save the device state 460 * before going into suspended state. If the device is going into low power 461 * state with only with runtime PM ops, then no explicit handling is needed 462 * for the devices which have NoSoftRst-. 463 */ 464 static const struct dev_pm_ops vfio_pci_core_pm_ops = { 465 SET_RUNTIME_PM_OPS(vfio_pci_core_runtime_suspend, 466 vfio_pci_core_runtime_resume, 467 NULL) 468 }; 469 470 int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) 471 { 472 struct pci_dev *pdev = vdev->pdev; 473 int ret; 474 u16 cmd; 475 u8 msix_pos; 476 477 if (!disable_idle_d3) { 478 ret = pm_runtime_resume_and_get(&pdev->dev); 479 if (ret < 0) 480 return ret; 481 } 482 483 /* Don't allow our initial saved state to include busmaster */ 484 pci_clear_master(pdev); 485 486 ret = pci_enable_device(pdev); 487 if (ret) 488 goto out_power; 489 490 /* If reset fails because of the device lock, fail this path entirely */ 491 ret = pci_try_reset_function(pdev); 492 if (ret == -EAGAIN) 493 goto out_disable_device; 494 495 vdev->reset_works = !ret; 496 pci_save_state(pdev); 497 vdev->pci_saved_state = pci_store_saved_state(pdev); 498 if (!vdev->pci_saved_state) 499 pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__); 500 501 if (likely(!nointxmask)) { 502 if (vfio_pci_nointx(pdev)) { 503 pci_info(pdev, "Masking broken INTx support\n"); 504 vdev->nointx = true; 505 pci_intx(pdev, 0); 506 } else 507 vdev->pci_2_3 = pci_intx_mask_supported(pdev); 508 } 509 510 pci_read_config_word(pdev, PCI_COMMAND, &cmd); 511 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) { 512 cmd &= ~PCI_COMMAND_INTX_DISABLE; 513 pci_write_config_word(pdev, PCI_COMMAND, cmd); 514 } 515 516 ret = vfio_pci_zdev_open_device(vdev); 517 if (ret) 518 goto out_free_state; 519 520 ret = vfio_config_init(vdev); 521 if (ret) 522 goto out_free_zdev; 523 524 msix_pos = pdev->msix_cap; 525 if (msix_pos) { 526 u16 flags; 527 u32 table; 528 529 pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags); 530 pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table); 531 532 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR; 533 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET; 534 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16; 535 vdev->has_dyn_msix = pci_msix_can_alloc_dyn(pdev); 536 } else { 537 vdev->msix_bar = 0xFF; 538 vdev->has_dyn_msix = false; 539 } 540 541 if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev)) 542 vdev->has_vga = true; 543 544 545 return 0; 546 547 out_free_zdev: 548 vfio_pci_zdev_close_device(vdev); 549 out_free_state: 550 kfree(vdev->pci_saved_state); 551 vdev->pci_saved_state = NULL; 552 out_disable_device: 553 pci_disable_device(pdev); 554 out_power: 555 if (!disable_idle_d3) 556 pm_runtime_put(&pdev->dev); 557 return ret; 558 } 559 EXPORT_SYMBOL_GPL(vfio_pci_core_enable); 560 561 void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) 562 { 563 struct pci_dev *pdev = vdev->pdev; 564 struct vfio_pci_dummy_resource *dummy_res, *tmp; 565 struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp; 566 int i, bar; 567 568 /* For needs_reset */ 569 lockdep_assert_held(&vdev->vdev.dev_set->lock); 570 571 /* 572 * This function can be invoked while the power state is non-D0. 573 * This non-D0 power state can be with or without runtime PM. 574 * vfio_pci_runtime_pm_exit() will internally increment the usage 575 * count corresponding to pm_runtime_put() called during low power 576 * feature entry and then pm_runtime_resume() will wake up the device, 577 * if the device has already gone into the suspended state. Otherwise, 578 * the vfio_pci_set_power_state() will change the device power state 579 * to D0. 580 */ 581 vfio_pci_runtime_pm_exit(vdev); 582 pm_runtime_resume(&pdev->dev); 583 584 /* 585 * This function calls __pci_reset_function_locked() which internally 586 * can use pci_pm_reset() for the function reset. pci_pm_reset() will 587 * fail if the power state is non-D0. Also, for the devices which 588 * have NoSoftRst-, the reset function can cause the PCI config space 589 * reset without restoring the original state (saved locally in 590 * 'vdev->pm_save'). 591 */ 592 vfio_pci_set_power_state(vdev, PCI_D0); 593 594 /* Stop the device from further DMA */ 595 pci_clear_master(pdev); 596 597 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE | 598 VFIO_IRQ_SET_ACTION_TRIGGER, 599 vdev->irq_type, 0, 0, NULL); 600 601 /* Device closed, don't need mutex here */ 602 list_for_each_entry_safe(ioeventfd, ioeventfd_tmp, 603 &vdev->ioeventfds_list, next) { 604 vfio_virqfd_disable(&ioeventfd->virqfd); 605 list_del(&ioeventfd->next); 606 kfree(ioeventfd); 607 } 608 vdev->ioeventfds_nr = 0; 609 610 vdev->virq_disabled = false; 611 612 for (i = 0; i < vdev->num_regions; i++) 613 vdev->region[i].ops->release(vdev, &vdev->region[i]); 614 615 vdev->num_regions = 0; 616 kfree(vdev->region); 617 vdev->region = NULL; /* don't krealloc a freed pointer */ 618 619 vfio_config_free(vdev); 620 621 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 622 bar = i + PCI_STD_RESOURCES; 623 if (!vdev->barmap[bar]) 624 continue; 625 pci_iounmap(pdev, vdev->barmap[bar]); 626 pci_release_selected_regions(pdev, 1 << bar); 627 vdev->barmap[bar] = NULL; 628 } 629 630 list_for_each_entry_safe(dummy_res, tmp, 631 &vdev->dummy_resources_list, res_next) { 632 list_del(&dummy_res->res_next); 633 release_resource(&dummy_res->resource); 634 kfree(dummy_res); 635 } 636 637 vdev->needs_reset = true; 638 639 vfio_pci_zdev_close_device(vdev); 640 641 /* 642 * If we have saved state, restore it. If we can reset the device, 643 * even better. Resetting with current state seems better than 644 * nothing, but saving and restoring current state without reset 645 * is just busy work. 646 */ 647 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) { 648 pci_info(pdev, "%s: Couldn't reload saved state\n", __func__); 649 650 if (!vdev->reset_works) 651 goto out; 652 653 pci_save_state(pdev); 654 } 655 656 /* 657 * Disable INTx and MSI, presumably to avoid spurious interrupts 658 * during reset. Stolen from pci_reset_function() 659 */ 660 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 661 662 /* 663 * Try to get the locks ourselves to prevent a deadlock. The 664 * success of this is dependent on being able to lock the device, 665 * which is not always possible. 666 * We can not use the "try" reset interface here, which will 667 * overwrite the previously restored configuration information. 668 */ 669 if (vdev->reset_works && pci_dev_trylock(pdev)) { 670 if (!__pci_reset_function_locked(pdev)) 671 vdev->needs_reset = false; 672 pci_dev_unlock(pdev); 673 } 674 675 pci_restore_state(pdev); 676 out: 677 pci_disable_device(pdev); 678 679 vfio_pci_dev_set_try_reset(vdev->vdev.dev_set); 680 681 /* Put the pm-runtime usage counter acquired during enable */ 682 if (!disable_idle_d3) 683 pm_runtime_put(&pdev->dev); 684 } 685 EXPORT_SYMBOL_GPL(vfio_pci_core_disable); 686 687 void vfio_pci_core_close_device(struct vfio_device *core_vdev) 688 { 689 struct vfio_pci_core_device *vdev = 690 container_of(core_vdev, struct vfio_pci_core_device, vdev); 691 692 if (vdev->sriov_pf_core_dev) { 693 mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock); 694 WARN_ON(!vdev->sriov_pf_core_dev->vf_token->users); 695 vdev->sriov_pf_core_dev->vf_token->users--; 696 mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock); 697 } 698 #if IS_ENABLED(CONFIG_EEH) 699 eeh_dev_release(vdev->pdev); 700 #endif 701 vfio_pci_core_disable(vdev); 702 703 mutex_lock(&vdev->igate); 704 if (vdev->err_trigger) { 705 eventfd_ctx_put(vdev->err_trigger); 706 vdev->err_trigger = NULL; 707 } 708 if (vdev->req_trigger) { 709 eventfd_ctx_put(vdev->req_trigger); 710 vdev->req_trigger = NULL; 711 } 712 mutex_unlock(&vdev->igate); 713 } 714 EXPORT_SYMBOL_GPL(vfio_pci_core_close_device); 715 716 void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev) 717 { 718 vfio_pci_probe_mmaps(vdev); 719 #if IS_ENABLED(CONFIG_EEH) 720 eeh_dev_open(vdev->pdev); 721 #endif 722 723 if (vdev->sriov_pf_core_dev) { 724 mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock); 725 vdev->sriov_pf_core_dev->vf_token->users++; 726 mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock); 727 } 728 } 729 EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable); 730 731 static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type) 732 { 733 if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) { 734 u8 pin; 735 736 if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || 737 vdev->nointx || vdev->pdev->is_virtfn) 738 return 0; 739 740 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin); 741 742 return pin ? 1 : 0; 743 } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) { 744 u8 pos; 745 u16 flags; 746 747 pos = vdev->pdev->msi_cap; 748 if (pos) { 749 pci_read_config_word(vdev->pdev, 750 pos + PCI_MSI_FLAGS, &flags); 751 return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1); 752 } 753 } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) { 754 u8 pos; 755 u16 flags; 756 757 pos = vdev->pdev->msix_cap; 758 if (pos) { 759 pci_read_config_word(vdev->pdev, 760 pos + PCI_MSIX_FLAGS, &flags); 761 762 return (flags & PCI_MSIX_FLAGS_QSIZE) + 1; 763 } 764 } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) { 765 if (pci_is_pcie(vdev->pdev)) 766 return 1; 767 } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) { 768 return 1; 769 } 770 771 return 0; 772 } 773 774 static int vfio_pci_count_devs(struct pci_dev *pdev, void *data) 775 { 776 (*(int *)data)++; 777 return 0; 778 } 779 780 struct vfio_pci_fill_info { 781 struct vfio_device *vdev; 782 struct vfio_pci_dependent_device *devices; 783 int nr_devices; 784 u32 count; 785 u32 flags; 786 }; 787 788 static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data) 789 { 790 struct vfio_pci_dependent_device *info; 791 struct vfio_pci_fill_info *fill = data; 792 793 /* The topology changed since we counted devices */ 794 if (fill->count >= fill->nr_devices) 795 return -EAGAIN; 796 797 info = &fill->devices[fill->count++]; 798 info->segment = pci_domain_nr(pdev->bus); 799 info->bus = pdev->bus->number; 800 info->devfn = pdev->devfn; 801 802 if (fill->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID) { 803 struct iommufd_ctx *iommufd = vfio_iommufd_device_ictx(fill->vdev); 804 struct vfio_device_set *dev_set = fill->vdev->dev_set; 805 struct vfio_device *vdev; 806 807 /* 808 * hot-reset requires all affected devices be represented in 809 * the dev_set. 810 */ 811 vdev = vfio_find_device_in_devset(dev_set, &pdev->dev); 812 if (!vdev) { 813 info->devid = VFIO_PCI_DEVID_NOT_OWNED; 814 } else { 815 int id = vfio_iommufd_get_dev_id(vdev, iommufd); 816 817 if (id > 0) 818 info->devid = id; 819 else if (id == -ENOENT) 820 info->devid = VFIO_PCI_DEVID_OWNED; 821 else 822 info->devid = VFIO_PCI_DEVID_NOT_OWNED; 823 } 824 /* If devid is VFIO_PCI_DEVID_NOT_OWNED, clear owned flag. */ 825 if (info->devid == VFIO_PCI_DEVID_NOT_OWNED) 826 fill->flags &= ~VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED; 827 } else { 828 struct iommu_group *iommu_group; 829 830 iommu_group = iommu_group_get(&pdev->dev); 831 if (!iommu_group) 832 return -EPERM; /* Cannot reset non-isolated devices */ 833 834 info->group_id = iommu_group_id(iommu_group); 835 iommu_group_put(iommu_group); 836 } 837 838 return 0; 839 } 840 841 struct vfio_pci_group_info { 842 int count; 843 struct file **files; 844 }; 845 846 static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot) 847 { 848 for (; pdev; pdev = pdev->bus->self) 849 if (pdev->bus == slot->bus) 850 return (pdev->slot == slot); 851 return false; 852 } 853 854 struct vfio_pci_walk_info { 855 int (*fn)(struct pci_dev *pdev, void *data); 856 void *data; 857 struct pci_dev *pdev; 858 bool slot; 859 int ret; 860 }; 861 862 static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data) 863 { 864 struct vfio_pci_walk_info *walk = data; 865 866 if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot)) 867 walk->ret = walk->fn(pdev, walk->data); 868 869 return walk->ret; 870 } 871 872 static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev, 873 int (*fn)(struct pci_dev *, 874 void *data), void *data, 875 bool slot) 876 { 877 struct vfio_pci_walk_info walk = { 878 .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0, 879 }; 880 881 pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk); 882 883 return walk.ret; 884 } 885 886 static int msix_mmappable_cap(struct vfio_pci_core_device *vdev, 887 struct vfio_info_cap *caps) 888 { 889 struct vfio_info_cap_header header = { 890 .id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE, 891 .version = 1 892 }; 893 894 return vfio_info_add_capability(caps, &header, sizeof(header)); 895 } 896 897 int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, 898 unsigned int type, unsigned int subtype, 899 const struct vfio_pci_regops *ops, 900 size_t size, u32 flags, void *data) 901 { 902 struct vfio_pci_region *region; 903 904 region = krealloc(vdev->region, 905 (vdev->num_regions + 1) * sizeof(*region), 906 GFP_KERNEL_ACCOUNT); 907 if (!region) 908 return -ENOMEM; 909 910 vdev->region = region; 911 vdev->region[vdev->num_regions].type = type; 912 vdev->region[vdev->num_regions].subtype = subtype; 913 vdev->region[vdev->num_regions].ops = ops; 914 vdev->region[vdev->num_regions].size = size; 915 vdev->region[vdev->num_regions].flags = flags; 916 vdev->region[vdev->num_regions].data = data; 917 918 vdev->num_regions++; 919 920 return 0; 921 } 922 EXPORT_SYMBOL_GPL(vfio_pci_core_register_dev_region); 923 924 static int vfio_pci_info_atomic_cap(struct vfio_pci_core_device *vdev, 925 struct vfio_info_cap *caps) 926 { 927 struct vfio_device_info_cap_pci_atomic_comp cap = { 928 .header.id = VFIO_DEVICE_INFO_CAP_PCI_ATOMIC_COMP, 929 .header.version = 1 930 }; 931 struct pci_dev *pdev = pci_physfn(vdev->pdev); 932 u32 devcap2; 933 934 pcie_capability_read_dword(pdev, PCI_EXP_DEVCAP2, &devcap2); 935 936 if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP32) && 937 !pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32)) 938 cap.flags |= VFIO_PCI_ATOMIC_COMP32; 939 940 if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP64) && 941 !pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64)) 942 cap.flags |= VFIO_PCI_ATOMIC_COMP64; 943 944 if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP128) && 945 !pci_enable_atomic_ops_to_root(pdev, 946 PCI_EXP_DEVCAP2_ATOMIC_COMP128)) 947 cap.flags |= VFIO_PCI_ATOMIC_COMP128; 948 949 if (!cap.flags) 950 return -ENODEV; 951 952 return vfio_info_add_capability(caps, &cap.header, sizeof(cap)); 953 } 954 955 static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev, 956 struct vfio_device_info __user *arg) 957 { 958 unsigned long minsz = offsetofend(struct vfio_device_info, num_irqs); 959 struct vfio_device_info info = {}; 960 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 961 int ret; 962 963 if (copy_from_user(&info, arg, minsz)) 964 return -EFAULT; 965 966 if (info.argsz < minsz) 967 return -EINVAL; 968 969 minsz = min_t(size_t, info.argsz, sizeof(info)); 970 971 info.flags = VFIO_DEVICE_FLAGS_PCI; 972 973 if (vdev->reset_works) 974 info.flags |= VFIO_DEVICE_FLAGS_RESET; 975 976 info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; 977 info.num_irqs = VFIO_PCI_NUM_IRQS; 978 979 ret = vfio_pci_info_zdev_add_caps(vdev, &caps); 980 if (ret && ret != -ENODEV) { 981 pci_warn(vdev->pdev, 982 "Failed to setup zPCI info capabilities\n"); 983 return ret; 984 } 985 986 ret = vfio_pci_info_atomic_cap(vdev, &caps); 987 if (ret && ret != -ENODEV) { 988 pci_warn(vdev->pdev, 989 "Failed to setup AtomicOps info capability\n"); 990 return ret; 991 } 992 993 if (caps.size) { 994 info.flags |= VFIO_DEVICE_FLAGS_CAPS; 995 if (info.argsz < sizeof(info) + caps.size) { 996 info.argsz = sizeof(info) + caps.size; 997 } else { 998 vfio_info_cap_shift(&caps, sizeof(info)); 999 if (copy_to_user(arg + 1, caps.buf, caps.size)) { 1000 kfree(caps.buf); 1001 return -EFAULT; 1002 } 1003 info.cap_offset = sizeof(*arg); 1004 } 1005 1006 kfree(caps.buf); 1007 } 1008 1009 return copy_to_user(arg, &info, minsz) ? -EFAULT : 0; 1010 } 1011 1012 static int vfio_pci_ioctl_get_region_info(struct vfio_pci_core_device *vdev, 1013 struct vfio_region_info __user *arg) 1014 { 1015 unsigned long minsz = offsetofend(struct vfio_region_info, offset); 1016 struct pci_dev *pdev = vdev->pdev; 1017 struct vfio_region_info info; 1018 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 1019 int i, ret; 1020 1021 if (copy_from_user(&info, arg, minsz)) 1022 return -EFAULT; 1023 1024 if (info.argsz < minsz) 1025 return -EINVAL; 1026 1027 switch (info.index) { 1028 case VFIO_PCI_CONFIG_REGION_INDEX: 1029 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1030 info.size = pdev->cfg_size; 1031 info.flags = VFIO_REGION_INFO_FLAG_READ | 1032 VFIO_REGION_INFO_FLAG_WRITE; 1033 break; 1034 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1035 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1036 info.size = pci_resource_len(pdev, info.index); 1037 if (!info.size) { 1038 info.flags = 0; 1039 break; 1040 } 1041 1042 info.flags = VFIO_REGION_INFO_FLAG_READ | 1043 VFIO_REGION_INFO_FLAG_WRITE; 1044 if (vdev->bar_mmap_supported[info.index]) { 1045 info.flags |= VFIO_REGION_INFO_FLAG_MMAP; 1046 if (info.index == vdev->msix_bar) { 1047 ret = msix_mmappable_cap(vdev, &caps); 1048 if (ret) 1049 return ret; 1050 } 1051 } 1052 1053 break; 1054 case VFIO_PCI_ROM_REGION_INDEX: { 1055 void __iomem *io; 1056 size_t size; 1057 u16 cmd; 1058 1059 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1060 info.flags = 0; 1061 1062 /* Report the BAR size, not the ROM size */ 1063 info.size = pci_resource_len(pdev, info.index); 1064 if (!info.size) { 1065 /* Shadow ROMs appear as PCI option ROMs */ 1066 if (pdev->resource[PCI_ROM_RESOURCE].flags & 1067 IORESOURCE_ROM_SHADOW) 1068 info.size = 0x20000; 1069 else 1070 break; 1071 } 1072 1073 /* 1074 * Is it really there? Enable memory decode for implicit access 1075 * in pci_map_rom(). 1076 */ 1077 cmd = vfio_pci_memory_lock_and_enable(vdev); 1078 io = pci_map_rom(pdev, &size); 1079 if (io) { 1080 info.flags = VFIO_REGION_INFO_FLAG_READ; 1081 pci_unmap_rom(pdev, io); 1082 } else { 1083 info.size = 0; 1084 } 1085 vfio_pci_memory_unlock_and_restore(vdev, cmd); 1086 1087 break; 1088 } 1089 case VFIO_PCI_VGA_REGION_INDEX: 1090 if (!vdev->has_vga) 1091 return -EINVAL; 1092 1093 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1094 info.size = 0xc0000; 1095 info.flags = VFIO_REGION_INFO_FLAG_READ | 1096 VFIO_REGION_INFO_FLAG_WRITE; 1097 1098 break; 1099 default: { 1100 struct vfio_region_info_cap_type cap_type = { 1101 .header.id = VFIO_REGION_INFO_CAP_TYPE, 1102 .header.version = 1 1103 }; 1104 1105 if (info.index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) 1106 return -EINVAL; 1107 info.index = array_index_nospec( 1108 info.index, VFIO_PCI_NUM_REGIONS + vdev->num_regions); 1109 1110 i = info.index - VFIO_PCI_NUM_REGIONS; 1111 1112 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1113 info.size = vdev->region[i].size; 1114 info.flags = vdev->region[i].flags; 1115 1116 cap_type.type = vdev->region[i].type; 1117 cap_type.subtype = vdev->region[i].subtype; 1118 1119 ret = vfio_info_add_capability(&caps, &cap_type.header, 1120 sizeof(cap_type)); 1121 if (ret) 1122 return ret; 1123 1124 if (vdev->region[i].ops->add_capability) { 1125 ret = vdev->region[i].ops->add_capability( 1126 vdev, &vdev->region[i], &caps); 1127 if (ret) 1128 return ret; 1129 } 1130 } 1131 } 1132 1133 if (caps.size) { 1134 info.flags |= VFIO_REGION_INFO_FLAG_CAPS; 1135 if (info.argsz < sizeof(info) + caps.size) { 1136 info.argsz = sizeof(info) + caps.size; 1137 info.cap_offset = 0; 1138 } else { 1139 vfio_info_cap_shift(&caps, sizeof(info)); 1140 if (copy_to_user(arg + 1, caps.buf, caps.size)) { 1141 kfree(caps.buf); 1142 return -EFAULT; 1143 } 1144 info.cap_offset = sizeof(*arg); 1145 } 1146 1147 kfree(caps.buf); 1148 } 1149 1150 return copy_to_user(arg, &info, minsz) ? -EFAULT : 0; 1151 } 1152 1153 static int vfio_pci_ioctl_get_irq_info(struct vfio_pci_core_device *vdev, 1154 struct vfio_irq_info __user *arg) 1155 { 1156 unsigned long minsz = offsetofend(struct vfio_irq_info, count); 1157 struct vfio_irq_info info; 1158 1159 if (copy_from_user(&info, arg, minsz)) 1160 return -EFAULT; 1161 1162 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 1163 return -EINVAL; 1164 1165 switch (info.index) { 1166 case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX: 1167 case VFIO_PCI_REQ_IRQ_INDEX: 1168 break; 1169 case VFIO_PCI_ERR_IRQ_INDEX: 1170 if (pci_is_pcie(vdev->pdev)) 1171 break; 1172 fallthrough; 1173 default: 1174 return -EINVAL; 1175 } 1176 1177 info.flags = VFIO_IRQ_INFO_EVENTFD; 1178 1179 info.count = vfio_pci_get_irq_count(vdev, info.index); 1180 1181 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 1182 info.flags |= 1183 (VFIO_IRQ_INFO_MASKABLE | VFIO_IRQ_INFO_AUTOMASKED); 1184 else if (info.index != VFIO_PCI_MSIX_IRQ_INDEX || !vdev->has_dyn_msix) 1185 info.flags |= VFIO_IRQ_INFO_NORESIZE; 1186 1187 return copy_to_user(arg, &info, minsz) ? -EFAULT : 0; 1188 } 1189 1190 static int vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device *vdev, 1191 struct vfio_irq_set __user *arg) 1192 { 1193 unsigned long minsz = offsetofend(struct vfio_irq_set, count); 1194 struct vfio_irq_set hdr; 1195 u8 *data = NULL; 1196 int max, ret = 0; 1197 size_t data_size = 0; 1198 1199 if (copy_from_user(&hdr, arg, minsz)) 1200 return -EFAULT; 1201 1202 max = vfio_pci_get_irq_count(vdev, hdr.index); 1203 1204 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, VFIO_PCI_NUM_IRQS, 1205 &data_size); 1206 if (ret) 1207 return ret; 1208 1209 if (data_size) { 1210 data = memdup_user(&arg->data, data_size); 1211 if (IS_ERR(data)) 1212 return PTR_ERR(data); 1213 } 1214 1215 mutex_lock(&vdev->igate); 1216 1217 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, hdr.start, 1218 hdr.count, data); 1219 1220 mutex_unlock(&vdev->igate); 1221 kfree(data); 1222 1223 return ret; 1224 } 1225 1226 static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev, 1227 void __user *arg) 1228 { 1229 int ret; 1230 1231 if (!vdev->reset_works) 1232 return -EINVAL; 1233 1234 vfio_pci_zap_and_down_write_memory_lock(vdev); 1235 1236 /* 1237 * This function can be invoked while the power state is non-D0. If 1238 * pci_try_reset_function() has been called while the power state is 1239 * non-D0, then pci_try_reset_function() will internally set the power 1240 * state to D0 without vfio driver involvement. For the devices which 1241 * have NoSoftRst-, the reset function can cause the PCI config space 1242 * reset without restoring the original state (saved locally in 1243 * 'vdev->pm_save'). 1244 */ 1245 vfio_pci_set_power_state(vdev, PCI_D0); 1246 1247 ret = pci_try_reset_function(vdev->pdev); 1248 up_write(&vdev->memory_lock); 1249 1250 return ret; 1251 } 1252 1253 static int vfio_pci_ioctl_get_pci_hot_reset_info( 1254 struct vfio_pci_core_device *vdev, 1255 struct vfio_pci_hot_reset_info __user *arg) 1256 { 1257 unsigned long minsz = 1258 offsetofend(struct vfio_pci_hot_reset_info, count); 1259 struct vfio_pci_dependent_device *devices = NULL; 1260 struct vfio_pci_hot_reset_info hdr; 1261 struct vfio_pci_fill_info fill = {}; 1262 bool slot = false; 1263 int ret, count; 1264 1265 if (copy_from_user(&hdr, arg, minsz)) 1266 return -EFAULT; 1267 1268 if (hdr.argsz < minsz) 1269 return -EINVAL; 1270 1271 hdr.flags = 0; 1272 1273 /* Can we do a slot or bus reset or neither? */ 1274 if (!pci_probe_reset_slot(vdev->pdev->slot)) 1275 slot = true; 1276 else if (pci_probe_reset_bus(vdev->pdev->bus)) 1277 return -ENODEV; 1278 1279 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs, 1280 &count, slot); 1281 if (ret) 1282 return ret; 1283 1284 if (WARN_ON(!count)) /* Should always be at least one */ 1285 return -ERANGE; 1286 1287 if (count > (hdr.argsz - sizeof(hdr)) / sizeof(*devices)) { 1288 hdr.count = count; 1289 ret = -ENOSPC; 1290 goto header; 1291 } 1292 1293 devices = kcalloc(count, sizeof(*devices), GFP_KERNEL); 1294 if (!devices) 1295 return -ENOMEM; 1296 1297 fill.devices = devices; 1298 fill.nr_devices = count; 1299 fill.vdev = &vdev->vdev; 1300 1301 if (vfio_device_cdev_opened(&vdev->vdev)) 1302 fill.flags |= VFIO_PCI_HOT_RESET_FLAG_DEV_ID | 1303 VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED; 1304 1305 mutex_lock(&vdev->vdev.dev_set->lock); 1306 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_fill_devs, 1307 &fill, slot); 1308 mutex_unlock(&vdev->vdev.dev_set->lock); 1309 if (ret) 1310 goto out; 1311 1312 if (copy_to_user(arg->devices, devices, 1313 sizeof(*devices) * fill.count)) { 1314 ret = -EFAULT; 1315 goto out; 1316 } 1317 1318 hdr.count = fill.count; 1319 hdr.flags = fill.flags; 1320 1321 header: 1322 if (copy_to_user(arg, &hdr, minsz)) 1323 ret = -EFAULT; 1324 out: 1325 kfree(devices); 1326 return ret; 1327 } 1328 1329 static int 1330 vfio_pci_ioctl_pci_hot_reset_groups(struct vfio_pci_core_device *vdev, 1331 int array_count, bool slot, 1332 struct vfio_pci_hot_reset __user *arg) 1333 { 1334 int32_t *group_fds; 1335 struct file **files; 1336 struct vfio_pci_group_info info; 1337 int file_idx, count = 0, ret = 0; 1338 1339 /* 1340 * We can't let userspace give us an arbitrarily large buffer to copy, 1341 * so verify how many we think there could be. Note groups can have 1342 * multiple devices so one group per device is the max. 1343 */ 1344 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs, 1345 &count, slot); 1346 if (ret) 1347 return ret; 1348 1349 if (array_count > count) 1350 return -EINVAL; 1351 1352 group_fds = kcalloc(array_count, sizeof(*group_fds), GFP_KERNEL); 1353 files = kcalloc(array_count, sizeof(*files), GFP_KERNEL); 1354 if (!group_fds || !files) { 1355 kfree(group_fds); 1356 kfree(files); 1357 return -ENOMEM; 1358 } 1359 1360 if (copy_from_user(group_fds, arg->group_fds, 1361 array_count * sizeof(*group_fds))) { 1362 kfree(group_fds); 1363 kfree(files); 1364 return -EFAULT; 1365 } 1366 1367 /* 1368 * Get the group file for each fd to ensure the group is held across 1369 * the reset 1370 */ 1371 for (file_idx = 0; file_idx < array_count; file_idx++) { 1372 struct file *file = fget(group_fds[file_idx]); 1373 1374 if (!file) { 1375 ret = -EBADF; 1376 break; 1377 } 1378 1379 /* Ensure the FD is a vfio group FD.*/ 1380 if (!vfio_file_is_group(file)) { 1381 fput(file); 1382 ret = -EINVAL; 1383 break; 1384 } 1385 1386 files[file_idx] = file; 1387 } 1388 1389 kfree(group_fds); 1390 1391 /* release reference to groups on error */ 1392 if (ret) 1393 goto hot_reset_release; 1394 1395 info.count = array_count; 1396 info.files = files; 1397 1398 ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info, NULL); 1399 1400 hot_reset_release: 1401 for (file_idx--; file_idx >= 0; file_idx--) 1402 fput(files[file_idx]); 1403 1404 kfree(files); 1405 return ret; 1406 } 1407 1408 static int vfio_pci_ioctl_pci_hot_reset(struct vfio_pci_core_device *vdev, 1409 struct vfio_pci_hot_reset __user *arg) 1410 { 1411 unsigned long minsz = offsetofend(struct vfio_pci_hot_reset, count); 1412 struct vfio_pci_hot_reset hdr; 1413 bool slot = false; 1414 1415 if (copy_from_user(&hdr, arg, minsz)) 1416 return -EFAULT; 1417 1418 if (hdr.argsz < minsz || hdr.flags) 1419 return -EINVAL; 1420 1421 /* zero-length array is only for cdev opened devices */ 1422 if (!!hdr.count == vfio_device_cdev_opened(&vdev->vdev)) 1423 return -EINVAL; 1424 1425 /* Can we do a slot or bus reset or neither? */ 1426 if (!pci_probe_reset_slot(vdev->pdev->slot)) 1427 slot = true; 1428 else if (pci_probe_reset_bus(vdev->pdev->bus)) 1429 return -ENODEV; 1430 1431 if (hdr.count) 1432 return vfio_pci_ioctl_pci_hot_reset_groups(vdev, hdr.count, slot, arg); 1433 1434 return vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, NULL, 1435 vfio_iommufd_device_ictx(&vdev->vdev)); 1436 } 1437 1438 static int vfio_pci_ioctl_ioeventfd(struct vfio_pci_core_device *vdev, 1439 struct vfio_device_ioeventfd __user *arg) 1440 { 1441 unsigned long minsz = offsetofend(struct vfio_device_ioeventfd, fd); 1442 struct vfio_device_ioeventfd ioeventfd; 1443 int count; 1444 1445 if (copy_from_user(&ioeventfd, arg, minsz)) 1446 return -EFAULT; 1447 1448 if (ioeventfd.argsz < minsz) 1449 return -EINVAL; 1450 1451 if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK) 1452 return -EINVAL; 1453 1454 count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK; 1455 1456 if (hweight8(count) != 1 || ioeventfd.fd < -1) 1457 return -EINVAL; 1458 1459 return vfio_pci_ioeventfd(vdev, ioeventfd.offset, ioeventfd.data, count, 1460 ioeventfd.fd); 1461 } 1462 1463 long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, 1464 unsigned long arg) 1465 { 1466 struct vfio_pci_core_device *vdev = 1467 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1468 void __user *uarg = (void __user *)arg; 1469 1470 switch (cmd) { 1471 case VFIO_DEVICE_GET_INFO: 1472 return vfio_pci_ioctl_get_info(vdev, uarg); 1473 case VFIO_DEVICE_GET_IRQ_INFO: 1474 return vfio_pci_ioctl_get_irq_info(vdev, uarg); 1475 case VFIO_DEVICE_GET_PCI_HOT_RESET_INFO: 1476 return vfio_pci_ioctl_get_pci_hot_reset_info(vdev, uarg); 1477 case VFIO_DEVICE_GET_REGION_INFO: 1478 return vfio_pci_ioctl_get_region_info(vdev, uarg); 1479 case VFIO_DEVICE_IOEVENTFD: 1480 return vfio_pci_ioctl_ioeventfd(vdev, uarg); 1481 case VFIO_DEVICE_PCI_HOT_RESET: 1482 return vfio_pci_ioctl_pci_hot_reset(vdev, uarg); 1483 case VFIO_DEVICE_RESET: 1484 return vfio_pci_ioctl_reset(vdev, uarg); 1485 case VFIO_DEVICE_SET_IRQS: 1486 return vfio_pci_ioctl_set_irqs(vdev, uarg); 1487 default: 1488 return -ENOTTY; 1489 } 1490 } 1491 EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl); 1492 1493 static int vfio_pci_core_feature_token(struct vfio_device *device, u32 flags, 1494 uuid_t __user *arg, size_t argsz) 1495 { 1496 struct vfio_pci_core_device *vdev = 1497 container_of(device, struct vfio_pci_core_device, vdev); 1498 uuid_t uuid; 1499 int ret; 1500 1501 if (!vdev->vf_token) 1502 return -ENOTTY; 1503 /* 1504 * We do not support GET of the VF Token UUID as this could 1505 * expose the token of the previous device user. 1506 */ 1507 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 1508 sizeof(uuid)); 1509 if (ret != 1) 1510 return ret; 1511 1512 if (copy_from_user(&uuid, arg, sizeof(uuid))) 1513 return -EFAULT; 1514 1515 mutex_lock(&vdev->vf_token->lock); 1516 uuid_copy(&vdev->vf_token->uuid, &uuid); 1517 mutex_unlock(&vdev->vf_token->lock); 1518 return 0; 1519 } 1520 1521 int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags, 1522 void __user *arg, size_t argsz) 1523 { 1524 switch (flags & VFIO_DEVICE_FEATURE_MASK) { 1525 case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY: 1526 return vfio_pci_core_pm_entry(device, flags, arg, argsz); 1527 case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP: 1528 return vfio_pci_core_pm_entry_with_wakeup(device, flags, 1529 arg, argsz); 1530 case VFIO_DEVICE_FEATURE_LOW_POWER_EXIT: 1531 return vfio_pci_core_pm_exit(device, flags, arg, argsz); 1532 case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN: 1533 return vfio_pci_core_feature_token(device, flags, arg, argsz); 1534 default: 1535 return -ENOTTY; 1536 } 1537 } 1538 EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl_feature); 1539 1540 static ssize_t vfio_pci_rw(struct vfio_pci_core_device *vdev, char __user *buf, 1541 size_t count, loff_t *ppos, bool iswrite) 1542 { 1543 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 1544 int ret; 1545 1546 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) 1547 return -EINVAL; 1548 1549 ret = pm_runtime_resume_and_get(&vdev->pdev->dev); 1550 if (ret) { 1551 pci_info_ratelimited(vdev->pdev, "runtime resume failed %d\n", 1552 ret); 1553 return -EIO; 1554 } 1555 1556 switch (index) { 1557 case VFIO_PCI_CONFIG_REGION_INDEX: 1558 ret = vfio_pci_config_rw(vdev, buf, count, ppos, iswrite); 1559 break; 1560 1561 case VFIO_PCI_ROM_REGION_INDEX: 1562 if (iswrite) 1563 ret = -EINVAL; 1564 else 1565 ret = vfio_pci_bar_rw(vdev, buf, count, ppos, false); 1566 break; 1567 1568 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1569 ret = vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite); 1570 break; 1571 1572 case VFIO_PCI_VGA_REGION_INDEX: 1573 ret = vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite); 1574 break; 1575 1576 default: 1577 index -= VFIO_PCI_NUM_REGIONS; 1578 ret = vdev->region[index].ops->rw(vdev, buf, 1579 count, ppos, iswrite); 1580 break; 1581 } 1582 1583 pm_runtime_put(&vdev->pdev->dev); 1584 return ret; 1585 } 1586 1587 ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, 1588 size_t count, loff_t *ppos) 1589 { 1590 struct vfio_pci_core_device *vdev = 1591 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1592 1593 if (!count) 1594 return 0; 1595 1596 return vfio_pci_rw(vdev, buf, count, ppos, false); 1597 } 1598 EXPORT_SYMBOL_GPL(vfio_pci_core_read); 1599 1600 ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, 1601 size_t count, loff_t *ppos) 1602 { 1603 struct vfio_pci_core_device *vdev = 1604 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1605 1606 if (!count) 1607 return 0; 1608 1609 return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true); 1610 } 1611 EXPORT_SYMBOL_GPL(vfio_pci_core_write); 1612 1613 static void vfio_pci_zap_bars(struct vfio_pci_core_device *vdev) 1614 { 1615 struct vfio_device *core_vdev = &vdev->vdev; 1616 loff_t start = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX); 1617 loff_t end = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX); 1618 loff_t len = end - start; 1619 1620 unmap_mapping_range(core_vdev->inode->i_mapping, start, len, true); 1621 } 1622 1623 void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev) 1624 { 1625 down_write(&vdev->memory_lock); 1626 vfio_pci_zap_bars(vdev); 1627 } 1628 1629 u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev) 1630 { 1631 u16 cmd; 1632 1633 down_write(&vdev->memory_lock); 1634 pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd); 1635 if (!(cmd & PCI_COMMAND_MEMORY)) 1636 pci_write_config_word(vdev->pdev, PCI_COMMAND, 1637 cmd | PCI_COMMAND_MEMORY); 1638 1639 return cmd; 1640 } 1641 1642 void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, u16 cmd) 1643 { 1644 pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd); 1645 up_write(&vdev->memory_lock); 1646 } 1647 1648 static unsigned long vma_to_pfn(struct vm_area_struct *vma) 1649 { 1650 struct vfio_pci_core_device *vdev = vma->vm_private_data; 1651 int index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 1652 u64 pgoff; 1653 1654 pgoff = vma->vm_pgoff & 1655 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 1656 1657 return (pci_resource_start(vdev->pdev, index) >> PAGE_SHIFT) + pgoff; 1658 } 1659 1660 static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf) 1661 { 1662 struct vm_area_struct *vma = vmf->vma; 1663 struct vfio_pci_core_device *vdev = vma->vm_private_data; 1664 unsigned long pfn, pgoff = vmf->pgoff - vma->vm_pgoff; 1665 unsigned long addr = vma->vm_start; 1666 vm_fault_t ret = VM_FAULT_SIGBUS; 1667 1668 pfn = vma_to_pfn(vma); 1669 1670 down_read(&vdev->memory_lock); 1671 1672 if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev)) 1673 goto out_unlock; 1674 1675 ret = vmf_insert_pfn(vma, vmf->address, pfn + pgoff); 1676 if (ret & VM_FAULT_ERROR) 1677 goto out_unlock; 1678 1679 /* 1680 * Pre-fault the remainder of the vma, abort further insertions and 1681 * supress error if fault is encountered during pre-fault. 1682 */ 1683 for (; addr < vma->vm_end; addr += PAGE_SIZE, pfn++) { 1684 if (addr == vmf->address) 1685 continue; 1686 1687 if (vmf_insert_pfn(vma, addr, pfn) & VM_FAULT_ERROR) 1688 break; 1689 } 1690 1691 out_unlock: 1692 up_read(&vdev->memory_lock); 1693 1694 return ret; 1695 } 1696 1697 static const struct vm_operations_struct vfio_pci_mmap_ops = { 1698 .fault = vfio_pci_mmap_fault, 1699 }; 1700 1701 int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma) 1702 { 1703 struct vfio_pci_core_device *vdev = 1704 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1705 struct pci_dev *pdev = vdev->pdev; 1706 unsigned int index; 1707 u64 phys_len, req_len, pgoff, req_start; 1708 int ret; 1709 1710 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 1711 1712 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) 1713 return -EINVAL; 1714 if (vma->vm_end < vma->vm_start) 1715 return -EINVAL; 1716 if ((vma->vm_flags & VM_SHARED) == 0) 1717 return -EINVAL; 1718 if (index >= VFIO_PCI_NUM_REGIONS) { 1719 int regnum = index - VFIO_PCI_NUM_REGIONS; 1720 struct vfio_pci_region *region = vdev->region + regnum; 1721 1722 if (region->ops && region->ops->mmap && 1723 (region->flags & VFIO_REGION_INFO_FLAG_MMAP)) 1724 return region->ops->mmap(vdev, region, vma); 1725 return -EINVAL; 1726 } 1727 if (index >= VFIO_PCI_ROM_REGION_INDEX) 1728 return -EINVAL; 1729 if (!vdev->bar_mmap_supported[index]) 1730 return -EINVAL; 1731 1732 phys_len = PAGE_ALIGN(pci_resource_len(pdev, index)); 1733 req_len = vma->vm_end - vma->vm_start; 1734 pgoff = vma->vm_pgoff & 1735 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 1736 req_start = pgoff << PAGE_SHIFT; 1737 1738 if (req_start + req_len > phys_len) 1739 return -EINVAL; 1740 1741 /* 1742 * Even though we don't make use of the barmap for the mmap, 1743 * we need to request the region and the barmap tracks that. 1744 */ 1745 if (!vdev->barmap[index]) { 1746 ret = pci_request_selected_regions(pdev, 1747 1 << index, "vfio-pci"); 1748 if (ret) 1749 return ret; 1750 1751 vdev->barmap[index] = pci_iomap(pdev, index, 0); 1752 if (!vdev->barmap[index]) { 1753 pci_release_selected_regions(pdev, 1 << index); 1754 return -ENOMEM; 1755 } 1756 } 1757 1758 vma->vm_private_data = vdev; 1759 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 1760 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); 1761 1762 /* 1763 * Set vm_flags now, they should not be changed in the fault handler. 1764 * We want the same flags and page protection (decrypted above) as 1765 * io_remap_pfn_range() would set. 1766 * 1767 * VM_ALLOW_ANY_UNCACHED: The VMA flag is implemented for ARM64, 1768 * allowing KVM stage 2 device mapping attributes to use Normal-NC 1769 * rather than DEVICE_nGnRE, which allows guest mappings 1770 * supporting write-combining attributes (WC). ARM does not 1771 * architecturally guarantee this is safe, and indeed some MMIO 1772 * regions like the GICv2 VCPU interface can trigger uncontained 1773 * faults if Normal-NC is used. 1774 * 1775 * To safely use VFIO in KVM the platform must guarantee full 1776 * safety in the guest where no action taken against a MMIO 1777 * mapping can trigger an uncontained failure. The assumption is 1778 * that most VFIO PCI platforms support this for both mapping types, 1779 * at least in common flows, based on some expectations of how 1780 * PCI IP is integrated. Hence VM_ALLOW_ANY_UNCACHED is set in 1781 * the VMA flags. 1782 */ 1783 vm_flags_set(vma, VM_ALLOW_ANY_UNCACHED | VM_IO | VM_PFNMAP | 1784 VM_DONTEXPAND | VM_DONTDUMP); 1785 vma->vm_ops = &vfio_pci_mmap_ops; 1786 1787 return 0; 1788 } 1789 EXPORT_SYMBOL_GPL(vfio_pci_core_mmap); 1790 1791 void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count) 1792 { 1793 struct vfio_pci_core_device *vdev = 1794 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1795 struct pci_dev *pdev = vdev->pdev; 1796 1797 mutex_lock(&vdev->igate); 1798 1799 if (vdev->req_trigger) { 1800 if (!(count % 10)) 1801 pci_notice_ratelimited(pdev, 1802 "Relaying device request to user (#%u)\n", 1803 count); 1804 eventfd_signal(vdev->req_trigger); 1805 } else if (count == 0) { 1806 pci_warn(pdev, 1807 "No device request channel registered, blocked until released by user\n"); 1808 } 1809 1810 mutex_unlock(&vdev->igate); 1811 } 1812 EXPORT_SYMBOL_GPL(vfio_pci_core_request); 1813 1814 static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev, 1815 bool vf_token, uuid_t *uuid) 1816 { 1817 /* 1818 * There's always some degree of trust or collaboration between SR-IOV 1819 * PF and VFs, even if just that the PF hosts the SR-IOV capability and 1820 * can disrupt VFs with a reset, but often the PF has more explicit 1821 * access to deny service to the VF or access data passed through the 1822 * VF. We therefore require an opt-in via a shared VF token (UUID) to 1823 * represent this trust. This both prevents that a VF driver might 1824 * assume the PF driver is a trusted, in-kernel driver, and also that 1825 * a PF driver might be replaced with a rogue driver, unknown to in-use 1826 * VF drivers. 1827 * 1828 * Therefore when presented with a VF, if the PF is a vfio device and 1829 * it is bound to the vfio-pci driver, the user needs to provide a VF 1830 * token to access the device, in the form of appending a vf_token to 1831 * the device name, for example: 1832 * 1833 * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3" 1834 * 1835 * When presented with a PF which has VFs in use, the user must also 1836 * provide the current VF token to prove collaboration with existing 1837 * VF users. If VFs are not in use, the VF token provided for the PF 1838 * device will act to set the VF token. 1839 * 1840 * If the VF token is provided but unused, an error is generated. 1841 */ 1842 if (vdev->pdev->is_virtfn) { 1843 struct vfio_pci_core_device *pf_vdev = vdev->sriov_pf_core_dev; 1844 bool match; 1845 1846 if (!pf_vdev) { 1847 if (!vf_token) 1848 return 0; /* PF is not vfio-pci, no VF token */ 1849 1850 pci_info_ratelimited(vdev->pdev, 1851 "VF token incorrectly provided, PF not bound to vfio-pci\n"); 1852 return -EINVAL; 1853 } 1854 1855 if (!vf_token) { 1856 pci_info_ratelimited(vdev->pdev, 1857 "VF token required to access device\n"); 1858 return -EACCES; 1859 } 1860 1861 mutex_lock(&pf_vdev->vf_token->lock); 1862 match = uuid_equal(uuid, &pf_vdev->vf_token->uuid); 1863 mutex_unlock(&pf_vdev->vf_token->lock); 1864 1865 if (!match) { 1866 pci_info_ratelimited(vdev->pdev, 1867 "Incorrect VF token provided for device\n"); 1868 return -EACCES; 1869 } 1870 } else if (vdev->vf_token) { 1871 mutex_lock(&vdev->vf_token->lock); 1872 if (vdev->vf_token->users) { 1873 if (!vf_token) { 1874 mutex_unlock(&vdev->vf_token->lock); 1875 pci_info_ratelimited(vdev->pdev, 1876 "VF token required to access device\n"); 1877 return -EACCES; 1878 } 1879 1880 if (!uuid_equal(uuid, &vdev->vf_token->uuid)) { 1881 mutex_unlock(&vdev->vf_token->lock); 1882 pci_info_ratelimited(vdev->pdev, 1883 "Incorrect VF token provided for device\n"); 1884 return -EACCES; 1885 } 1886 } else if (vf_token) { 1887 uuid_copy(&vdev->vf_token->uuid, uuid); 1888 } 1889 1890 mutex_unlock(&vdev->vf_token->lock); 1891 } else if (vf_token) { 1892 pci_info_ratelimited(vdev->pdev, 1893 "VF token incorrectly provided, not a PF or VF\n"); 1894 return -EINVAL; 1895 } 1896 1897 return 0; 1898 } 1899 1900 #define VF_TOKEN_ARG "vf_token=" 1901 1902 int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf) 1903 { 1904 struct vfio_pci_core_device *vdev = 1905 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1906 bool vf_token = false; 1907 uuid_t uuid; 1908 int ret; 1909 1910 if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev)))) 1911 return 0; /* No match */ 1912 1913 if (strlen(buf) > strlen(pci_name(vdev->pdev))) { 1914 buf += strlen(pci_name(vdev->pdev)); 1915 1916 if (*buf != ' ') 1917 return 0; /* No match: non-whitespace after name */ 1918 1919 while (*buf) { 1920 if (*buf == ' ') { 1921 buf++; 1922 continue; 1923 } 1924 1925 if (!vf_token && !strncmp(buf, VF_TOKEN_ARG, 1926 strlen(VF_TOKEN_ARG))) { 1927 buf += strlen(VF_TOKEN_ARG); 1928 1929 if (strlen(buf) < UUID_STRING_LEN) 1930 return -EINVAL; 1931 1932 ret = uuid_parse(buf, &uuid); 1933 if (ret) 1934 return ret; 1935 1936 vf_token = true; 1937 buf += UUID_STRING_LEN; 1938 } else { 1939 /* Unknown/duplicate option */ 1940 return -EINVAL; 1941 } 1942 } 1943 } 1944 1945 ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid); 1946 if (ret) 1947 return ret; 1948 1949 return 1; /* Match */ 1950 } 1951 EXPORT_SYMBOL_GPL(vfio_pci_core_match); 1952 1953 static int vfio_pci_bus_notifier(struct notifier_block *nb, 1954 unsigned long action, void *data) 1955 { 1956 struct vfio_pci_core_device *vdev = container_of(nb, 1957 struct vfio_pci_core_device, nb); 1958 struct device *dev = data; 1959 struct pci_dev *pdev = to_pci_dev(dev); 1960 struct pci_dev *physfn = pci_physfn(pdev); 1961 1962 if (action == BUS_NOTIFY_ADD_DEVICE && 1963 pdev->is_virtfn && physfn == vdev->pdev) { 1964 pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n", 1965 pci_name(pdev)); 1966 pdev->driver_override = kasprintf(GFP_KERNEL, "%s", 1967 vdev->vdev.ops->name); 1968 WARN_ON(!pdev->driver_override); 1969 } else if (action == BUS_NOTIFY_BOUND_DRIVER && 1970 pdev->is_virtfn && physfn == vdev->pdev) { 1971 struct pci_driver *drv = pci_dev_driver(pdev); 1972 1973 if (drv && drv != pci_dev_driver(vdev->pdev)) 1974 pci_warn(vdev->pdev, 1975 "VF %s bound to driver %s while PF bound to driver %s\n", 1976 pci_name(pdev), drv->name, 1977 pci_dev_driver(vdev->pdev)->name); 1978 } 1979 1980 return 0; 1981 } 1982 1983 static int vfio_pci_vf_init(struct vfio_pci_core_device *vdev) 1984 { 1985 struct pci_dev *pdev = vdev->pdev; 1986 struct vfio_pci_core_device *cur; 1987 struct pci_dev *physfn; 1988 int ret; 1989 1990 if (pdev->is_virtfn) { 1991 /* 1992 * If this VF was created by our vfio_pci_core_sriov_configure() 1993 * then we can find the PF vfio_pci_core_device now, and due to 1994 * the locking in pci_disable_sriov() it cannot change until 1995 * this VF device driver is removed. 1996 */ 1997 physfn = pci_physfn(vdev->pdev); 1998 mutex_lock(&vfio_pci_sriov_pfs_mutex); 1999 list_for_each_entry(cur, &vfio_pci_sriov_pfs, sriov_pfs_item) { 2000 if (cur->pdev == physfn) { 2001 vdev->sriov_pf_core_dev = cur; 2002 break; 2003 } 2004 } 2005 mutex_unlock(&vfio_pci_sriov_pfs_mutex); 2006 return 0; 2007 } 2008 2009 /* Not a SRIOV PF */ 2010 if (!pdev->is_physfn) 2011 return 0; 2012 2013 vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL); 2014 if (!vdev->vf_token) 2015 return -ENOMEM; 2016 2017 mutex_init(&vdev->vf_token->lock); 2018 uuid_gen(&vdev->vf_token->uuid); 2019 2020 vdev->nb.notifier_call = vfio_pci_bus_notifier; 2021 ret = bus_register_notifier(&pci_bus_type, &vdev->nb); 2022 if (ret) { 2023 kfree(vdev->vf_token); 2024 return ret; 2025 } 2026 return 0; 2027 } 2028 2029 static void vfio_pci_vf_uninit(struct vfio_pci_core_device *vdev) 2030 { 2031 if (!vdev->vf_token) 2032 return; 2033 2034 bus_unregister_notifier(&pci_bus_type, &vdev->nb); 2035 WARN_ON(vdev->vf_token->users); 2036 mutex_destroy(&vdev->vf_token->lock); 2037 kfree(vdev->vf_token); 2038 } 2039 2040 static int vfio_pci_vga_init(struct vfio_pci_core_device *vdev) 2041 { 2042 struct pci_dev *pdev = vdev->pdev; 2043 int ret; 2044 2045 if (!vfio_pci_is_vga(pdev)) 2046 return 0; 2047 2048 ret = aperture_remove_conflicting_pci_devices(pdev, vdev->vdev.ops->name); 2049 if (ret) 2050 return ret; 2051 2052 ret = vga_client_register(pdev, vfio_pci_set_decode); 2053 if (ret) 2054 return ret; 2055 vga_set_legacy_decoding(pdev, vfio_pci_set_decode(pdev, false)); 2056 return 0; 2057 } 2058 2059 static void vfio_pci_vga_uninit(struct vfio_pci_core_device *vdev) 2060 { 2061 struct pci_dev *pdev = vdev->pdev; 2062 2063 if (!vfio_pci_is_vga(pdev)) 2064 return; 2065 vga_client_unregister(pdev); 2066 vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | 2067 VGA_RSRC_LEGACY_IO | 2068 VGA_RSRC_LEGACY_MEM); 2069 } 2070 2071 int vfio_pci_core_init_dev(struct vfio_device *core_vdev) 2072 { 2073 struct vfio_pci_core_device *vdev = 2074 container_of(core_vdev, struct vfio_pci_core_device, vdev); 2075 2076 vdev->pdev = to_pci_dev(core_vdev->dev); 2077 vdev->irq_type = VFIO_PCI_NUM_IRQS; 2078 mutex_init(&vdev->igate); 2079 spin_lock_init(&vdev->irqlock); 2080 mutex_init(&vdev->ioeventfds_lock); 2081 INIT_LIST_HEAD(&vdev->dummy_resources_list); 2082 INIT_LIST_HEAD(&vdev->ioeventfds_list); 2083 INIT_LIST_HEAD(&vdev->sriov_pfs_item); 2084 init_rwsem(&vdev->memory_lock); 2085 xa_init(&vdev->ctx); 2086 2087 return 0; 2088 } 2089 EXPORT_SYMBOL_GPL(vfio_pci_core_init_dev); 2090 2091 void vfio_pci_core_release_dev(struct vfio_device *core_vdev) 2092 { 2093 struct vfio_pci_core_device *vdev = 2094 container_of(core_vdev, struct vfio_pci_core_device, vdev); 2095 2096 mutex_destroy(&vdev->igate); 2097 mutex_destroy(&vdev->ioeventfds_lock); 2098 kfree(vdev->region); 2099 kfree(vdev->pm_save); 2100 } 2101 EXPORT_SYMBOL_GPL(vfio_pci_core_release_dev); 2102 2103 int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) 2104 { 2105 struct pci_dev *pdev = vdev->pdev; 2106 struct device *dev = &pdev->dev; 2107 int ret; 2108 2109 /* Drivers must set the vfio_pci_core_device to their drvdata */ 2110 if (WARN_ON(vdev != dev_get_drvdata(dev))) 2111 return -EINVAL; 2112 2113 if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL) 2114 return -EINVAL; 2115 2116 if (vdev->vdev.mig_ops) { 2117 if (!(vdev->vdev.mig_ops->migration_get_state && 2118 vdev->vdev.mig_ops->migration_set_state && 2119 vdev->vdev.mig_ops->migration_get_data_size) || 2120 !(vdev->vdev.migration_flags & VFIO_MIGRATION_STOP_COPY)) 2121 return -EINVAL; 2122 } 2123 2124 if (vdev->vdev.log_ops && !(vdev->vdev.log_ops->log_start && 2125 vdev->vdev.log_ops->log_stop && 2126 vdev->vdev.log_ops->log_read_and_clear)) 2127 return -EINVAL; 2128 2129 /* 2130 * Prevent binding to PFs with VFs enabled, the VFs might be in use 2131 * by the host or other users. We cannot capture the VFs if they 2132 * already exist, nor can we track VF users. Disabling SR-IOV here 2133 * would initiate removing the VFs, which would unbind the driver, 2134 * which is prone to blocking if that VF is also in use by vfio-pci. 2135 * Just reject these PFs and let the user sort it out. 2136 */ 2137 if (pci_num_vf(pdev)) { 2138 pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n"); 2139 return -EBUSY; 2140 } 2141 2142 if (pci_is_root_bus(pdev->bus)) { 2143 ret = vfio_assign_device_set(&vdev->vdev, vdev); 2144 } else if (!pci_probe_reset_slot(pdev->slot)) { 2145 ret = vfio_assign_device_set(&vdev->vdev, pdev->slot); 2146 } else { 2147 /* 2148 * If there is no slot reset support for this device, the whole 2149 * bus needs to be grouped together to support bus-wide resets. 2150 */ 2151 ret = vfio_assign_device_set(&vdev->vdev, pdev->bus); 2152 } 2153 2154 if (ret) 2155 return ret; 2156 ret = vfio_pci_vf_init(vdev); 2157 if (ret) 2158 return ret; 2159 ret = vfio_pci_vga_init(vdev); 2160 if (ret) 2161 goto out_vf; 2162 2163 vfio_pci_probe_power_state(vdev); 2164 2165 /* 2166 * pci-core sets the device power state to an unknown value at 2167 * bootup and after being removed from a driver. The only 2168 * transition it allows from this unknown state is to D0, which 2169 * typically happens when a driver calls pci_enable_device(). 2170 * We're not ready to enable the device yet, but we do want to 2171 * be able to get to D3. Therefore first do a D0 transition 2172 * before enabling runtime PM. 2173 */ 2174 vfio_pci_set_power_state(vdev, PCI_D0); 2175 2176 dev->driver->pm = &vfio_pci_core_pm_ops; 2177 pm_runtime_allow(dev); 2178 if (!disable_idle_d3) 2179 pm_runtime_put(dev); 2180 2181 ret = vfio_register_group_dev(&vdev->vdev); 2182 if (ret) 2183 goto out_power; 2184 return 0; 2185 2186 out_power: 2187 if (!disable_idle_d3) 2188 pm_runtime_get_noresume(dev); 2189 2190 pm_runtime_forbid(dev); 2191 out_vf: 2192 vfio_pci_vf_uninit(vdev); 2193 return ret; 2194 } 2195 EXPORT_SYMBOL_GPL(vfio_pci_core_register_device); 2196 2197 void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev) 2198 { 2199 vfio_pci_core_sriov_configure(vdev, 0); 2200 2201 vfio_unregister_group_dev(&vdev->vdev); 2202 2203 vfio_pci_vf_uninit(vdev); 2204 vfio_pci_vga_uninit(vdev); 2205 2206 if (!disable_idle_d3) 2207 pm_runtime_get_noresume(&vdev->pdev->dev); 2208 2209 pm_runtime_forbid(&vdev->pdev->dev); 2210 } 2211 EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device); 2212 2213 pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, 2214 pci_channel_state_t state) 2215 { 2216 struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev); 2217 2218 mutex_lock(&vdev->igate); 2219 2220 if (vdev->err_trigger) 2221 eventfd_signal(vdev->err_trigger); 2222 2223 mutex_unlock(&vdev->igate); 2224 2225 return PCI_ERS_RESULT_CAN_RECOVER; 2226 } 2227 EXPORT_SYMBOL_GPL(vfio_pci_core_aer_err_detected); 2228 2229 int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev, 2230 int nr_virtfn) 2231 { 2232 struct pci_dev *pdev = vdev->pdev; 2233 int ret = 0; 2234 2235 device_lock_assert(&pdev->dev); 2236 2237 if (nr_virtfn) { 2238 mutex_lock(&vfio_pci_sriov_pfs_mutex); 2239 /* 2240 * The thread that adds the vdev to the list is the only thread 2241 * that gets to call pci_enable_sriov() and we will only allow 2242 * it to be called once without going through 2243 * pci_disable_sriov() 2244 */ 2245 if (!list_empty(&vdev->sriov_pfs_item)) { 2246 ret = -EINVAL; 2247 goto out_unlock; 2248 } 2249 list_add_tail(&vdev->sriov_pfs_item, &vfio_pci_sriov_pfs); 2250 mutex_unlock(&vfio_pci_sriov_pfs_mutex); 2251 2252 /* 2253 * The PF power state should always be higher than the VF power 2254 * state. The PF can be in low power state either with runtime 2255 * power management (when there is no user) or PCI_PM_CTRL 2256 * register write by the user. If PF is in the low power state, 2257 * then change the power state to D0 first before enabling 2258 * SR-IOV. Also, this function can be called at any time, and 2259 * userspace PCI_PM_CTRL write can race against this code path, 2260 * so protect the same with 'memory_lock'. 2261 */ 2262 ret = pm_runtime_resume_and_get(&pdev->dev); 2263 if (ret) 2264 goto out_del; 2265 2266 down_write(&vdev->memory_lock); 2267 vfio_pci_set_power_state(vdev, PCI_D0); 2268 ret = pci_enable_sriov(pdev, nr_virtfn); 2269 up_write(&vdev->memory_lock); 2270 if (ret) { 2271 pm_runtime_put(&pdev->dev); 2272 goto out_del; 2273 } 2274 return nr_virtfn; 2275 } 2276 2277 if (pci_num_vf(pdev)) { 2278 pci_disable_sriov(pdev); 2279 pm_runtime_put(&pdev->dev); 2280 } 2281 2282 out_del: 2283 mutex_lock(&vfio_pci_sriov_pfs_mutex); 2284 list_del_init(&vdev->sriov_pfs_item); 2285 out_unlock: 2286 mutex_unlock(&vfio_pci_sriov_pfs_mutex); 2287 return ret; 2288 } 2289 EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure); 2290 2291 const struct pci_error_handlers vfio_pci_core_err_handlers = { 2292 .error_detected = vfio_pci_core_aer_err_detected, 2293 }; 2294 EXPORT_SYMBOL_GPL(vfio_pci_core_err_handlers); 2295 2296 static bool vfio_dev_in_groups(struct vfio_device *vdev, 2297 struct vfio_pci_group_info *groups) 2298 { 2299 unsigned int i; 2300 2301 if (!groups) 2302 return false; 2303 2304 for (i = 0; i < groups->count; i++) 2305 if (vfio_file_has_dev(groups->files[i], vdev)) 2306 return true; 2307 return false; 2308 } 2309 2310 static int vfio_pci_is_device_in_set(struct pci_dev *pdev, void *data) 2311 { 2312 struct vfio_device_set *dev_set = data; 2313 2314 return vfio_find_device_in_devset(dev_set, &pdev->dev) ? 0 : -ENODEV; 2315 } 2316 2317 /* 2318 * vfio-core considers a group to be viable and will create a vfio_device even 2319 * if some devices are bound to drivers like pci-stub or pcieport. Here we 2320 * require all PCI devices to be inside our dev_set since that ensures they stay 2321 * put and that every driver controlling the device can co-ordinate with the 2322 * device reset. 2323 * 2324 * Returns the pci_dev to pass to pci_reset_bus() if every PCI device to be 2325 * reset is inside the dev_set, and pci_reset_bus() can succeed. NULL otherwise. 2326 */ 2327 static struct pci_dev * 2328 vfio_pci_dev_set_resettable(struct vfio_device_set *dev_set) 2329 { 2330 struct pci_dev *pdev; 2331 2332 lockdep_assert_held(&dev_set->lock); 2333 2334 /* 2335 * By definition all PCI devices in the dev_set share the same PCI 2336 * reset, so any pci_dev will have the same outcomes for 2337 * pci_probe_reset_*() and pci_reset_bus(). 2338 */ 2339 pdev = list_first_entry(&dev_set->device_list, 2340 struct vfio_pci_core_device, 2341 vdev.dev_set_list)->pdev; 2342 2343 /* pci_reset_bus() is supported */ 2344 if (pci_probe_reset_slot(pdev->slot) && pci_probe_reset_bus(pdev->bus)) 2345 return NULL; 2346 2347 if (vfio_pci_for_each_slot_or_bus(pdev, vfio_pci_is_device_in_set, 2348 dev_set, 2349 !pci_probe_reset_slot(pdev->slot))) 2350 return NULL; 2351 return pdev; 2352 } 2353 2354 static int vfio_pci_dev_set_pm_runtime_get(struct vfio_device_set *dev_set) 2355 { 2356 struct vfio_pci_core_device *cur; 2357 int ret; 2358 2359 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) { 2360 ret = pm_runtime_resume_and_get(&cur->pdev->dev); 2361 if (ret) 2362 goto unwind; 2363 } 2364 2365 return 0; 2366 2367 unwind: 2368 list_for_each_entry_continue_reverse(cur, &dev_set->device_list, 2369 vdev.dev_set_list) 2370 pm_runtime_put(&cur->pdev->dev); 2371 2372 return ret; 2373 } 2374 2375 static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set, 2376 struct vfio_pci_group_info *groups, 2377 struct iommufd_ctx *iommufd_ctx) 2378 { 2379 struct vfio_pci_core_device *vdev; 2380 struct pci_dev *pdev; 2381 int ret; 2382 2383 mutex_lock(&dev_set->lock); 2384 2385 pdev = vfio_pci_dev_set_resettable(dev_set); 2386 if (!pdev) { 2387 ret = -EINVAL; 2388 goto err_unlock; 2389 } 2390 2391 /* 2392 * Some of the devices in the dev_set can be in the runtime suspended 2393 * state. Increment the usage count for all the devices in the dev_set 2394 * before reset and decrement the same after reset. 2395 */ 2396 ret = vfio_pci_dev_set_pm_runtime_get(dev_set); 2397 if (ret) 2398 goto err_unlock; 2399 2400 list_for_each_entry(vdev, &dev_set->device_list, vdev.dev_set_list) { 2401 bool owned; 2402 2403 /* 2404 * Test whether all the affected devices can be reset by the 2405 * user. 2406 * 2407 * If called from a group opened device and the user provides 2408 * a set of groups, all the devices in the dev_set should be 2409 * contained by the set of groups provided by the user. 2410 * 2411 * If called from a cdev opened device and the user provides 2412 * a zero-length array, all the devices in the dev_set must 2413 * be bound to the same iommufd_ctx as the input iommufd_ctx. 2414 * If there is any device that has not been bound to any 2415 * iommufd_ctx yet, check if its iommu_group has any device 2416 * bound to the input iommufd_ctx. Such devices can be 2417 * considered owned by the input iommufd_ctx as the device 2418 * cannot be owned by another iommufd_ctx when its iommu_group 2419 * is owned. 2420 * 2421 * Otherwise, reset is not allowed. 2422 */ 2423 if (iommufd_ctx) { 2424 int devid = vfio_iommufd_get_dev_id(&vdev->vdev, 2425 iommufd_ctx); 2426 2427 owned = (devid > 0 || devid == -ENOENT); 2428 } else { 2429 owned = vfio_dev_in_groups(&vdev->vdev, groups); 2430 } 2431 2432 if (!owned) { 2433 ret = -EINVAL; 2434 break; 2435 } 2436 2437 /* 2438 * Take the memory write lock for each device and zap BAR 2439 * mappings to prevent the user accessing the device while in 2440 * reset. Locking multiple devices is prone to deadlock, 2441 * runaway and unwind if we hit contention. 2442 */ 2443 if (!down_write_trylock(&vdev->memory_lock)) { 2444 ret = -EBUSY; 2445 break; 2446 } 2447 2448 vfio_pci_zap_bars(vdev); 2449 } 2450 2451 if (!list_entry_is_head(vdev, 2452 &dev_set->device_list, vdev.dev_set_list)) { 2453 vdev = list_prev_entry(vdev, vdev.dev_set_list); 2454 goto err_undo; 2455 } 2456 2457 /* 2458 * The pci_reset_bus() will reset all the devices in the bus. 2459 * The power state can be non-D0 for some of the devices in the bus. 2460 * For these devices, the pci_reset_bus() will internally set 2461 * the power state to D0 without vfio driver involvement. 2462 * For the devices which have NoSoftRst-, the reset function can 2463 * cause the PCI config space reset without restoring the original 2464 * state (saved locally in 'vdev->pm_save'). 2465 */ 2466 list_for_each_entry(vdev, &dev_set->device_list, vdev.dev_set_list) 2467 vfio_pci_set_power_state(vdev, PCI_D0); 2468 2469 ret = pci_reset_bus(pdev); 2470 2471 vdev = list_last_entry(&dev_set->device_list, 2472 struct vfio_pci_core_device, vdev.dev_set_list); 2473 2474 err_undo: 2475 list_for_each_entry_from_reverse(vdev, &dev_set->device_list, 2476 vdev.dev_set_list) 2477 up_write(&vdev->memory_lock); 2478 2479 list_for_each_entry(vdev, &dev_set->device_list, vdev.dev_set_list) 2480 pm_runtime_put(&vdev->pdev->dev); 2481 2482 err_unlock: 2483 mutex_unlock(&dev_set->lock); 2484 return ret; 2485 } 2486 2487 static bool vfio_pci_dev_set_needs_reset(struct vfio_device_set *dev_set) 2488 { 2489 struct vfio_pci_core_device *cur; 2490 bool needs_reset = false; 2491 2492 /* No other VFIO device in the set can be open. */ 2493 if (vfio_device_set_open_count(dev_set) > 1) 2494 return false; 2495 2496 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) 2497 needs_reset |= cur->needs_reset; 2498 return needs_reset; 2499 } 2500 2501 /* 2502 * If a bus or slot reset is available for the provided dev_set and: 2503 * - All of the devices affected by that bus or slot reset are unused 2504 * - At least one of the affected devices is marked dirty via 2505 * needs_reset (such as by lack of FLR support) 2506 * Then attempt to perform that bus or slot reset. 2507 */ 2508 static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set) 2509 { 2510 struct vfio_pci_core_device *cur; 2511 struct pci_dev *pdev; 2512 bool reset_done = false; 2513 2514 if (!vfio_pci_dev_set_needs_reset(dev_set)) 2515 return; 2516 2517 pdev = vfio_pci_dev_set_resettable(dev_set); 2518 if (!pdev) 2519 return; 2520 2521 /* 2522 * Some of the devices in the bus can be in the runtime suspended 2523 * state. Increment the usage count for all the devices in the dev_set 2524 * before reset and decrement the same after reset. 2525 */ 2526 if (!disable_idle_d3 && vfio_pci_dev_set_pm_runtime_get(dev_set)) 2527 return; 2528 2529 if (!pci_reset_bus(pdev)) 2530 reset_done = true; 2531 2532 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) { 2533 if (reset_done) 2534 cur->needs_reset = false; 2535 2536 if (!disable_idle_d3) 2537 pm_runtime_put(&cur->pdev->dev); 2538 } 2539 } 2540 2541 void vfio_pci_core_set_params(bool is_nointxmask, bool is_disable_vga, 2542 bool is_disable_idle_d3) 2543 { 2544 nointxmask = is_nointxmask; 2545 disable_vga = is_disable_vga; 2546 disable_idle_d3 = is_disable_idle_d3; 2547 } 2548 EXPORT_SYMBOL_GPL(vfio_pci_core_set_params); 2549 2550 static void vfio_pci_core_cleanup(void) 2551 { 2552 vfio_pci_uninit_perm_bits(); 2553 } 2554 2555 static int __init vfio_pci_core_init(void) 2556 { 2557 /* Allocate shared config space permission data used by all devices */ 2558 return vfio_pci_init_perm_bits(); 2559 } 2560 2561 module_init(vfio_pci_core_init); 2562 module_exit(vfio_pci_core_cleanup); 2563 2564 MODULE_LICENSE("GPL v2"); 2565 MODULE_AUTHOR(DRIVER_AUTHOR); 2566 MODULE_DESCRIPTION(DRIVER_DESC); 2567