1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 4 * Author: Alex Williamson <alex.williamson@redhat.com> 5 * 6 * Derived from original vfio: 7 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 8 * Author: Tom Lyon, pugs@cisco.com 9 */ 10 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 13 #include <linux/device.h> 14 #include <linux/eventfd.h> 15 #include <linux/file.h> 16 #include <linux/interrupt.h> 17 #include <linux/iommu.h> 18 #include <linux/module.h> 19 #include <linux/mutex.h> 20 #include <linux/notifier.h> 21 #include <linux/pci.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/slab.h> 24 #include <linux/types.h> 25 #include <linux/uaccess.h> 26 #include <linux/vgaarb.h> 27 #include <linux/nospec.h> 28 #include <linux/sched/mm.h> 29 30 #include <linux/vfio_pci_core.h> 31 32 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 33 #define DRIVER_DESC "core driver for VFIO based PCI devices" 34 35 static bool nointxmask; 36 static bool disable_vga; 37 static bool disable_idle_d3; 38 39 /* List of PF's that vfio_pci_core_sriov_configure() has been called on */ 40 static DEFINE_MUTEX(vfio_pci_sriov_pfs_mutex); 41 static LIST_HEAD(vfio_pci_sriov_pfs); 42 43 static inline bool vfio_vga_disabled(void) 44 { 45 #ifdef CONFIG_VFIO_PCI_VGA 46 return disable_vga; 47 #else 48 return true; 49 #endif 50 } 51 52 /* 53 * Our VGA arbiter participation is limited since we don't know anything 54 * about the device itself. However, if the device is the only VGA device 55 * downstream of a bridge and VFIO VGA support is disabled, then we can 56 * safely return legacy VGA IO and memory as not decoded since the user 57 * has no way to get to it and routing can be disabled externally at the 58 * bridge. 59 */ 60 static unsigned int vfio_pci_set_decode(struct pci_dev *pdev, bool single_vga) 61 { 62 struct pci_dev *tmp = NULL; 63 unsigned char max_busnr; 64 unsigned int decodes; 65 66 if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus)) 67 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | 68 VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM; 69 70 max_busnr = pci_bus_max_busnr(pdev->bus); 71 decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 72 73 while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) { 74 if (tmp == pdev || 75 pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) || 76 pci_is_root_bus(tmp->bus)) 77 continue; 78 79 if (tmp->bus->number >= pdev->bus->number && 80 tmp->bus->number <= max_busnr) { 81 pci_dev_put(tmp); 82 decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM; 83 break; 84 } 85 } 86 87 return decodes; 88 } 89 90 static void vfio_pci_probe_mmaps(struct vfio_pci_core_device *vdev) 91 { 92 struct resource *res; 93 int i; 94 struct vfio_pci_dummy_resource *dummy_res; 95 96 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 97 int bar = i + PCI_STD_RESOURCES; 98 99 res = &vdev->pdev->resource[bar]; 100 101 if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP)) 102 goto no_mmap; 103 104 if (!(res->flags & IORESOURCE_MEM)) 105 goto no_mmap; 106 107 /* 108 * The PCI core shouldn't set up a resource with a 109 * type but zero size. But there may be bugs that 110 * cause us to do that. 111 */ 112 if (!resource_size(res)) 113 goto no_mmap; 114 115 if (resource_size(res) >= PAGE_SIZE) { 116 vdev->bar_mmap_supported[bar] = true; 117 continue; 118 } 119 120 if (!(res->start & ~PAGE_MASK)) { 121 /* 122 * Add a dummy resource to reserve the remainder 123 * of the exclusive page in case that hot-add 124 * device's bar is assigned into it. 125 */ 126 dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL); 127 if (dummy_res == NULL) 128 goto no_mmap; 129 130 dummy_res->resource.name = "vfio sub-page reserved"; 131 dummy_res->resource.start = res->end + 1; 132 dummy_res->resource.end = res->start + PAGE_SIZE - 1; 133 dummy_res->resource.flags = res->flags; 134 if (request_resource(res->parent, 135 &dummy_res->resource)) { 136 kfree(dummy_res); 137 goto no_mmap; 138 } 139 dummy_res->index = bar; 140 list_add(&dummy_res->res_next, 141 &vdev->dummy_resources_list); 142 vdev->bar_mmap_supported[bar] = true; 143 continue; 144 } 145 /* 146 * Here we don't handle the case when the BAR is not page 147 * aligned because we can't expect the BAR will be 148 * assigned into the same location in a page in guest 149 * when we passthrough the BAR. And it's hard to access 150 * this BAR in userspace because we have no way to get 151 * the BAR's location in a page. 152 */ 153 no_mmap: 154 vdev->bar_mmap_supported[bar] = false; 155 } 156 } 157 158 struct vfio_pci_group_info; 159 static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set); 160 static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set, 161 struct vfio_pci_group_info *groups); 162 163 /* 164 * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND 165 * _and_ the ability detect when the device is asserting INTx via PCI_STATUS. 166 * If a device implements the former but not the latter we would typically 167 * expect broken_intx_masking be set and require an exclusive interrupt. 168 * However since we do have control of the device's ability to assert INTx, 169 * we can instead pretend that the device does not implement INTx, virtualizing 170 * the pin register to report zero and maintaining DisINTx set on the host. 171 */ 172 static bool vfio_pci_nointx(struct pci_dev *pdev) 173 { 174 switch (pdev->vendor) { 175 case PCI_VENDOR_ID_INTEL: 176 switch (pdev->device) { 177 /* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */ 178 case 0x1572: 179 case 0x1574: 180 case 0x1580 ... 0x1581: 181 case 0x1583 ... 0x158b: 182 case 0x37d0 ... 0x37d2: 183 /* X550 */ 184 case 0x1563: 185 return true; 186 default: 187 return false; 188 } 189 } 190 191 return false; 192 } 193 194 static void vfio_pci_probe_power_state(struct vfio_pci_core_device *vdev) 195 { 196 struct pci_dev *pdev = vdev->pdev; 197 u16 pmcsr; 198 199 if (!pdev->pm_cap) 200 return; 201 202 pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr); 203 204 vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET); 205 } 206 207 /* 208 * pci_set_power_state() wrapper handling devices which perform a soft reset on 209 * D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev, 210 * restore when returned to D0. Saved separately from pci_saved_state for use 211 * by PM capability emulation and separately from pci_dev internal saved state 212 * to avoid it being overwritten and consumed around other resets. 213 */ 214 int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t state) 215 { 216 struct pci_dev *pdev = vdev->pdev; 217 bool needs_restore = false, needs_save = false; 218 int ret; 219 220 if (vdev->needs_pm_restore) { 221 if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) { 222 pci_save_state(pdev); 223 needs_save = true; 224 } 225 226 if (pdev->current_state >= PCI_D3hot && state <= PCI_D0) 227 needs_restore = true; 228 } 229 230 ret = pci_set_power_state(pdev, state); 231 232 if (!ret) { 233 /* D3 might be unsupported via quirk, skip unless in D3 */ 234 if (needs_save && pdev->current_state >= PCI_D3hot) { 235 /* 236 * The current PCI state will be saved locally in 237 * 'pm_save' during the D3hot transition. When the 238 * device state is changed to D0 again with the current 239 * function, then pci_store_saved_state() will restore 240 * the state and will free the memory pointed by 241 * 'pm_save'. There are few cases where the PCI power 242 * state can be changed to D0 without the involvement 243 * of the driver. For these cases, free the earlier 244 * allocated memory first before overwriting 'pm_save' 245 * to prevent the memory leak. 246 */ 247 kfree(vdev->pm_save); 248 vdev->pm_save = pci_store_saved_state(pdev); 249 } else if (needs_restore) { 250 pci_load_and_free_saved_state(pdev, &vdev->pm_save); 251 pci_restore_state(pdev); 252 } 253 } 254 255 return ret; 256 } 257 258 int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) 259 { 260 struct pci_dev *pdev = vdev->pdev; 261 int ret; 262 u16 cmd; 263 u8 msix_pos; 264 265 vfio_pci_set_power_state(vdev, PCI_D0); 266 267 /* Don't allow our initial saved state to include busmaster */ 268 pci_clear_master(pdev); 269 270 ret = pci_enable_device(pdev); 271 if (ret) 272 return ret; 273 274 /* If reset fails because of the device lock, fail this path entirely */ 275 ret = pci_try_reset_function(pdev); 276 if (ret == -EAGAIN) { 277 pci_disable_device(pdev); 278 return ret; 279 } 280 281 vdev->reset_works = !ret; 282 pci_save_state(pdev); 283 vdev->pci_saved_state = pci_store_saved_state(pdev); 284 if (!vdev->pci_saved_state) 285 pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__); 286 287 if (likely(!nointxmask)) { 288 if (vfio_pci_nointx(pdev)) { 289 pci_info(pdev, "Masking broken INTx support\n"); 290 vdev->nointx = true; 291 pci_intx(pdev, 0); 292 } else 293 vdev->pci_2_3 = pci_intx_mask_supported(pdev); 294 } 295 296 pci_read_config_word(pdev, PCI_COMMAND, &cmd); 297 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) { 298 cmd &= ~PCI_COMMAND_INTX_DISABLE; 299 pci_write_config_word(pdev, PCI_COMMAND, cmd); 300 } 301 302 ret = vfio_config_init(vdev); 303 if (ret) { 304 kfree(vdev->pci_saved_state); 305 vdev->pci_saved_state = NULL; 306 pci_disable_device(pdev); 307 return ret; 308 } 309 310 msix_pos = pdev->msix_cap; 311 if (msix_pos) { 312 u16 flags; 313 u32 table; 314 315 pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags); 316 pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table); 317 318 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR; 319 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET; 320 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16; 321 } else 322 vdev->msix_bar = 0xFF; 323 324 if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev)) 325 vdev->has_vga = true; 326 327 328 return 0; 329 } 330 EXPORT_SYMBOL_GPL(vfio_pci_core_enable); 331 332 void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) 333 { 334 struct pci_dev *pdev = vdev->pdev; 335 struct vfio_pci_dummy_resource *dummy_res, *tmp; 336 struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp; 337 int i, bar; 338 339 /* For needs_reset */ 340 lockdep_assert_held(&vdev->vdev.dev_set->lock); 341 342 /* 343 * This function can be invoked while the power state is non-D0. 344 * This function calls __pci_reset_function_locked() which internally 345 * can use pci_pm_reset() for the function reset. pci_pm_reset() will 346 * fail if the power state is non-D0. Also, for the devices which 347 * have NoSoftRst-, the reset function can cause the PCI config space 348 * reset without restoring the original state (saved locally in 349 * 'vdev->pm_save'). 350 */ 351 vfio_pci_set_power_state(vdev, PCI_D0); 352 353 /* Stop the device from further DMA */ 354 pci_clear_master(pdev); 355 356 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE | 357 VFIO_IRQ_SET_ACTION_TRIGGER, 358 vdev->irq_type, 0, 0, NULL); 359 360 /* Device closed, don't need mutex here */ 361 list_for_each_entry_safe(ioeventfd, ioeventfd_tmp, 362 &vdev->ioeventfds_list, next) { 363 vfio_virqfd_disable(&ioeventfd->virqfd); 364 list_del(&ioeventfd->next); 365 kfree(ioeventfd); 366 } 367 vdev->ioeventfds_nr = 0; 368 369 vdev->virq_disabled = false; 370 371 for (i = 0; i < vdev->num_regions; i++) 372 vdev->region[i].ops->release(vdev, &vdev->region[i]); 373 374 vdev->num_regions = 0; 375 kfree(vdev->region); 376 vdev->region = NULL; /* don't krealloc a freed pointer */ 377 378 vfio_config_free(vdev); 379 380 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 381 bar = i + PCI_STD_RESOURCES; 382 if (!vdev->barmap[bar]) 383 continue; 384 pci_iounmap(pdev, vdev->barmap[bar]); 385 pci_release_selected_regions(pdev, 1 << bar); 386 vdev->barmap[bar] = NULL; 387 } 388 389 list_for_each_entry_safe(dummy_res, tmp, 390 &vdev->dummy_resources_list, res_next) { 391 list_del(&dummy_res->res_next); 392 release_resource(&dummy_res->resource); 393 kfree(dummy_res); 394 } 395 396 vdev->needs_reset = true; 397 398 /* 399 * If we have saved state, restore it. If we can reset the device, 400 * even better. Resetting with current state seems better than 401 * nothing, but saving and restoring current state without reset 402 * is just busy work. 403 */ 404 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) { 405 pci_info(pdev, "%s: Couldn't reload saved state\n", __func__); 406 407 if (!vdev->reset_works) 408 goto out; 409 410 pci_save_state(pdev); 411 } 412 413 /* 414 * Disable INTx and MSI, presumably to avoid spurious interrupts 415 * during reset. Stolen from pci_reset_function() 416 */ 417 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 418 419 /* 420 * Try to get the locks ourselves to prevent a deadlock. The 421 * success of this is dependent on being able to lock the device, 422 * which is not always possible. 423 * We can not use the "try" reset interface here, which will 424 * overwrite the previously restored configuration information. 425 */ 426 if (vdev->reset_works && pci_dev_trylock(pdev)) { 427 if (!__pci_reset_function_locked(pdev)) 428 vdev->needs_reset = false; 429 pci_dev_unlock(pdev); 430 } 431 432 pci_restore_state(pdev); 433 out: 434 pci_disable_device(pdev); 435 436 if (!vfio_pci_dev_set_try_reset(vdev->vdev.dev_set) && !disable_idle_d3) 437 vfio_pci_set_power_state(vdev, PCI_D3hot); 438 } 439 EXPORT_SYMBOL_GPL(vfio_pci_core_disable); 440 441 void vfio_pci_core_close_device(struct vfio_device *core_vdev) 442 { 443 struct vfio_pci_core_device *vdev = 444 container_of(core_vdev, struct vfio_pci_core_device, vdev); 445 446 if (vdev->sriov_pf_core_dev) { 447 mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock); 448 WARN_ON(!vdev->sriov_pf_core_dev->vf_token->users); 449 vdev->sriov_pf_core_dev->vf_token->users--; 450 mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock); 451 } 452 vfio_spapr_pci_eeh_release(vdev->pdev); 453 vfio_pci_core_disable(vdev); 454 455 mutex_lock(&vdev->igate); 456 if (vdev->err_trigger) { 457 eventfd_ctx_put(vdev->err_trigger); 458 vdev->err_trigger = NULL; 459 } 460 if (vdev->req_trigger) { 461 eventfd_ctx_put(vdev->req_trigger); 462 vdev->req_trigger = NULL; 463 } 464 mutex_unlock(&vdev->igate); 465 } 466 EXPORT_SYMBOL_GPL(vfio_pci_core_close_device); 467 468 void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev) 469 { 470 vfio_pci_probe_mmaps(vdev); 471 vfio_spapr_pci_eeh_open(vdev->pdev); 472 473 if (vdev->sriov_pf_core_dev) { 474 mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock); 475 vdev->sriov_pf_core_dev->vf_token->users++; 476 mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock); 477 } 478 } 479 EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable); 480 481 static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type) 482 { 483 if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) { 484 u8 pin; 485 486 if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || 487 vdev->nointx || vdev->pdev->is_virtfn) 488 return 0; 489 490 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin); 491 492 return pin ? 1 : 0; 493 } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) { 494 u8 pos; 495 u16 flags; 496 497 pos = vdev->pdev->msi_cap; 498 if (pos) { 499 pci_read_config_word(vdev->pdev, 500 pos + PCI_MSI_FLAGS, &flags); 501 return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1); 502 } 503 } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) { 504 u8 pos; 505 u16 flags; 506 507 pos = vdev->pdev->msix_cap; 508 if (pos) { 509 pci_read_config_word(vdev->pdev, 510 pos + PCI_MSIX_FLAGS, &flags); 511 512 return (flags & PCI_MSIX_FLAGS_QSIZE) + 1; 513 } 514 } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) { 515 if (pci_is_pcie(vdev->pdev)) 516 return 1; 517 } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) { 518 return 1; 519 } 520 521 return 0; 522 } 523 524 static int vfio_pci_count_devs(struct pci_dev *pdev, void *data) 525 { 526 (*(int *)data)++; 527 return 0; 528 } 529 530 struct vfio_pci_fill_info { 531 int max; 532 int cur; 533 struct vfio_pci_dependent_device *devices; 534 }; 535 536 static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data) 537 { 538 struct vfio_pci_fill_info *fill = data; 539 struct iommu_group *iommu_group; 540 541 if (fill->cur == fill->max) 542 return -EAGAIN; /* Something changed, try again */ 543 544 iommu_group = iommu_group_get(&pdev->dev); 545 if (!iommu_group) 546 return -EPERM; /* Cannot reset non-isolated devices */ 547 548 fill->devices[fill->cur].group_id = iommu_group_id(iommu_group); 549 fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus); 550 fill->devices[fill->cur].bus = pdev->bus->number; 551 fill->devices[fill->cur].devfn = pdev->devfn; 552 fill->cur++; 553 iommu_group_put(iommu_group); 554 return 0; 555 } 556 557 struct vfio_pci_group_info { 558 int count; 559 struct vfio_group **groups; 560 }; 561 562 static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot) 563 { 564 for (; pdev; pdev = pdev->bus->self) 565 if (pdev->bus == slot->bus) 566 return (pdev->slot == slot); 567 return false; 568 } 569 570 struct vfio_pci_walk_info { 571 int (*fn)(struct pci_dev *pdev, void *data); 572 void *data; 573 struct pci_dev *pdev; 574 bool slot; 575 int ret; 576 }; 577 578 static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data) 579 { 580 struct vfio_pci_walk_info *walk = data; 581 582 if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot)) 583 walk->ret = walk->fn(pdev, walk->data); 584 585 return walk->ret; 586 } 587 588 static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev, 589 int (*fn)(struct pci_dev *, 590 void *data), void *data, 591 bool slot) 592 { 593 struct vfio_pci_walk_info walk = { 594 .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0, 595 }; 596 597 pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk); 598 599 return walk.ret; 600 } 601 602 static int msix_mmappable_cap(struct vfio_pci_core_device *vdev, 603 struct vfio_info_cap *caps) 604 { 605 struct vfio_info_cap_header header = { 606 .id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE, 607 .version = 1 608 }; 609 610 return vfio_info_add_capability(caps, &header, sizeof(header)); 611 } 612 613 int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev, 614 unsigned int type, unsigned int subtype, 615 const struct vfio_pci_regops *ops, 616 size_t size, u32 flags, void *data) 617 { 618 struct vfio_pci_region *region; 619 620 region = krealloc(vdev->region, 621 (vdev->num_regions + 1) * sizeof(*region), 622 GFP_KERNEL); 623 if (!region) 624 return -ENOMEM; 625 626 vdev->region = region; 627 vdev->region[vdev->num_regions].type = type; 628 vdev->region[vdev->num_regions].subtype = subtype; 629 vdev->region[vdev->num_regions].ops = ops; 630 vdev->region[vdev->num_regions].size = size; 631 vdev->region[vdev->num_regions].flags = flags; 632 vdev->region[vdev->num_regions].data = data; 633 634 vdev->num_regions++; 635 636 return 0; 637 } 638 EXPORT_SYMBOL_GPL(vfio_pci_register_dev_region); 639 640 long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, 641 unsigned long arg) 642 { 643 struct vfio_pci_core_device *vdev = 644 container_of(core_vdev, struct vfio_pci_core_device, vdev); 645 unsigned long minsz; 646 647 if (cmd == VFIO_DEVICE_GET_INFO) { 648 struct vfio_device_info info; 649 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 650 unsigned long capsz; 651 int ret; 652 653 minsz = offsetofend(struct vfio_device_info, num_irqs); 654 655 /* For backward compatibility, cannot require this */ 656 capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset); 657 658 if (copy_from_user(&info, (void __user *)arg, minsz)) 659 return -EFAULT; 660 661 if (info.argsz < minsz) 662 return -EINVAL; 663 664 if (info.argsz >= capsz) { 665 minsz = capsz; 666 info.cap_offset = 0; 667 } 668 669 info.flags = VFIO_DEVICE_FLAGS_PCI; 670 671 if (vdev->reset_works) 672 info.flags |= VFIO_DEVICE_FLAGS_RESET; 673 674 info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; 675 info.num_irqs = VFIO_PCI_NUM_IRQS; 676 677 ret = vfio_pci_info_zdev_add_caps(vdev, &caps); 678 if (ret && ret != -ENODEV) { 679 pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n"); 680 return ret; 681 } 682 683 if (caps.size) { 684 info.flags |= VFIO_DEVICE_FLAGS_CAPS; 685 if (info.argsz < sizeof(info) + caps.size) { 686 info.argsz = sizeof(info) + caps.size; 687 } else { 688 vfio_info_cap_shift(&caps, sizeof(info)); 689 if (copy_to_user((void __user *)arg + 690 sizeof(info), caps.buf, 691 caps.size)) { 692 kfree(caps.buf); 693 return -EFAULT; 694 } 695 info.cap_offset = sizeof(info); 696 } 697 698 kfree(caps.buf); 699 } 700 701 return copy_to_user((void __user *)arg, &info, minsz) ? 702 -EFAULT : 0; 703 704 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 705 struct pci_dev *pdev = vdev->pdev; 706 struct vfio_region_info info; 707 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 708 int i, ret; 709 710 minsz = offsetofend(struct vfio_region_info, offset); 711 712 if (copy_from_user(&info, (void __user *)arg, minsz)) 713 return -EFAULT; 714 715 if (info.argsz < minsz) 716 return -EINVAL; 717 718 switch (info.index) { 719 case VFIO_PCI_CONFIG_REGION_INDEX: 720 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 721 info.size = pdev->cfg_size; 722 info.flags = VFIO_REGION_INFO_FLAG_READ | 723 VFIO_REGION_INFO_FLAG_WRITE; 724 break; 725 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 726 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 727 info.size = pci_resource_len(pdev, info.index); 728 if (!info.size) { 729 info.flags = 0; 730 break; 731 } 732 733 info.flags = VFIO_REGION_INFO_FLAG_READ | 734 VFIO_REGION_INFO_FLAG_WRITE; 735 if (vdev->bar_mmap_supported[info.index]) { 736 info.flags |= VFIO_REGION_INFO_FLAG_MMAP; 737 if (info.index == vdev->msix_bar) { 738 ret = msix_mmappable_cap(vdev, &caps); 739 if (ret) 740 return ret; 741 } 742 } 743 744 break; 745 case VFIO_PCI_ROM_REGION_INDEX: 746 { 747 void __iomem *io; 748 size_t size; 749 u16 cmd; 750 751 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 752 info.flags = 0; 753 754 /* Report the BAR size, not the ROM size */ 755 info.size = pci_resource_len(pdev, info.index); 756 if (!info.size) { 757 /* Shadow ROMs appear as PCI option ROMs */ 758 if (pdev->resource[PCI_ROM_RESOURCE].flags & 759 IORESOURCE_ROM_SHADOW) 760 info.size = 0x20000; 761 else 762 break; 763 } 764 765 /* 766 * Is it really there? Enable memory decode for 767 * implicit access in pci_map_rom(). 768 */ 769 cmd = vfio_pci_memory_lock_and_enable(vdev); 770 io = pci_map_rom(pdev, &size); 771 if (io) { 772 info.flags = VFIO_REGION_INFO_FLAG_READ; 773 pci_unmap_rom(pdev, io); 774 } else { 775 info.size = 0; 776 } 777 vfio_pci_memory_unlock_and_restore(vdev, cmd); 778 779 break; 780 } 781 case VFIO_PCI_VGA_REGION_INDEX: 782 if (!vdev->has_vga) 783 return -EINVAL; 784 785 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 786 info.size = 0xc0000; 787 info.flags = VFIO_REGION_INFO_FLAG_READ | 788 VFIO_REGION_INFO_FLAG_WRITE; 789 790 break; 791 default: 792 { 793 struct vfio_region_info_cap_type cap_type = { 794 .header.id = VFIO_REGION_INFO_CAP_TYPE, 795 .header.version = 1 }; 796 797 if (info.index >= 798 VFIO_PCI_NUM_REGIONS + vdev->num_regions) 799 return -EINVAL; 800 info.index = array_index_nospec(info.index, 801 VFIO_PCI_NUM_REGIONS + 802 vdev->num_regions); 803 804 i = info.index - VFIO_PCI_NUM_REGIONS; 805 806 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 807 info.size = vdev->region[i].size; 808 info.flags = vdev->region[i].flags; 809 810 cap_type.type = vdev->region[i].type; 811 cap_type.subtype = vdev->region[i].subtype; 812 813 ret = vfio_info_add_capability(&caps, &cap_type.header, 814 sizeof(cap_type)); 815 if (ret) 816 return ret; 817 818 if (vdev->region[i].ops->add_capability) { 819 ret = vdev->region[i].ops->add_capability(vdev, 820 &vdev->region[i], &caps); 821 if (ret) 822 return ret; 823 } 824 } 825 } 826 827 if (caps.size) { 828 info.flags |= VFIO_REGION_INFO_FLAG_CAPS; 829 if (info.argsz < sizeof(info) + caps.size) { 830 info.argsz = sizeof(info) + caps.size; 831 info.cap_offset = 0; 832 } else { 833 vfio_info_cap_shift(&caps, sizeof(info)); 834 if (copy_to_user((void __user *)arg + 835 sizeof(info), caps.buf, 836 caps.size)) { 837 kfree(caps.buf); 838 return -EFAULT; 839 } 840 info.cap_offset = sizeof(info); 841 } 842 843 kfree(caps.buf); 844 } 845 846 return copy_to_user((void __user *)arg, &info, minsz) ? 847 -EFAULT : 0; 848 849 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 850 struct vfio_irq_info info; 851 852 minsz = offsetofend(struct vfio_irq_info, count); 853 854 if (copy_from_user(&info, (void __user *)arg, minsz)) 855 return -EFAULT; 856 857 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 858 return -EINVAL; 859 860 switch (info.index) { 861 case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX: 862 case VFIO_PCI_REQ_IRQ_INDEX: 863 break; 864 case VFIO_PCI_ERR_IRQ_INDEX: 865 if (pci_is_pcie(vdev->pdev)) 866 break; 867 fallthrough; 868 default: 869 return -EINVAL; 870 } 871 872 info.flags = VFIO_IRQ_INFO_EVENTFD; 873 874 info.count = vfio_pci_get_irq_count(vdev, info.index); 875 876 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 877 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 878 VFIO_IRQ_INFO_AUTOMASKED); 879 else 880 info.flags |= VFIO_IRQ_INFO_NORESIZE; 881 882 return copy_to_user((void __user *)arg, &info, minsz) ? 883 -EFAULT : 0; 884 885 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 886 struct vfio_irq_set hdr; 887 u8 *data = NULL; 888 int max, ret = 0; 889 size_t data_size = 0; 890 891 minsz = offsetofend(struct vfio_irq_set, count); 892 893 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 894 return -EFAULT; 895 896 max = vfio_pci_get_irq_count(vdev, hdr.index); 897 898 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, 899 VFIO_PCI_NUM_IRQS, &data_size); 900 if (ret) 901 return ret; 902 903 if (data_size) { 904 data = memdup_user((void __user *)(arg + minsz), 905 data_size); 906 if (IS_ERR(data)) 907 return PTR_ERR(data); 908 } 909 910 mutex_lock(&vdev->igate); 911 912 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, 913 hdr.start, hdr.count, data); 914 915 mutex_unlock(&vdev->igate); 916 kfree(data); 917 918 return ret; 919 920 } else if (cmd == VFIO_DEVICE_RESET) { 921 int ret; 922 923 if (!vdev->reset_works) 924 return -EINVAL; 925 926 vfio_pci_zap_and_down_write_memory_lock(vdev); 927 928 /* 929 * This function can be invoked while the power state is non-D0. 930 * If pci_try_reset_function() has been called while the power 931 * state is non-D0, then pci_try_reset_function() will 932 * internally set the power state to D0 without vfio driver 933 * involvement. For the devices which have NoSoftRst-, the 934 * reset function can cause the PCI config space reset without 935 * restoring the original state (saved locally in 936 * 'vdev->pm_save'). 937 */ 938 vfio_pci_set_power_state(vdev, PCI_D0); 939 940 ret = pci_try_reset_function(vdev->pdev); 941 up_write(&vdev->memory_lock); 942 943 return ret; 944 945 } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) { 946 struct vfio_pci_hot_reset_info hdr; 947 struct vfio_pci_fill_info fill = { 0 }; 948 struct vfio_pci_dependent_device *devices = NULL; 949 bool slot = false; 950 int ret = 0; 951 952 minsz = offsetofend(struct vfio_pci_hot_reset_info, count); 953 954 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 955 return -EFAULT; 956 957 if (hdr.argsz < minsz) 958 return -EINVAL; 959 960 hdr.flags = 0; 961 962 /* Can we do a slot or bus reset or neither? */ 963 if (!pci_probe_reset_slot(vdev->pdev->slot)) 964 slot = true; 965 else if (pci_probe_reset_bus(vdev->pdev->bus)) 966 return -ENODEV; 967 968 /* How many devices are affected? */ 969 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 970 vfio_pci_count_devs, 971 &fill.max, slot); 972 if (ret) 973 return ret; 974 975 WARN_ON(!fill.max); /* Should always be at least one */ 976 977 /* 978 * If there's enough space, fill it now, otherwise return 979 * -ENOSPC and the number of devices affected. 980 */ 981 if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) { 982 ret = -ENOSPC; 983 hdr.count = fill.max; 984 goto reset_info_exit; 985 } 986 987 devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL); 988 if (!devices) 989 return -ENOMEM; 990 991 fill.devices = devices; 992 993 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 994 vfio_pci_fill_devs, 995 &fill, slot); 996 997 /* 998 * If a device was removed between counting and filling, 999 * we may come up short of fill.max. If a device was 1000 * added, we'll have a return of -EAGAIN above. 1001 */ 1002 if (!ret) 1003 hdr.count = fill.cur; 1004 1005 reset_info_exit: 1006 if (copy_to_user((void __user *)arg, &hdr, minsz)) 1007 ret = -EFAULT; 1008 1009 if (!ret) { 1010 if (copy_to_user((void __user *)(arg + minsz), devices, 1011 hdr.count * sizeof(*devices))) 1012 ret = -EFAULT; 1013 } 1014 1015 kfree(devices); 1016 return ret; 1017 1018 } else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) { 1019 struct vfio_pci_hot_reset hdr; 1020 int32_t *group_fds; 1021 struct vfio_group **groups; 1022 struct vfio_pci_group_info info; 1023 bool slot = false; 1024 int group_idx, count = 0, ret = 0; 1025 1026 minsz = offsetofend(struct vfio_pci_hot_reset, count); 1027 1028 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1029 return -EFAULT; 1030 1031 if (hdr.argsz < minsz || hdr.flags) 1032 return -EINVAL; 1033 1034 /* Can we do a slot or bus reset or neither? */ 1035 if (!pci_probe_reset_slot(vdev->pdev->slot)) 1036 slot = true; 1037 else if (pci_probe_reset_bus(vdev->pdev->bus)) 1038 return -ENODEV; 1039 1040 /* 1041 * We can't let userspace give us an arbitrarily large 1042 * buffer to copy, so verify how many we think there 1043 * could be. Note groups can have multiple devices so 1044 * one group per device is the max. 1045 */ 1046 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 1047 vfio_pci_count_devs, 1048 &count, slot); 1049 if (ret) 1050 return ret; 1051 1052 /* Somewhere between 1 and count is OK */ 1053 if (!hdr.count || hdr.count > count) 1054 return -EINVAL; 1055 1056 group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL); 1057 groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL); 1058 if (!group_fds || !groups) { 1059 kfree(group_fds); 1060 kfree(groups); 1061 return -ENOMEM; 1062 } 1063 1064 if (copy_from_user(group_fds, (void __user *)(arg + minsz), 1065 hdr.count * sizeof(*group_fds))) { 1066 kfree(group_fds); 1067 kfree(groups); 1068 return -EFAULT; 1069 } 1070 1071 /* 1072 * For each group_fd, get the group through the vfio external 1073 * user interface and store the group and iommu ID. This 1074 * ensures the group is held across the reset. 1075 */ 1076 for (group_idx = 0; group_idx < hdr.count; group_idx++) { 1077 struct vfio_group *group; 1078 struct fd f = fdget(group_fds[group_idx]); 1079 if (!f.file) { 1080 ret = -EBADF; 1081 break; 1082 } 1083 1084 group = vfio_group_get_external_user(f.file); 1085 fdput(f); 1086 if (IS_ERR(group)) { 1087 ret = PTR_ERR(group); 1088 break; 1089 } 1090 1091 groups[group_idx] = group; 1092 } 1093 1094 kfree(group_fds); 1095 1096 /* release reference to groups on error */ 1097 if (ret) 1098 goto hot_reset_release; 1099 1100 info.count = hdr.count; 1101 info.groups = groups; 1102 1103 ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info); 1104 1105 hot_reset_release: 1106 for (group_idx--; group_idx >= 0; group_idx--) 1107 vfio_group_put_external_user(groups[group_idx]); 1108 1109 kfree(groups); 1110 return ret; 1111 } else if (cmd == VFIO_DEVICE_IOEVENTFD) { 1112 struct vfio_device_ioeventfd ioeventfd; 1113 int count; 1114 1115 minsz = offsetofend(struct vfio_device_ioeventfd, fd); 1116 1117 if (copy_from_user(&ioeventfd, (void __user *)arg, minsz)) 1118 return -EFAULT; 1119 1120 if (ioeventfd.argsz < minsz) 1121 return -EINVAL; 1122 1123 if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK) 1124 return -EINVAL; 1125 1126 count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK; 1127 1128 if (hweight8(count) != 1 || ioeventfd.fd < -1) 1129 return -EINVAL; 1130 1131 return vfio_pci_ioeventfd(vdev, ioeventfd.offset, 1132 ioeventfd.data, count, ioeventfd.fd); 1133 } 1134 return -ENOTTY; 1135 } 1136 EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl); 1137 1138 static int vfio_pci_core_feature_token(struct vfio_device *device, u32 flags, 1139 void __user *arg, size_t argsz) 1140 { 1141 struct vfio_pci_core_device *vdev = 1142 container_of(device, struct vfio_pci_core_device, vdev); 1143 uuid_t uuid; 1144 int ret; 1145 1146 if (!vdev->vf_token) 1147 return -ENOTTY; 1148 /* 1149 * We do not support GET of the VF Token UUID as this could 1150 * expose the token of the previous device user. 1151 */ 1152 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 1153 sizeof(uuid)); 1154 if (ret != 1) 1155 return ret; 1156 1157 if (copy_from_user(&uuid, arg, sizeof(uuid))) 1158 return -EFAULT; 1159 1160 mutex_lock(&vdev->vf_token->lock); 1161 uuid_copy(&vdev->vf_token->uuid, &uuid); 1162 mutex_unlock(&vdev->vf_token->lock); 1163 return 0; 1164 } 1165 1166 int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags, 1167 void __user *arg, size_t argsz) 1168 { 1169 switch (flags & VFIO_DEVICE_FEATURE_MASK) { 1170 case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN: 1171 return vfio_pci_core_feature_token(device, flags, arg, argsz); 1172 default: 1173 return -ENOTTY; 1174 } 1175 } 1176 EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl_feature); 1177 1178 static ssize_t vfio_pci_rw(struct vfio_pci_core_device *vdev, char __user *buf, 1179 size_t count, loff_t *ppos, bool iswrite) 1180 { 1181 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 1182 1183 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) 1184 return -EINVAL; 1185 1186 switch (index) { 1187 case VFIO_PCI_CONFIG_REGION_INDEX: 1188 return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite); 1189 1190 case VFIO_PCI_ROM_REGION_INDEX: 1191 if (iswrite) 1192 return -EINVAL; 1193 return vfio_pci_bar_rw(vdev, buf, count, ppos, false); 1194 1195 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1196 return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite); 1197 1198 case VFIO_PCI_VGA_REGION_INDEX: 1199 return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite); 1200 default: 1201 index -= VFIO_PCI_NUM_REGIONS; 1202 return vdev->region[index].ops->rw(vdev, buf, 1203 count, ppos, iswrite); 1204 } 1205 1206 return -EINVAL; 1207 } 1208 1209 ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, 1210 size_t count, loff_t *ppos) 1211 { 1212 struct vfio_pci_core_device *vdev = 1213 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1214 1215 if (!count) 1216 return 0; 1217 1218 return vfio_pci_rw(vdev, buf, count, ppos, false); 1219 } 1220 EXPORT_SYMBOL_GPL(vfio_pci_core_read); 1221 1222 ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, 1223 size_t count, loff_t *ppos) 1224 { 1225 struct vfio_pci_core_device *vdev = 1226 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1227 1228 if (!count) 1229 return 0; 1230 1231 return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true); 1232 } 1233 EXPORT_SYMBOL_GPL(vfio_pci_core_write); 1234 1235 /* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */ 1236 static int vfio_pci_zap_and_vma_lock(struct vfio_pci_core_device *vdev, bool try) 1237 { 1238 struct vfio_pci_mmap_vma *mmap_vma, *tmp; 1239 1240 /* 1241 * Lock ordering: 1242 * vma_lock is nested under mmap_lock for vm_ops callback paths. 1243 * The memory_lock semaphore is used by both code paths calling 1244 * into this function to zap vmas and the vm_ops.fault callback 1245 * to protect the memory enable state of the device. 1246 * 1247 * When zapping vmas we need to maintain the mmap_lock => vma_lock 1248 * ordering, which requires using vma_lock to walk vma_list to 1249 * acquire an mm, then dropping vma_lock to get the mmap_lock and 1250 * reacquiring vma_lock. This logic is derived from similar 1251 * requirements in uverbs_user_mmap_disassociate(). 1252 * 1253 * mmap_lock must always be the top-level lock when it is taken. 1254 * Therefore we can only hold the memory_lock write lock when 1255 * vma_list is empty, as we'd need to take mmap_lock to clear 1256 * entries. vma_list can only be guaranteed empty when holding 1257 * vma_lock, thus memory_lock is nested under vma_lock. 1258 * 1259 * This enables the vm_ops.fault callback to acquire vma_lock, 1260 * followed by memory_lock read lock, while already holding 1261 * mmap_lock without risk of deadlock. 1262 */ 1263 while (1) { 1264 struct mm_struct *mm = NULL; 1265 1266 if (try) { 1267 if (!mutex_trylock(&vdev->vma_lock)) 1268 return 0; 1269 } else { 1270 mutex_lock(&vdev->vma_lock); 1271 } 1272 while (!list_empty(&vdev->vma_list)) { 1273 mmap_vma = list_first_entry(&vdev->vma_list, 1274 struct vfio_pci_mmap_vma, 1275 vma_next); 1276 mm = mmap_vma->vma->vm_mm; 1277 if (mmget_not_zero(mm)) 1278 break; 1279 1280 list_del(&mmap_vma->vma_next); 1281 kfree(mmap_vma); 1282 mm = NULL; 1283 } 1284 if (!mm) 1285 return 1; 1286 mutex_unlock(&vdev->vma_lock); 1287 1288 if (try) { 1289 if (!mmap_read_trylock(mm)) { 1290 mmput(mm); 1291 return 0; 1292 } 1293 } else { 1294 mmap_read_lock(mm); 1295 } 1296 if (try) { 1297 if (!mutex_trylock(&vdev->vma_lock)) { 1298 mmap_read_unlock(mm); 1299 mmput(mm); 1300 return 0; 1301 } 1302 } else { 1303 mutex_lock(&vdev->vma_lock); 1304 } 1305 list_for_each_entry_safe(mmap_vma, tmp, 1306 &vdev->vma_list, vma_next) { 1307 struct vm_area_struct *vma = mmap_vma->vma; 1308 1309 if (vma->vm_mm != mm) 1310 continue; 1311 1312 list_del(&mmap_vma->vma_next); 1313 kfree(mmap_vma); 1314 1315 zap_vma_ptes(vma, vma->vm_start, 1316 vma->vm_end - vma->vm_start); 1317 } 1318 mutex_unlock(&vdev->vma_lock); 1319 mmap_read_unlock(mm); 1320 mmput(mm); 1321 } 1322 } 1323 1324 void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev) 1325 { 1326 vfio_pci_zap_and_vma_lock(vdev, false); 1327 down_write(&vdev->memory_lock); 1328 mutex_unlock(&vdev->vma_lock); 1329 } 1330 1331 u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev) 1332 { 1333 u16 cmd; 1334 1335 down_write(&vdev->memory_lock); 1336 pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd); 1337 if (!(cmd & PCI_COMMAND_MEMORY)) 1338 pci_write_config_word(vdev->pdev, PCI_COMMAND, 1339 cmd | PCI_COMMAND_MEMORY); 1340 1341 return cmd; 1342 } 1343 1344 void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, u16 cmd) 1345 { 1346 pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd); 1347 up_write(&vdev->memory_lock); 1348 } 1349 1350 /* Caller holds vma_lock */ 1351 static int __vfio_pci_add_vma(struct vfio_pci_core_device *vdev, 1352 struct vm_area_struct *vma) 1353 { 1354 struct vfio_pci_mmap_vma *mmap_vma; 1355 1356 mmap_vma = kmalloc(sizeof(*mmap_vma), GFP_KERNEL); 1357 if (!mmap_vma) 1358 return -ENOMEM; 1359 1360 mmap_vma->vma = vma; 1361 list_add(&mmap_vma->vma_next, &vdev->vma_list); 1362 1363 return 0; 1364 } 1365 1366 /* 1367 * Zap mmaps on open so that we can fault them in on access and therefore 1368 * our vma_list only tracks mappings accessed since last zap. 1369 */ 1370 static void vfio_pci_mmap_open(struct vm_area_struct *vma) 1371 { 1372 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); 1373 } 1374 1375 static void vfio_pci_mmap_close(struct vm_area_struct *vma) 1376 { 1377 struct vfio_pci_core_device *vdev = vma->vm_private_data; 1378 struct vfio_pci_mmap_vma *mmap_vma; 1379 1380 mutex_lock(&vdev->vma_lock); 1381 list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) { 1382 if (mmap_vma->vma == vma) { 1383 list_del(&mmap_vma->vma_next); 1384 kfree(mmap_vma); 1385 break; 1386 } 1387 } 1388 mutex_unlock(&vdev->vma_lock); 1389 } 1390 1391 static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf) 1392 { 1393 struct vm_area_struct *vma = vmf->vma; 1394 struct vfio_pci_core_device *vdev = vma->vm_private_data; 1395 struct vfio_pci_mmap_vma *mmap_vma; 1396 vm_fault_t ret = VM_FAULT_NOPAGE; 1397 1398 mutex_lock(&vdev->vma_lock); 1399 down_read(&vdev->memory_lock); 1400 1401 if (!__vfio_pci_memory_enabled(vdev)) { 1402 ret = VM_FAULT_SIGBUS; 1403 goto up_out; 1404 } 1405 1406 /* 1407 * We populate the whole vma on fault, so we need to test whether 1408 * the vma has already been mapped, such as for concurrent faults 1409 * to the same vma. io_remap_pfn_range() will trigger a BUG_ON if 1410 * we ask it to fill the same range again. 1411 */ 1412 list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) { 1413 if (mmap_vma->vma == vma) 1414 goto up_out; 1415 } 1416 1417 if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 1418 vma->vm_end - vma->vm_start, 1419 vma->vm_page_prot)) { 1420 ret = VM_FAULT_SIGBUS; 1421 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); 1422 goto up_out; 1423 } 1424 1425 if (__vfio_pci_add_vma(vdev, vma)) { 1426 ret = VM_FAULT_OOM; 1427 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); 1428 } 1429 1430 up_out: 1431 up_read(&vdev->memory_lock); 1432 mutex_unlock(&vdev->vma_lock); 1433 return ret; 1434 } 1435 1436 static const struct vm_operations_struct vfio_pci_mmap_ops = { 1437 .open = vfio_pci_mmap_open, 1438 .close = vfio_pci_mmap_close, 1439 .fault = vfio_pci_mmap_fault, 1440 }; 1441 1442 int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma) 1443 { 1444 struct vfio_pci_core_device *vdev = 1445 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1446 struct pci_dev *pdev = vdev->pdev; 1447 unsigned int index; 1448 u64 phys_len, req_len, pgoff, req_start; 1449 int ret; 1450 1451 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 1452 1453 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) 1454 return -EINVAL; 1455 if (vma->vm_end < vma->vm_start) 1456 return -EINVAL; 1457 if ((vma->vm_flags & VM_SHARED) == 0) 1458 return -EINVAL; 1459 if (index >= VFIO_PCI_NUM_REGIONS) { 1460 int regnum = index - VFIO_PCI_NUM_REGIONS; 1461 struct vfio_pci_region *region = vdev->region + regnum; 1462 1463 if (region->ops && region->ops->mmap && 1464 (region->flags & VFIO_REGION_INFO_FLAG_MMAP)) 1465 return region->ops->mmap(vdev, region, vma); 1466 return -EINVAL; 1467 } 1468 if (index >= VFIO_PCI_ROM_REGION_INDEX) 1469 return -EINVAL; 1470 if (!vdev->bar_mmap_supported[index]) 1471 return -EINVAL; 1472 1473 phys_len = PAGE_ALIGN(pci_resource_len(pdev, index)); 1474 req_len = vma->vm_end - vma->vm_start; 1475 pgoff = vma->vm_pgoff & 1476 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 1477 req_start = pgoff << PAGE_SHIFT; 1478 1479 if (req_start + req_len > phys_len) 1480 return -EINVAL; 1481 1482 /* 1483 * Even though we don't make use of the barmap for the mmap, 1484 * we need to request the region and the barmap tracks that. 1485 */ 1486 if (!vdev->barmap[index]) { 1487 ret = pci_request_selected_regions(pdev, 1488 1 << index, "vfio-pci"); 1489 if (ret) 1490 return ret; 1491 1492 vdev->barmap[index] = pci_iomap(pdev, index, 0); 1493 if (!vdev->barmap[index]) { 1494 pci_release_selected_regions(pdev, 1 << index); 1495 return -ENOMEM; 1496 } 1497 } 1498 1499 vma->vm_private_data = vdev; 1500 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 1501 vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff; 1502 1503 /* 1504 * See remap_pfn_range(), called from vfio_pci_fault() but we can't 1505 * change vm_flags within the fault handler. Set them now. 1506 */ 1507 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 1508 vma->vm_ops = &vfio_pci_mmap_ops; 1509 1510 return 0; 1511 } 1512 EXPORT_SYMBOL_GPL(vfio_pci_core_mmap); 1513 1514 void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count) 1515 { 1516 struct vfio_pci_core_device *vdev = 1517 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1518 struct pci_dev *pdev = vdev->pdev; 1519 1520 mutex_lock(&vdev->igate); 1521 1522 if (vdev->req_trigger) { 1523 if (!(count % 10)) 1524 pci_notice_ratelimited(pdev, 1525 "Relaying device request to user (#%u)\n", 1526 count); 1527 eventfd_signal(vdev->req_trigger, 1); 1528 } else if (count == 0) { 1529 pci_warn(pdev, 1530 "No device request channel registered, blocked until released by user\n"); 1531 } 1532 1533 mutex_unlock(&vdev->igate); 1534 } 1535 EXPORT_SYMBOL_GPL(vfio_pci_core_request); 1536 1537 static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev, 1538 bool vf_token, uuid_t *uuid) 1539 { 1540 /* 1541 * There's always some degree of trust or collaboration between SR-IOV 1542 * PF and VFs, even if just that the PF hosts the SR-IOV capability and 1543 * can disrupt VFs with a reset, but often the PF has more explicit 1544 * access to deny service to the VF or access data passed through the 1545 * VF. We therefore require an opt-in via a shared VF token (UUID) to 1546 * represent this trust. This both prevents that a VF driver might 1547 * assume the PF driver is a trusted, in-kernel driver, and also that 1548 * a PF driver might be replaced with a rogue driver, unknown to in-use 1549 * VF drivers. 1550 * 1551 * Therefore when presented with a VF, if the PF is a vfio device and 1552 * it is bound to the vfio-pci driver, the user needs to provide a VF 1553 * token to access the device, in the form of appending a vf_token to 1554 * the device name, for example: 1555 * 1556 * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3" 1557 * 1558 * When presented with a PF which has VFs in use, the user must also 1559 * provide the current VF token to prove collaboration with existing 1560 * VF users. If VFs are not in use, the VF token provided for the PF 1561 * device will act to set the VF token. 1562 * 1563 * If the VF token is provided but unused, an error is generated. 1564 */ 1565 if (vdev->pdev->is_virtfn) { 1566 struct vfio_pci_core_device *pf_vdev = vdev->sriov_pf_core_dev; 1567 bool match; 1568 1569 if (!pf_vdev) { 1570 if (!vf_token) 1571 return 0; /* PF is not vfio-pci, no VF token */ 1572 1573 pci_info_ratelimited(vdev->pdev, 1574 "VF token incorrectly provided, PF not bound to vfio-pci\n"); 1575 return -EINVAL; 1576 } 1577 1578 if (!vf_token) { 1579 pci_info_ratelimited(vdev->pdev, 1580 "VF token required to access device\n"); 1581 return -EACCES; 1582 } 1583 1584 mutex_lock(&pf_vdev->vf_token->lock); 1585 match = uuid_equal(uuid, &pf_vdev->vf_token->uuid); 1586 mutex_unlock(&pf_vdev->vf_token->lock); 1587 1588 if (!match) { 1589 pci_info_ratelimited(vdev->pdev, 1590 "Incorrect VF token provided for device\n"); 1591 return -EACCES; 1592 } 1593 } else if (vdev->vf_token) { 1594 mutex_lock(&vdev->vf_token->lock); 1595 if (vdev->vf_token->users) { 1596 if (!vf_token) { 1597 mutex_unlock(&vdev->vf_token->lock); 1598 pci_info_ratelimited(vdev->pdev, 1599 "VF token required to access device\n"); 1600 return -EACCES; 1601 } 1602 1603 if (!uuid_equal(uuid, &vdev->vf_token->uuid)) { 1604 mutex_unlock(&vdev->vf_token->lock); 1605 pci_info_ratelimited(vdev->pdev, 1606 "Incorrect VF token provided for device\n"); 1607 return -EACCES; 1608 } 1609 } else if (vf_token) { 1610 uuid_copy(&vdev->vf_token->uuid, uuid); 1611 } 1612 1613 mutex_unlock(&vdev->vf_token->lock); 1614 } else if (vf_token) { 1615 pci_info_ratelimited(vdev->pdev, 1616 "VF token incorrectly provided, not a PF or VF\n"); 1617 return -EINVAL; 1618 } 1619 1620 return 0; 1621 } 1622 1623 #define VF_TOKEN_ARG "vf_token=" 1624 1625 int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf) 1626 { 1627 struct vfio_pci_core_device *vdev = 1628 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1629 bool vf_token = false; 1630 uuid_t uuid; 1631 int ret; 1632 1633 if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev)))) 1634 return 0; /* No match */ 1635 1636 if (strlen(buf) > strlen(pci_name(vdev->pdev))) { 1637 buf += strlen(pci_name(vdev->pdev)); 1638 1639 if (*buf != ' ') 1640 return 0; /* No match: non-whitespace after name */ 1641 1642 while (*buf) { 1643 if (*buf == ' ') { 1644 buf++; 1645 continue; 1646 } 1647 1648 if (!vf_token && !strncmp(buf, VF_TOKEN_ARG, 1649 strlen(VF_TOKEN_ARG))) { 1650 buf += strlen(VF_TOKEN_ARG); 1651 1652 if (strlen(buf) < UUID_STRING_LEN) 1653 return -EINVAL; 1654 1655 ret = uuid_parse(buf, &uuid); 1656 if (ret) 1657 return ret; 1658 1659 vf_token = true; 1660 buf += UUID_STRING_LEN; 1661 } else { 1662 /* Unknown/duplicate option */ 1663 return -EINVAL; 1664 } 1665 } 1666 } 1667 1668 ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid); 1669 if (ret) 1670 return ret; 1671 1672 return 1; /* Match */ 1673 } 1674 EXPORT_SYMBOL_GPL(vfio_pci_core_match); 1675 1676 static int vfio_pci_bus_notifier(struct notifier_block *nb, 1677 unsigned long action, void *data) 1678 { 1679 struct vfio_pci_core_device *vdev = container_of(nb, 1680 struct vfio_pci_core_device, nb); 1681 struct device *dev = data; 1682 struct pci_dev *pdev = to_pci_dev(dev); 1683 struct pci_dev *physfn = pci_physfn(pdev); 1684 1685 if (action == BUS_NOTIFY_ADD_DEVICE && 1686 pdev->is_virtfn && physfn == vdev->pdev) { 1687 pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n", 1688 pci_name(pdev)); 1689 pdev->driver_override = kasprintf(GFP_KERNEL, "%s", 1690 vdev->vdev.ops->name); 1691 } else if (action == BUS_NOTIFY_BOUND_DRIVER && 1692 pdev->is_virtfn && physfn == vdev->pdev) { 1693 struct pci_driver *drv = pci_dev_driver(pdev); 1694 1695 if (drv && drv != pci_dev_driver(vdev->pdev)) 1696 pci_warn(vdev->pdev, 1697 "VF %s bound to driver %s while PF bound to driver %s\n", 1698 pci_name(pdev), drv->name, 1699 pci_dev_driver(vdev->pdev)->name); 1700 } 1701 1702 return 0; 1703 } 1704 1705 static int vfio_pci_vf_init(struct vfio_pci_core_device *vdev) 1706 { 1707 struct pci_dev *pdev = vdev->pdev; 1708 struct vfio_pci_core_device *cur; 1709 struct pci_dev *physfn; 1710 int ret; 1711 1712 if (pdev->is_virtfn) { 1713 /* 1714 * If this VF was created by our vfio_pci_core_sriov_configure() 1715 * then we can find the PF vfio_pci_core_device now, and due to 1716 * the locking in pci_disable_sriov() it cannot change until 1717 * this VF device driver is removed. 1718 */ 1719 physfn = pci_physfn(vdev->pdev); 1720 mutex_lock(&vfio_pci_sriov_pfs_mutex); 1721 list_for_each_entry(cur, &vfio_pci_sriov_pfs, sriov_pfs_item) { 1722 if (cur->pdev == physfn) { 1723 vdev->sriov_pf_core_dev = cur; 1724 break; 1725 } 1726 } 1727 mutex_unlock(&vfio_pci_sriov_pfs_mutex); 1728 return 0; 1729 } 1730 1731 /* Not a SRIOV PF */ 1732 if (!pdev->is_physfn) 1733 return 0; 1734 1735 vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL); 1736 if (!vdev->vf_token) 1737 return -ENOMEM; 1738 1739 mutex_init(&vdev->vf_token->lock); 1740 uuid_gen(&vdev->vf_token->uuid); 1741 1742 vdev->nb.notifier_call = vfio_pci_bus_notifier; 1743 ret = bus_register_notifier(&pci_bus_type, &vdev->nb); 1744 if (ret) { 1745 kfree(vdev->vf_token); 1746 return ret; 1747 } 1748 return 0; 1749 } 1750 1751 static void vfio_pci_vf_uninit(struct vfio_pci_core_device *vdev) 1752 { 1753 if (!vdev->vf_token) 1754 return; 1755 1756 bus_unregister_notifier(&pci_bus_type, &vdev->nb); 1757 WARN_ON(vdev->vf_token->users); 1758 mutex_destroy(&vdev->vf_token->lock); 1759 kfree(vdev->vf_token); 1760 } 1761 1762 static int vfio_pci_vga_init(struct vfio_pci_core_device *vdev) 1763 { 1764 struct pci_dev *pdev = vdev->pdev; 1765 int ret; 1766 1767 if (!vfio_pci_is_vga(pdev)) 1768 return 0; 1769 1770 ret = vga_client_register(pdev, vfio_pci_set_decode); 1771 if (ret) 1772 return ret; 1773 vga_set_legacy_decoding(pdev, vfio_pci_set_decode(pdev, false)); 1774 return 0; 1775 } 1776 1777 static void vfio_pci_vga_uninit(struct vfio_pci_core_device *vdev) 1778 { 1779 struct pci_dev *pdev = vdev->pdev; 1780 1781 if (!vfio_pci_is_vga(pdev)) 1782 return; 1783 vga_client_unregister(pdev); 1784 vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | 1785 VGA_RSRC_LEGACY_IO | 1786 VGA_RSRC_LEGACY_MEM); 1787 } 1788 1789 void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev, 1790 struct pci_dev *pdev, 1791 const struct vfio_device_ops *vfio_pci_ops) 1792 { 1793 vfio_init_group_dev(&vdev->vdev, &pdev->dev, vfio_pci_ops); 1794 vdev->pdev = pdev; 1795 vdev->irq_type = VFIO_PCI_NUM_IRQS; 1796 mutex_init(&vdev->igate); 1797 spin_lock_init(&vdev->irqlock); 1798 mutex_init(&vdev->ioeventfds_lock); 1799 INIT_LIST_HEAD(&vdev->dummy_resources_list); 1800 INIT_LIST_HEAD(&vdev->ioeventfds_list); 1801 mutex_init(&vdev->vma_lock); 1802 INIT_LIST_HEAD(&vdev->vma_list); 1803 INIT_LIST_HEAD(&vdev->sriov_pfs_item); 1804 init_rwsem(&vdev->memory_lock); 1805 } 1806 EXPORT_SYMBOL_GPL(vfio_pci_core_init_device); 1807 1808 void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev) 1809 { 1810 mutex_destroy(&vdev->igate); 1811 mutex_destroy(&vdev->ioeventfds_lock); 1812 mutex_destroy(&vdev->vma_lock); 1813 vfio_uninit_group_dev(&vdev->vdev); 1814 kfree(vdev->region); 1815 kfree(vdev->pm_save); 1816 } 1817 EXPORT_SYMBOL_GPL(vfio_pci_core_uninit_device); 1818 1819 int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) 1820 { 1821 struct pci_dev *pdev = vdev->pdev; 1822 int ret; 1823 1824 if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL) 1825 return -EINVAL; 1826 1827 /* 1828 * Prevent binding to PFs with VFs enabled, the VFs might be in use 1829 * by the host or other users. We cannot capture the VFs if they 1830 * already exist, nor can we track VF users. Disabling SR-IOV here 1831 * would initiate removing the VFs, which would unbind the driver, 1832 * which is prone to blocking if that VF is also in use by vfio-pci. 1833 * Just reject these PFs and let the user sort it out. 1834 */ 1835 if (pci_num_vf(pdev)) { 1836 pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n"); 1837 return -EBUSY; 1838 } 1839 1840 if (pci_is_root_bus(pdev->bus)) { 1841 ret = vfio_assign_device_set(&vdev->vdev, vdev); 1842 } else if (!pci_probe_reset_slot(pdev->slot)) { 1843 ret = vfio_assign_device_set(&vdev->vdev, pdev->slot); 1844 } else { 1845 /* 1846 * If there is no slot reset support for this device, the whole 1847 * bus needs to be grouped together to support bus-wide resets. 1848 */ 1849 ret = vfio_assign_device_set(&vdev->vdev, pdev->bus); 1850 } 1851 1852 if (ret) 1853 return ret; 1854 ret = vfio_pci_vf_init(vdev); 1855 if (ret) 1856 return ret; 1857 ret = vfio_pci_vga_init(vdev); 1858 if (ret) 1859 goto out_vf; 1860 1861 vfio_pci_probe_power_state(vdev); 1862 1863 if (!disable_idle_d3) { 1864 /* 1865 * pci-core sets the device power state to an unknown value at 1866 * bootup and after being removed from a driver. The only 1867 * transition it allows from this unknown state is to D0, which 1868 * typically happens when a driver calls pci_enable_device(). 1869 * We're not ready to enable the device yet, but we do want to 1870 * be able to get to D3. Therefore first do a D0 transition 1871 * before going to D3. 1872 */ 1873 vfio_pci_set_power_state(vdev, PCI_D0); 1874 vfio_pci_set_power_state(vdev, PCI_D3hot); 1875 } 1876 1877 ret = vfio_register_group_dev(&vdev->vdev); 1878 if (ret) 1879 goto out_power; 1880 return 0; 1881 1882 out_power: 1883 if (!disable_idle_d3) 1884 vfio_pci_set_power_state(vdev, PCI_D0); 1885 out_vf: 1886 vfio_pci_vf_uninit(vdev); 1887 return ret; 1888 } 1889 EXPORT_SYMBOL_GPL(vfio_pci_core_register_device); 1890 1891 void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev) 1892 { 1893 struct pci_dev *pdev = vdev->pdev; 1894 1895 vfio_pci_core_sriov_configure(pdev, 0); 1896 1897 vfio_unregister_group_dev(&vdev->vdev); 1898 1899 vfio_pci_vf_uninit(vdev); 1900 vfio_pci_vga_uninit(vdev); 1901 1902 if (!disable_idle_d3) 1903 vfio_pci_set_power_state(vdev, PCI_D0); 1904 } 1905 EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device); 1906 1907 pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, 1908 pci_channel_state_t state) 1909 { 1910 struct vfio_pci_core_device *vdev; 1911 struct vfio_device *device; 1912 1913 device = vfio_device_get_from_dev(&pdev->dev); 1914 if (device == NULL) 1915 return PCI_ERS_RESULT_DISCONNECT; 1916 1917 vdev = container_of(device, struct vfio_pci_core_device, vdev); 1918 1919 mutex_lock(&vdev->igate); 1920 1921 if (vdev->err_trigger) 1922 eventfd_signal(vdev->err_trigger, 1); 1923 1924 mutex_unlock(&vdev->igate); 1925 1926 vfio_device_put(device); 1927 1928 return PCI_ERS_RESULT_CAN_RECOVER; 1929 } 1930 EXPORT_SYMBOL_GPL(vfio_pci_core_aer_err_detected); 1931 1932 int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn) 1933 { 1934 struct vfio_pci_core_device *vdev; 1935 struct vfio_device *device; 1936 int ret = 0; 1937 1938 device_lock_assert(&pdev->dev); 1939 1940 device = vfio_device_get_from_dev(&pdev->dev); 1941 if (!device) 1942 return -ENODEV; 1943 1944 vdev = container_of(device, struct vfio_pci_core_device, vdev); 1945 1946 if (nr_virtfn) { 1947 mutex_lock(&vfio_pci_sriov_pfs_mutex); 1948 /* 1949 * The thread that adds the vdev to the list is the only thread 1950 * that gets to call pci_enable_sriov() and we will only allow 1951 * it to be called once without going through 1952 * pci_disable_sriov() 1953 */ 1954 if (!list_empty(&vdev->sriov_pfs_item)) { 1955 ret = -EINVAL; 1956 goto out_unlock; 1957 } 1958 list_add_tail(&vdev->sriov_pfs_item, &vfio_pci_sriov_pfs); 1959 mutex_unlock(&vfio_pci_sriov_pfs_mutex); 1960 ret = pci_enable_sriov(pdev, nr_virtfn); 1961 if (ret) 1962 goto out_del; 1963 ret = nr_virtfn; 1964 goto out_put; 1965 } 1966 1967 pci_disable_sriov(pdev); 1968 1969 out_del: 1970 mutex_lock(&vfio_pci_sriov_pfs_mutex); 1971 list_del_init(&vdev->sriov_pfs_item); 1972 out_unlock: 1973 mutex_unlock(&vfio_pci_sriov_pfs_mutex); 1974 out_put: 1975 vfio_device_put(device); 1976 return ret; 1977 } 1978 EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure); 1979 1980 const struct pci_error_handlers vfio_pci_core_err_handlers = { 1981 .error_detected = vfio_pci_core_aer_err_detected, 1982 }; 1983 EXPORT_SYMBOL_GPL(vfio_pci_core_err_handlers); 1984 1985 static bool vfio_dev_in_groups(struct vfio_pci_core_device *vdev, 1986 struct vfio_pci_group_info *groups) 1987 { 1988 unsigned int i; 1989 1990 for (i = 0; i < groups->count; i++) 1991 if (groups->groups[i] == vdev->vdev.group) 1992 return true; 1993 return false; 1994 } 1995 1996 static int vfio_pci_is_device_in_set(struct pci_dev *pdev, void *data) 1997 { 1998 struct vfio_device_set *dev_set = data; 1999 struct vfio_device *cur; 2000 2001 list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 2002 if (cur->dev == &pdev->dev) 2003 return 0; 2004 return -EBUSY; 2005 } 2006 2007 /* 2008 * vfio-core considers a group to be viable and will create a vfio_device even 2009 * if some devices are bound to drivers like pci-stub or pcieport. Here we 2010 * require all PCI devices to be inside our dev_set since that ensures they stay 2011 * put and that every driver controlling the device can co-ordinate with the 2012 * device reset. 2013 * 2014 * Returns the pci_dev to pass to pci_reset_bus() if every PCI device to be 2015 * reset is inside the dev_set, and pci_reset_bus() can succeed. NULL otherwise. 2016 */ 2017 static struct pci_dev * 2018 vfio_pci_dev_set_resettable(struct vfio_device_set *dev_set) 2019 { 2020 struct pci_dev *pdev; 2021 2022 lockdep_assert_held(&dev_set->lock); 2023 2024 /* 2025 * By definition all PCI devices in the dev_set share the same PCI 2026 * reset, so any pci_dev will have the same outcomes for 2027 * pci_probe_reset_*() and pci_reset_bus(). 2028 */ 2029 pdev = list_first_entry(&dev_set->device_list, 2030 struct vfio_pci_core_device, 2031 vdev.dev_set_list)->pdev; 2032 2033 /* pci_reset_bus() is supported */ 2034 if (pci_probe_reset_slot(pdev->slot) && pci_probe_reset_bus(pdev->bus)) 2035 return NULL; 2036 2037 if (vfio_pci_for_each_slot_or_bus(pdev, vfio_pci_is_device_in_set, 2038 dev_set, 2039 !pci_probe_reset_slot(pdev->slot))) 2040 return NULL; 2041 return pdev; 2042 } 2043 2044 /* 2045 * We need to get memory_lock for each device, but devices can share mmap_lock, 2046 * therefore we need to zap and hold the vma_lock for each device, and only then 2047 * get each memory_lock. 2048 */ 2049 static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set, 2050 struct vfio_pci_group_info *groups) 2051 { 2052 struct vfio_pci_core_device *cur_mem; 2053 struct vfio_pci_core_device *cur_vma; 2054 struct vfio_pci_core_device *cur; 2055 struct pci_dev *pdev; 2056 bool is_mem = true; 2057 int ret; 2058 2059 mutex_lock(&dev_set->lock); 2060 cur_mem = list_first_entry(&dev_set->device_list, 2061 struct vfio_pci_core_device, 2062 vdev.dev_set_list); 2063 2064 pdev = vfio_pci_dev_set_resettable(dev_set); 2065 if (!pdev) { 2066 ret = -EINVAL; 2067 goto err_unlock; 2068 } 2069 2070 list_for_each_entry(cur_vma, &dev_set->device_list, vdev.dev_set_list) { 2071 /* 2072 * Test whether all the affected devices are contained by the 2073 * set of groups provided by the user. 2074 */ 2075 if (!vfio_dev_in_groups(cur_vma, groups)) { 2076 ret = -EINVAL; 2077 goto err_undo; 2078 } 2079 2080 /* 2081 * Locking multiple devices is prone to deadlock, runaway and 2082 * unwind if we hit contention. 2083 */ 2084 if (!vfio_pci_zap_and_vma_lock(cur_vma, true)) { 2085 ret = -EBUSY; 2086 goto err_undo; 2087 } 2088 } 2089 cur_vma = NULL; 2090 2091 list_for_each_entry(cur_mem, &dev_set->device_list, vdev.dev_set_list) { 2092 if (!down_write_trylock(&cur_mem->memory_lock)) { 2093 ret = -EBUSY; 2094 goto err_undo; 2095 } 2096 mutex_unlock(&cur_mem->vma_lock); 2097 } 2098 cur_mem = NULL; 2099 2100 /* 2101 * The pci_reset_bus() will reset all the devices in the bus. 2102 * The power state can be non-D0 for some of the devices in the bus. 2103 * For these devices, the pci_reset_bus() will internally set 2104 * the power state to D0 without vfio driver involvement. 2105 * For the devices which have NoSoftRst-, the reset function can 2106 * cause the PCI config space reset without restoring the original 2107 * state (saved locally in 'vdev->pm_save'). 2108 */ 2109 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) 2110 vfio_pci_set_power_state(cur, PCI_D0); 2111 2112 ret = pci_reset_bus(pdev); 2113 2114 err_undo: 2115 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) { 2116 if (cur == cur_mem) 2117 is_mem = false; 2118 if (cur == cur_vma) 2119 break; 2120 if (is_mem) 2121 up_write(&cur->memory_lock); 2122 else 2123 mutex_unlock(&cur->vma_lock); 2124 } 2125 err_unlock: 2126 mutex_unlock(&dev_set->lock); 2127 return ret; 2128 } 2129 2130 static bool vfio_pci_dev_set_needs_reset(struct vfio_device_set *dev_set) 2131 { 2132 struct vfio_pci_core_device *cur; 2133 bool needs_reset = false; 2134 2135 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) { 2136 /* No VFIO device in the set can have an open device FD */ 2137 if (cur->vdev.open_count) 2138 return false; 2139 needs_reset |= cur->needs_reset; 2140 } 2141 return needs_reset; 2142 } 2143 2144 /* 2145 * If a bus or slot reset is available for the provided dev_set and: 2146 * - All of the devices affected by that bus or slot reset are unused 2147 * - At least one of the affected devices is marked dirty via 2148 * needs_reset (such as by lack of FLR support) 2149 * Then attempt to perform that bus or slot reset. 2150 * Returns true if the dev_set was reset. 2151 */ 2152 static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set) 2153 { 2154 struct vfio_pci_core_device *cur; 2155 struct pci_dev *pdev; 2156 int ret; 2157 2158 if (!vfio_pci_dev_set_needs_reset(dev_set)) 2159 return false; 2160 2161 pdev = vfio_pci_dev_set_resettable(dev_set); 2162 if (!pdev) 2163 return false; 2164 2165 /* 2166 * The pci_reset_bus() will reset all the devices in the bus. 2167 * The power state can be non-D0 for some of the devices in the bus. 2168 * For these devices, the pci_reset_bus() will internally set 2169 * the power state to D0 without vfio driver involvement. 2170 * For the devices which have NoSoftRst-, the reset function can 2171 * cause the PCI config space reset without restoring the original 2172 * state (saved locally in 'vdev->pm_save'). 2173 */ 2174 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) 2175 vfio_pci_set_power_state(cur, PCI_D0); 2176 2177 ret = pci_reset_bus(pdev); 2178 if (ret) 2179 return false; 2180 2181 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) { 2182 cur->needs_reset = false; 2183 if (!disable_idle_d3) 2184 vfio_pci_set_power_state(cur, PCI_D3hot); 2185 } 2186 return true; 2187 } 2188 2189 void vfio_pci_core_set_params(bool is_nointxmask, bool is_disable_vga, 2190 bool is_disable_idle_d3) 2191 { 2192 nointxmask = is_nointxmask; 2193 disable_vga = is_disable_vga; 2194 disable_idle_d3 = is_disable_idle_d3; 2195 } 2196 EXPORT_SYMBOL_GPL(vfio_pci_core_set_params); 2197 2198 static void vfio_pci_core_cleanup(void) 2199 { 2200 vfio_pci_uninit_perm_bits(); 2201 } 2202 2203 static int __init vfio_pci_core_init(void) 2204 { 2205 /* Allocate shared config space permission data used by all devices */ 2206 return vfio_pci_init_perm_bits(); 2207 } 2208 2209 module_init(vfio_pci_core_init); 2210 module_exit(vfio_pci_core_cleanup); 2211 2212 MODULE_LICENSE("GPL v2"); 2213 MODULE_AUTHOR(DRIVER_AUTHOR); 2214 MODULE_DESCRIPTION(DRIVER_DESC); 2215