1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 4 * Author: Alex Williamson <alex.williamson@redhat.com> 5 * 6 * Derived from original vfio: 7 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 8 * Author: Tom Lyon, pugs@cisco.com 9 */ 10 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 13 #include <linux/device.h> 14 #include <linux/eventfd.h> 15 #include <linux/file.h> 16 #include <linux/interrupt.h> 17 #include <linux/iommu.h> 18 #include <linux/module.h> 19 #include <linux/mutex.h> 20 #include <linux/notifier.h> 21 #include <linux/pci.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/slab.h> 24 #include <linux/types.h> 25 #include <linux/uaccess.h> 26 #include <linux/vgaarb.h> 27 #include <linux/nospec.h> 28 #include <linux/sched/mm.h> 29 30 #include <linux/vfio_pci_core.h> 31 32 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 33 #define DRIVER_DESC "core driver for VFIO based PCI devices" 34 35 static bool nointxmask; 36 static bool disable_vga; 37 static bool disable_idle_d3; 38 39 static inline bool vfio_vga_disabled(void) 40 { 41 #ifdef CONFIG_VFIO_PCI_VGA 42 return disable_vga; 43 #else 44 return true; 45 #endif 46 } 47 48 /* 49 * Our VGA arbiter participation is limited since we don't know anything 50 * about the device itself. However, if the device is the only VGA device 51 * downstream of a bridge and VFIO VGA support is disabled, then we can 52 * safely return legacy VGA IO and memory as not decoded since the user 53 * has no way to get to it and routing can be disabled externally at the 54 * bridge. 55 */ 56 static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga) 57 { 58 struct vfio_pci_core_device *vdev = opaque; 59 struct pci_dev *tmp = NULL, *pdev = vdev->pdev; 60 unsigned char max_busnr; 61 unsigned int decodes; 62 63 if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus)) 64 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | 65 VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM; 66 67 max_busnr = pci_bus_max_busnr(pdev->bus); 68 decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 69 70 while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) { 71 if (tmp == pdev || 72 pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) || 73 pci_is_root_bus(tmp->bus)) 74 continue; 75 76 if (tmp->bus->number >= pdev->bus->number && 77 tmp->bus->number <= max_busnr) { 78 pci_dev_put(tmp); 79 decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM; 80 break; 81 } 82 } 83 84 return decodes; 85 } 86 87 static void vfio_pci_probe_mmaps(struct vfio_pci_core_device *vdev) 88 { 89 struct resource *res; 90 int i; 91 struct vfio_pci_dummy_resource *dummy_res; 92 93 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 94 int bar = i + PCI_STD_RESOURCES; 95 96 res = &vdev->pdev->resource[bar]; 97 98 if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP)) 99 goto no_mmap; 100 101 if (!(res->flags & IORESOURCE_MEM)) 102 goto no_mmap; 103 104 /* 105 * The PCI core shouldn't set up a resource with a 106 * type but zero size. But there may be bugs that 107 * cause us to do that. 108 */ 109 if (!resource_size(res)) 110 goto no_mmap; 111 112 if (resource_size(res) >= PAGE_SIZE) { 113 vdev->bar_mmap_supported[bar] = true; 114 continue; 115 } 116 117 if (!(res->start & ~PAGE_MASK)) { 118 /* 119 * Add a dummy resource to reserve the remainder 120 * of the exclusive page in case that hot-add 121 * device's bar is assigned into it. 122 */ 123 dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL); 124 if (dummy_res == NULL) 125 goto no_mmap; 126 127 dummy_res->resource.name = "vfio sub-page reserved"; 128 dummy_res->resource.start = res->end + 1; 129 dummy_res->resource.end = res->start + PAGE_SIZE - 1; 130 dummy_res->resource.flags = res->flags; 131 if (request_resource(res->parent, 132 &dummy_res->resource)) { 133 kfree(dummy_res); 134 goto no_mmap; 135 } 136 dummy_res->index = bar; 137 list_add(&dummy_res->res_next, 138 &vdev->dummy_resources_list); 139 vdev->bar_mmap_supported[bar] = true; 140 continue; 141 } 142 /* 143 * Here we don't handle the case when the BAR is not page 144 * aligned because we can't expect the BAR will be 145 * assigned into the same location in a page in guest 146 * when we passthrough the BAR. And it's hard to access 147 * this BAR in userspace because we have no way to get 148 * the BAR's location in a page. 149 */ 150 no_mmap: 151 vdev->bar_mmap_supported[bar] = false; 152 } 153 } 154 155 struct vfio_pci_group_info; 156 static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set); 157 static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set, 158 struct vfio_pci_group_info *groups); 159 160 /* 161 * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND 162 * _and_ the ability detect when the device is asserting INTx via PCI_STATUS. 163 * If a device implements the former but not the latter we would typically 164 * expect broken_intx_masking be set and require an exclusive interrupt. 165 * However since we do have control of the device's ability to assert INTx, 166 * we can instead pretend that the device does not implement INTx, virtualizing 167 * the pin register to report zero and maintaining DisINTx set on the host. 168 */ 169 static bool vfio_pci_nointx(struct pci_dev *pdev) 170 { 171 switch (pdev->vendor) { 172 case PCI_VENDOR_ID_INTEL: 173 switch (pdev->device) { 174 /* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */ 175 case 0x1572: 176 case 0x1574: 177 case 0x1580 ... 0x1581: 178 case 0x1583 ... 0x158b: 179 case 0x37d0 ... 0x37d2: 180 /* X550 */ 181 case 0x1563: 182 return true; 183 default: 184 return false; 185 } 186 } 187 188 return false; 189 } 190 191 static void vfio_pci_probe_power_state(struct vfio_pci_core_device *vdev) 192 { 193 struct pci_dev *pdev = vdev->pdev; 194 u16 pmcsr; 195 196 if (!pdev->pm_cap) 197 return; 198 199 pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr); 200 201 vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET); 202 } 203 204 /* 205 * pci_set_power_state() wrapper handling devices which perform a soft reset on 206 * D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev, 207 * restore when returned to D0. Saved separately from pci_saved_state for use 208 * by PM capability emulation and separately from pci_dev internal saved state 209 * to avoid it being overwritten and consumed around other resets. 210 */ 211 int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t state) 212 { 213 struct pci_dev *pdev = vdev->pdev; 214 bool needs_restore = false, needs_save = false; 215 int ret; 216 217 if (vdev->needs_pm_restore) { 218 if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) { 219 pci_save_state(pdev); 220 needs_save = true; 221 } 222 223 if (pdev->current_state >= PCI_D3hot && state <= PCI_D0) 224 needs_restore = true; 225 } 226 227 ret = pci_set_power_state(pdev, state); 228 229 if (!ret) { 230 /* D3 might be unsupported via quirk, skip unless in D3 */ 231 if (needs_save && pdev->current_state >= PCI_D3hot) { 232 vdev->pm_save = pci_store_saved_state(pdev); 233 } else if (needs_restore) { 234 pci_load_and_free_saved_state(pdev, &vdev->pm_save); 235 pci_restore_state(pdev); 236 } 237 } 238 239 return ret; 240 } 241 242 int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) 243 { 244 struct pci_dev *pdev = vdev->pdev; 245 int ret; 246 u16 cmd; 247 u8 msix_pos; 248 249 vfio_pci_set_power_state(vdev, PCI_D0); 250 251 /* Don't allow our initial saved state to include busmaster */ 252 pci_clear_master(pdev); 253 254 ret = pci_enable_device(pdev); 255 if (ret) 256 return ret; 257 258 /* If reset fails because of the device lock, fail this path entirely */ 259 ret = pci_try_reset_function(pdev); 260 if (ret == -EAGAIN) { 261 pci_disable_device(pdev); 262 return ret; 263 } 264 265 vdev->reset_works = !ret; 266 pci_save_state(pdev); 267 vdev->pci_saved_state = pci_store_saved_state(pdev); 268 if (!vdev->pci_saved_state) 269 pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__); 270 271 if (likely(!nointxmask)) { 272 if (vfio_pci_nointx(pdev)) { 273 pci_info(pdev, "Masking broken INTx support\n"); 274 vdev->nointx = true; 275 pci_intx(pdev, 0); 276 } else 277 vdev->pci_2_3 = pci_intx_mask_supported(pdev); 278 } 279 280 pci_read_config_word(pdev, PCI_COMMAND, &cmd); 281 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) { 282 cmd &= ~PCI_COMMAND_INTX_DISABLE; 283 pci_write_config_word(pdev, PCI_COMMAND, cmd); 284 } 285 286 ret = vfio_config_init(vdev); 287 if (ret) { 288 kfree(vdev->pci_saved_state); 289 vdev->pci_saved_state = NULL; 290 pci_disable_device(pdev); 291 return ret; 292 } 293 294 msix_pos = pdev->msix_cap; 295 if (msix_pos) { 296 u16 flags; 297 u32 table; 298 299 pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags); 300 pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table); 301 302 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR; 303 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET; 304 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16; 305 } else 306 vdev->msix_bar = 0xFF; 307 308 if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev)) 309 vdev->has_vga = true; 310 311 312 return 0; 313 } 314 EXPORT_SYMBOL_GPL(vfio_pci_core_enable); 315 316 void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) 317 { 318 struct pci_dev *pdev = vdev->pdev; 319 struct vfio_pci_dummy_resource *dummy_res, *tmp; 320 struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp; 321 int i, bar; 322 323 /* For needs_reset */ 324 lockdep_assert_held(&vdev->vdev.dev_set->lock); 325 326 /* Stop the device from further DMA */ 327 pci_clear_master(pdev); 328 329 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE | 330 VFIO_IRQ_SET_ACTION_TRIGGER, 331 vdev->irq_type, 0, 0, NULL); 332 333 /* Device closed, don't need mutex here */ 334 list_for_each_entry_safe(ioeventfd, ioeventfd_tmp, 335 &vdev->ioeventfds_list, next) { 336 vfio_virqfd_disable(&ioeventfd->virqfd); 337 list_del(&ioeventfd->next); 338 kfree(ioeventfd); 339 } 340 vdev->ioeventfds_nr = 0; 341 342 vdev->virq_disabled = false; 343 344 for (i = 0; i < vdev->num_regions; i++) 345 vdev->region[i].ops->release(vdev, &vdev->region[i]); 346 347 vdev->num_regions = 0; 348 kfree(vdev->region); 349 vdev->region = NULL; /* don't krealloc a freed pointer */ 350 351 vfio_config_free(vdev); 352 353 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 354 bar = i + PCI_STD_RESOURCES; 355 if (!vdev->barmap[bar]) 356 continue; 357 pci_iounmap(pdev, vdev->barmap[bar]); 358 pci_release_selected_regions(pdev, 1 << bar); 359 vdev->barmap[bar] = NULL; 360 } 361 362 list_for_each_entry_safe(dummy_res, tmp, 363 &vdev->dummy_resources_list, res_next) { 364 list_del(&dummy_res->res_next); 365 release_resource(&dummy_res->resource); 366 kfree(dummy_res); 367 } 368 369 vdev->needs_reset = true; 370 371 /* 372 * If we have saved state, restore it. If we can reset the device, 373 * even better. Resetting with current state seems better than 374 * nothing, but saving and restoring current state without reset 375 * is just busy work. 376 */ 377 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) { 378 pci_info(pdev, "%s: Couldn't reload saved state\n", __func__); 379 380 if (!vdev->reset_works) 381 goto out; 382 383 pci_save_state(pdev); 384 } 385 386 /* 387 * Disable INTx and MSI, presumably to avoid spurious interrupts 388 * during reset. Stolen from pci_reset_function() 389 */ 390 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 391 392 /* 393 * Try to get the locks ourselves to prevent a deadlock. The 394 * success of this is dependent on being able to lock the device, 395 * which is not always possible. 396 * We can not use the "try" reset interface here, which will 397 * overwrite the previously restored configuration information. 398 */ 399 if (vdev->reset_works && pci_dev_trylock(pdev)) { 400 if (!__pci_reset_function_locked(pdev)) 401 vdev->needs_reset = false; 402 pci_dev_unlock(pdev); 403 } 404 405 pci_restore_state(pdev); 406 out: 407 pci_disable_device(pdev); 408 409 if (!vfio_pci_dev_set_try_reset(vdev->vdev.dev_set) && !disable_idle_d3) 410 vfio_pci_set_power_state(vdev, PCI_D3hot); 411 } 412 EXPORT_SYMBOL_GPL(vfio_pci_core_disable); 413 414 static struct vfio_pci_core_device *get_pf_vdev(struct vfio_pci_core_device *vdev) 415 { 416 struct pci_dev *physfn = pci_physfn(vdev->pdev); 417 struct vfio_device *pf_dev; 418 419 if (!vdev->pdev->is_virtfn) 420 return NULL; 421 422 pf_dev = vfio_device_get_from_dev(&physfn->dev); 423 if (!pf_dev) 424 return NULL; 425 426 if (pci_dev_driver(physfn) != pci_dev_driver(vdev->pdev)) { 427 vfio_device_put(pf_dev); 428 return NULL; 429 } 430 431 return container_of(pf_dev, struct vfio_pci_core_device, vdev); 432 } 433 434 static void vfio_pci_vf_token_user_add(struct vfio_pci_core_device *vdev, int val) 435 { 436 struct vfio_pci_core_device *pf_vdev = get_pf_vdev(vdev); 437 438 if (!pf_vdev) 439 return; 440 441 mutex_lock(&pf_vdev->vf_token->lock); 442 pf_vdev->vf_token->users += val; 443 WARN_ON(pf_vdev->vf_token->users < 0); 444 mutex_unlock(&pf_vdev->vf_token->lock); 445 446 vfio_device_put(&pf_vdev->vdev); 447 } 448 449 void vfio_pci_core_close_device(struct vfio_device *core_vdev) 450 { 451 struct vfio_pci_core_device *vdev = 452 container_of(core_vdev, struct vfio_pci_core_device, vdev); 453 454 vfio_pci_vf_token_user_add(vdev, -1); 455 vfio_spapr_pci_eeh_release(vdev->pdev); 456 vfio_pci_core_disable(vdev); 457 458 mutex_lock(&vdev->igate); 459 if (vdev->err_trigger) { 460 eventfd_ctx_put(vdev->err_trigger); 461 vdev->err_trigger = NULL; 462 } 463 if (vdev->req_trigger) { 464 eventfd_ctx_put(vdev->req_trigger); 465 vdev->req_trigger = NULL; 466 } 467 mutex_unlock(&vdev->igate); 468 } 469 EXPORT_SYMBOL_GPL(vfio_pci_core_close_device); 470 471 void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev) 472 { 473 vfio_pci_probe_mmaps(vdev); 474 vfio_spapr_pci_eeh_open(vdev->pdev); 475 vfio_pci_vf_token_user_add(vdev, 1); 476 } 477 EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable); 478 479 static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type) 480 { 481 if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) { 482 u8 pin; 483 484 if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || 485 vdev->nointx || vdev->pdev->is_virtfn) 486 return 0; 487 488 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin); 489 490 return pin ? 1 : 0; 491 } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) { 492 u8 pos; 493 u16 flags; 494 495 pos = vdev->pdev->msi_cap; 496 if (pos) { 497 pci_read_config_word(vdev->pdev, 498 pos + PCI_MSI_FLAGS, &flags); 499 return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1); 500 } 501 } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) { 502 u8 pos; 503 u16 flags; 504 505 pos = vdev->pdev->msix_cap; 506 if (pos) { 507 pci_read_config_word(vdev->pdev, 508 pos + PCI_MSIX_FLAGS, &flags); 509 510 return (flags & PCI_MSIX_FLAGS_QSIZE) + 1; 511 } 512 } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) { 513 if (pci_is_pcie(vdev->pdev)) 514 return 1; 515 } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) { 516 return 1; 517 } 518 519 return 0; 520 } 521 522 static int vfio_pci_count_devs(struct pci_dev *pdev, void *data) 523 { 524 (*(int *)data)++; 525 return 0; 526 } 527 528 struct vfio_pci_fill_info { 529 int max; 530 int cur; 531 struct vfio_pci_dependent_device *devices; 532 }; 533 534 static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data) 535 { 536 struct vfio_pci_fill_info *fill = data; 537 struct iommu_group *iommu_group; 538 539 if (fill->cur == fill->max) 540 return -EAGAIN; /* Something changed, try again */ 541 542 iommu_group = iommu_group_get(&pdev->dev); 543 if (!iommu_group) 544 return -EPERM; /* Cannot reset non-isolated devices */ 545 546 fill->devices[fill->cur].group_id = iommu_group_id(iommu_group); 547 fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus); 548 fill->devices[fill->cur].bus = pdev->bus->number; 549 fill->devices[fill->cur].devfn = pdev->devfn; 550 fill->cur++; 551 iommu_group_put(iommu_group); 552 return 0; 553 } 554 555 struct vfio_pci_group_info { 556 int count; 557 struct vfio_group **groups; 558 }; 559 560 static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot) 561 { 562 for (; pdev; pdev = pdev->bus->self) 563 if (pdev->bus == slot->bus) 564 return (pdev->slot == slot); 565 return false; 566 } 567 568 struct vfio_pci_walk_info { 569 int (*fn)(struct pci_dev *, void *data); 570 void *data; 571 struct pci_dev *pdev; 572 bool slot; 573 int ret; 574 }; 575 576 static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data) 577 { 578 struct vfio_pci_walk_info *walk = data; 579 580 if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot)) 581 walk->ret = walk->fn(pdev, walk->data); 582 583 return walk->ret; 584 } 585 586 static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev, 587 int (*fn)(struct pci_dev *, 588 void *data), void *data, 589 bool slot) 590 { 591 struct vfio_pci_walk_info walk = { 592 .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0, 593 }; 594 595 pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk); 596 597 return walk.ret; 598 } 599 600 static int msix_mmappable_cap(struct vfio_pci_core_device *vdev, 601 struct vfio_info_cap *caps) 602 { 603 struct vfio_info_cap_header header = { 604 .id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE, 605 .version = 1 606 }; 607 608 return vfio_info_add_capability(caps, &header, sizeof(header)); 609 } 610 611 int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev, 612 unsigned int type, unsigned int subtype, 613 const struct vfio_pci_regops *ops, 614 size_t size, u32 flags, void *data) 615 { 616 struct vfio_pci_region *region; 617 618 region = krealloc(vdev->region, 619 (vdev->num_regions + 1) * sizeof(*region), 620 GFP_KERNEL); 621 if (!region) 622 return -ENOMEM; 623 624 vdev->region = region; 625 vdev->region[vdev->num_regions].type = type; 626 vdev->region[vdev->num_regions].subtype = subtype; 627 vdev->region[vdev->num_regions].ops = ops; 628 vdev->region[vdev->num_regions].size = size; 629 vdev->region[vdev->num_regions].flags = flags; 630 vdev->region[vdev->num_regions].data = data; 631 632 vdev->num_regions++; 633 634 return 0; 635 } 636 EXPORT_SYMBOL_GPL(vfio_pci_register_dev_region); 637 638 long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, 639 unsigned long arg) 640 { 641 struct vfio_pci_core_device *vdev = 642 container_of(core_vdev, struct vfio_pci_core_device, vdev); 643 unsigned long minsz; 644 645 if (cmd == VFIO_DEVICE_GET_INFO) { 646 struct vfio_device_info info; 647 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 648 unsigned long capsz; 649 int ret; 650 651 minsz = offsetofend(struct vfio_device_info, num_irqs); 652 653 /* For backward compatibility, cannot require this */ 654 capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset); 655 656 if (copy_from_user(&info, (void __user *)arg, minsz)) 657 return -EFAULT; 658 659 if (info.argsz < minsz) 660 return -EINVAL; 661 662 if (info.argsz >= capsz) { 663 minsz = capsz; 664 info.cap_offset = 0; 665 } 666 667 info.flags = VFIO_DEVICE_FLAGS_PCI; 668 669 if (vdev->reset_works) 670 info.flags |= VFIO_DEVICE_FLAGS_RESET; 671 672 info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; 673 info.num_irqs = VFIO_PCI_NUM_IRQS; 674 675 ret = vfio_pci_info_zdev_add_caps(vdev, &caps); 676 if (ret && ret != -ENODEV) { 677 pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n"); 678 return ret; 679 } 680 681 if (caps.size) { 682 info.flags |= VFIO_DEVICE_FLAGS_CAPS; 683 if (info.argsz < sizeof(info) + caps.size) { 684 info.argsz = sizeof(info) + caps.size; 685 } else { 686 vfio_info_cap_shift(&caps, sizeof(info)); 687 if (copy_to_user((void __user *)arg + 688 sizeof(info), caps.buf, 689 caps.size)) { 690 kfree(caps.buf); 691 return -EFAULT; 692 } 693 info.cap_offset = sizeof(info); 694 } 695 696 kfree(caps.buf); 697 } 698 699 return copy_to_user((void __user *)arg, &info, minsz) ? 700 -EFAULT : 0; 701 702 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 703 struct pci_dev *pdev = vdev->pdev; 704 struct vfio_region_info info; 705 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 706 int i, ret; 707 708 minsz = offsetofend(struct vfio_region_info, offset); 709 710 if (copy_from_user(&info, (void __user *)arg, minsz)) 711 return -EFAULT; 712 713 if (info.argsz < minsz) 714 return -EINVAL; 715 716 switch (info.index) { 717 case VFIO_PCI_CONFIG_REGION_INDEX: 718 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 719 info.size = pdev->cfg_size; 720 info.flags = VFIO_REGION_INFO_FLAG_READ | 721 VFIO_REGION_INFO_FLAG_WRITE; 722 break; 723 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 724 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 725 info.size = pci_resource_len(pdev, info.index); 726 if (!info.size) { 727 info.flags = 0; 728 break; 729 } 730 731 info.flags = VFIO_REGION_INFO_FLAG_READ | 732 VFIO_REGION_INFO_FLAG_WRITE; 733 if (vdev->bar_mmap_supported[info.index]) { 734 info.flags |= VFIO_REGION_INFO_FLAG_MMAP; 735 if (info.index == vdev->msix_bar) { 736 ret = msix_mmappable_cap(vdev, &caps); 737 if (ret) 738 return ret; 739 } 740 } 741 742 break; 743 case VFIO_PCI_ROM_REGION_INDEX: 744 { 745 void __iomem *io; 746 size_t size; 747 u16 cmd; 748 749 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 750 info.flags = 0; 751 752 /* Report the BAR size, not the ROM size */ 753 info.size = pci_resource_len(pdev, info.index); 754 if (!info.size) { 755 /* Shadow ROMs appear as PCI option ROMs */ 756 if (pdev->resource[PCI_ROM_RESOURCE].flags & 757 IORESOURCE_ROM_SHADOW) 758 info.size = 0x20000; 759 else 760 break; 761 } 762 763 /* 764 * Is it really there? Enable memory decode for 765 * implicit access in pci_map_rom(). 766 */ 767 cmd = vfio_pci_memory_lock_and_enable(vdev); 768 io = pci_map_rom(pdev, &size); 769 if (io) { 770 info.flags = VFIO_REGION_INFO_FLAG_READ; 771 pci_unmap_rom(pdev, io); 772 } else { 773 info.size = 0; 774 } 775 vfio_pci_memory_unlock_and_restore(vdev, cmd); 776 777 break; 778 } 779 case VFIO_PCI_VGA_REGION_INDEX: 780 if (!vdev->has_vga) 781 return -EINVAL; 782 783 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 784 info.size = 0xc0000; 785 info.flags = VFIO_REGION_INFO_FLAG_READ | 786 VFIO_REGION_INFO_FLAG_WRITE; 787 788 break; 789 default: 790 { 791 struct vfio_region_info_cap_type cap_type = { 792 .header.id = VFIO_REGION_INFO_CAP_TYPE, 793 .header.version = 1 }; 794 795 if (info.index >= 796 VFIO_PCI_NUM_REGIONS + vdev->num_regions) 797 return -EINVAL; 798 info.index = array_index_nospec(info.index, 799 VFIO_PCI_NUM_REGIONS + 800 vdev->num_regions); 801 802 i = info.index - VFIO_PCI_NUM_REGIONS; 803 804 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 805 info.size = vdev->region[i].size; 806 info.flags = vdev->region[i].flags; 807 808 cap_type.type = vdev->region[i].type; 809 cap_type.subtype = vdev->region[i].subtype; 810 811 ret = vfio_info_add_capability(&caps, &cap_type.header, 812 sizeof(cap_type)); 813 if (ret) 814 return ret; 815 816 if (vdev->region[i].ops->add_capability) { 817 ret = vdev->region[i].ops->add_capability(vdev, 818 &vdev->region[i], &caps); 819 if (ret) 820 return ret; 821 } 822 } 823 } 824 825 if (caps.size) { 826 info.flags |= VFIO_REGION_INFO_FLAG_CAPS; 827 if (info.argsz < sizeof(info) + caps.size) { 828 info.argsz = sizeof(info) + caps.size; 829 info.cap_offset = 0; 830 } else { 831 vfio_info_cap_shift(&caps, sizeof(info)); 832 if (copy_to_user((void __user *)arg + 833 sizeof(info), caps.buf, 834 caps.size)) { 835 kfree(caps.buf); 836 return -EFAULT; 837 } 838 info.cap_offset = sizeof(info); 839 } 840 841 kfree(caps.buf); 842 } 843 844 return copy_to_user((void __user *)arg, &info, minsz) ? 845 -EFAULT : 0; 846 847 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 848 struct vfio_irq_info info; 849 850 minsz = offsetofend(struct vfio_irq_info, count); 851 852 if (copy_from_user(&info, (void __user *)arg, minsz)) 853 return -EFAULT; 854 855 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 856 return -EINVAL; 857 858 switch (info.index) { 859 case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX: 860 case VFIO_PCI_REQ_IRQ_INDEX: 861 break; 862 case VFIO_PCI_ERR_IRQ_INDEX: 863 if (pci_is_pcie(vdev->pdev)) 864 break; 865 fallthrough; 866 default: 867 return -EINVAL; 868 } 869 870 info.flags = VFIO_IRQ_INFO_EVENTFD; 871 872 info.count = vfio_pci_get_irq_count(vdev, info.index); 873 874 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 875 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 876 VFIO_IRQ_INFO_AUTOMASKED); 877 else 878 info.flags |= VFIO_IRQ_INFO_NORESIZE; 879 880 return copy_to_user((void __user *)arg, &info, minsz) ? 881 -EFAULT : 0; 882 883 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 884 struct vfio_irq_set hdr; 885 u8 *data = NULL; 886 int max, ret = 0; 887 size_t data_size = 0; 888 889 minsz = offsetofend(struct vfio_irq_set, count); 890 891 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 892 return -EFAULT; 893 894 max = vfio_pci_get_irq_count(vdev, hdr.index); 895 896 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, 897 VFIO_PCI_NUM_IRQS, &data_size); 898 if (ret) 899 return ret; 900 901 if (data_size) { 902 data = memdup_user((void __user *)(arg + minsz), 903 data_size); 904 if (IS_ERR(data)) 905 return PTR_ERR(data); 906 } 907 908 mutex_lock(&vdev->igate); 909 910 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, 911 hdr.start, hdr.count, data); 912 913 mutex_unlock(&vdev->igate); 914 kfree(data); 915 916 return ret; 917 918 } else if (cmd == VFIO_DEVICE_RESET) { 919 int ret; 920 921 if (!vdev->reset_works) 922 return -EINVAL; 923 924 vfio_pci_zap_and_down_write_memory_lock(vdev); 925 ret = pci_try_reset_function(vdev->pdev); 926 up_write(&vdev->memory_lock); 927 928 return ret; 929 930 } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) { 931 struct vfio_pci_hot_reset_info hdr; 932 struct vfio_pci_fill_info fill = { 0 }; 933 struct vfio_pci_dependent_device *devices = NULL; 934 bool slot = false; 935 int ret = 0; 936 937 minsz = offsetofend(struct vfio_pci_hot_reset_info, count); 938 939 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 940 return -EFAULT; 941 942 if (hdr.argsz < minsz) 943 return -EINVAL; 944 945 hdr.flags = 0; 946 947 /* Can we do a slot or bus reset or neither? */ 948 if (!pci_probe_reset_slot(vdev->pdev->slot)) 949 slot = true; 950 else if (pci_probe_reset_bus(vdev->pdev->bus)) 951 return -ENODEV; 952 953 /* How many devices are affected? */ 954 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 955 vfio_pci_count_devs, 956 &fill.max, slot); 957 if (ret) 958 return ret; 959 960 WARN_ON(!fill.max); /* Should always be at least one */ 961 962 /* 963 * If there's enough space, fill it now, otherwise return 964 * -ENOSPC and the number of devices affected. 965 */ 966 if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) { 967 ret = -ENOSPC; 968 hdr.count = fill.max; 969 goto reset_info_exit; 970 } 971 972 devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL); 973 if (!devices) 974 return -ENOMEM; 975 976 fill.devices = devices; 977 978 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 979 vfio_pci_fill_devs, 980 &fill, slot); 981 982 /* 983 * If a device was removed between counting and filling, 984 * we may come up short of fill.max. If a device was 985 * added, we'll have a return of -EAGAIN above. 986 */ 987 if (!ret) 988 hdr.count = fill.cur; 989 990 reset_info_exit: 991 if (copy_to_user((void __user *)arg, &hdr, minsz)) 992 ret = -EFAULT; 993 994 if (!ret) { 995 if (copy_to_user((void __user *)(arg + minsz), devices, 996 hdr.count * sizeof(*devices))) 997 ret = -EFAULT; 998 } 999 1000 kfree(devices); 1001 return ret; 1002 1003 } else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) { 1004 struct vfio_pci_hot_reset hdr; 1005 int32_t *group_fds; 1006 struct vfio_group **groups; 1007 struct vfio_pci_group_info info; 1008 bool slot = false; 1009 int group_idx, count = 0, ret = 0; 1010 1011 minsz = offsetofend(struct vfio_pci_hot_reset, count); 1012 1013 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1014 return -EFAULT; 1015 1016 if (hdr.argsz < minsz || hdr.flags) 1017 return -EINVAL; 1018 1019 /* Can we do a slot or bus reset or neither? */ 1020 if (!pci_probe_reset_slot(vdev->pdev->slot)) 1021 slot = true; 1022 else if (pci_probe_reset_bus(vdev->pdev->bus)) 1023 return -ENODEV; 1024 1025 /* 1026 * We can't let userspace give us an arbitrarily large 1027 * buffer to copy, so verify how many we think there 1028 * could be. Note groups can have multiple devices so 1029 * one group per device is the max. 1030 */ 1031 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 1032 vfio_pci_count_devs, 1033 &count, slot); 1034 if (ret) 1035 return ret; 1036 1037 /* Somewhere between 1 and count is OK */ 1038 if (!hdr.count || hdr.count > count) 1039 return -EINVAL; 1040 1041 group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL); 1042 groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL); 1043 if (!group_fds || !groups) { 1044 kfree(group_fds); 1045 kfree(groups); 1046 return -ENOMEM; 1047 } 1048 1049 if (copy_from_user(group_fds, (void __user *)(arg + minsz), 1050 hdr.count * sizeof(*group_fds))) { 1051 kfree(group_fds); 1052 kfree(groups); 1053 return -EFAULT; 1054 } 1055 1056 /* 1057 * For each group_fd, get the group through the vfio external 1058 * user interface and store the group and iommu ID. This 1059 * ensures the group is held across the reset. 1060 */ 1061 for (group_idx = 0; group_idx < hdr.count; group_idx++) { 1062 struct vfio_group *group; 1063 struct fd f = fdget(group_fds[group_idx]); 1064 if (!f.file) { 1065 ret = -EBADF; 1066 break; 1067 } 1068 1069 group = vfio_group_get_external_user(f.file); 1070 fdput(f); 1071 if (IS_ERR(group)) { 1072 ret = PTR_ERR(group); 1073 break; 1074 } 1075 1076 groups[group_idx] = group; 1077 } 1078 1079 kfree(group_fds); 1080 1081 /* release reference to groups on error */ 1082 if (ret) 1083 goto hot_reset_release; 1084 1085 info.count = hdr.count; 1086 info.groups = groups; 1087 1088 ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info); 1089 1090 hot_reset_release: 1091 for (group_idx--; group_idx >= 0; group_idx--) 1092 vfio_group_put_external_user(groups[group_idx]); 1093 1094 kfree(groups); 1095 return ret; 1096 } else if (cmd == VFIO_DEVICE_IOEVENTFD) { 1097 struct vfio_device_ioeventfd ioeventfd; 1098 int count; 1099 1100 minsz = offsetofend(struct vfio_device_ioeventfd, fd); 1101 1102 if (copy_from_user(&ioeventfd, (void __user *)arg, minsz)) 1103 return -EFAULT; 1104 1105 if (ioeventfd.argsz < minsz) 1106 return -EINVAL; 1107 1108 if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK) 1109 return -EINVAL; 1110 1111 count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK; 1112 1113 if (hweight8(count) != 1 || ioeventfd.fd < -1) 1114 return -EINVAL; 1115 1116 return vfio_pci_ioeventfd(vdev, ioeventfd.offset, 1117 ioeventfd.data, count, ioeventfd.fd); 1118 } else if (cmd == VFIO_DEVICE_FEATURE) { 1119 struct vfio_device_feature feature; 1120 uuid_t uuid; 1121 1122 minsz = offsetofend(struct vfio_device_feature, flags); 1123 1124 if (copy_from_user(&feature, (void __user *)arg, minsz)) 1125 return -EFAULT; 1126 1127 if (feature.argsz < minsz) 1128 return -EINVAL; 1129 1130 /* Check unknown flags */ 1131 if (feature.flags & ~(VFIO_DEVICE_FEATURE_MASK | 1132 VFIO_DEVICE_FEATURE_SET | 1133 VFIO_DEVICE_FEATURE_GET | 1134 VFIO_DEVICE_FEATURE_PROBE)) 1135 return -EINVAL; 1136 1137 /* GET & SET are mutually exclusive except with PROBE */ 1138 if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) && 1139 (feature.flags & VFIO_DEVICE_FEATURE_SET) && 1140 (feature.flags & VFIO_DEVICE_FEATURE_GET)) 1141 return -EINVAL; 1142 1143 switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) { 1144 case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN: 1145 if (!vdev->vf_token) 1146 return -ENOTTY; 1147 1148 /* 1149 * We do not support GET of the VF Token UUID as this 1150 * could expose the token of the previous device user. 1151 */ 1152 if (feature.flags & VFIO_DEVICE_FEATURE_GET) 1153 return -EINVAL; 1154 1155 if (feature.flags & VFIO_DEVICE_FEATURE_PROBE) 1156 return 0; 1157 1158 /* Don't SET unless told to do so */ 1159 if (!(feature.flags & VFIO_DEVICE_FEATURE_SET)) 1160 return -EINVAL; 1161 1162 if (feature.argsz < minsz + sizeof(uuid)) 1163 return -EINVAL; 1164 1165 if (copy_from_user(&uuid, (void __user *)(arg + minsz), 1166 sizeof(uuid))) 1167 return -EFAULT; 1168 1169 mutex_lock(&vdev->vf_token->lock); 1170 uuid_copy(&vdev->vf_token->uuid, &uuid); 1171 mutex_unlock(&vdev->vf_token->lock); 1172 1173 return 0; 1174 default: 1175 return -ENOTTY; 1176 } 1177 } 1178 1179 return -ENOTTY; 1180 } 1181 EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl); 1182 1183 static ssize_t vfio_pci_rw(struct vfio_pci_core_device *vdev, char __user *buf, 1184 size_t count, loff_t *ppos, bool iswrite) 1185 { 1186 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 1187 1188 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) 1189 return -EINVAL; 1190 1191 switch (index) { 1192 case VFIO_PCI_CONFIG_REGION_INDEX: 1193 return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite); 1194 1195 case VFIO_PCI_ROM_REGION_INDEX: 1196 if (iswrite) 1197 return -EINVAL; 1198 return vfio_pci_bar_rw(vdev, buf, count, ppos, false); 1199 1200 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1201 return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite); 1202 1203 case VFIO_PCI_VGA_REGION_INDEX: 1204 return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite); 1205 default: 1206 index -= VFIO_PCI_NUM_REGIONS; 1207 return vdev->region[index].ops->rw(vdev, buf, 1208 count, ppos, iswrite); 1209 } 1210 1211 return -EINVAL; 1212 } 1213 1214 ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, 1215 size_t count, loff_t *ppos) 1216 { 1217 struct vfio_pci_core_device *vdev = 1218 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1219 1220 if (!count) 1221 return 0; 1222 1223 return vfio_pci_rw(vdev, buf, count, ppos, false); 1224 } 1225 EXPORT_SYMBOL_GPL(vfio_pci_core_read); 1226 1227 ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, 1228 size_t count, loff_t *ppos) 1229 { 1230 struct vfio_pci_core_device *vdev = 1231 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1232 1233 if (!count) 1234 return 0; 1235 1236 return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true); 1237 } 1238 EXPORT_SYMBOL_GPL(vfio_pci_core_write); 1239 1240 /* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */ 1241 static int vfio_pci_zap_and_vma_lock(struct vfio_pci_core_device *vdev, bool try) 1242 { 1243 struct vfio_pci_mmap_vma *mmap_vma, *tmp; 1244 1245 /* 1246 * Lock ordering: 1247 * vma_lock is nested under mmap_lock for vm_ops callback paths. 1248 * The memory_lock semaphore is used by both code paths calling 1249 * into this function to zap vmas and the vm_ops.fault callback 1250 * to protect the memory enable state of the device. 1251 * 1252 * When zapping vmas we need to maintain the mmap_lock => vma_lock 1253 * ordering, which requires using vma_lock to walk vma_list to 1254 * acquire an mm, then dropping vma_lock to get the mmap_lock and 1255 * reacquiring vma_lock. This logic is derived from similar 1256 * requirements in uverbs_user_mmap_disassociate(). 1257 * 1258 * mmap_lock must always be the top-level lock when it is taken. 1259 * Therefore we can only hold the memory_lock write lock when 1260 * vma_list is empty, as we'd need to take mmap_lock to clear 1261 * entries. vma_list can only be guaranteed empty when holding 1262 * vma_lock, thus memory_lock is nested under vma_lock. 1263 * 1264 * This enables the vm_ops.fault callback to acquire vma_lock, 1265 * followed by memory_lock read lock, while already holding 1266 * mmap_lock without risk of deadlock. 1267 */ 1268 while (1) { 1269 struct mm_struct *mm = NULL; 1270 1271 if (try) { 1272 if (!mutex_trylock(&vdev->vma_lock)) 1273 return 0; 1274 } else { 1275 mutex_lock(&vdev->vma_lock); 1276 } 1277 while (!list_empty(&vdev->vma_list)) { 1278 mmap_vma = list_first_entry(&vdev->vma_list, 1279 struct vfio_pci_mmap_vma, 1280 vma_next); 1281 mm = mmap_vma->vma->vm_mm; 1282 if (mmget_not_zero(mm)) 1283 break; 1284 1285 list_del(&mmap_vma->vma_next); 1286 kfree(mmap_vma); 1287 mm = NULL; 1288 } 1289 if (!mm) 1290 return 1; 1291 mutex_unlock(&vdev->vma_lock); 1292 1293 if (try) { 1294 if (!mmap_read_trylock(mm)) { 1295 mmput(mm); 1296 return 0; 1297 } 1298 } else { 1299 mmap_read_lock(mm); 1300 } 1301 if (try) { 1302 if (!mutex_trylock(&vdev->vma_lock)) { 1303 mmap_read_unlock(mm); 1304 mmput(mm); 1305 return 0; 1306 } 1307 } else { 1308 mutex_lock(&vdev->vma_lock); 1309 } 1310 list_for_each_entry_safe(mmap_vma, tmp, 1311 &vdev->vma_list, vma_next) { 1312 struct vm_area_struct *vma = mmap_vma->vma; 1313 1314 if (vma->vm_mm != mm) 1315 continue; 1316 1317 list_del(&mmap_vma->vma_next); 1318 kfree(mmap_vma); 1319 1320 zap_vma_ptes(vma, vma->vm_start, 1321 vma->vm_end - vma->vm_start); 1322 } 1323 mutex_unlock(&vdev->vma_lock); 1324 mmap_read_unlock(mm); 1325 mmput(mm); 1326 } 1327 } 1328 1329 void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev) 1330 { 1331 vfio_pci_zap_and_vma_lock(vdev, false); 1332 down_write(&vdev->memory_lock); 1333 mutex_unlock(&vdev->vma_lock); 1334 } 1335 1336 u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev) 1337 { 1338 u16 cmd; 1339 1340 down_write(&vdev->memory_lock); 1341 pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd); 1342 if (!(cmd & PCI_COMMAND_MEMORY)) 1343 pci_write_config_word(vdev->pdev, PCI_COMMAND, 1344 cmd | PCI_COMMAND_MEMORY); 1345 1346 return cmd; 1347 } 1348 1349 void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, u16 cmd) 1350 { 1351 pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd); 1352 up_write(&vdev->memory_lock); 1353 } 1354 1355 /* Caller holds vma_lock */ 1356 static int __vfio_pci_add_vma(struct vfio_pci_core_device *vdev, 1357 struct vm_area_struct *vma) 1358 { 1359 struct vfio_pci_mmap_vma *mmap_vma; 1360 1361 mmap_vma = kmalloc(sizeof(*mmap_vma), GFP_KERNEL); 1362 if (!mmap_vma) 1363 return -ENOMEM; 1364 1365 mmap_vma->vma = vma; 1366 list_add(&mmap_vma->vma_next, &vdev->vma_list); 1367 1368 return 0; 1369 } 1370 1371 /* 1372 * Zap mmaps on open so that we can fault them in on access and therefore 1373 * our vma_list only tracks mappings accessed since last zap. 1374 */ 1375 static void vfio_pci_mmap_open(struct vm_area_struct *vma) 1376 { 1377 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); 1378 } 1379 1380 static void vfio_pci_mmap_close(struct vm_area_struct *vma) 1381 { 1382 struct vfio_pci_core_device *vdev = vma->vm_private_data; 1383 struct vfio_pci_mmap_vma *mmap_vma; 1384 1385 mutex_lock(&vdev->vma_lock); 1386 list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) { 1387 if (mmap_vma->vma == vma) { 1388 list_del(&mmap_vma->vma_next); 1389 kfree(mmap_vma); 1390 break; 1391 } 1392 } 1393 mutex_unlock(&vdev->vma_lock); 1394 } 1395 1396 static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf) 1397 { 1398 struct vm_area_struct *vma = vmf->vma; 1399 struct vfio_pci_core_device *vdev = vma->vm_private_data; 1400 struct vfio_pci_mmap_vma *mmap_vma; 1401 vm_fault_t ret = VM_FAULT_NOPAGE; 1402 1403 mutex_lock(&vdev->vma_lock); 1404 down_read(&vdev->memory_lock); 1405 1406 if (!__vfio_pci_memory_enabled(vdev)) { 1407 ret = VM_FAULT_SIGBUS; 1408 goto up_out; 1409 } 1410 1411 /* 1412 * We populate the whole vma on fault, so we need to test whether 1413 * the vma has already been mapped, such as for concurrent faults 1414 * to the same vma. io_remap_pfn_range() will trigger a BUG_ON if 1415 * we ask it to fill the same range again. 1416 */ 1417 list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) { 1418 if (mmap_vma->vma == vma) 1419 goto up_out; 1420 } 1421 1422 if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 1423 vma->vm_end - vma->vm_start, 1424 vma->vm_page_prot)) { 1425 ret = VM_FAULT_SIGBUS; 1426 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); 1427 goto up_out; 1428 } 1429 1430 if (__vfio_pci_add_vma(vdev, vma)) { 1431 ret = VM_FAULT_OOM; 1432 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); 1433 } 1434 1435 up_out: 1436 up_read(&vdev->memory_lock); 1437 mutex_unlock(&vdev->vma_lock); 1438 return ret; 1439 } 1440 1441 static const struct vm_operations_struct vfio_pci_mmap_ops = { 1442 .open = vfio_pci_mmap_open, 1443 .close = vfio_pci_mmap_close, 1444 .fault = vfio_pci_mmap_fault, 1445 }; 1446 1447 int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma) 1448 { 1449 struct vfio_pci_core_device *vdev = 1450 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1451 struct pci_dev *pdev = vdev->pdev; 1452 unsigned int index; 1453 u64 phys_len, req_len, pgoff, req_start; 1454 int ret; 1455 1456 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 1457 1458 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) 1459 return -EINVAL; 1460 if (vma->vm_end < vma->vm_start) 1461 return -EINVAL; 1462 if ((vma->vm_flags & VM_SHARED) == 0) 1463 return -EINVAL; 1464 if (index >= VFIO_PCI_NUM_REGIONS) { 1465 int regnum = index - VFIO_PCI_NUM_REGIONS; 1466 struct vfio_pci_region *region = vdev->region + regnum; 1467 1468 if (region->ops && region->ops->mmap && 1469 (region->flags & VFIO_REGION_INFO_FLAG_MMAP)) 1470 return region->ops->mmap(vdev, region, vma); 1471 return -EINVAL; 1472 } 1473 if (index >= VFIO_PCI_ROM_REGION_INDEX) 1474 return -EINVAL; 1475 if (!vdev->bar_mmap_supported[index]) 1476 return -EINVAL; 1477 1478 phys_len = PAGE_ALIGN(pci_resource_len(pdev, index)); 1479 req_len = vma->vm_end - vma->vm_start; 1480 pgoff = vma->vm_pgoff & 1481 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 1482 req_start = pgoff << PAGE_SHIFT; 1483 1484 if (req_start + req_len > phys_len) 1485 return -EINVAL; 1486 1487 /* 1488 * Even though we don't make use of the barmap for the mmap, 1489 * we need to request the region and the barmap tracks that. 1490 */ 1491 if (!vdev->barmap[index]) { 1492 ret = pci_request_selected_regions(pdev, 1493 1 << index, "vfio-pci"); 1494 if (ret) 1495 return ret; 1496 1497 vdev->barmap[index] = pci_iomap(pdev, index, 0); 1498 if (!vdev->barmap[index]) { 1499 pci_release_selected_regions(pdev, 1 << index); 1500 return -ENOMEM; 1501 } 1502 } 1503 1504 vma->vm_private_data = vdev; 1505 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 1506 vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff; 1507 1508 /* 1509 * See remap_pfn_range(), called from vfio_pci_fault() but we can't 1510 * change vm_flags within the fault handler. Set them now. 1511 */ 1512 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 1513 vma->vm_ops = &vfio_pci_mmap_ops; 1514 1515 return 0; 1516 } 1517 EXPORT_SYMBOL_GPL(vfio_pci_core_mmap); 1518 1519 void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count) 1520 { 1521 struct vfio_pci_core_device *vdev = 1522 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1523 struct pci_dev *pdev = vdev->pdev; 1524 1525 mutex_lock(&vdev->igate); 1526 1527 if (vdev->req_trigger) { 1528 if (!(count % 10)) 1529 pci_notice_ratelimited(pdev, 1530 "Relaying device request to user (#%u)\n", 1531 count); 1532 eventfd_signal(vdev->req_trigger, 1); 1533 } else if (count == 0) { 1534 pci_warn(pdev, 1535 "No device request channel registered, blocked until released by user\n"); 1536 } 1537 1538 mutex_unlock(&vdev->igate); 1539 } 1540 EXPORT_SYMBOL_GPL(vfio_pci_core_request); 1541 1542 static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev, 1543 bool vf_token, uuid_t *uuid) 1544 { 1545 /* 1546 * There's always some degree of trust or collaboration between SR-IOV 1547 * PF and VFs, even if just that the PF hosts the SR-IOV capability and 1548 * can disrupt VFs with a reset, but often the PF has more explicit 1549 * access to deny service to the VF or access data passed through the 1550 * VF. We therefore require an opt-in via a shared VF token (UUID) to 1551 * represent this trust. This both prevents that a VF driver might 1552 * assume the PF driver is a trusted, in-kernel driver, and also that 1553 * a PF driver might be replaced with a rogue driver, unknown to in-use 1554 * VF drivers. 1555 * 1556 * Therefore when presented with a VF, if the PF is a vfio device and 1557 * it is bound to the vfio-pci driver, the user needs to provide a VF 1558 * token to access the device, in the form of appending a vf_token to 1559 * the device name, for example: 1560 * 1561 * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3" 1562 * 1563 * When presented with a PF which has VFs in use, the user must also 1564 * provide the current VF token to prove collaboration with existing 1565 * VF users. If VFs are not in use, the VF token provided for the PF 1566 * device will act to set the VF token. 1567 * 1568 * If the VF token is provided but unused, an error is generated. 1569 */ 1570 if (!vdev->pdev->is_virtfn && !vdev->vf_token && !vf_token) 1571 return 0; /* No VF token provided or required */ 1572 1573 if (vdev->pdev->is_virtfn) { 1574 struct vfio_pci_core_device *pf_vdev = get_pf_vdev(vdev); 1575 bool match; 1576 1577 if (!pf_vdev) { 1578 if (!vf_token) 1579 return 0; /* PF is not vfio-pci, no VF token */ 1580 1581 pci_info_ratelimited(vdev->pdev, 1582 "VF token incorrectly provided, PF not bound to vfio-pci\n"); 1583 return -EINVAL; 1584 } 1585 1586 if (!vf_token) { 1587 vfio_device_put(&pf_vdev->vdev); 1588 pci_info_ratelimited(vdev->pdev, 1589 "VF token required to access device\n"); 1590 return -EACCES; 1591 } 1592 1593 mutex_lock(&pf_vdev->vf_token->lock); 1594 match = uuid_equal(uuid, &pf_vdev->vf_token->uuid); 1595 mutex_unlock(&pf_vdev->vf_token->lock); 1596 1597 vfio_device_put(&pf_vdev->vdev); 1598 1599 if (!match) { 1600 pci_info_ratelimited(vdev->pdev, 1601 "Incorrect VF token provided for device\n"); 1602 return -EACCES; 1603 } 1604 } else if (vdev->vf_token) { 1605 mutex_lock(&vdev->vf_token->lock); 1606 if (vdev->vf_token->users) { 1607 if (!vf_token) { 1608 mutex_unlock(&vdev->vf_token->lock); 1609 pci_info_ratelimited(vdev->pdev, 1610 "VF token required to access device\n"); 1611 return -EACCES; 1612 } 1613 1614 if (!uuid_equal(uuid, &vdev->vf_token->uuid)) { 1615 mutex_unlock(&vdev->vf_token->lock); 1616 pci_info_ratelimited(vdev->pdev, 1617 "Incorrect VF token provided for device\n"); 1618 return -EACCES; 1619 } 1620 } else if (vf_token) { 1621 uuid_copy(&vdev->vf_token->uuid, uuid); 1622 } 1623 1624 mutex_unlock(&vdev->vf_token->lock); 1625 } else if (vf_token) { 1626 pci_info_ratelimited(vdev->pdev, 1627 "VF token incorrectly provided, not a PF or VF\n"); 1628 return -EINVAL; 1629 } 1630 1631 return 0; 1632 } 1633 1634 #define VF_TOKEN_ARG "vf_token=" 1635 1636 int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf) 1637 { 1638 struct vfio_pci_core_device *vdev = 1639 container_of(core_vdev, struct vfio_pci_core_device, vdev); 1640 bool vf_token = false; 1641 uuid_t uuid; 1642 int ret; 1643 1644 if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev)))) 1645 return 0; /* No match */ 1646 1647 if (strlen(buf) > strlen(pci_name(vdev->pdev))) { 1648 buf += strlen(pci_name(vdev->pdev)); 1649 1650 if (*buf != ' ') 1651 return 0; /* No match: non-whitespace after name */ 1652 1653 while (*buf) { 1654 if (*buf == ' ') { 1655 buf++; 1656 continue; 1657 } 1658 1659 if (!vf_token && !strncmp(buf, VF_TOKEN_ARG, 1660 strlen(VF_TOKEN_ARG))) { 1661 buf += strlen(VF_TOKEN_ARG); 1662 1663 if (strlen(buf) < UUID_STRING_LEN) 1664 return -EINVAL; 1665 1666 ret = uuid_parse(buf, &uuid); 1667 if (ret) 1668 return ret; 1669 1670 vf_token = true; 1671 buf += UUID_STRING_LEN; 1672 } else { 1673 /* Unknown/duplicate option */ 1674 return -EINVAL; 1675 } 1676 } 1677 } 1678 1679 ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid); 1680 if (ret) 1681 return ret; 1682 1683 return 1; /* Match */ 1684 } 1685 EXPORT_SYMBOL_GPL(vfio_pci_core_match); 1686 1687 static int vfio_pci_bus_notifier(struct notifier_block *nb, 1688 unsigned long action, void *data) 1689 { 1690 struct vfio_pci_core_device *vdev = container_of(nb, 1691 struct vfio_pci_core_device, nb); 1692 struct device *dev = data; 1693 struct pci_dev *pdev = to_pci_dev(dev); 1694 struct pci_dev *physfn = pci_physfn(pdev); 1695 1696 if (action == BUS_NOTIFY_ADD_DEVICE && 1697 pdev->is_virtfn && physfn == vdev->pdev) { 1698 pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n", 1699 pci_name(pdev)); 1700 pdev->driver_override = kasprintf(GFP_KERNEL, "%s", 1701 vdev->vdev.ops->name); 1702 } else if (action == BUS_NOTIFY_BOUND_DRIVER && 1703 pdev->is_virtfn && physfn == vdev->pdev) { 1704 struct pci_driver *drv = pci_dev_driver(pdev); 1705 1706 if (drv && drv != pci_dev_driver(vdev->pdev)) 1707 pci_warn(vdev->pdev, 1708 "VF %s bound to driver %s while PF bound to driver %s\n", 1709 pci_name(pdev), drv->name, 1710 pci_dev_driver(vdev->pdev)->name); 1711 } 1712 1713 return 0; 1714 } 1715 1716 static int vfio_pci_vf_init(struct vfio_pci_core_device *vdev) 1717 { 1718 struct pci_dev *pdev = vdev->pdev; 1719 int ret; 1720 1721 if (!pdev->is_physfn) 1722 return 0; 1723 1724 vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL); 1725 if (!vdev->vf_token) 1726 return -ENOMEM; 1727 1728 mutex_init(&vdev->vf_token->lock); 1729 uuid_gen(&vdev->vf_token->uuid); 1730 1731 vdev->nb.notifier_call = vfio_pci_bus_notifier; 1732 ret = bus_register_notifier(&pci_bus_type, &vdev->nb); 1733 if (ret) { 1734 kfree(vdev->vf_token); 1735 return ret; 1736 } 1737 return 0; 1738 } 1739 1740 static void vfio_pci_vf_uninit(struct vfio_pci_core_device *vdev) 1741 { 1742 if (!vdev->vf_token) 1743 return; 1744 1745 bus_unregister_notifier(&pci_bus_type, &vdev->nb); 1746 WARN_ON(vdev->vf_token->users); 1747 mutex_destroy(&vdev->vf_token->lock); 1748 kfree(vdev->vf_token); 1749 } 1750 1751 static int vfio_pci_vga_init(struct vfio_pci_core_device *vdev) 1752 { 1753 struct pci_dev *pdev = vdev->pdev; 1754 int ret; 1755 1756 if (!vfio_pci_is_vga(pdev)) 1757 return 0; 1758 1759 ret = vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode); 1760 if (ret) 1761 return ret; 1762 vga_set_legacy_decoding(pdev, vfio_pci_set_vga_decode(vdev, false)); 1763 return 0; 1764 } 1765 1766 static void vfio_pci_vga_uninit(struct vfio_pci_core_device *vdev) 1767 { 1768 struct pci_dev *pdev = vdev->pdev; 1769 1770 if (!vfio_pci_is_vga(pdev)) 1771 return; 1772 vga_client_register(pdev, NULL, NULL, NULL); 1773 vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | 1774 VGA_RSRC_LEGACY_IO | 1775 VGA_RSRC_LEGACY_MEM); 1776 } 1777 1778 void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev, 1779 struct pci_dev *pdev, 1780 const struct vfio_device_ops *vfio_pci_ops) 1781 { 1782 vfio_init_group_dev(&vdev->vdev, &pdev->dev, vfio_pci_ops); 1783 vdev->pdev = pdev; 1784 vdev->irq_type = VFIO_PCI_NUM_IRQS; 1785 mutex_init(&vdev->igate); 1786 spin_lock_init(&vdev->irqlock); 1787 mutex_init(&vdev->ioeventfds_lock); 1788 INIT_LIST_HEAD(&vdev->dummy_resources_list); 1789 INIT_LIST_HEAD(&vdev->ioeventfds_list); 1790 mutex_init(&vdev->vma_lock); 1791 INIT_LIST_HEAD(&vdev->vma_list); 1792 init_rwsem(&vdev->memory_lock); 1793 } 1794 EXPORT_SYMBOL_GPL(vfio_pci_core_init_device); 1795 1796 void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev) 1797 { 1798 mutex_destroy(&vdev->igate); 1799 mutex_destroy(&vdev->ioeventfds_lock); 1800 mutex_destroy(&vdev->vma_lock); 1801 vfio_uninit_group_dev(&vdev->vdev); 1802 kfree(vdev->region); 1803 kfree(vdev->pm_save); 1804 } 1805 EXPORT_SYMBOL_GPL(vfio_pci_core_uninit_device); 1806 1807 int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) 1808 { 1809 struct pci_dev *pdev = vdev->pdev; 1810 struct iommu_group *group; 1811 int ret; 1812 1813 if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL) 1814 return -EINVAL; 1815 1816 /* 1817 * Prevent binding to PFs with VFs enabled, the VFs might be in use 1818 * by the host or other users. We cannot capture the VFs if they 1819 * already exist, nor can we track VF users. Disabling SR-IOV here 1820 * would initiate removing the VFs, which would unbind the driver, 1821 * which is prone to blocking if that VF is also in use by vfio-pci. 1822 * Just reject these PFs and let the user sort it out. 1823 */ 1824 if (pci_num_vf(pdev)) { 1825 pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n"); 1826 return -EBUSY; 1827 } 1828 1829 group = vfio_iommu_group_get(&pdev->dev); 1830 if (!group) 1831 return -EINVAL; 1832 1833 if (pci_is_root_bus(pdev->bus)) { 1834 ret = vfio_assign_device_set(&vdev->vdev, vdev); 1835 } else if (!pci_probe_reset_slot(pdev->slot)) { 1836 ret = vfio_assign_device_set(&vdev->vdev, pdev->slot); 1837 } else { 1838 /* 1839 * If there is no slot reset support for this device, the whole 1840 * bus needs to be grouped together to support bus-wide resets. 1841 */ 1842 ret = vfio_assign_device_set(&vdev->vdev, pdev->bus); 1843 } 1844 1845 if (ret) 1846 goto out_group_put; 1847 ret = vfio_pci_vf_init(vdev); 1848 if (ret) 1849 goto out_group_put; 1850 ret = vfio_pci_vga_init(vdev); 1851 if (ret) 1852 goto out_vf; 1853 1854 vfio_pci_probe_power_state(vdev); 1855 1856 if (!disable_idle_d3) { 1857 /* 1858 * pci-core sets the device power state to an unknown value at 1859 * bootup and after being removed from a driver. The only 1860 * transition it allows from this unknown state is to D0, which 1861 * typically happens when a driver calls pci_enable_device(). 1862 * We're not ready to enable the device yet, but we do want to 1863 * be able to get to D3. Therefore first do a D0 transition 1864 * before going to D3. 1865 */ 1866 vfio_pci_set_power_state(vdev, PCI_D0); 1867 vfio_pci_set_power_state(vdev, PCI_D3hot); 1868 } 1869 1870 ret = vfio_register_group_dev(&vdev->vdev); 1871 if (ret) 1872 goto out_power; 1873 return 0; 1874 1875 out_power: 1876 if (!disable_idle_d3) 1877 vfio_pci_set_power_state(vdev, PCI_D0); 1878 out_vf: 1879 vfio_pci_vf_uninit(vdev); 1880 out_group_put: 1881 vfio_iommu_group_put(group, &pdev->dev); 1882 return ret; 1883 } 1884 EXPORT_SYMBOL_GPL(vfio_pci_core_register_device); 1885 1886 void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev) 1887 { 1888 struct pci_dev *pdev = vdev->pdev; 1889 1890 pci_disable_sriov(pdev); 1891 1892 vfio_unregister_group_dev(&vdev->vdev); 1893 1894 vfio_pci_vf_uninit(vdev); 1895 vfio_pci_vga_uninit(vdev); 1896 1897 vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev); 1898 1899 if (!disable_idle_d3) 1900 vfio_pci_set_power_state(vdev, PCI_D0); 1901 } 1902 EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device); 1903 1904 static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev, 1905 pci_channel_state_t state) 1906 { 1907 struct vfio_pci_core_device *vdev; 1908 struct vfio_device *device; 1909 1910 device = vfio_device_get_from_dev(&pdev->dev); 1911 if (device == NULL) 1912 return PCI_ERS_RESULT_DISCONNECT; 1913 1914 vdev = container_of(device, struct vfio_pci_core_device, vdev); 1915 1916 mutex_lock(&vdev->igate); 1917 1918 if (vdev->err_trigger) 1919 eventfd_signal(vdev->err_trigger, 1); 1920 1921 mutex_unlock(&vdev->igate); 1922 1923 vfio_device_put(device); 1924 1925 return PCI_ERS_RESULT_CAN_RECOVER; 1926 } 1927 1928 int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn) 1929 { 1930 struct vfio_device *device; 1931 int ret = 0; 1932 1933 device = vfio_device_get_from_dev(&pdev->dev); 1934 if (!device) 1935 return -ENODEV; 1936 1937 if (nr_virtfn == 0) 1938 pci_disable_sriov(pdev); 1939 else 1940 ret = pci_enable_sriov(pdev, nr_virtfn); 1941 1942 vfio_device_put(device); 1943 1944 return ret < 0 ? ret : nr_virtfn; 1945 } 1946 EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure); 1947 1948 const struct pci_error_handlers vfio_pci_core_err_handlers = { 1949 .error_detected = vfio_pci_aer_err_detected, 1950 }; 1951 EXPORT_SYMBOL_GPL(vfio_pci_core_err_handlers); 1952 1953 static bool vfio_dev_in_groups(struct vfio_pci_core_device *vdev, 1954 struct vfio_pci_group_info *groups) 1955 { 1956 unsigned int i; 1957 1958 for (i = 0; i < groups->count; i++) 1959 if (groups->groups[i] == vdev->vdev.group) 1960 return true; 1961 return false; 1962 } 1963 1964 static int vfio_pci_is_device_in_set(struct pci_dev *pdev, void *data) 1965 { 1966 struct vfio_device_set *dev_set = data; 1967 struct vfio_device *cur; 1968 1969 list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 1970 if (cur->dev == &pdev->dev) 1971 return 0; 1972 return -EBUSY; 1973 } 1974 1975 /* 1976 * vfio-core considers a group to be viable and will create a vfio_device even 1977 * if some devices are bound to drivers like pci-stub or pcieport. Here we 1978 * require all PCI devices to be inside our dev_set since that ensures they stay 1979 * put and that every driver controlling the device can co-ordinate with the 1980 * device reset. 1981 * 1982 * Returns the pci_dev to pass to pci_reset_bus() if every PCI device to be 1983 * reset is inside the dev_set, and pci_reset_bus() can succeed. NULL otherwise. 1984 */ 1985 static struct pci_dev * 1986 vfio_pci_dev_set_resettable(struct vfio_device_set *dev_set) 1987 { 1988 struct pci_dev *pdev; 1989 1990 lockdep_assert_held(&dev_set->lock); 1991 1992 /* 1993 * By definition all PCI devices in the dev_set share the same PCI 1994 * reset, so any pci_dev will have the same outcomes for 1995 * pci_probe_reset_*() and pci_reset_bus(). 1996 */ 1997 pdev = list_first_entry(&dev_set->device_list, 1998 struct vfio_pci_core_device, 1999 vdev.dev_set_list)->pdev; 2000 2001 /* pci_reset_bus() is supported */ 2002 if (pci_probe_reset_slot(pdev->slot) && pci_probe_reset_bus(pdev->bus)) 2003 return NULL; 2004 2005 if (vfio_pci_for_each_slot_or_bus(pdev, vfio_pci_is_device_in_set, 2006 dev_set, 2007 !pci_probe_reset_slot(pdev->slot))) 2008 return NULL; 2009 return pdev; 2010 } 2011 2012 /* 2013 * We need to get memory_lock for each device, but devices can share mmap_lock, 2014 * therefore we need to zap and hold the vma_lock for each device, and only then 2015 * get each memory_lock. 2016 */ 2017 static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set, 2018 struct vfio_pci_group_info *groups) 2019 { 2020 struct vfio_pci_core_device *cur_mem; 2021 struct vfio_pci_core_device *cur_vma; 2022 struct vfio_pci_core_device *cur; 2023 struct pci_dev *pdev; 2024 bool is_mem = true; 2025 int ret; 2026 2027 mutex_lock(&dev_set->lock); 2028 cur_mem = list_first_entry(&dev_set->device_list, 2029 struct vfio_pci_core_device, 2030 vdev.dev_set_list); 2031 2032 pdev = vfio_pci_dev_set_resettable(dev_set); 2033 if (!pdev) { 2034 ret = -EINVAL; 2035 goto err_unlock; 2036 } 2037 2038 list_for_each_entry(cur_vma, &dev_set->device_list, vdev.dev_set_list) { 2039 /* 2040 * Test whether all the affected devices are contained by the 2041 * set of groups provided by the user. 2042 */ 2043 if (!vfio_dev_in_groups(cur_vma, groups)) { 2044 ret = -EINVAL; 2045 goto err_undo; 2046 } 2047 2048 /* 2049 * Locking multiple devices is prone to deadlock, runaway and 2050 * unwind if we hit contention. 2051 */ 2052 if (!vfio_pci_zap_and_vma_lock(cur_vma, true)) { 2053 ret = -EBUSY; 2054 goto err_undo; 2055 } 2056 } 2057 cur_vma = NULL; 2058 2059 list_for_each_entry(cur_mem, &dev_set->device_list, vdev.dev_set_list) { 2060 if (!down_write_trylock(&cur_mem->memory_lock)) { 2061 ret = -EBUSY; 2062 goto err_undo; 2063 } 2064 mutex_unlock(&cur_mem->vma_lock); 2065 } 2066 cur_mem = NULL; 2067 2068 ret = pci_reset_bus(pdev); 2069 2070 err_undo: 2071 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) { 2072 if (cur == cur_mem) 2073 is_mem = false; 2074 if (cur == cur_vma) 2075 break; 2076 if (is_mem) 2077 up_write(&cur->memory_lock); 2078 else 2079 mutex_unlock(&cur->vma_lock); 2080 } 2081 err_unlock: 2082 mutex_unlock(&dev_set->lock); 2083 return ret; 2084 } 2085 2086 static bool vfio_pci_dev_set_needs_reset(struct vfio_device_set *dev_set) 2087 { 2088 struct vfio_pci_core_device *cur; 2089 bool needs_reset = false; 2090 2091 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) { 2092 /* No VFIO device in the set can have an open device FD */ 2093 if (cur->vdev.open_count) 2094 return false; 2095 needs_reset |= cur->needs_reset; 2096 } 2097 return needs_reset; 2098 } 2099 2100 /* 2101 * If a bus or slot reset is available for the provided dev_set and: 2102 * - All of the devices affected by that bus or slot reset are unused 2103 * - At least one of the affected devices is marked dirty via 2104 * needs_reset (such as by lack of FLR support) 2105 * Then attempt to perform that bus or slot reset. 2106 * Returns true if the dev_set was reset. 2107 */ 2108 static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set) 2109 { 2110 struct vfio_pci_core_device *cur; 2111 struct pci_dev *pdev; 2112 int ret; 2113 2114 if (!vfio_pci_dev_set_needs_reset(dev_set)) 2115 return false; 2116 2117 pdev = vfio_pci_dev_set_resettable(dev_set); 2118 if (!pdev) 2119 return false; 2120 2121 ret = pci_reset_bus(pdev); 2122 if (ret) 2123 return false; 2124 2125 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) { 2126 cur->needs_reset = false; 2127 if (!disable_idle_d3) 2128 vfio_pci_set_power_state(cur, PCI_D3hot); 2129 } 2130 return true; 2131 } 2132 2133 void vfio_pci_core_set_params(bool is_nointxmask, bool is_disable_vga, 2134 bool is_disable_idle_d3) 2135 { 2136 nointxmask = is_nointxmask; 2137 disable_vga = is_disable_vga; 2138 disable_idle_d3 = is_disable_idle_d3; 2139 } 2140 EXPORT_SYMBOL_GPL(vfio_pci_core_set_params); 2141 2142 static void vfio_pci_core_cleanup(void) 2143 { 2144 vfio_pci_uninit_perm_bits(); 2145 } 2146 2147 static int __init vfio_pci_core_init(void) 2148 { 2149 /* Allocate shared config space permission data used by all devices */ 2150 return vfio_pci_init_perm_bits(); 2151 } 2152 2153 module_init(vfio_pci_core_init); 2154 module_exit(vfio_pci_core_cleanup); 2155 2156 MODULE_LICENSE("GPL v2"); 2157 MODULE_AUTHOR(DRIVER_AUTHOR); 2158 MODULE_DESCRIPTION(DRIVER_DESC); 2159