1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO core 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 * 8 * Derived from original vfio: 9 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 10 * Author: Tom Lyon, pugs@cisco.com 11 */ 12 13 #include <linux/cdev.h> 14 #include <linux/compat.h> 15 #include <linux/device.h> 16 #include <linux/fs.h> 17 #include <linux/idr.h> 18 #include <linux/iommu.h> 19 #if IS_ENABLED(CONFIG_KVM) 20 #include <linux/kvm_host.h> 21 #endif 22 #include <linux/list.h> 23 #include <linux/miscdevice.h> 24 #include <linux/module.h> 25 #include <linux/mount.h> 26 #include <linux/mutex.h> 27 #include <linux/pci.h> 28 #include <linux/pseudo_fs.h> 29 #include <linux/rwsem.h> 30 #include <linux/sched.h> 31 #include <linux/seq_file.h> 32 #include <linux/slab.h> 33 #include <linux/stat.h> 34 #include <linux/string.h> 35 #include <linux/uaccess.h> 36 #include <linux/vfio.h> 37 #include <linux/wait.h> 38 #include <linux/sched/signal.h> 39 #include <linux/pm_runtime.h> 40 #include <linux/interval_tree.h> 41 #include <linux/iova_bitmap.h> 42 #include <linux/iommufd.h> 43 #include "vfio.h" 44 45 #define DRIVER_VERSION "0.3" 46 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 47 #define DRIVER_DESC "VFIO - User Level meta-driver" 48 49 #define VFIO_MAGIC 0x5646494f /* "VFIO" */ 50 51 static struct vfio { 52 struct ida device_ida; 53 struct vfsmount *vfs_mount; 54 int fs_count; 55 } vfio; 56 57 #ifdef CONFIG_VFIO_NOIOMMU 58 bool vfio_noiommu __read_mostly; 59 module_param_named(enable_unsafe_noiommu_mode, 60 vfio_noiommu, bool, S_IRUGO | S_IWUSR); 61 MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)"); 62 #endif 63 64 static DEFINE_XARRAY(vfio_device_set_xa); 65 66 static char *vfio_device_devnode(const struct device *dev, umode_t *mode) 67 { 68 return kasprintf(GFP_KERNEL, "vfio/devices/%s", dev_name(dev)); 69 } 70 71 static const struct class vfio_device_class = { 72 .name = "vfio-dev", 73 .devnode = vfio_device_devnode 74 }; 75 76 int vfio_assign_device_set(struct vfio_device *device, void *set_id) 77 { 78 unsigned long idx = (unsigned long)set_id; 79 struct vfio_device_set *new_dev_set; 80 struct vfio_device_set *dev_set; 81 82 if (WARN_ON(!set_id)) 83 return -EINVAL; 84 85 /* 86 * Atomically acquire a singleton object in the xarray for this set_id 87 */ 88 xa_lock(&vfio_device_set_xa); 89 dev_set = xa_load(&vfio_device_set_xa, idx); 90 if (dev_set) 91 goto found_get_ref; 92 xa_unlock(&vfio_device_set_xa); 93 94 new_dev_set = kzalloc_obj(*new_dev_set); 95 if (!new_dev_set) 96 return -ENOMEM; 97 mutex_init(&new_dev_set->lock); 98 INIT_LIST_HEAD(&new_dev_set->device_list); 99 new_dev_set->set_id = set_id; 100 101 xa_lock(&vfio_device_set_xa); 102 dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set, 103 GFP_KERNEL); 104 if (!dev_set) { 105 dev_set = new_dev_set; 106 goto found_get_ref; 107 } 108 109 kfree(new_dev_set); 110 if (xa_is_err(dev_set)) { 111 xa_unlock(&vfio_device_set_xa); 112 return xa_err(dev_set); 113 } 114 115 found_get_ref: 116 dev_set->device_count++; 117 xa_unlock(&vfio_device_set_xa); 118 mutex_lock(&dev_set->lock); 119 device->dev_set = dev_set; 120 list_add_tail(&device->dev_set_list, &dev_set->device_list); 121 mutex_unlock(&dev_set->lock); 122 return 0; 123 } 124 EXPORT_SYMBOL_GPL(vfio_assign_device_set); 125 126 static void vfio_release_device_set(struct vfio_device *device) 127 { 128 struct vfio_device_set *dev_set = device->dev_set; 129 130 if (!dev_set) 131 return; 132 133 mutex_lock(&dev_set->lock); 134 list_del(&device->dev_set_list); 135 mutex_unlock(&dev_set->lock); 136 137 xa_lock(&vfio_device_set_xa); 138 if (!--dev_set->device_count) { 139 __xa_erase(&vfio_device_set_xa, 140 (unsigned long)dev_set->set_id); 141 mutex_destroy(&dev_set->lock); 142 kfree(dev_set); 143 } 144 xa_unlock(&vfio_device_set_xa); 145 } 146 147 unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set) 148 { 149 struct vfio_device *cur; 150 unsigned int open_count = 0; 151 152 lockdep_assert_held(&dev_set->lock); 153 154 list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 155 open_count += cur->open_count; 156 return open_count; 157 } 158 EXPORT_SYMBOL_GPL(vfio_device_set_open_count); 159 160 struct vfio_device * 161 vfio_find_device_in_devset(struct vfio_device_set *dev_set, 162 struct device *dev) 163 { 164 struct vfio_device *cur; 165 166 lockdep_assert_held(&dev_set->lock); 167 168 list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 169 if (cur->dev == dev) 170 return cur; 171 return NULL; 172 } 173 EXPORT_SYMBOL_GPL(vfio_find_device_in_devset); 174 175 /* 176 * Device objects - create, release, get, put, search 177 */ 178 /* Device reference always implies a group reference */ 179 void vfio_device_put_registration(struct vfio_device *device) 180 { 181 if (refcount_dec_and_test(&device->refcount)) 182 complete(&device->comp); 183 } 184 EXPORT_SYMBOL_GPL(vfio_device_put_registration); 185 186 bool vfio_device_try_get_registration(struct vfio_device *device) 187 { 188 return refcount_inc_not_zero(&device->refcount); 189 } 190 EXPORT_SYMBOL_GPL(vfio_device_try_get_registration); 191 192 /* 193 * VFIO driver API 194 */ 195 /* Release helper called by vfio_put_device() */ 196 static void vfio_device_release(struct device *dev) 197 { 198 struct vfio_device *device = 199 container_of(dev, struct vfio_device, device); 200 201 vfio_release_device_set(device); 202 ida_free(&vfio.device_ida, device->index); 203 204 if (device->ops->release) 205 device->ops->release(device); 206 207 iput(device->inode); 208 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count); 209 kvfree(device); 210 } 211 212 static int vfio_init_device(struct vfio_device *device, struct device *dev, 213 const struct vfio_device_ops *ops); 214 215 /* 216 * Allocate and initialize vfio_device so it can be registered to vfio 217 * core. 218 * 219 * Drivers should use the wrapper vfio_alloc_device() for allocation. 220 * @size is the size of the structure to be allocated, including any 221 * private data used by the driver. 222 * 223 * Driver may provide an @init callback to cover device private data. 224 * 225 * Use vfio_put_device() to release the structure after success return. 226 */ 227 struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, 228 const struct vfio_device_ops *ops) 229 { 230 struct vfio_device *device; 231 int ret; 232 233 if (WARN_ON(size < sizeof(struct vfio_device))) 234 return ERR_PTR(-EINVAL); 235 236 device = kvzalloc(size, GFP_KERNEL); 237 if (!device) 238 return ERR_PTR(-ENOMEM); 239 240 ret = vfio_init_device(device, dev, ops); 241 if (ret) 242 goto out_free; 243 return device; 244 245 out_free: 246 kvfree(device); 247 return ERR_PTR(ret); 248 } 249 EXPORT_SYMBOL_GPL(_vfio_alloc_device); 250 251 static int vfio_fs_init_fs_context(struct fs_context *fc) 252 { 253 return init_pseudo(fc, VFIO_MAGIC) ? 0 : -ENOMEM; 254 } 255 256 static struct file_system_type vfio_fs_type = { 257 .name = "vfio", 258 .owner = THIS_MODULE, 259 .init_fs_context = vfio_fs_init_fs_context, 260 .kill_sb = kill_anon_super, 261 }; 262 263 static struct inode *vfio_fs_inode_new(void) 264 { 265 struct inode *inode; 266 int ret; 267 268 ret = simple_pin_fs(&vfio_fs_type, &vfio.vfs_mount, &vfio.fs_count); 269 if (ret) 270 return ERR_PTR(ret); 271 272 inode = alloc_anon_inode(vfio.vfs_mount->mnt_sb); 273 if (IS_ERR(inode)) 274 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count); 275 276 return inode; 277 } 278 279 /* 280 * Initialize a vfio_device so it can be registered to vfio core. 281 */ 282 static int vfio_init_device(struct vfio_device *device, struct device *dev, 283 const struct vfio_device_ops *ops) 284 { 285 int ret; 286 287 ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL); 288 if (ret < 0) { 289 dev_dbg(dev, "Error to alloc index\n"); 290 return ret; 291 } 292 293 device->index = ret; 294 init_completion(&device->comp); 295 device->dev = dev; 296 device->ops = ops; 297 device->inode = vfio_fs_inode_new(); 298 if (IS_ERR(device->inode)) { 299 ret = PTR_ERR(device->inode); 300 goto out_inode; 301 } 302 303 if (ops->init) { 304 ret = ops->init(device); 305 if (ret) 306 goto out_uninit; 307 } 308 309 device_initialize(&device->device); 310 device->device.release = vfio_device_release; 311 device->device.class = &vfio_device_class; 312 device->device.parent = device->dev; 313 return 0; 314 315 out_uninit: 316 iput(device->inode); 317 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count); 318 out_inode: 319 vfio_release_device_set(device); 320 ida_free(&vfio.device_ida, device->index); 321 return ret; 322 } 323 324 static int __vfio_register_dev(struct vfio_device *device, 325 enum vfio_group_type type) 326 { 327 int ret; 328 329 if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) && 330 (!device->ops->bind_iommufd || 331 !device->ops->unbind_iommufd || 332 !device->ops->attach_ioas || 333 !device->ops->detach_ioas))) 334 return -EINVAL; 335 336 /* 337 * If the driver doesn't specify a set then the device is added to a 338 * singleton set just for itself. 339 */ 340 if (!device->dev_set) 341 vfio_assign_device_set(device, device); 342 343 ret = dev_set_name(&device->device, "vfio%d", device->index); 344 if (ret) 345 return ret; 346 347 ret = vfio_device_set_group(device, type); 348 if (ret) 349 return ret; 350 351 /* 352 * VFIO always sets IOMMU_CACHE because we offer no way for userspace to 353 * restore cache coherency. It has to be checked here because it is only 354 * valid for cases where we are using iommu groups. 355 */ 356 if (type == VFIO_IOMMU && !vfio_device_is_noiommu(device) && 357 !device_iommu_capable(device->dev, IOMMU_CAP_CACHE_COHERENCY)) { 358 ret = -EINVAL; 359 goto err_out; 360 } 361 362 ret = vfio_device_add(device); 363 if (ret) 364 goto err_out; 365 366 /* Refcounting can't start until the driver calls register */ 367 refcount_set(&device->refcount, 1); 368 369 vfio_device_group_register(device); 370 vfio_device_debugfs_init(device); 371 372 return 0; 373 err_out: 374 vfio_device_remove_group(device); 375 return ret; 376 } 377 378 int vfio_register_group_dev(struct vfio_device *device) 379 { 380 return __vfio_register_dev(device, VFIO_IOMMU); 381 } 382 EXPORT_SYMBOL_GPL(vfio_register_group_dev); 383 384 /* 385 * Register a virtual device without IOMMU backing. The user of this 386 * device must not be able to directly trigger unmediated DMA. 387 */ 388 int vfio_register_emulated_iommu_dev(struct vfio_device *device) 389 { 390 return __vfio_register_dev(device, VFIO_EMULATED_IOMMU); 391 } 392 EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev); 393 394 /* 395 * Decrement the device reference count and wait for the device to be 396 * removed. Open file descriptors for the device... */ 397 void vfio_unregister_group_dev(struct vfio_device *device) 398 { 399 unsigned int i = 0; 400 bool interrupted = false; 401 long rc; 402 403 /* 404 * Prevent new device opened by userspace via the 405 * VFIO_GROUP_GET_DEVICE_FD in the group path. 406 */ 407 vfio_device_group_unregister(device); 408 409 /* 410 * Remove debugfs before device_del(), which releases devres. Some 411 * debugfs entries are created with debugfs_create_devm_seqfile() and 412 * therefore rely on devres-managed inode private data. 413 */ 414 vfio_device_debugfs_exit(device); 415 416 /* 417 * Balances vfio_device_add() in register path, also prevents 418 * new device opened by userspace in the cdev path. 419 */ 420 vfio_device_del(device); 421 422 vfio_device_put_registration(device); 423 rc = try_wait_for_completion(&device->comp); 424 while (rc <= 0) { 425 if (device->ops->request) 426 device->ops->request(device, i++); 427 428 if (interrupted) { 429 rc = wait_for_completion_timeout(&device->comp, 430 HZ * 10); 431 } else { 432 rc = wait_for_completion_interruptible_timeout( 433 &device->comp, HZ * 10); 434 if (rc < 0) { 435 interrupted = true; 436 dev_warn(device->dev, 437 "Device is currently in use, task" 438 " \"%s\" (%d) " 439 "blocked until device is released", 440 current->comm, task_pid_nr(current)); 441 } 442 } 443 } 444 445 /* Balances vfio_device_set_group in register path */ 446 vfio_device_remove_group(device); 447 } 448 EXPORT_SYMBOL_GPL(vfio_unregister_group_dev); 449 450 #if IS_ENABLED(CONFIG_KVM) 451 void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm) 452 { 453 void (*pfn)(struct kvm *kvm); 454 bool (*fn)(struct kvm *kvm); 455 bool ret; 456 457 lockdep_assert_held(&device->dev_set->lock); 458 459 if (!kvm) 460 return; 461 462 pfn = symbol_get(kvm_put_kvm); 463 if (WARN_ON(!pfn)) 464 return; 465 466 fn = symbol_get(kvm_get_kvm_safe); 467 if (WARN_ON(!fn)) { 468 symbol_put(kvm_put_kvm); 469 return; 470 } 471 472 ret = fn(kvm); 473 symbol_put(kvm_get_kvm_safe); 474 if (!ret) { 475 symbol_put(kvm_put_kvm); 476 return; 477 } 478 479 device->put_kvm = pfn; 480 device->kvm = kvm; 481 } 482 483 void vfio_device_put_kvm(struct vfio_device *device) 484 { 485 lockdep_assert_held(&device->dev_set->lock); 486 487 if (!device->kvm) 488 return; 489 490 if (WARN_ON(!device->put_kvm)) 491 goto clear; 492 493 device->put_kvm(device->kvm); 494 device->put_kvm = NULL; 495 symbol_put(kvm_put_kvm); 496 497 clear: 498 device->kvm = NULL; 499 } 500 #endif 501 502 /* true if the vfio_device has open_device() called but not close_device() */ 503 static bool vfio_assert_device_open(struct vfio_device *device) 504 { 505 return !WARN_ON_ONCE(!READ_ONCE(device->open_count)); 506 } 507 508 struct vfio_device_file * 509 vfio_allocate_device_file(struct vfio_device *device) 510 { 511 struct vfio_device_file *df; 512 513 df = kzalloc_obj(*df, GFP_KERNEL_ACCOUNT); 514 if (!df) 515 return ERR_PTR(-ENOMEM); 516 517 df->device = device; 518 spin_lock_init(&df->kvm_ref_lock); 519 520 return df; 521 } 522 523 static int vfio_df_device_first_open(struct vfio_device_file *df) 524 { 525 struct vfio_device *device = df->device; 526 struct iommufd_ctx *iommufd = df->iommufd; 527 int ret; 528 529 lockdep_assert_held(&device->dev_set->lock); 530 531 if (!try_module_get(device->dev->driver->owner)) 532 return -ENODEV; 533 534 if (iommufd) 535 ret = vfio_df_iommufd_bind(df); 536 else 537 ret = vfio_device_group_use_iommu(device); 538 if (ret) 539 goto err_module_put; 540 541 if (device->ops->open_device) { 542 ret = device->ops->open_device(device); 543 if (ret) 544 goto err_unuse_iommu; 545 } 546 return 0; 547 548 err_unuse_iommu: 549 if (iommufd) 550 vfio_df_iommufd_unbind(df); 551 else 552 vfio_device_group_unuse_iommu(device); 553 err_module_put: 554 module_put(device->dev->driver->owner); 555 return ret; 556 } 557 558 static void vfio_df_device_last_close(struct vfio_device_file *df) 559 { 560 struct vfio_device *device = df->device; 561 struct iommufd_ctx *iommufd = df->iommufd; 562 563 lockdep_assert_held(&device->dev_set->lock); 564 565 if (device->ops->close_device) 566 device->ops->close_device(device); 567 if (iommufd) 568 vfio_df_iommufd_unbind(df); 569 else 570 vfio_device_group_unuse_iommu(device); 571 device->precopy_info_v2 = 0; 572 module_put(device->dev->driver->owner); 573 } 574 575 int vfio_df_open(struct vfio_device_file *df) 576 { 577 struct vfio_device *device = df->device; 578 int ret = 0; 579 580 lockdep_assert_held(&device->dev_set->lock); 581 582 /* 583 * Only the group path allows the device to be opened multiple 584 * times. The device cdev path doesn't have a secure way for it. 585 */ 586 if (device->open_count != 0 && !df->group) 587 return -EINVAL; 588 589 device->open_count++; 590 if (device->open_count == 1) { 591 ret = vfio_df_device_first_open(df); 592 if (ret) 593 device->open_count--; 594 } 595 596 return ret; 597 } 598 599 void vfio_df_close(struct vfio_device_file *df) 600 { 601 struct vfio_device *device = df->device; 602 603 lockdep_assert_held(&device->dev_set->lock); 604 605 if (!vfio_assert_device_open(device)) 606 return; 607 if (device->open_count == 1) 608 vfio_df_device_last_close(df); 609 device->open_count--; 610 } 611 612 /* 613 * Wrapper around pm_runtime_resume_and_get(). 614 * Return error code on failure or 0 on success. 615 */ 616 static inline int vfio_device_pm_runtime_get(struct vfio_device *device) 617 { 618 struct device *dev = device->dev; 619 620 if (dev->driver && dev->driver->pm) { 621 int ret; 622 623 ret = pm_runtime_resume_and_get(dev); 624 if (ret) { 625 dev_info_ratelimited(dev, 626 "vfio: runtime resume failed %d\n", ret); 627 return -EIO; 628 } 629 } 630 631 return 0; 632 } 633 634 /* 635 * Wrapper around pm_runtime_put(). 636 */ 637 static inline void vfio_device_pm_runtime_put(struct vfio_device *device) 638 { 639 struct device *dev = device->dev; 640 641 if (dev->driver && dev->driver->pm) 642 pm_runtime_put(dev); 643 } 644 645 /* 646 * VFIO Device fd 647 */ 648 static int vfio_device_fops_release(struct inode *inode, struct file *filep) 649 { 650 struct vfio_device_file *df = filep->private_data; 651 struct vfio_device *device = df->device; 652 653 if (df->group) 654 vfio_df_group_close(df); 655 else 656 vfio_df_unbind_iommufd(df); 657 658 vfio_device_put_registration(device); 659 660 kfree(df); 661 662 return 0; 663 } 664 665 /* 666 * vfio_mig_get_next_state - Compute the next step in the FSM 667 * @cur_fsm - The current state the device is in 668 * @new_fsm - The target state to reach 669 * @next_fsm - Pointer to the next step to get to new_fsm 670 * 671 * Return 0 upon success, otherwise -errno 672 * Upon success the next step in the state progression between cur_fsm and 673 * new_fsm will be set in next_fsm. 674 * 675 * This breaks down requests for combination transitions into smaller steps and 676 * returns the next step to get to new_fsm. The function may need to be called 677 * multiple times before reaching new_fsm. 678 * 679 */ 680 int vfio_mig_get_next_state(struct vfio_device *device, 681 enum vfio_device_mig_state cur_fsm, 682 enum vfio_device_mig_state new_fsm, 683 enum vfio_device_mig_state *next_fsm) 684 { 685 enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 }; 686 /* 687 * The coding in this table requires the driver to implement the 688 * following FSM arcs: 689 * RESUMING -> STOP 690 * STOP -> RESUMING 691 * STOP -> STOP_COPY 692 * STOP_COPY -> STOP 693 * 694 * If P2P is supported then the driver must also implement these FSM 695 * arcs: 696 * RUNNING -> RUNNING_P2P 697 * RUNNING_P2P -> RUNNING 698 * RUNNING_P2P -> STOP 699 * STOP -> RUNNING_P2P 700 * 701 * If precopy is supported then the driver must support these additional 702 * FSM arcs: 703 * RUNNING -> PRE_COPY 704 * PRE_COPY -> RUNNING 705 * PRE_COPY -> STOP_COPY 706 * However, if precopy and P2P are supported together then the driver 707 * must support these additional arcs beyond the P2P arcs above: 708 * PRE_COPY -> RUNNING 709 * PRE_COPY -> PRE_COPY_P2P 710 * PRE_COPY_P2P -> PRE_COPY 711 * PRE_COPY_P2P -> RUNNING_P2P 712 * PRE_COPY_P2P -> STOP_COPY 713 * RUNNING -> PRE_COPY 714 * RUNNING_P2P -> PRE_COPY_P2P 715 * 716 * Without P2P and precopy the driver must implement: 717 * RUNNING -> STOP 718 * STOP -> RUNNING 719 * 720 * The coding will step through multiple states for some combination 721 * transitions; if all optional features are supported, this means the 722 * following ones: 723 * PRE_COPY -> PRE_COPY_P2P -> STOP_COPY 724 * PRE_COPY -> RUNNING -> RUNNING_P2P 725 * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP 726 * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING 727 * PRE_COPY_P2P -> RUNNING_P2P -> RUNNING 728 * PRE_COPY_P2P -> RUNNING_P2P -> STOP 729 * PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING 730 * RESUMING -> STOP -> RUNNING_P2P 731 * RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P 732 * RESUMING -> STOP -> RUNNING_P2P -> RUNNING 733 * RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 734 * RESUMING -> STOP -> STOP_COPY 735 * RUNNING -> RUNNING_P2P -> PRE_COPY_P2P 736 * RUNNING -> RUNNING_P2P -> STOP 737 * RUNNING -> RUNNING_P2P -> STOP -> RESUMING 738 * RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY 739 * RUNNING_P2P -> RUNNING -> PRE_COPY 740 * RUNNING_P2P -> STOP -> RESUMING 741 * RUNNING_P2P -> STOP -> STOP_COPY 742 * STOP -> RUNNING_P2P -> PRE_COPY_P2P 743 * STOP -> RUNNING_P2P -> RUNNING 744 * STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 745 * STOP_COPY -> STOP -> RESUMING 746 * STOP_COPY -> STOP -> RUNNING_P2P 747 * STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING 748 * 749 * The following transitions are blocked: 750 * STOP_COPY -> PRE_COPY 751 * STOP_COPY -> PRE_COPY_P2P 752 */ 753 static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = { 754 [VFIO_DEVICE_STATE_STOP] = { 755 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 756 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 757 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 758 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 759 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 760 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 761 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 762 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 763 }, 764 [VFIO_DEVICE_STATE_RUNNING] = { 765 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 766 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 767 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 768 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 769 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 770 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 771 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 772 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 773 }, 774 [VFIO_DEVICE_STATE_PRE_COPY] = { 775 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING, 776 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 777 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 778 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 779 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 780 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING, 781 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING, 782 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 783 }, 784 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = { 785 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 786 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 787 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 788 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 789 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 790 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 791 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 792 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 793 }, 794 [VFIO_DEVICE_STATE_STOP_COPY] = { 795 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 796 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 797 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 798 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 799 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 800 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 801 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 802 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 803 }, 804 [VFIO_DEVICE_STATE_RESUMING] = { 805 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 806 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 807 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP, 808 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP, 809 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 810 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 811 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 812 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 813 }, 814 [VFIO_DEVICE_STATE_RUNNING_P2P] = { 815 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 816 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 817 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING, 818 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 819 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 820 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 821 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 822 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 823 }, 824 [VFIO_DEVICE_STATE_ERROR] = { 825 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR, 826 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR, 827 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 828 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 829 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR, 830 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR, 831 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR, 832 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 833 }, 834 }; 835 836 static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = { 837 [VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY, 838 [VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY, 839 [VFIO_DEVICE_STATE_PRE_COPY] = 840 VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY, 841 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY | 842 VFIO_MIGRATION_P2P | 843 VFIO_MIGRATION_PRE_COPY, 844 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY, 845 [VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY, 846 [VFIO_DEVICE_STATE_RUNNING_P2P] = 847 VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P, 848 [VFIO_DEVICE_STATE_ERROR] = ~0U, 849 }; 850 851 if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 852 (state_flags_table[cur_fsm] & device->migration_flags) != 853 state_flags_table[cur_fsm])) 854 return -EINVAL; 855 856 if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 857 (state_flags_table[new_fsm] & device->migration_flags) != 858 state_flags_table[new_fsm]) 859 return -EINVAL; 860 861 /* 862 * Arcs touching optional and unsupported states are skipped over. The 863 * driver will instead see an arc from the original state to the next 864 * logical state, as per the above comment. 865 */ 866 *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm]; 867 while (*next_fsm != VFIO_DEVICE_STATE_ERROR && 868 (state_flags_table[*next_fsm] & device->migration_flags) != 869 state_flags_table[*next_fsm]) 870 *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm]; 871 872 return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL; 873 } 874 EXPORT_SYMBOL_GPL(vfio_mig_get_next_state); 875 876 /* 877 * Convert the drivers's struct file into a FD number and return it to userspace 878 */ 879 static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg, 880 struct vfio_device_feature_mig_state *mig) 881 { 882 int ret; 883 int fd; 884 885 fd = get_unused_fd_flags(O_CLOEXEC); 886 if (fd < 0) { 887 ret = fd; 888 goto out_fput; 889 } 890 891 mig->data_fd = fd; 892 if (copy_to_user(arg, mig, sizeof(*mig))) { 893 ret = -EFAULT; 894 goto out_put_unused; 895 } 896 fd_install(fd, filp); 897 return 0; 898 899 out_put_unused: 900 put_unused_fd(fd); 901 out_fput: 902 fput(filp); 903 return ret; 904 } 905 906 static int 907 vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device, 908 u32 flags, void __user *arg, 909 size_t argsz) 910 { 911 size_t minsz = 912 offsetofend(struct vfio_device_feature_mig_state, data_fd); 913 struct vfio_device_feature_mig_state mig; 914 struct file *filp = NULL; 915 int ret; 916 917 if (!device->mig_ops) 918 return -ENOTTY; 919 920 ret = vfio_check_feature(flags, argsz, 921 VFIO_DEVICE_FEATURE_SET | 922 VFIO_DEVICE_FEATURE_GET, 923 sizeof(mig)); 924 if (ret != 1) 925 return ret; 926 927 if (copy_from_user(&mig, arg, minsz)) 928 return -EFAULT; 929 930 if (flags & VFIO_DEVICE_FEATURE_GET) { 931 enum vfio_device_mig_state curr_state; 932 933 ret = device->mig_ops->migration_get_state(device, 934 &curr_state); 935 if (ret) 936 return ret; 937 mig.device_state = curr_state; 938 goto out_copy; 939 } 940 941 /* Handle the VFIO_DEVICE_FEATURE_SET */ 942 filp = device->mig_ops->migration_set_state(device, mig.device_state); 943 if (IS_ERR(filp) || !filp) 944 goto out_copy; 945 946 return vfio_ioct_mig_return_fd(filp, arg, &mig); 947 out_copy: 948 mig.data_fd = -1; 949 if (copy_to_user(arg, &mig, sizeof(mig))) 950 return -EFAULT; 951 if (IS_ERR(filp)) 952 return PTR_ERR(filp); 953 return 0; 954 } 955 956 static int 957 vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device, 958 u32 flags, void __user *arg, 959 size_t argsz) 960 { 961 struct vfio_device_feature_mig_data_size data_size = {}; 962 unsigned long stop_copy_length; 963 int ret; 964 965 if (!device->mig_ops) 966 return -ENOTTY; 967 968 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 969 sizeof(data_size)); 970 if (ret != 1) 971 return ret; 972 973 ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length); 974 if (ret) 975 return ret; 976 977 data_size.stop_copy_length = stop_copy_length; 978 if (copy_to_user(arg, &data_size, sizeof(data_size))) 979 return -EFAULT; 980 981 return 0; 982 } 983 984 static int 985 vfio_ioctl_device_feature_migration_precopy_info_v2(struct vfio_device *device, 986 u32 flags, size_t argsz) 987 { 988 int ret; 989 990 if (!(device->migration_flags & VFIO_MIGRATION_PRE_COPY)) 991 return -EINVAL; 992 993 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0); 994 if (ret != 1) 995 return ret; 996 997 device->precopy_info_v2 = 1; 998 return 0; 999 } 1000 1001 static int vfio_ioctl_device_feature_migration(struct vfio_device *device, 1002 u32 flags, void __user *arg, 1003 size_t argsz) 1004 { 1005 struct vfio_device_feature_migration mig = { 1006 .flags = device->migration_flags, 1007 }; 1008 int ret; 1009 1010 if (!device->mig_ops) 1011 return -ENOTTY; 1012 1013 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 1014 sizeof(mig)); 1015 if (ret != 1) 1016 return ret; 1017 if (copy_to_user(arg, &mig, sizeof(mig))) 1018 return -EFAULT; 1019 return 0; 1020 } 1021 1022 void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes, 1023 u32 req_nodes) 1024 { 1025 struct interval_tree_node *prev, *curr, *comb_start, *comb_end; 1026 unsigned long min_gap, curr_gap; 1027 1028 /* Special shortcut when a single range is required */ 1029 if (req_nodes == 1) { 1030 unsigned long last; 1031 1032 comb_start = interval_tree_iter_first(root, 0, ULONG_MAX); 1033 1034 /* Empty list */ 1035 if (WARN_ON_ONCE(!comb_start)) 1036 return; 1037 1038 curr = comb_start; 1039 while (curr) { 1040 last = curr->last; 1041 prev = curr; 1042 curr = interval_tree_iter_next(curr, 0, ULONG_MAX); 1043 if (prev != comb_start) 1044 interval_tree_remove(prev, root); 1045 } 1046 comb_start->last = last; 1047 return; 1048 } 1049 1050 /* Combine ranges which have the smallest gap */ 1051 while (cur_nodes > req_nodes) { 1052 prev = NULL; 1053 min_gap = ULONG_MAX; 1054 curr = interval_tree_iter_first(root, 0, ULONG_MAX); 1055 while (curr) { 1056 if (prev) { 1057 curr_gap = curr->start - prev->last; 1058 if (curr_gap < min_gap) { 1059 min_gap = curr_gap; 1060 comb_start = prev; 1061 comb_end = curr; 1062 } 1063 } 1064 prev = curr; 1065 curr = interval_tree_iter_next(curr, 0, ULONG_MAX); 1066 } 1067 1068 /* Empty list or no nodes to combine */ 1069 if (WARN_ON_ONCE(min_gap == ULONG_MAX)) 1070 break; 1071 1072 comb_start->last = comb_end->last; 1073 interval_tree_remove(comb_end, root); 1074 cur_nodes--; 1075 } 1076 } 1077 EXPORT_SYMBOL_GPL(vfio_combine_iova_ranges); 1078 1079 /* Ranges should fit into a single kernel page */ 1080 #define LOG_MAX_RANGES \ 1081 (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range)) 1082 1083 static int 1084 vfio_ioctl_device_feature_logging_start(struct vfio_device *device, 1085 u32 flags, void __user *arg, 1086 size_t argsz) 1087 { 1088 size_t minsz = 1089 offsetofend(struct vfio_device_feature_dma_logging_control, 1090 ranges); 1091 struct vfio_device_feature_dma_logging_range __user *ranges; 1092 struct vfio_device_feature_dma_logging_control control; 1093 struct vfio_device_feature_dma_logging_range range; 1094 struct rb_root_cached root = RB_ROOT_CACHED; 1095 struct interval_tree_node *nodes; 1096 u64 iova_end; 1097 u32 nnodes; 1098 int i, ret; 1099 1100 if (!device->log_ops) 1101 return -ENOTTY; 1102 1103 ret = vfio_check_feature(flags, argsz, 1104 VFIO_DEVICE_FEATURE_SET, 1105 sizeof(control)); 1106 if (ret != 1) 1107 return ret; 1108 1109 if (copy_from_user(&control, arg, minsz)) 1110 return -EFAULT; 1111 1112 nnodes = control.num_ranges; 1113 if (!nnodes) 1114 return -EINVAL; 1115 1116 if (nnodes > LOG_MAX_RANGES) 1117 return -E2BIG; 1118 1119 ranges = u64_to_user_ptr(control.ranges); 1120 nodes = kmalloc_objs(struct interval_tree_node, nnodes); 1121 if (!nodes) 1122 return -ENOMEM; 1123 1124 for (i = 0; i < nnodes; i++) { 1125 if (copy_from_user(&range, &ranges[i], sizeof(range))) { 1126 ret = -EFAULT; 1127 goto end; 1128 } 1129 if (!IS_ALIGNED(range.iova, control.page_size) || 1130 !IS_ALIGNED(range.length, control.page_size)) { 1131 ret = -EINVAL; 1132 goto end; 1133 } 1134 1135 if (check_add_overflow(range.iova, range.length, &iova_end) || 1136 iova_end > ULONG_MAX) { 1137 ret = -EOVERFLOW; 1138 goto end; 1139 } 1140 1141 nodes[i].start = range.iova; 1142 nodes[i].last = range.iova + range.length - 1; 1143 if (interval_tree_iter_first(&root, nodes[i].start, 1144 nodes[i].last)) { 1145 /* Range overlapping */ 1146 ret = -EINVAL; 1147 goto end; 1148 } 1149 interval_tree_insert(nodes + i, &root); 1150 } 1151 1152 ret = device->log_ops->log_start(device, &root, nnodes, 1153 &control.page_size); 1154 if (ret) 1155 goto end; 1156 1157 if (copy_to_user(arg, &control, sizeof(control))) { 1158 ret = -EFAULT; 1159 device->log_ops->log_stop(device); 1160 } 1161 1162 end: 1163 kfree(nodes); 1164 return ret; 1165 } 1166 1167 static int 1168 vfio_ioctl_device_feature_logging_stop(struct vfio_device *device, 1169 u32 flags, void __user *arg, 1170 size_t argsz) 1171 { 1172 int ret; 1173 1174 if (!device->log_ops) 1175 return -ENOTTY; 1176 1177 ret = vfio_check_feature(flags, argsz, 1178 VFIO_DEVICE_FEATURE_SET, 0); 1179 if (ret != 1) 1180 return ret; 1181 1182 return device->log_ops->log_stop(device); 1183 } 1184 1185 static int vfio_device_log_read_and_clear(struct iova_bitmap *iter, 1186 unsigned long iova, size_t length, 1187 void *opaque) 1188 { 1189 struct vfio_device *device = opaque; 1190 1191 return device->log_ops->log_read_and_clear(device, iova, length, iter); 1192 } 1193 1194 static int 1195 vfio_ioctl_device_feature_logging_report(struct vfio_device *device, 1196 u32 flags, void __user *arg, 1197 size_t argsz) 1198 { 1199 size_t minsz = 1200 offsetofend(struct vfio_device_feature_dma_logging_report, 1201 bitmap); 1202 struct vfio_device_feature_dma_logging_report report; 1203 struct iova_bitmap *iter; 1204 u64 iova_end; 1205 int ret; 1206 1207 if (!device->log_ops) 1208 return -ENOTTY; 1209 1210 ret = vfio_check_feature(flags, argsz, 1211 VFIO_DEVICE_FEATURE_GET, 1212 sizeof(report)); 1213 if (ret != 1) 1214 return ret; 1215 1216 if (copy_from_user(&report, arg, minsz)) 1217 return -EFAULT; 1218 1219 if (report.page_size < SZ_4K || !is_power_of_2(report.page_size)) 1220 return -EINVAL; 1221 1222 if (check_add_overflow(report.iova, report.length, &iova_end) || 1223 iova_end > ULONG_MAX) 1224 return -EOVERFLOW; 1225 1226 iter = iova_bitmap_alloc(report.iova, report.length, 1227 report.page_size, 1228 u64_to_user_ptr(report.bitmap)); 1229 if (IS_ERR(iter)) 1230 return PTR_ERR(iter); 1231 1232 ret = iova_bitmap_for_each(iter, device, 1233 vfio_device_log_read_and_clear); 1234 1235 iova_bitmap_free(iter); 1236 return ret; 1237 } 1238 1239 static int vfio_ioctl_device_feature(struct vfio_device *device, 1240 struct vfio_device_feature __user *arg) 1241 { 1242 size_t minsz = offsetofend(struct vfio_device_feature, flags); 1243 struct vfio_device_feature feature; 1244 1245 if (copy_from_user(&feature, arg, minsz)) 1246 return -EFAULT; 1247 1248 if (feature.argsz < minsz) 1249 return -EINVAL; 1250 1251 /* Check unknown flags */ 1252 if (feature.flags & 1253 ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET | 1254 VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE)) 1255 return -EINVAL; 1256 1257 /* GET & SET are mutually exclusive except with PROBE */ 1258 if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) && 1259 (feature.flags & VFIO_DEVICE_FEATURE_SET) && 1260 (feature.flags & VFIO_DEVICE_FEATURE_GET)) 1261 return -EINVAL; 1262 1263 switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) { 1264 case VFIO_DEVICE_FEATURE_MIGRATION: 1265 return vfio_ioctl_device_feature_migration( 1266 device, feature.flags, arg->data, 1267 feature.argsz - minsz); 1268 case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE: 1269 return vfio_ioctl_device_feature_mig_device_state( 1270 device, feature.flags, arg->data, 1271 feature.argsz - minsz); 1272 case VFIO_DEVICE_FEATURE_DMA_LOGGING_START: 1273 return vfio_ioctl_device_feature_logging_start( 1274 device, feature.flags, arg->data, 1275 feature.argsz - minsz); 1276 case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP: 1277 return vfio_ioctl_device_feature_logging_stop( 1278 device, feature.flags, arg->data, 1279 feature.argsz - minsz); 1280 case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT: 1281 return vfio_ioctl_device_feature_logging_report( 1282 device, feature.flags, arg->data, 1283 feature.argsz - minsz); 1284 case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE: 1285 return vfio_ioctl_device_feature_migration_data_size( 1286 device, feature.flags, arg->data, 1287 feature.argsz - minsz); 1288 case VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2: 1289 return vfio_ioctl_device_feature_migration_precopy_info_v2( 1290 device, feature.flags, feature.argsz - minsz); 1291 default: 1292 if (unlikely(!device->ops->device_feature)) 1293 return -ENOTTY; 1294 return device->ops->device_feature(device, feature.flags, 1295 arg->data, 1296 feature.argsz - minsz); 1297 } 1298 } 1299 1300 static long vfio_get_region_info(struct vfio_device *device, 1301 struct vfio_region_info __user *arg) 1302 { 1303 unsigned long minsz = offsetofend(struct vfio_region_info, offset); 1304 struct vfio_region_info info = {}; 1305 struct vfio_info_cap caps = {}; 1306 int ret; 1307 1308 if (unlikely(!device->ops->get_region_info_caps)) 1309 return -EINVAL; 1310 1311 if (copy_from_user(&info, arg, minsz)) 1312 return -EFAULT; 1313 if (info.argsz < minsz) 1314 return -EINVAL; 1315 1316 ret = device->ops->get_region_info_caps(device, &info, &caps); 1317 if (ret) 1318 goto out_free; 1319 1320 if (caps.size) { 1321 info.flags |= VFIO_REGION_INFO_FLAG_CAPS; 1322 if (info.argsz < sizeof(info) + caps.size) { 1323 info.argsz = sizeof(info) + caps.size; 1324 info.cap_offset = 0; 1325 } else { 1326 vfio_info_cap_shift(&caps, sizeof(info)); 1327 if (copy_to_user(arg + 1, caps.buf, caps.size)) { 1328 ret = -EFAULT; 1329 goto out_free; 1330 } 1331 info.cap_offset = sizeof(info); 1332 } 1333 } 1334 1335 if (copy_to_user(arg, &info, minsz)){ 1336 ret = -EFAULT; 1337 goto out_free; 1338 } 1339 1340 out_free: 1341 kfree(caps.buf); 1342 return ret; 1343 } 1344 1345 static long vfio_device_fops_unl_ioctl(struct file *filep, 1346 unsigned int cmd, unsigned long arg) 1347 { 1348 struct vfio_device_file *df = filep->private_data; 1349 struct vfio_device *device = df->device; 1350 void __user *uptr = (void __user *)arg; 1351 int ret; 1352 1353 if (cmd == VFIO_DEVICE_BIND_IOMMUFD) 1354 return vfio_df_ioctl_bind_iommufd(df, uptr); 1355 1356 /* Paired with smp_store_release() following vfio_df_open() */ 1357 if (!smp_load_acquire(&df->access_granted)) 1358 return -EINVAL; 1359 1360 ret = vfio_device_pm_runtime_get(device); 1361 if (ret) 1362 return ret; 1363 1364 /* cdev only ioctls */ 1365 if (IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) && !df->group) { 1366 switch (cmd) { 1367 case VFIO_DEVICE_ATTACH_IOMMUFD_PT: 1368 ret = vfio_df_ioctl_attach_pt(df, uptr); 1369 goto out; 1370 1371 case VFIO_DEVICE_DETACH_IOMMUFD_PT: 1372 ret = vfio_df_ioctl_detach_pt(df, uptr); 1373 goto out; 1374 } 1375 } 1376 1377 switch (cmd) { 1378 case VFIO_DEVICE_FEATURE: 1379 ret = vfio_ioctl_device_feature(device, uptr); 1380 break; 1381 1382 case VFIO_DEVICE_GET_REGION_INFO: 1383 ret = vfio_get_region_info(device, uptr); 1384 break; 1385 1386 default: 1387 if (unlikely(!device->ops->ioctl)) 1388 ret = -EINVAL; 1389 else 1390 ret = device->ops->ioctl(device, cmd, arg); 1391 break; 1392 } 1393 out: 1394 vfio_device_pm_runtime_put(device); 1395 return ret; 1396 } 1397 1398 static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf, 1399 size_t count, loff_t *ppos) 1400 { 1401 struct vfio_device_file *df = filep->private_data; 1402 struct vfio_device *device = df->device; 1403 1404 /* Paired with smp_store_release() following vfio_df_open() */ 1405 if (!smp_load_acquire(&df->access_granted)) 1406 return -EINVAL; 1407 1408 if (unlikely(!device->ops->read)) 1409 return -EINVAL; 1410 1411 return device->ops->read(device, buf, count, ppos); 1412 } 1413 1414 static ssize_t vfio_device_fops_write(struct file *filep, 1415 const char __user *buf, 1416 size_t count, loff_t *ppos) 1417 { 1418 struct vfio_device_file *df = filep->private_data; 1419 struct vfio_device *device = df->device; 1420 1421 /* Paired with smp_store_release() following vfio_df_open() */ 1422 if (!smp_load_acquire(&df->access_granted)) 1423 return -EINVAL; 1424 1425 if (unlikely(!device->ops->write)) 1426 return -EINVAL; 1427 1428 return device->ops->write(device, buf, count, ppos); 1429 } 1430 1431 static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma) 1432 { 1433 struct vfio_device_file *df = filep->private_data; 1434 struct vfio_device *device = df->device; 1435 1436 /* Paired with smp_store_release() following vfio_df_open() */ 1437 if (!smp_load_acquire(&df->access_granted)) 1438 return -EINVAL; 1439 1440 if (unlikely(!device->ops->mmap)) 1441 return -EINVAL; 1442 1443 return device->ops->mmap(device, vma); 1444 } 1445 1446 #ifdef CONFIG_PROC_FS 1447 static void vfio_device_show_fdinfo(struct seq_file *m, struct file *filep) 1448 { 1449 char *path; 1450 struct vfio_device_file *df = filep->private_data; 1451 struct vfio_device *device = df->device; 1452 1453 path = kobject_get_path(&device->dev->kobj, GFP_KERNEL); 1454 if (!path) 1455 return; 1456 1457 seq_printf(m, "vfio-device-syspath: /sys%s\n", path); 1458 kfree(path); 1459 } 1460 #endif 1461 1462 const struct file_operations vfio_device_fops = { 1463 .owner = THIS_MODULE, 1464 .open = vfio_device_fops_cdev_open, 1465 .release = vfio_device_fops_release, 1466 .read = vfio_device_fops_read, 1467 .write = vfio_device_fops_write, 1468 .unlocked_ioctl = vfio_device_fops_unl_ioctl, 1469 .compat_ioctl = compat_ptr_ioctl, 1470 .mmap = vfio_device_fops_mmap, 1471 #ifdef CONFIG_PROC_FS 1472 .show_fdinfo = vfio_device_show_fdinfo, 1473 #endif 1474 }; 1475 1476 static struct vfio_device *vfio_device_from_file(struct file *file) 1477 { 1478 struct vfio_device_file *df = file->private_data; 1479 1480 if (file->f_op != &vfio_device_fops) 1481 return NULL; 1482 return df->device; 1483 } 1484 1485 /** 1486 * vfio_file_is_valid - True if the file is valid vfio file 1487 * @file: VFIO group file or VFIO device file 1488 */ 1489 bool vfio_file_is_valid(struct file *file) 1490 { 1491 return vfio_group_from_file(file) || 1492 vfio_device_from_file(file); 1493 } 1494 EXPORT_SYMBOL_GPL(vfio_file_is_valid); 1495 1496 /** 1497 * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file 1498 * is always CPU cache coherent 1499 * @file: VFIO group file or VFIO device file 1500 * 1501 * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop 1502 * bit in DMA transactions. A return of false indicates that the user has 1503 * rights to access additional instructions such as wbinvd on x86. 1504 */ 1505 bool vfio_file_enforced_coherent(struct file *file) 1506 { 1507 struct vfio_device *device; 1508 struct vfio_group *group; 1509 1510 group = vfio_group_from_file(file); 1511 if (group) 1512 return vfio_group_enforced_coherent(group); 1513 1514 device = vfio_device_from_file(file); 1515 if (device) 1516 return device_iommu_capable(device->dev, 1517 IOMMU_CAP_ENFORCE_CACHE_COHERENCY); 1518 1519 return true; 1520 } 1521 EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent); 1522 1523 static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm) 1524 { 1525 struct vfio_device_file *df = file->private_data; 1526 1527 /* 1528 * The kvm is first recorded in the vfio_device_file, and will 1529 * be propagated to vfio_device::kvm when the file is bound to 1530 * iommufd successfully in the vfio device cdev path. 1531 */ 1532 spin_lock(&df->kvm_ref_lock); 1533 df->kvm = kvm; 1534 spin_unlock(&df->kvm_ref_lock); 1535 } 1536 1537 /** 1538 * vfio_file_set_kvm - Link a kvm with VFIO drivers 1539 * @file: VFIO group file or VFIO device file 1540 * @kvm: KVM to link 1541 * 1542 * When a VFIO device is first opened the KVM will be available in 1543 * device->kvm if one was associated with the file. 1544 */ 1545 void vfio_file_set_kvm(struct file *file, struct kvm *kvm) 1546 { 1547 struct vfio_group *group; 1548 1549 group = vfio_group_from_file(file); 1550 if (group) 1551 vfio_group_set_kvm(group, kvm); 1552 1553 if (vfio_device_from_file(file)) 1554 vfio_device_file_set_kvm(file, kvm); 1555 } 1556 EXPORT_SYMBOL_GPL(vfio_file_set_kvm); 1557 1558 /* 1559 * Sub-module support 1560 */ 1561 /* 1562 * Helper for managing a buffer of info chain capabilities, allocate or 1563 * reallocate a buffer with additional @size, filling in @id and @version 1564 * of the capability. A pointer to the new capability is returned. 1565 * 1566 * NB. The chain is based at the head of the buffer, so new entries are 1567 * added to the tail, vfio_info_cap_shift() should be called to fixup the 1568 * next offsets prior to copying to the user buffer. 1569 */ 1570 struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps, 1571 size_t size, u16 id, u16 version) 1572 { 1573 void *buf; 1574 struct vfio_info_cap_header *header, *tmp; 1575 1576 /* Ensure that the next capability struct will be aligned */ 1577 size = ALIGN(size, sizeof(u64)); 1578 1579 buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL); 1580 if (!buf) { 1581 kfree(caps->buf); 1582 caps->buf = NULL; 1583 caps->size = 0; 1584 return ERR_PTR(-ENOMEM); 1585 } 1586 1587 caps->buf = buf; 1588 header = buf + caps->size; 1589 1590 /* Eventually copied to user buffer, zero */ 1591 memset(header, 0, size); 1592 1593 header->id = id; 1594 header->version = version; 1595 1596 /* Add to the end of the capability chain */ 1597 for (tmp = buf; tmp->next; tmp = buf + tmp->next) 1598 ; /* nothing */ 1599 1600 tmp->next = caps->size; 1601 caps->size += size; 1602 1603 return header; 1604 } 1605 EXPORT_SYMBOL_GPL(vfio_info_cap_add); 1606 1607 void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset) 1608 { 1609 struct vfio_info_cap_header *tmp; 1610 void *buf = (void *)caps->buf; 1611 1612 /* Capability structs should start with proper alignment */ 1613 WARN_ON(!IS_ALIGNED(offset, sizeof(u64))); 1614 1615 for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset) 1616 tmp->next += offset; 1617 } 1618 EXPORT_SYMBOL(vfio_info_cap_shift); 1619 1620 int vfio_info_add_capability(struct vfio_info_cap *caps, 1621 struct vfio_info_cap_header *cap, size_t size) 1622 { 1623 struct vfio_info_cap_header *header; 1624 1625 header = vfio_info_cap_add(caps, size, cap->id, cap->version); 1626 if (IS_ERR(header)) 1627 return PTR_ERR(header); 1628 1629 memcpy(header + 1, cap + 1, size - sizeof(*header)); 1630 1631 return 0; 1632 } 1633 EXPORT_SYMBOL(vfio_info_add_capability); 1634 1635 int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs, 1636 int max_irq_type, size_t *data_size) 1637 { 1638 unsigned long minsz; 1639 size_t size; 1640 1641 minsz = offsetofend(struct vfio_irq_set, count); 1642 1643 if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) || 1644 (hdr->count >= (U32_MAX - hdr->start)) || 1645 (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK | 1646 VFIO_IRQ_SET_ACTION_TYPE_MASK))) 1647 return -EINVAL; 1648 1649 if (data_size) 1650 *data_size = 0; 1651 1652 if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs) 1653 return -EINVAL; 1654 1655 switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 1656 case VFIO_IRQ_SET_DATA_NONE: 1657 size = 0; 1658 break; 1659 case VFIO_IRQ_SET_DATA_BOOL: 1660 size = sizeof(uint8_t); 1661 break; 1662 case VFIO_IRQ_SET_DATA_EVENTFD: 1663 size = sizeof(int32_t); 1664 break; 1665 default: 1666 return -EINVAL; 1667 } 1668 1669 if (size) { 1670 if (hdr->argsz - minsz < hdr->count * size) 1671 return -EINVAL; 1672 1673 if (!data_size) 1674 return -EINVAL; 1675 1676 *data_size = hdr->count * size; 1677 } 1678 1679 return 0; 1680 } 1681 EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare); 1682 1683 /* 1684 * Pin contiguous user pages and return their associated host pages for local 1685 * domain only. 1686 * @device [in] : device 1687 * @iova [in] : starting IOVA of user pages to be pinned. 1688 * @npage [in] : count of pages to be pinned. This count should not 1689 * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 1690 * @prot [in] : protection flags 1691 * @pages[out] : array of host pages 1692 * Return error or number of pages pinned. 1693 * 1694 * A driver may only call this function if the vfio_device was created 1695 * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages(). 1696 */ 1697 int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, 1698 int npage, int prot, struct page **pages) 1699 { 1700 /* group->container cannot change while a vfio device is open */ 1701 if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device))) 1702 return -EINVAL; 1703 if (!device->ops->dma_unmap) 1704 return -EINVAL; 1705 if (vfio_device_has_container(device)) 1706 return vfio_device_container_pin_pages(device, iova, 1707 npage, prot, pages); 1708 if (device->iommufd_access) { 1709 int ret; 1710 1711 if (iova > ULONG_MAX) 1712 return -EINVAL; 1713 /* 1714 * VFIO ignores the sub page offset, npages is from the start of 1715 * a PAGE_SIZE chunk of IOVA. The caller is expected to recover 1716 * the sub page offset by doing: 1717 * pages[0] + (iova % PAGE_SIZE) 1718 */ 1719 ret = iommufd_access_pin_pages( 1720 device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE), 1721 npage * PAGE_SIZE, pages, 1722 (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0); 1723 if (ret) 1724 return ret; 1725 return npage; 1726 } 1727 return -EINVAL; 1728 } 1729 EXPORT_SYMBOL(vfio_pin_pages); 1730 1731 /* 1732 * Unpin contiguous host pages for local domain only. 1733 * @device [in] : device 1734 * @iova [in] : starting address of user pages to be unpinned. 1735 * @npage [in] : count of pages to be unpinned. This count should not 1736 * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 1737 */ 1738 void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage) 1739 { 1740 if (WARN_ON(!vfio_assert_device_open(device))) 1741 return; 1742 if (WARN_ON(!device->ops->dma_unmap)) 1743 return; 1744 1745 if (vfio_device_has_container(device)) { 1746 vfio_device_container_unpin_pages(device, iova, npage); 1747 return; 1748 } 1749 if (device->iommufd_access) { 1750 if (WARN_ON(iova > ULONG_MAX)) 1751 return; 1752 iommufd_access_unpin_pages(device->iommufd_access, 1753 ALIGN_DOWN(iova, PAGE_SIZE), 1754 npage * PAGE_SIZE); 1755 return; 1756 } 1757 } 1758 EXPORT_SYMBOL(vfio_unpin_pages); 1759 1760 /* 1761 * This interface allows the CPUs to perform some sort of virtual DMA on 1762 * behalf of the device. 1763 * 1764 * CPUs read/write from/into a range of IOVAs pointing to user space memory 1765 * into/from a kernel buffer. 1766 * 1767 * As the read/write of user space memory is conducted via the CPUs and is 1768 * not a real device DMA, it is not necessary to pin the user space memory. 1769 * 1770 * @device [in] : VFIO device 1771 * @iova [in] : base IOVA of a user space buffer 1772 * @data [in] : pointer to kernel buffer 1773 * @len [in] : kernel buffer length 1774 * @write : indicate read or write 1775 * Return error code on failure or 0 on success. 1776 */ 1777 int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, 1778 size_t len, bool write) 1779 { 1780 if (!data || len <= 0 || !vfio_assert_device_open(device)) 1781 return -EINVAL; 1782 1783 if (vfio_device_has_container(device)) 1784 return vfio_device_container_dma_rw(device, iova, 1785 data, len, write); 1786 1787 if (device->iommufd_access) { 1788 unsigned int flags = 0; 1789 1790 if (iova > ULONG_MAX) 1791 return -EINVAL; 1792 1793 /* VFIO historically tries to auto-detect a kthread */ 1794 if (!current->mm) 1795 flags |= IOMMUFD_ACCESS_RW_KTHREAD; 1796 if (write) 1797 flags |= IOMMUFD_ACCESS_RW_WRITE; 1798 return iommufd_access_rw(device->iommufd_access, iova, data, 1799 len, flags); 1800 } 1801 return -EINVAL; 1802 } 1803 EXPORT_SYMBOL(vfio_dma_rw); 1804 1805 /* 1806 * Module/class support 1807 */ 1808 static int __init vfio_init(void) 1809 { 1810 int ret; 1811 1812 ida_init(&vfio.device_ida); 1813 1814 ret = vfio_group_init(); 1815 if (ret) 1816 return ret; 1817 1818 ret = vfio_virqfd_init(); 1819 if (ret) 1820 goto err_virqfd; 1821 1822 /* /sys/class/vfio-dev/vfioX */ 1823 ret = class_register(&vfio_device_class); 1824 if (ret) 1825 goto err_dev_class; 1826 1827 ret = vfio_cdev_init(); 1828 if (ret) 1829 goto err_alloc_dev_chrdev; 1830 1831 vfio_debugfs_create_root(); 1832 pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n"); 1833 return 0; 1834 1835 err_alloc_dev_chrdev: 1836 class_unregister(&vfio_device_class); 1837 err_dev_class: 1838 vfio_virqfd_exit(); 1839 err_virqfd: 1840 vfio_group_cleanup(); 1841 return ret; 1842 } 1843 1844 static void __exit vfio_cleanup(void) 1845 { 1846 vfio_debugfs_remove_root(); 1847 ida_destroy(&vfio.device_ida); 1848 vfio_cdev_cleanup(); 1849 class_unregister(&vfio_device_class); 1850 vfio_virqfd_exit(); 1851 vfio_group_cleanup(); 1852 xa_destroy(&vfio_device_set_xa); 1853 } 1854 1855 module_init(vfio_init); 1856 module_exit(vfio_cleanup); 1857 1858 MODULE_IMPORT_NS("IOMMUFD"); 1859 MODULE_VERSION(DRIVER_VERSION); 1860 MODULE_LICENSE("GPL v2"); 1861 MODULE_AUTHOR(DRIVER_AUTHOR); 1862 MODULE_DESCRIPTION(DRIVER_DESC); 1863 MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce"); 1864