1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO core 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 * 8 * Derived from original vfio: 9 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 10 * Author: Tom Lyon, pugs@cisco.com 11 */ 12 13 #include <linux/cdev.h> 14 #include <linux/compat.h> 15 #include <linux/device.h> 16 #include <linux/fs.h> 17 #include <linux/idr.h> 18 #include <linux/iommu.h> 19 #if IS_ENABLED(CONFIG_KVM) 20 #include <linux/kvm_host.h> 21 #endif 22 #include <linux/list.h> 23 #include <linux/miscdevice.h> 24 #include <linux/module.h> 25 #include <linux/mount.h> 26 #include <linux/mutex.h> 27 #include <linux/pci.h> 28 #include <linux/pseudo_fs.h> 29 #include <linux/rwsem.h> 30 #include <linux/sched.h> 31 #include <linux/seq_file.h> 32 #include <linux/slab.h> 33 #include <linux/stat.h> 34 #include <linux/string.h> 35 #include <linux/uaccess.h> 36 #include <linux/vfio.h> 37 #include <linux/wait.h> 38 #include <linux/sched/signal.h> 39 #include <linux/pm_runtime.h> 40 #include <linux/interval_tree.h> 41 #include <linux/iova_bitmap.h> 42 #include <linux/iommufd.h> 43 #include "vfio.h" 44 45 #define DRIVER_VERSION "0.3" 46 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 47 #define DRIVER_DESC "VFIO - User Level meta-driver" 48 49 #define VFIO_MAGIC 0x5646494f /* "VFIO" */ 50 51 static struct vfio { 52 struct ida device_ida; 53 struct vfsmount *vfs_mount; 54 int fs_count; 55 } vfio; 56 57 #ifdef CONFIG_VFIO_NOIOMMU 58 bool vfio_noiommu __read_mostly; 59 module_param_named(enable_unsafe_noiommu_mode, 60 vfio_noiommu, bool, S_IRUGO | S_IWUSR); 61 MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)"); 62 #endif 63 64 static DEFINE_XARRAY(vfio_device_set_xa); 65 66 static char *vfio_device_devnode(const struct device *dev, umode_t *mode) 67 { 68 return kasprintf(GFP_KERNEL, "vfio/devices/%s", dev_name(dev)); 69 } 70 71 static const struct class vfio_device_class = { 72 .name = "vfio-dev", 73 .devnode = vfio_device_devnode 74 }; 75 76 int vfio_assign_device_set(struct vfio_device *device, void *set_id) 77 { 78 unsigned long idx = (unsigned long)set_id; 79 struct vfio_device_set *new_dev_set; 80 struct vfio_device_set *dev_set; 81 82 if (WARN_ON(!set_id)) 83 return -EINVAL; 84 85 /* 86 * Atomically acquire a singleton object in the xarray for this set_id 87 */ 88 xa_lock(&vfio_device_set_xa); 89 dev_set = xa_load(&vfio_device_set_xa, idx); 90 if (dev_set) 91 goto found_get_ref; 92 xa_unlock(&vfio_device_set_xa); 93 94 new_dev_set = kzalloc_obj(*new_dev_set); 95 if (!new_dev_set) 96 return -ENOMEM; 97 mutex_init(&new_dev_set->lock); 98 INIT_LIST_HEAD(&new_dev_set->device_list); 99 new_dev_set->set_id = set_id; 100 101 xa_lock(&vfio_device_set_xa); 102 dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set, 103 GFP_KERNEL); 104 if (!dev_set) { 105 dev_set = new_dev_set; 106 goto found_get_ref; 107 } 108 109 kfree(new_dev_set); 110 if (xa_is_err(dev_set)) { 111 xa_unlock(&vfio_device_set_xa); 112 return xa_err(dev_set); 113 } 114 115 found_get_ref: 116 dev_set->device_count++; 117 xa_unlock(&vfio_device_set_xa); 118 mutex_lock(&dev_set->lock); 119 device->dev_set = dev_set; 120 list_add_tail(&device->dev_set_list, &dev_set->device_list); 121 mutex_unlock(&dev_set->lock); 122 return 0; 123 } 124 EXPORT_SYMBOL_GPL(vfio_assign_device_set); 125 126 static void vfio_release_device_set(struct vfio_device *device) 127 { 128 struct vfio_device_set *dev_set = device->dev_set; 129 130 if (!dev_set) 131 return; 132 133 mutex_lock(&dev_set->lock); 134 list_del(&device->dev_set_list); 135 mutex_unlock(&dev_set->lock); 136 137 xa_lock(&vfio_device_set_xa); 138 if (!--dev_set->device_count) { 139 __xa_erase(&vfio_device_set_xa, 140 (unsigned long)dev_set->set_id); 141 mutex_destroy(&dev_set->lock); 142 kfree(dev_set); 143 } 144 xa_unlock(&vfio_device_set_xa); 145 } 146 147 unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set) 148 { 149 struct vfio_device *cur; 150 unsigned int open_count = 0; 151 152 lockdep_assert_held(&dev_set->lock); 153 154 list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 155 open_count += cur->open_count; 156 return open_count; 157 } 158 EXPORT_SYMBOL_GPL(vfio_device_set_open_count); 159 160 struct vfio_device * 161 vfio_find_device_in_devset(struct vfio_device_set *dev_set, 162 struct device *dev) 163 { 164 struct vfio_device *cur; 165 166 lockdep_assert_held(&dev_set->lock); 167 168 list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 169 if (cur->dev == dev) 170 return cur; 171 return NULL; 172 } 173 EXPORT_SYMBOL_GPL(vfio_find_device_in_devset); 174 175 /* 176 * Device objects - create, release, get, put, search 177 */ 178 /* Device reference always implies a group reference */ 179 void vfio_device_put_registration(struct vfio_device *device) 180 { 181 if (refcount_dec_and_test(&device->refcount)) 182 complete(&device->comp); 183 } 184 EXPORT_SYMBOL_GPL(vfio_device_put_registration); 185 186 bool vfio_device_try_get_registration(struct vfio_device *device) 187 { 188 return refcount_inc_not_zero(&device->refcount); 189 } 190 EXPORT_SYMBOL_GPL(vfio_device_try_get_registration); 191 192 /* 193 * VFIO driver API 194 */ 195 /* Release helper called by vfio_put_device() */ 196 static void vfio_device_release(struct device *dev) 197 { 198 struct vfio_device *device = 199 container_of(dev, struct vfio_device, device); 200 201 vfio_release_device_set(device); 202 ida_free(&vfio.device_ida, device->index); 203 204 if (device->ops->release) 205 device->ops->release(device); 206 207 iput(device->inode); 208 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count); 209 kvfree(device); 210 } 211 212 static int vfio_init_device(struct vfio_device *device, struct device *dev, 213 const struct vfio_device_ops *ops); 214 215 /* 216 * Allocate and initialize vfio_device so it can be registered to vfio 217 * core. 218 * 219 * Drivers should use the wrapper vfio_alloc_device() for allocation. 220 * @size is the size of the structure to be allocated, including any 221 * private data used by the driver. 222 * 223 * Driver may provide an @init callback to cover device private data. 224 * 225 * Use vfio_put_device() to release the structure after success return. 226 */ 227 struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, 228 const struct vfio_device_ops *ops) 229 { 230 struct vfio_device *device; 231 int ret; 232 233 if (WARN_ON(size < sizeof(struct vfio_device))) 234 return ERR_PTR(-EINVAL); 235 236 device = kvzalloc(size, GFP_KERNEL); 237 if (!device) 238 return ERR_PTR(-ENOMEM); 239 240 ret = vfio_init_device(device, dev, ops); 241 if (ret) 242 goto out_free; 243 return device; 244 245 out_free: 246 kvfree(device); 247 return ERR_PTR(ret); 248 } 249 EXPORT_SYMBOL_GPL(_vfio_alloc_device); 250 251 static int vfio_fs_init_fs_context(struct fs_context *fc) 252 { 253 return init_pseudo(fc, VFIO_MAGIC) ? 0 : -ENOMEM; 254 } 255 256 static struct file_system_type vfio_fs_type = { 257 .name = "vfio", 258 .owner = THIS_MODULE, 259 .init_fs_context = vfio_fs_init_fs_context, 260 .kill_sb = kill_anon_super, 261 }; 262 263 static struct inode *vfio_fs_inode_new(void) 264 { 265 struct inode *inode; 266 int ret; 267 268 ret = simple_pin_fs(&vfio_fs_type, &vfio.vfs_mount, &vfio.fs_count); 269 if (ret) 270 return ERR_PTR(ret); 271 272 inode = alloc_anon_inode(vfio.vfs_mount->mnt_sb); 273 if (IS_ERR(inode)) 274 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count); 275 276 return inode; 277 } 278 279 /* 280 * Initialize a vfio_device so it can be registered to vfio core. 281 */ 282 static int vfio_init_device(struct vfio_device *device, struct device *dev, 283 const struct vfio_device_ops *ops) 284 { 285 int ret; 286 287 ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL); 288 if (ret < 0) { 289 dev_dbg(dev, "Error to alloc index\n"); 290 return ret; 291 } 292 293 device->index = ret; 294 init_completion(&device->comp); 295 device->dev = dev; 296 device->ops = ops; 297 device->inode = vfio_fs_inode_new(); 298 if (IS_ERR(device->inode)) { 299 ret = PTR_ERR(device->inode); 300 goto out_inode; 301 } 302 303 if (ops->init) { 304 ret = ops->init(device); 305 if (ret) 306 goto out_uninit; 307 } 308 309 device_initialize(&device->device); 310 device->device.release = vfio_device_release; 311 device->device.class = &vfio_device_class; 312 device->device.parent = device->dev; 313 return 0; 314 315 out_uninit: 316 iput(device->inode); 317 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count); 318 out_inode: 319 vfio_release_device_set(device); 320 ida_free(&vfio.device_ida, device->index); 321 return ret; 322 } 323 324 static int vfio_device_set_noiommu_and_name(struct vfio_device *device, enum vfio_group_type type) 325 { 326 if (IS_ENABLED(CONFIG_IOMMUFD_NOIOMMU) && vfio_noiommu && 327 !device->dev->iommu && type == VFIO_IOMMU) 328 device->noiommu = true; 329 330 /* 331 * device->noiommu records no-IOMMU support for the standalone cdev 332 * interface. VFIO_NOIOMMU enables both group and cdev no-IOMMU; when 333 * cdev no-IOMMU is available, device->noiommu is set before 334 * vfio_device_set_group(), so the cdev is named noiommu-vfio%d up 335 * front. If IOMMUFD_NOIOMMU is unavailable, no-IOMMU devices are 336 * limited to the group interface and do not receive a device cdev. 337 */ 338 return dev_set_name(&device->device, "%svfio%d", 339 device->noiommu ? "noiommu-" : "", device->index); 340 } 341 342 static int __vfio_register_dev(struct vfio_device *device, 343 enum vfio_group_type type) 344 { 345 int ret; 346 347 if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) && 348 (!device->ops->bind_iommufd || 349 !device->ops->unbind_iommufd || 350 !device->ops->attach_ioas || 351 !device->ops->detach_ioas))) 352 return -EINVAL; 353 354 /* 355 * If the driver doesn't specify a set then the device is added to a 356 * singleton set just for itself. 357 */ 358 if (!device->dev_set) 359 vfio_assign_device_set(device, device); 360 361 ret = vfio_device_set_noiommu_and_name(device, type); 362 if (ret) 363 return ret; 364 365 ret = vfio_device_set_group(device, type); 366 if (ret) 367 return ret; 368 369 if (vfio_device_is_noiommu(device) && IS_ENABLED(CONFIG_IOMMUFD_NOIOMMU)) { 370 add_taint(TAINT_USER, LOCKDEP_STILL_OK); 371 dev_warn(device->dev, 372 "Adding kernel taint for vfio-noiommu cdev\n"); 373 } 374 375 /* 376 * VFIO always sets IOMMU_CACHE because we offer no way for userspace to 377 * restore cache coherency. It has to be checked here because it is only 378 * valid for cases where we are using iommu groups. 379 */ 380 if (type == VFIO_IOMMU && !vfio_device_is_noiommu(device) && 381 !device_iommu_capable(device->dev, IOMMU_CAP_CACHE_COHERENCY)) { 382 ret = -EINVAL; 383 goto err_out; 384 } 385 386 ret = vfio_device_add(device); 387 if (ret) 388 goto err_out; 389 390 /* Refcounting can't start until the driver calls register */ 391 refcount_set(&device->refcount, 1); 392 393 vfio_device_group_register(device); 394 vfio_device_debugfs_init(device); 395 396 return 0; 397 err_out: 398 vfio_device_remove_group(device); 399 return ret; 400 } 401 402 int vfio_register_group_dev(struct vfio_device *device) 403 { 404 return __vfio_register_dev(device, VFIO_IOMMU); 405 } 406 EXPORT_SYMBOL_GPL(vfio_register_group_dev); 407 408 /* 409 * Register a virtual device without IOMMU backing. The user of this 410 * device must not be able to directly trigger unmediated DMA. 411 */ 412 int vfio_register_emulated_iommu_dev(struct vfio_device *device) 413 { 414 return __vfio_register_dev(device, VFIO_EMULATED_IOMMU); 415 } 416 EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev); 417 418 /* 419 * Decrement the device reference count and wait for the device to be 420 * removed. Open file descriptors for the device... */ 421 void vfio_unregister_group_dev(struct vfio_device *device) 422 { 423 unsigned int i = 0; 424 bool interrupted = false; 425 long rc; 426 427 /* 428 * Prevent new device opened by userspace via the 429 * VFIO_GROUP_GET_DEVICE_FD in the group path. 430 */ 431 vfio_device_group_unregister(device); 432 433 /* 434 * Remove debugfs before device_del(), which releases devres. Some 435 * debugfs entries are created with debugfs_create_devm_seqfile() and 436 * therefore rely on devres-managed inode private data. 437 */ 438 vfio_device_debugfs_exit(device); 439 440 /* 441 * Balances vfio_device_add() in register path, also prevents 442 * new device opened by userspace in the cdev path. 443 */ 444 vfio_device_del(device); 445 446 vfio_device_put_registration(device); 447 rc = try_wait_for_completion(&device->comp); 448 while (rc <= 0) { 449 if (device->ops->request) 450 device->ops->request(device, i++); 451 452 if (interrupted) { 453 rc = wait_for_completion_timeout(&device->comp, 454 HZ * 10); 455 } else { 456 rc = wait_for_completion_interruptible_timeout( 457 &device->comp, HZ * 10); 458 if (rc < 0) { 459 interrupted = true; 460 dev_warn(device->dev, 461 "Device is currently in use, task" 462 " \"%s\" (%d) " 463 "blocked until device is released", 464 current->comm, task_pid_nr(current)); 465 } 466 } 467 } 468 469 /* Balances vfio_device_set_group in register path */ 470 vfio_device_remove_group(device); 471 } 472 EXPORT_SYMBOL_GPL(vfio_unregister_group_dev); 473 474 #if IS_ENABLED(CONFIG_KVM) 475 void vfio_device_get_kvm(struct vfio_device *device, struct file *kvm) 476 { 477 lockdep_assert_held(&device->dev_set->lock); 478 479 if (!kvm) 480 return; 481 482 device->kvm = get_file(kvm); 483 } 484 485 void vfio_device_put_kvm(struct vfio_device *device) 486 { 487 lockdep_assert_held(&device->dev_set->lock); 488 489 if (!device->kvm) 490 return; 491 492 fput(device->kvm); 493 device->kvm = NULL; 494 } 495 #endif 496 497 /* true if the vfio_device has open_device() called but not close_device() */ 498 static bool vfio_assert_device_open(struct vfio_device *device) 499 { 500 return !WARN_ON_ONCE(!READ_ONCE(device->open_count)); 501 } 502 503 struct vfio_device_file * 504 vfio_allocate_device_file(struct vfio_device *device) 505 { 506 struct vfio_device_file *df; 507 508 df = kzalloc_obj(*df, GFP_KERNEL_ACCOUNT); 509 if (!df) 510 return ERR_PTR(-ENOMEM); 511 512 df->device = device; 513 spin_lock_init(&df->kvm_ref_lock); 514 515 return df; 516 } 517 518 static int vfio_df_device_first_open(struct vfio_device_file *df) 519 { 520 struct vfio_device *device = df->device; 521 struct iommufd_ctx *iommufd = df->iommufd; 522 int ret; 523 524 lockdep_assert_held(&device->dev_set->lock); 525 526 if (!try_module_get(device->dev->driver->owner)) 527 return -ENODEV; 528 529 if (iommufd) 530 ret = vfio_df_iommufd_bind(df); 531 else 532 ret = vfio_device_group_use_iommu(device); 533 if (ret) 534 goto err_module_put; 535 536 if (device->ops->open_device) { 537 ret = device->ops->open_device(device); 538 if (ret) 539 goto err_unuse_iommu; 540 } 541 return 0; 542 543 err_unuse_iommu: 544 if (iommufd) 545 vfio_df_iommufd_unbind(df); 546 else 547 vfio_device_group_unuse_iommu(device); 548 err_module_put: 549 module_put(device->dev->driver->owner); 550 return ret; 551 } 552 553 static void vfio_df_device_last_close(struct vfio_device_file *df) 554 { 555 struct vfio_device *device = df->device; 556 struct iommufd_ctx *iommufd = df->iommufd; 557 558 lockdep_assert_held(&device->dev_set->lock); 559 560 if (device->ops->close_device) 561 device->ops->close_device(device); 562 if (iommufd) 563 vfio_df_iommufd_unbind(df); 564 else 565 vfio_device_group_unuse_iommu(device); 566 device->precopy_info_v2 = 0; 567 module_put(device->dev->driver->owner); 568 } 569 570 int vfio_df_open(struct vfio_device_file *df) 571 { 572 struct vfio_device *device = df->device; 573 int ret = 0; 574 575 lockdep_assert_held(&device->dev_set->lock); 576 577 device->open_count++; 578 if (device->open_count == 1) { 579 ret = vfio_df_device_first_open(df); 580 if (ret) 581 device->open_count--; 582 } 583 584 return ret; 585 } 586 587 void vfio_df_close(struct vfio_device_file *df) 588 { 589 struct vfio_device *device = df->device; 590 591 lockdep_assert_held(&device->dev_set->lock); 592 593 if (!vfio_assert_device_open(device)) 594 return; 595 if (device->open_count == 1) 596 vfio_df_device_last_close(df); 597 device->open_count--; 598 } 599 600 /* 601 * Wrapper around pm_runtime_resume_and_get(). 602 * Return error code on failure or 0 on success. 603 */ 604 static inline int vfio_device_pm_runtime_get(struct vfio_device *device) 605 { 606 struct device *dev = device->dev; 607 608 if (dev->driver && dev->driver->pm) { 609 int ret; 610 611 ret = pm_runtime_resume_and_get(dev); 612 if (ret) { 613 dev_info_ratelimited(dev, 614 "vfio: runtime resume failed %d\n", ret); 615 return -EIO; 616 } 617 } 618 619 return 0; 620 } 621 622 /* 623 * Wrapper around pm_runtime_put(). 624 */ 625 static inline void vfio_device_pm_runtime_put(struct vfio_device *device) 626 { 627 struct device *dev = device->dev; 628 629 if (dev->driver && dev->driver->pm) 630 pm_runtime_put(dev); 631 } 632 633 /* 634 * VFIO Device fd 635 */ 636 static int vfio_device_fops_release(struct inode *inode, struct file *filep) 637 { 638 struct vfio_device_file *df = filep->private_data; 639 struct vfio_device *device = df->device; 640 641 if (df->group) 642 vfio_df_group_close(df); 643 else 644 vfio_df_unbind_iommufd(df); 645 646 vfio_device_put_registration(device); 647 648 kfree(df); 649 650 return 0; 651 } 652 653 /* 654 * vfio_mig_get_next_state - Compute the next step in the FSM 655 * @cur_fsm - The current state the device is in 656 * @new_fsm - The target state to reach 657 * @next_fsm - Pointer to the next step to get to new_fsm 658 * 659 * Return 0 upon success, otherwise -errno 660 * Upon success the next step in the state progression between cur_fsm and 661 * new_fsm will be set in next_fsm. 662 * 663 * This breaks down requests for combination transitions into smaller steps and 664 * returns the next step to get to new_fsm. The function may need to be called 665 * multiple times before reaching new_fsm. 666 * 667 */ 668 int vfio_mig_get_next_state(struct vfio_device *device, 669 enum vfio_device_mig_state cur_fsm, 670 enum vfio_device_mig_state new_fsm, 671 enum vfio_device_mig_state *next_fsm) 672 { 673 enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 }; 674 /* 675 * The coding in this table requires the driver to implement the 676 * following FSM arcs: 677 * RESUMING -> STOP 678 * STOP -> RESUMING 679 * STOP -> STOP_COPY 680 * STOP_COPY -> STOP 681 * 682 * If P2P is supported then the driver must also implement these FSM 683 * arcs: 684 * RUNNING -> RUNNING_P2P 685 * RUNNING_P2P -> RUNNING 686 * RUNNING_P2P -> STOP 687 * STOP -> RUNNING_P2P 688 * 689 * If precopy is supported then the driver must support these additional 690 * FSM arcs: 691 * RUNNING -> PRE_COPY 692 * PRE_COPY -> RUNNING 693 * PRE_COPY -> STOP_COPY 694 * However, if precopy and P2P are supported together then the driver 695 * must support these additional arcs beyond the P2P arcs above: 696 * PRE_COPY -> RUNNING 697 * PRE_COPY -> PRE_COPY_P2P 698 * PRE_COPY_P2P -> PRE_COPY 699 * PRE_COPY_P2P -> RUNNING_P2P 700 * PRE_COPY_P2P -> STOP_COPY 701 * RUNNING -> PRE_COPY 702 * RUNNING_P2P -> PRE_COPY_P2P 703 * 704 * Without P2P and precopy the driver must implement: 705 * RUNNING -> STOP 706 * STOP -> RUNNING 707 * 708 * The coding will step through multiple states for some combination 709 * transitions; if all optional features are supported, this means the 710 * following ones: 711 * PRE_COPY -> PRE_COPY_P2P -> STOP_COPY 712 * PRE_COPY -> RUNNING -> RUNNING_P2P 713 * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP 714 * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING 715 * PRE_COPY_P2P -> RUNNING_P2P -> RUNNING 716 * PRE_COPY_P2P -> RUNNING_P2P -> STOP 717 * PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING 718 * RESUMING -> STOP -> RUNNING_P2P 719 * RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P 720 * RESUMING -> STOP -> RUNNING_P2P -> RUNNING 721 * RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 722 * RESUMING -> STOP -> STOP_COPY 723 * RUNNING -> RUNNING_P2P -> PRE_COPY_P2P 724 * RUNNING -> RUNNING_P2P -> STOP 725 * RUNNING -> RUNNING_P2P -> STOP -> RESUMING 726 * RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY 727 * RUNNING_P2P -> RUNNING -> PRE_COPY 728 * RUNNING_P2P -> STOP -> RESUMING 729 * RUNNING_P2P -> STOP -> STOP_COPY 730 * STOP -> RUNNING_P2P -> PRE_COPY_P2P 731 * STOP -> RUNNING_P2P -> RUNNING 732 * STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 733 * STOP_COPY -> STOP -> RESUMING 734 * STOP_COPY -> STOP -> RUNNING_P2P 735 * STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING 736 * 737 * The following transitions are blocked: 738 * STOP_COPY -> PRE_COPY 739 * STOP_COPY -> PRE_COPY_P2P 740 */ 741 static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = { 742 [VFIO_DEVICE_STATE_STOP] = { 743 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 744 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 745 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 746 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 747 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 748 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 749 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 750 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 751 }, 752 [VFIO_DEVICE_STATE_RUNNING] = { 753 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 754 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 755 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 756 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 757 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 758 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 759 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 760 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 761 }, 762 [VFIO_DEVICE_STATE_PRE_COPY] = { 763 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING, 764 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 765 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 766 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 767 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 768 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING, 769 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING, 770 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 771 }, 772 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = { 773 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 774 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 775 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 776 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 777 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 778 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 779 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 780 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 781 }, 782 [VFIO_DEVICE_STATE_STOP_COPY] = { 783 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 784 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 785 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 786 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 787 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 788 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 789 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 790 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 791 }, 792 [VFIO_DEVICE_STATE_RESUMING] = { 793 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 794 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 795 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP, 796 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP, 797 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 798 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 799 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 800 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 801 }, 802 [VFIO_DEVICE_STATE_RUNNING_P2P] = { 803 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 804 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 805 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING, 806 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 807 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 808 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 809 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 810 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 811 }, 812 [VFIO_DEVICE_STATE_ERROR] = { 813 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR, 814 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR, 815 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 816 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 817 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR, 818 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR, 819 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR, 820 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 821 }, 822 }; 823 824 static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = { 825 [VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY, 826 [VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY, 827 [VFIO_DEVICE_STATE_PRE_COPY] = 828 VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY, 829 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY | 830 VFIO_MIGRATION_P2P | 831 VFIO_MIGRATION_PRE_COPY, 832 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY, 833 [VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY, 834 [VFIO_DEVICE_STATE_RUNNING_P2P] = 835 VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P, 836 [VFIO_DEVICE_STATE_ERROR] = ~0U, 837 }; 838 839 if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 840 (state_flags_table[cur_fsm] & device->migration_flags) != 841 state_flags_table[cur_fsm])) 842 return -EINVAL; 843 844 if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 845 (state_flags_table[new_fsm] & device->migration_flags) != 846 state_flags_table[new_fsm]) 847 return -EINVAL; 848 849 /* 850 * Arcs touching optional and unsupported states are skipped over. The 851 * driver will instead see an arc from the original state to the next 852 * logical state, as per the above comment. 853 */ 854 *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm]; 855 while (*next_fsm != VFIO_DEVICE_STATE_ERROR && 856 (state_flags_table[*next_fsm] & device->migration_flags) != 857 state_flags_table[*next_fsm]) 858 *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm]; 859 860 return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL; 861 } 862 EXPORT_SYMBOL_GPL(vfio_mig_get_next_state); 863 864 /* 865 * Convert the drivers's struct file into a FD number and return it to userspace 866 */ 867 static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg, 868 struct vfio_device_feature_mig_state *mig) 869 { 870 int ret; 871 int fd; 872 873 fd = get_unused_fd_flags(O_CLOEXEC); 874 if (fd < 0) { 875 ret = fd; 876 goto out_fput; 877 } 878 879 mig->data_fd = fd; 880 if (copy_to_user(arg, mig, sizeof(*mig))) { 881 ret = -EFAULT; 882 goto out_put_unused; 883 } 884 fd_install(fd, filp); 885 return 0; 886 887 out_put_unused: 888 put_unused_fd(fd); 889 out_fput: 890 fput(filp); 891 return ret; 892 } 893 894 static int 895 vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device, 896 u32 flags, void __user *arg, 897 size_t argsz) 898 { 899 size_t minsz = 900 offsetofend(struct vfio_device_feature_mig_state, data_fd); 901 struct vfio_device_feature_mig_state mig; 902 struct file *filp = NULL; 903 int ret; 904 905 if (!device->mig_ops) 906 return -ENOTTY; 907 908 ret = vfio_check_feature(flags, argsz, 909 VFIO_DEVICE_FEATURE_SET | 910 VFIO_DEVICE_FEATURE_GET, 911 sizeof(mig)); 912 if (ret != 1) 913 return ret; 914 915 if (copy_from_user(&mig, arg, minsz)) 916 return -EFAULT; 917 918 if (flags & VFIO_DEVICE_FEATURE_GET) { 919 enum vfio_device_mig_state curr_state; 920 921 ret = device->mig_ops->migration_get_state(device, 922 &curr_state); 923 if (ret) 924 return ret; 925 mig.device_state = curr_state; 926 goto out_copy; 927 } 928 929 /* Handle the VFIO_DEVICE_FEATURE_SET */ 930 filp = device->mig_ops->migration_set_state(device, mig.device_state); 931 if (IS_ERR(filp) || !filp) 932 goto out_copy; 933 934 return vfio_ioct_mig_return_fd(filp, arg, &mig); 935 out_copy: 936 mig.data_fd = -1; 937 if (copy_to_user(arg, &mig, sizeof(mig))) 938 return -EFAULT; 939 if (IS_ERR(filp)) 940 return PTR_ERR(filp); 941 return 0; 942 } 943 944 static int 945 vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device, 946 u32 flags, void __user *arg, 947 size_t argsz) 948 { 949 struct vfio_device_feature_mig_data_size data_size = {}; 950 unsigned long stop_copy_length; 951 int ret; 952 953 if (!device->mig_ops) 954 return -ENOTTY; 955 956 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 957 sizeof(data_size)); 958 if (ret != 1) 959 return ret; 960 961 ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length); 962 if (ret) 963 return ret; 964 965 data_size.stop_copy_length = stop_copy_length; 966 if (copy_to_user(arg, &data_size, sizeof(data_size))) 967 return -EFAULT; 968 969 return 0; 970 } 971 972 static int 973 vfio_ioctl_device_feature_migration_precopy_info_v2(struct vfio_device *device, 974 u32 flags, size_t argsz) 975 { 976 int ret; 977 978 if (!(device->migration_flags & VFIO_MIGRATION_PRE_COPY)) 979 return -EINVAL; 980 981 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0); 982 if (ret != 1) 983 return ret; 984 985 device->precopy_info_v2 = 1; 986 return 0; 987 } 988 989 static int vfio_ioctl_device_feature_migration(struct vfio_device *device, 990 u32 flags, void __user *arg, 991 size_t argsz) 992 { 993 struct vfio_device_feature_migration mig = { 994 .flags = device->migration_flags, 995 }; 996 int ret; 997 998 if (!device->mig_ops) 999 return -ENOTTY; 1000 1001 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 1002 sizeof(mig)); 1003 if (ret != 1) 1004 return ret; 1005 if (copy_to_user(arg, &mig, sizeof(mig))) 1006 return -EFAULT; 1007 return 0; 1008 } 1009 1010 void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes, 1011 u32 req_nodes) 1012 { 1013 struct interval_tree_node *prev, *curr, *comb_start, *comb_end; 1014 unsigned long min_gap, curr_gap; 1015 1016 /* Special shortcut when a single range is required */ 1017 if (req_nodes == 1) { 1018 unsigned long last; 1019 1020 comb_start = interval_tree_iter_first(root, 0, ULONG_MAX); 1021 1022 /* Empty list */ 1023 if (WARN_ON_ONCE(!comb_start)) 1024 return; 1025 1026 curr = comb_start; 1027 while (curr) { 1028 last = curr->last; 1029 prev = curr; 1030 curr = interval_tree_iter_next(curr, 0, ULONG_MAX); 1031 if (prev != comb_start) 1032 interval_tree_remove(prev, root); 1033 } 1034 comb_start->last = last; 1035 return; 1036 } 1037 1038 /* Combine ranges which have the smallest gap */ 1039 while (cur_nodes > req_nodes) { 1040 prev = NULL; 1041 min_gap = ULONG_MAX; 1042 curr = interval_tree_iter_first(root, 0, ULONG_MAX); 1043 while (curr) { 1044 if (prev) { 1045 curr_gap = curr->start - prev->last; 1046 if (curr_gap < min_gap) { 1047 min_gap = curr_gap; 1048 comb_start = prev; 1049 comb_end = curr; 1050 } 1051 } 1052 prev = curr; 1053 curr = interval_tree_iter_next(curr, 0, ULONG_MAX); 1054 } 1055 1056 /* Empty list or no nodes to combine */ 1057 if (WARN_ON_ONCE(min_gap == ULONG_MAX)) 1058 break; 1059 1060 comb_start->last = comb_end->last; 1061 interval_tree_remove(comb_end, root); 1062 cur_nodes--; 1063 } 1064 } 1065 EXPORT_SYMBOL_GPL(vfio_combine_iova_ranges); 1066 1067 /* Ranges should fit into a single kernel page */ 1068 #define LOG_MAX_RANGES \ 1069 (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range)) 1070 1071 static int 1072 vfio_ioctl_device_feature_logging_start(struct vfio_device *device, 1073 u32 flags, void __user *arg, 1074 size_t argsz) 1075 { 1076 size_t minsz = 1077 offsetofend(struct vfio_device_feature_dma_logging_control, 1078 ranges); 1079 struct vfio_device_feature_dma_logging_range __user *ranges; 1080 struct vfio_device_feature_dma_logging_control control; 1081 struct vfio_device_feature_dma_logging_range range; 1082 struct rb_root_cached root = RB_ROOT_CACHED; 1083 struct interval_tree_node *nodes; 1084 u64 iova_end; 1085 u32 nnodes; 1086 int i, ret; 1087 1088 if (!device->log_ops) 1089 return -ENOTTY; 1090 1091 ret = vfio_check_feature(flags, argsz, 1092 VFIO_DEVICE_FEATURE_SET, 1093 sizeof(control)); 1094 if (ret != 1) 1095 return ret; 1096 1097 if (copy_from_user(&control, arg, minsz)) 1098 return -EFAULT; 1099 1100 nnodes = control.num_ranges; 1101 if (!nnodes) 1102 return -EINVAL; 1103 1104 if (nnodes > LOG_MAX_RANGES) 1105 return -E2BIG; 1106 1107 ranges = u64_to_user_ptr(control.ranges); 1108 nodes = kmalloc_objs(struct interval_tree_node, nnodes); 1109 if (!nodes) 1110 return -ENOMEM; 1111 1112 for (i = 0; i < nnodes; i++) { 1113 if (copy_from_user(&range, &ranges[i], sizeof(range))) { 1114 ret = -EFAULT; 1115 goto end; 1116 } 1117 if (!IS_ALIGNED(range.iova, control.page_size) || 1118 !IS_ALIGNED(range.length, control.page_size)) { 1119 ret = -EINVAL; 1120 goto end; 1121 } 1122 1123 if (check_add_overflow(range.iova, range.length, &iova_end) || 1124 iova_end > ULONG_MAX) { 1125 ret = -EOVERFLOW; 1126 goto end; 1127 } 1128 1129 nodes[i].start = range.iova; 1130 nodes[i].last = range.iova + range.length - 1; 1131 if (interval_tree_iter_first(&root, nodes[i].start, 1132 nodes[i].last)) { 1133 /* Range overlapping */ 1134 ret = -EINVAL; 1135 goto end; 1136 } 1137 interval_tree_insert(nodes + i, &root); 1138 } 1139 1140 ret = device->log_ops->log_start(device, &root, nnodes, 1141 &control.page_size); 1142 if (ret) 1143 goto end; 1144 1145 if (copy_to_user(arg, &control, sizeof(control))) { 1146 ret = -EFAULT; 1147 device->log_ops->log_stop(device); 1148 } 1149 1150 end: 1151 kfree(nodes); 1152 return ret; 1153 } 1154 1155 static int 1156 vfio_ioctl_device_feature_logging_stop(struct vfio_device *device, 1157 u32 flags, void __user *arg, 1158 size_t argsz) 1159 { 1160 int ret; 1161 1162 if (!device->log_ops) 1163 return -ENOTTY; 1164 1165 ret = vfio_check_feature(flags, argsz, 1166 VFIO_DEVICE_FEATURE_SET, 0); 1167 if (ret != 1) 1168 return ret; 1169 1170 return device->log_ops->log_stop(device); 1171 } 1172 1173 static int vfio_device_log_read_and_clear(struct iova_bitmap *iter, 1174 unsigned long iova, size_t length, 1175 void *opaque) 1176 { 1177 struct vfio_device *device = opaque; 1178 1179 return device->log_ops->log_read_and_clear(device, iova, length, iter); 1180 } 1181 1182 static int 1183 vfio_ioctl_device_feature_logging_report(struct vfio_device *device, 1184 u32 flags, void __user *arg, 1185 size_t argsz) 1186 { 1187 size_t minsz = 1188 offsetofend(struct vfio_device_feature_dma_logging_report, 1189 bitmap); 1190 struct vfio_device_feature_dma_logging_report report; 1191 struct iova_bitmap *iter; 1192 u64 iova_end; 1193 int ret; 1194 1195 if (!device->log_ops) 1196 return -ENOTTY; 1197 1198 ret = vfio_check_feature(flags, argsz, 1199 VFIO_DEVICE_FEATURE_GET, 1200 sizeof(report)); 1201 if (ret != 1) 1202 return ret; 1203 1204 if (copy_from_user(&report, arg, minsz)) 1205 return -EFAULT; 1206 1207 if (report.page_size < SZ_4K || !is_power_of_2(report.page_size)) 1208 return -EINVAL; 1209 1210 if (check_add_overflow(report.iova, report.length, &iova_end) || 1211 iova_end > ULONG_MAX) 1212 return -EOVERFLOW; 1213 1214 iter = iova_bitmap_alloc(report.iova, report.length, 1215 report.page_size, 1216 u64_to_user_ptr(report.bitmap)); 1217 if (IS_ERR(iter)) 1218 return PTR_ERR(iter); 1219 1220 ret = iova_bitmap_for_each(iter, device, 1221 vfio_device_log_read_and_clear); 1222 1223 iova_bitmap_free(iter); 1224 return ret; 1225 } 1226 1227 static int vfio_ioctl_device_feature(struct vfio_device *device, 1228 struct vfio_device_feature __user *arg) 1229 { 1230 size_t minsz = offsetofend(struct vfio_device_feature, flags); 1231 struct vfio_device_feature feature; 1232 1233 if (copy_from_user(&feature, arg, minsz)) 1234 return -EFAULT; 1235 1236 if (feature.argsz < minsz) 1237 return -EINVAL; 1238 1239 /* Check unknown flags */ 1240 if (feature.flags & 1241 ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET | 1242 VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE)) 1243 return -EINVAL; 1244 1245 /* GET & SET are mutually exclusive except with PROBE */ 1246 if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) && 1247 (feature.flags & VFIO_DEVICE_FEATURE_SET) && 1248 (feature.flags & VFIO_DEVICE_FEATURE_GET)) 1249 return -EINVAL; 1250 1251 switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) { 1252 case VFIO_DEVICE_FEATURE_MIGRATION: 1253 return vfio_ioctl_device_feature_migration( 1254 device, feature.flags, arg->data, 1255 feature.argsz - minsz); 1256 case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE: 1257 return vfio_ioctl_device_feature_mig_device_state( 1258 device, feature.flags, arg->data, 1259 feature.argsz - minsz); 1260 case VFIO_DEVICE_FEATURE_DMA_LOGGING_START: 1261 return vfio_ioctl_device_feature_logging_start( 1262 device, feature.flags, arg->data, 1263 feature.argsz - minsz); 1264 case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP: 1265 return vfio_ioctl_device_feature_logging_stop( 1266 device, feature.flags, arg->data, 1267 feature.argsz - minsz); 1268 case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT: 1269 return vfio_ioctl_device_feature_logging_report( 1270 device, feature.flags, arg->data, 1271 feature.argsz - minsz); 1272 case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE: 1273 return vfio_ioctl_device_feature_migration_data_size( 1274 device, feature.flags, arg->data, 1275 feature.argsz - minsz); 1276 case VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2: 1277 return vfio_ioctl_device_feature_migration_precopy_info_v2( 1278 device, feature.flags, feature.argsz - minsz); 1279 default: 1280 if (unlikely(!device->ops->device_feature)) 1281 return -ENOTTY; 1282 return device->ops->device_feature(device, feature.flags, 1283 arg->data, 1284 feature.argsz - minsz); 1285 } 1286 } 1287 1288 static long vfio_get_region_info(struct vfio_device *device, 1289 struct vfio_region_info __user *arg) 1290 { 1291 unsigned long minsz = offsetofend(struct vfio_region_info, offset); 1292 struct vfio_region_info info = {}; 1293 struct vfio_info_cap caps = {}; 1294 int ret; 1295 1296 if (unlikely(!device->ops->get_region_info_caps)) 1297 return -EINVAL; 1298 1299 if (copy_from_user(&info, arg, minsz)) 1300 return -EFAULT; 1301 if (info.argsz < minsz) 1302 return -EINVAL; 1303 1304 ret = device->ops->get_region_info_caps(device, &info, &caps); 1305 if (ret) 1306 goto out_free; 1307 1308 if (caps.size) { 1309 info.flags |= VFIO_REGION_INFO_FLAG_CAPS; 1310 if (info.argsz < sizeof(info) + caps.size) { 1311 info.argsz = sizeof(info) + caps.size; 1312 info.cap_offset = 0; 1313 } else { 1314 vfio_info_cap_shift(&caps, sizeof(info)); 1315 if (copy_to_user(arg + 1, caps.buf, caps.size)) { 1316 ret = -EFAULT; 1317 goto out_free; 1318 } 1319 info.cap_offset = sizeof(info); 1320 } 1321 } 1322 1323 if (copy_to_user(arg, &info, minsz)){ 1324 ret = -EFAULT; 1325 goto out_free; 1326 } 1327 1328 out_free: 1329 kfree(caps.buf); 1330 return ret; 1331 } 1332 1333 static long vfio_device_fops_unl_ioctl(struct file *filep, 1334 unsigned int cmd, unsigned long arg) 1335 { 1336 struct vfio_device_file *df = filep->private_data; 1337 struct vfio_device *device = df->device; 1338 void __user *uptr = (void __user *)arg; 1339 int ret; 1340 1341 if (cmd == VFIO_DEVICE_BIND_IOMMUFD) 1342 return vfio_df_ioctl_bind_iommufd(df, uptr); 1343 1344 /* Paired with smp_store_release() following vfio_df_open() */ 1345 if (!smp_load_acquire(&df->access_granted)) 1346 return -EINVAL; 1347 1348 ret = vfio_device_pm_runtime_get(device); 1349 if (ret) 1350 return ret; 1351 1352 /* cdev only ioctls */ 1353 if (IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) && !df->group) { 1354 switch (cmd) { 1355 case VFIO_DEVICE_ATTACH_IOMMUFD_PT: 1356 ret = vfio_df_ioctl_attach_pt(df, uptr); 1357 goto out; 1358 1359 case VFIO_DEVICE_DETACH_IOMMUFD_PT: 1360 ret = vfio_df_ioctl_detach_pt(df, uptr); 1361 goto out; 1362 } 1363 } 1364 1365 switch (cmd) { 1366 case VFIO_DEVICE_FEATURE: 1367 ret = vfio_ioctl_device_feature(device, uptr); 1368 break; 1369 1370 case VFIO_DEVICE_GET_REGION_INFO: 1371 ret = vfio_get_region_info(device, uptr); 1372 break; 1373 1374 default: 1375 if (unlikely(!device->ops->ioctl)) 1376 ret = -EINVAL; 1377 else 1378 ret = device->ops->ioctl(device, cmd, arg); 1379 break; 1380 } 1381 out: 1382 vfio_device_pm_runtime_put(device); 1383 return ret; 1384 } 1385 1386 static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf, 1387 size_t count, loff_t *ppos) 1388 { 1389 struct vfio_device_file *df = filep->private_data; 1390 struct vfio_device *device = df->device; 1391 1392 /* Paired with smp_store_release() following vfio_df_open() */ 1393 if (!smp_load_acquire(&df->access_granted)) 1394 return -EINVAL; 1395 1396 if (unlikely(!device->ops->read)) 1397 return -EINVAL; 1398 1399 return device->ops->read(device, buf, count, ppos); 1400 } 1401 1402 static ssize_t vfio_device_fops_write(struct file *filep, 1403 const char __user *buf, 1404 size_t count, loff_t *ppos) 1405 { 1406 struct vfio_device_file *df = filep->private_data; 1407 struct vfio_device *device = df->device; 1408 1409 /* Paired with smp_store_release() following vfio_df_open() */ 1410 if (!smp_load_acquire(&df->access_granted)) 1411 return -EINVAL; 1412 1413 if (unlikely(!device->ops->write)) 1414 return -EINVAL; 1415 1416 return device->ops->write(device, buf, count, ppos); 1417 } 1418 1419 static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma) 1420 { 1421 struct vfio_device_file *df = filep->private_data; 1422 struct vfio_device *device = df->device; 1423 1424 /* Paired with smp_store_release() following vfio_df_open() */ 1425 if (!smp_load_acquire(&df->access_granted)) 1426 return -EINVAL; 1427 1428 if (unlikely(!device->ops->mmap)) 1429 return -EINVAL; 1430 1431 return device->ops->mmap(device, vma); 1432 } 1433 1434 #ifdef CONFIG_PROC_FS 1435 static void vfio_device_show_fdinfo(struct seq_file *m, struct file *filep) 1436 { 1437 char *path; 1438 struct vfio_device_file *df = filep->private_data; 1439 struct vfio_device *device = df->device; 1440 1441 path = kobject_get_path(&device->dev->kobj, GFP_KERNEL); 1442 if (!path) 1443 return; 1444 1445 seq_printf(m, "vfio-device-syspath: /sys%s\n", path); 1446 kfree(path); 1447 } 1448 #endif 1449 1450 const struct file_operations vfio_device_fops = { 1451 .owner = THIS_MODULE, 1452 .open = vfio_device_fops_cdev_open, 1453 .release = vfio_device_fops_release, 1454 .read = vfio_device_fops_read, 1455 .write = vfio_device_fops_write, 1456 .unlocked_ioctl = vfio_device_fops_unl_ioctl, 1457 .compat_ioctl = compat_ptr_ioctl, 1458 .mmap = vfio_device_fops_mmap, 1459 #ifdef CONFIG_PROC_FS 1460 .show_fdinfo = vfio_device_show_fdinfo, 1461 #endif 1462 }; 1463 1464 static struct vfio_device *vfio_device_from_file(struct file *file) 1465 { 1466 struct vfio_device_file *df = file->private_data; 1467 1468 if (file->f_op != &vfio_device_fops) 1469 return NULL; 1470 return df->device; 1471 } 1472 1473 /** 1474 * vfio_file_is_valid - True if the file is valid vfio file 1475 * @file: VFIO group file or VFIO device file 1476 */ 1477 bool vfio_file_is_valid(struct file *file) 1478 { 1479 return vfio_group_from_file(file) || 1480 vfio_device_from_file(file); 1481 } 1482 EXPORT_SYMBOL_GPL(vfio_file_is_valid); 1483 1484 /** 1485 * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file 1486 * is always CPU cache coherent 1487 * @file: VFIO group file or VFIO device file 1488 * 1489 * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop 1490 * bit in DMA transactions. A return of false indicates that the user has 1491 * rights to access additional instructions such as wbinvd on x86. 1492 */ 1493 bool vfio_file_enforced_coherent(struct file *file) 1494 { 1495 struct vfio_device *device; 1496 struct vfio_group *group; 1497 1498 group = vfio_group_from_file(file); 1499 if (group) 1500 return vfio_group_enforced_coherent(group); 1501 1502 device = vfio_device_from_file(file); 1503 if (device) 1504 return device_iommu_capable(device->dev, 1505 IOMMU_CAP_ENFORCE_CACHE_COHERENCY); 1506 1507 return true; 1508 } 1509 EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent); 1510 1511 static void vfio_device_file_set_kvm(struct file *file, struct file *kvm) 1512 { 1513 struct vfio_device_file *df = file->private_data; 1514 struct file *old; 1515 1516 if (kvm) 1517 get_file(kvm); 1518 1519 /* 1520 * The kvm is first recorded in the vfio_device_file, and will 1521 * be propagated to vfio_device::kvm when the file is bound to 1522 * iommufd successfully in the vfio device cdev path. 1523 */ 1524 spin_lock(&df->kvm_ref_lock); 1525 old = df->kvm; 1526 df->kvm = kvm; 1527 spin_unlock(&df->kvm_ref_lock); 1528 1529 if (old) 1530 fput(old); 1531 } 1532 1533 /** 1534 * vfio_file_set_kvm - Link a kvm with VFIO drivers 1535 * @vfio_file: VFIO group file or VFIO device file 1536 * @kvm: KVM file to link 1537 * 1538 * When a VFIO device is first opened the KVM will be available in 1539 * device->kvm if one was associated with the file. 1540 */ 1541 void vfio_file_set_kvm(struct file *vfio_file, struct file *kvm) 1542 { 1543 struct vfio_group *group; 1544 1545 group = vfio_group_from_file(vfio_file); 1546 if (group) 1547 vfio_group_set_kvm(group, kvm); 1548 1549 if (vfio_device_from_file(vfio_file)) 1550 vfio_device_file_set_kvm(vfio_file, kvm); 1551 } 1552 EXPORT_SYMBOL_GPL(vfio_file_set_kvm); 1553 1554 /* 1555 * Sub-module support 1556 */ 1557 /* 1558 * Helper for managing a buffer of info chain capabilities, allocate or 1559 * reallocate a buffer with additional @size, filling in @id and @version 1560 * of the capability. A pointer to the new capability is returned. 1561 * 1562 * NB. The chain is based at the head of the buffer, so new entries are 1563 * added to the tail, vfio_info_cap_shift() should be called to fixup the 1564 * next offsets prior to copying to the user buffer. 1565 */ 1566 struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps, 1567 size_t size, u16 id, u16 version) 1568 { 1569 void *buf; 1570 struct vfio_info_cap_header *header, *tmp; 1571 1572 /* Ensure that the next capability struct will be aligned */ 1573 size = ALIGN(size, sizeof(u64)); 1574 1575 buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL); 1576 if (!buf) { 1577 kfree(caps->buf); 1578 caps->buf = NULL; 1579 caps->size = 0; 1580 return ERR_PTR(-ENOMEM); 1581 } 1582 1583 caps->buf = buf; 1584 header = buf + caps->size; 1585 1586 /* Eventually copied to user buffer, zero */ 1587 memset(header, 0, size); 1588 1589 header->id = id; 1590 header->version = version; 1591 1592 /* Add to the end of the capability chain */ 1593 for (tmp = buf; tmp->next; tmp = buf + tmp->next) 1594 ; /* nothing */ 1595 1596 tmp->next = caps->size; 1597 caps->size += size; 1598 1599 return header; 1600 } 1601 EXPORT_SYMBOL_GPL(vfio_info_cap_add); 1602 1603 void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset) 1604 { 1605 struct vfio_info_cap_header *tmp; 1606 void *buf = (void *)caps->buf; 1607 1608 /* Capability structs should start with proper alignment */ 1609 WARN_ON(!IS_ALIGNED(offset, sizeof(u64))); 1610 1611 for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset) 1612 tmp->next += offset; 1613 } 1614 EXPORT_SYMBOL(vfio_info_cap_shift); 1615 1616 int vfio_info_add_capability(struct vfio_info_cap *caps, 1617 struct vfio_info_cap_header *cap, size_t size) 1618 { 1619 struct vfio_info_cap_header *header; 1620 1621 header = vfio_info_cap_add(caps, size, cap->id, cap->version); 1622 if (IS_ERR(header)) 1623 return PTR_ERR(header); 1624 1625 memcpy(header + 1, cap + 1, size - sizeof(*header)); 1626 1627 return 0; 1628 } 1629 EXPORT_SYMBOL(vfio_info_add_capability); 1630 1631 int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs, 1632 int max_irq_type, size_t *data_size) 1633 { 1634 unsigned long minsz; 1635 size_t size; 1636 1637 minsz = offsetofend(struct vfio_irq_set, count); 1638 1639 if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) || 1640 (hdr->count >= (U32_MAX - hdr->start)) || 1641 (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK | 1642 VFIO_IRQ_SET_ACTION_TYPE_MASK))) 1643 return -EINVAL; 1644 1645 if (data_size) 1646 *data_size = 0; 1647 1648 if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs) 1649 return -EINVAL; 1650 1651 switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 1652 case VFIO_IRQ_SET_DATA_NONE: 1653 size = 0; 1654 break; 1655 case VFIO_IRQ_SET_DATA_BOOL: 1656 size = sizeof(uint8_t); 1657 break; 1658 case VFIO_IRQ_SET_DATA_EVENTFD: 1659 size = sizeof(int32_t); 1660 break; 1661 default: 1662 return -EINVAL; 1663 } 1664 1665 if (size) { 1666 if (hdr->argsz - minsz < hdr->count * size) 1667 return -EINVAL; 1668 1669 if (!data_size) 1670 return -EINVAL; 1671 1672 *data_size = hdr->count * size; 1673 } 1674 1675 return 0; 1676 } 1677 EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare); 1678 1679 /* 1680 * Pin contiguous user pages and return their associated host pages for local 1681 * domain only. 1682 * @device [in] : device 1683 * @iova [in] : starting IOVA of user pages to be pinned. 1684 * @npage [in] : count of pages to be pinned. This count should not 1685 * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 1686 * @prot [in] : protection flags 1687 * @pages[out] : array of host pages 1688 * Return error or number of pages pinned. 1689 * 1690 * A driver may only call this function if the vfio_device was created 1691 * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages(). 1692 */ 1693 int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, 1694 int npage, int prot, struct page **pages) 1695 { 1696 /* group->container cannot change while a vfio device is open */ 1697 if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device))) 1698 return -EINVAL; 1699 if (!device->ops->dma_unmap) 1700 return -EINVAL; 1701 if (vfio_device_has_container(device)) 1702 return vfio_device_container_pin_pages(device, iova, 1703 npage, prot, pages); 1704 if (device->iommufd_access) { 1705 int ret; 1706 1707 if (iova > ULONG_MAX) 1708 return -EINVAL; 1709 /* 1710 * VFIO ignores the sub page offset, npages is from the start of 1711 * a PAGE_SIZE chunk of IOVA. The caller is expected to recover 1712 * the sub page offset by doing: 1713 * pages[0] + (iova % PAGE_SIZE) 1714 */ 1715 ret = iommufd_access_pin_pages( 1716 device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE), 1717 npage * PAGE_SIZE, pages, 1718 (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0); 1719 if (ret) 1720 return ret; 1721 return npage; 1722 } 1723 return -EINVAL; 1724 } 1725 EXPORT_SYMBOL(vfio_pin_pages); 1726 1727 /* 1728 * Unpin contiguous host pages for local domain only. 1729 * @device [in] : device 1730 * @iova [in] : starting address of user pages to be unpinned. 1731 * @npage [in] : count of pages to be unpinned. This count should not 1732 * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 1733 */ 1734 void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage) 1735 { 1736 if (WARN_ON(!vfio_assert_device_open(device))) 1737 return; 1738 if (WARN_ON(!device->ops->dma_unmap)) 1739 return; 1740 1741 if (vfio_device_has_container(device)) { 1742 vfio_device_container_unpin_pages(device, iova, npage); 1743 return; 1744 } 1745 if (device->iommufd_access) { 1746 if (WARN_ON(iova > ULONG_MAX)) 1747 return; 1748 iommufd_access_unpin_pages(device->iommufd_access, 1749 ALIGN_DOWN(iova, PAGE_SIZE), 1750 npage * PAGE_SIZE); 1751 return; 1752 } 1753 } 1754 EXPORT_SYMBOL(vfio_unpin_pages); 1755 1756 /* 1757 * This interface allows the CPUs to perform some sort of virtual DMA on 1758 * behalf of the device. 1759 * 1760 * CPUs read/write from/into a range of IOVAs pointing to user space memory 1761 * into/from a kernel buffer. 1762 * 1763 * As the read/write of user space memory is conducted via the CPUs and is 1764 * not a real device DMA, it is not necessary to pin the user space memory. 1765 * 1766 * @device [in] : VFIO device 1767 * @iova [in] : base IOVA of a user space buffer 1768 * @data [in] : pointer to kernel buffer 1769 * @len [in] : kernel buffer length 1770 * @write : indicate read or write 1771 * Return error code on failure or 0 on success. 1772 */ 1773 int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, 1774 size_t len, bool write) 1775 { 1776 if (!data || len <= 0 || !vfio_assert_device_open(device)) 1777 return -EINVAL; 1778 1779 if (vfio_device_has_container(device)) 1780 return vfio_device_container_dma_rw(device, iova, 1781 data, len, write); 1782 1783 if (device->iommufd_access) { 1784 unsigned int flags = 0; 1785 1786 if (iova > ULONG_MAX) 1787 return -EINVAL; 1788 1789 /* VFIO historically tries to auto-detect a kthread */ 1790 if (!current->mm) 1791 flags |= IOMMUFD_ACCESS_RW_KTHREAD; 1792 if (write) 1793 flags |= IOMMUFD_ACCESS_RW_WRITE; 1794 return iommufd_access_rw(device->iommufd_access, iova, data, 1795 len, flags); 1796 } 1797 return -EINVAL; 1798 } 1799 EXPORT_SYMBOL(vfio_dma_rw); 1800 1801 /* 1802 * Module/class support 1803 */ 1804 static int __init vfio_init(void) 1805 { 1806 int ret; 1807 1808 ida_init(&vfio.device_ida); 1809 1810 ret = vfio_group_init(); 1811 if (ret) 1812 return ret; 1813 1814 ret = vfio_virqfd_init(); 1815 if (ret) 1816 goto err_virqfd; 1817 1818 /* /sys/class/vfio-dev/vfioX */ 1819 ret = class_register(&vfio_device_class); 1820 if (ret) 1821 goto err_dev_class; 1822 1823 ret = vfio_cdev_init(); 1824 if (ret) 1825 goto err_alloc_dev_chrdev; 1826 1827 vfio_debugfs_create_root(); 1828 pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n"); 1829 return 0; 1830 1831 err_alloc_dev_chrdev: 1832 class_unregister(&vfio_device_class); 1833 err_dev_class: 1834 vfio_virqfd_exit(); 1835 err_virqfd: 1836 vfio_group_cleanup(); 1837 return ret; 1838 } 1839 1840 static void __exit vfio_cleanup(void) 1841 { 1842 vfio_debugfs_remove_root(); 1843 ida_destroy(&vfio.device_ida); 1844 vfio_cdev_cleanup(); 1845 class_unregister(&vfio_device_class); 1846 vfio_virqfd_exit(); 1847 vfio_group_cleanup(); 1848 xa_destroy(&vfio_device_set_xa); 1849 } 1850 1851 module_init(vfio_init); 1852 module_exit(vfio_cleanup); 1853 1854 MODULE_IMPORT_NS("IOMMUFD"); 1855 MODULE_VERSION(DRIVER_VERSION); 1856 MODULE_LICENSE("GPL v2"); 1857 MODULE_AUTHOR(DRIVER_AUTHOR); 1858 MODULE_DESCRIPTION(DRIVER_DESC); 1859 MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce"); 1860