1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO core 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 * 8 * Derived from original vfio: 9 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 10 * Author: Tom Lyon, pugs@cisco.com 11 */ 12 13 #include <linux/cdev.h> 14 #include <linux/compat.h> 15 #include <linux/device.h> 16 #include <linux/fs.h> 17 #include <linux/idr.h> 18 #include <linux/iommu.h> 19 #if IS_ENABLED(CONFIG_KVM) 20 #include <linux/kvm_host.h> 21 #endif 22 #include <linux/list.h> 23 #include <linux/miscdevice.h> 24 #include <linux/module.h> 25 #include <linux/mount.h> 26 #include <linux/mutex.h> 27 #include <linux/pci.h> 28 #include <linux/pseudo_fs.h> 29 #include <linux/rwsem.h> 30 #include <linux/sched.h> 31 #include <linux/seq_file.h> 32 #include <linux/slab.h> 33 #include <linux/stat.h> 34 #include <linux/string.h> 35 #include <linux/uaccess.h> 36 #include <linux/vfio.h> 37 #include <linux/wait.h> 38 #include <linux/sched/signal.h> 39 #include <linux/pm_runtime.h> 40 #include <linux/interval_tree.h> 41 #include <linux/iova_bitmap.h> 42 #include <linux/iommufd.h> 43 #include "vfio.h" 44 45 #define DRIVER_VERSION "0.3" 46 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 47 #define DRIVER_DESC "VFIO - User Level meta-driver" 48 49 #define VFIO_MAGIC 0x5646494f /* "VFIO" */ 50 51 static struct vfio { 52 struct class *device_class; 53 struct ida device_ida; 54 struct vfsmount *vfs_mount; 55 int fs_count; 56 } vfio; 57 58 #ifdef CONFIG_VFIO_NOIOMMU 59 bool vfio_noiommu __read_mostly; 60 module_param_named(enable_unsafe_noiommu_mode, 61 vfio_noiommu, bool, S_IRUGO | S_IWUSR); 62 MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)"); 63 #endif 64 65 static DEFINE_XARRAY(vfio_device_set_xa); 66 67 int vfio_assign_device_set(struct vfio_device *device, void *set_id) 68 { 69 unsigned long idx = (unsigned long)set_id; 70 struct vfio_device_set *new_dev_set; 71 struct vfio_device_set *dev_set; 72 73 if (WARN_ON(!set_id)) 74 return -EINVAL; 75 76 /* 77 * Atomically acquire a singleton object in the xarray for this set_id 78 */ 79 xa_lock(&vfio_device_set_xa); 80 dev_set = xa_load(&vfio_device_set_xa, idx); 81 if (dev_set) 82 goto found_get_ref; 83 xa_unlock(&vfio_device_set_xa); 84 85 new_dev_set = kzalloc(sizeof(*new_dev_set), GFP_KERNEL); 86 if (!new_dev_set) 87 return -ENOMEM; 88 mutex_init(&new_dev_set->lock); 89 INIT_LIST_HEAD(&new_dev_set->device_list); 90 new_dev_set->set_id = set_id; 91 92 xa_lock(&vfio_device_set_xa); 93 dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set, 94 GFP_KERNEL); 95 if (!dev_set) { 96 dev_set = new_dev_set; 97 goto found_get_ref; 98 } 99 100 kfree(new_dev_set); 101 if (xa_is_err(dev_set)) { 102 xa_unlock(&vfio_device_set_xa); 103 return xa_err(dev_set); 104 } 105 106 found_get_ref: 107 dev_set->device_count++; 108 xa_unlock(&vfio_device_set_xa); 109 mutex_lock(&dev_set->lock); 110 device->dev_set = dev_set; 111 list_add_tail(&device->dev_set_list, &dev_set->device_list); 112 mutex_unlock(&dev_set->lock); 113 return 0; 114 } 115 EXPORT_SYMBOL_GPL(vfio_assign_device_set); 116 117 static void vfio_release_device_set(struct vfio_device *device) 118 { 119 struct vfio_device_set *dev_set = device->dev_set; 120 121 if (!dev_set) 122 return; 123 124 mutex_lock(&dev_set->lock); 125 list_del(&device->dev_set_list); 126 mutex_unlock(&dev_set->lock); 127 128 xa_lock(&vfio_device_set_xa); 129 if (!--dev_set->device_count) { 130 __xa_erase(&vfio_device_set_xa, 131 (unsigned long)dev_set->set_id); 132 mutex_destroy(&dev_set->lock); 133 kfree(dev_set); 134 } 135 xa_unlock(&vfio_device_set_xa); 136 } 137 138 unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set) 139 { 140 struct vfio_device *cur; 141 unsigned int open_count = 0; 142 143 lockdep_assert_held(&dev_set->lock); 144 145 list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 146 open_count += cur->open_count; 147 return open_count; 148 } 149 EXPORT_SYMBOL_GPL(vfio_device_set_open_count); 150 151 struct vfio_device * 152 vfio_find_device_in_devset(struct vfio_device_set *dev_set, 153 struct device *dev) 154 { 155 struct vfio_device *cur; 156 157 lockdep_assert_held(&dev_set->lock); 158 159 list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 160 if (cur->dev == dev) 161 return cur; 162 return NULL; 163 } 164 EXPORT_SYMBOL_GPL(vfio_find_device_in_devset); 165 166 /* 167 * Device objects - create, release, get, put, search 168 */ 169 /* Device reference always implies a group reference */ 170 void vfio_device_put_registration(struct vfio_device *device) 171 { 172 if (refcount_dec_and_test(&device->refcount)) 173 complete(&device->comp); 174 } 175 176 bool vfio_device_try_get_registration(struct vfio_device *device) 177 { 178 return refcount_inc_not_zero(&device->refcount); 179 } 180 181 /* 182 * VFIO driver API 183 */ 184 /* Release helper called by vfio_put_device() */ 185 static void vfio_device_release(struct device *dev) 186 { 187 struct vfio_device *device = 188 container_of(dev, struct vfio_device, device); 189 190 vfio_release_device_set(device); 191 ida_free(&vfio.device_ida, device->index); 192 193 if (device->ops->release) 194 device->ops->release(device); 195 196 iput(device->inode); 197 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count); 198 kvfree(device); 199 } 200 201 static int vfio_init_device(struct vfio_device *device, struct device *dev, 202 const struct vfio_device_ops *ops); 203 204 /* 205 * Allocate and initialize vfio_device so it can be registered to vfio 206 * core. 207 * 208 * Drivers should use the wrapper vfio_alloc_device() for allocation. 209 * @size is the size of the structure to be allocated, including any 210 * private data used by the driver. 211 * 212 * Driver may provide an @init callback to cover device private data. 213 * 214 * Use vfio_put_device() to release the structure after success return. 215 */ 216 struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, 217 const struct vfio_device_ops *ops) 218 { 219 struct vfio_device *device; 220 int ret; 221 222 if (WARN_ON(size < sizeof(struct vfio_device))) 223 return ERR_PTR(-EINVAL); 224 225 device = kvzalloc(size, GFP_KERNEL); 226 if (!device) 227 return ERR_PTR(-ENOMEM); 228 229 ret = vfio_init_device(device, dev, ops); 230 if (ret) 231 goto out_free; 232 return device; 233 234 out_free: 235 kvfree(device); 236 return ERR_PTR(ret); 237 } 238 EXPORT_SYMBOL_GPL(_vfio_alloc_device); 239 240 static int vfio_fs_init_fs_context(struct fs_context *fc) 241 { 242 return init_pseudo(fc, VFIO_MAGIC) ? 0 : -ENOMEM; 243 } 244 245 static struct file_system_type vfio_fs_type = { 246 .name = "vfio", 247 .owner = THIS_MODULE, 248 .init_fs_context = vfio_fs_init_fs_context, 249 .kill_sb = kill_anon_super, 250 }; 251 252 static struct inode *vfio_fs_inode_new(void) 253 { 254 struct inode *inode; 255 int ret; 256 257 ret = simple_pin_fs(&vfio_fs_type, &vfio.vfs_mount, &vfio.fs_count); 258 if (ret) 259 return ERR_PTR(ret); 260 261 inode = alloc_anon_inode(vfio.vfs_mount->mnt_sb); 262 if (IS_ERR(inode)) 263 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count); 264 265 return inode; 266 } 267 268 /* 269 * Initialize a vfio_device so it can be registered to vfio core. 270 */ 271 static int vfio_init_device(struct vfio_device *device, struct device *dev, 272 const struct vfio_device_ops *ops) 273 { 274 int ret; 275 276 ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL); 277 if (ret < 0) { 278 dev_dbg(dev, "Error to alloc index\n"); 279 return ret; 280 } 281 282 device->index = ret; 283 init_completion(&device->comp); 284 device->dev = dev; 285 device->ops = ops; 286 device->inode = vfio_fs_inode_new(); 287 if (IS_ERR(device->inode)) { 288 ret = PTR_ERR(device->inode); 289 goto out_inode; 290 } 291 292 if (ops->init) { 293 ret = ops->init(device); 294 if (ret) 295 goto out_uninit; 296 } 297 298 device_initialize(&device->device); 299 device->device.release = vfio_device_release; 300 device->device.class = vfio.device_class; 301 device->device.parent = device->dev; 302 return 0; 303 304 out_uninit: 305 iput(device->inode); 306 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count); 307 out_inode: 308 vfio_release_device_set(device); 309 ida_free(&vfio.device_ida, device->index); 310 return ret; 311 } 312 313 static int __vfio_register_dev(struct vfio_device *device, 314 enum vfio_group_type type) 315 { 316 int ret; 317 318 if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) && 319 (!device->ops->bind_iommufd || 320 !device->ops->unbind_iommufd || 321 !device->ops->attach_ioas || 322 !device->ops->detach_ioas))) 323 return -EINVAL; 324 325 /* 326 * If the driver doesn't specify a set then the device is added to a 327 * singleton set just for itself. 328 */ 329 if (!device->dev_set) 330 vfio_assign_device_set(device, device); 331 332 ret = dev_set_name(&device->device, "vfio%d", device->index); 333 if (ret) 334 return ret; 335 336 ret = vfio_device_set_group(device, type); 337 if (ret) 338 return ret; 339 340 /* 341 * VFIO always sets IOMMU_CACHE because we offer no way for userspace to 342 * restore cache coherency. It has to be checked here because it is only 343 * valid for cases where we are using iommu groups. 344 */ 345 if (type == VFIO_IOMMU && !vfio_device_is_noiommu(device) && 346 !device_iommu_capable(device->dev, IOMMU_CAP_CACHE_COHERENCY)) { 347 ret = -EINVAL; 348 goto err_out; 349 } 350 351 ret = vfio_device_add(device); 352 if (ret) 353 goto err_out; 354 355 /* Refcounting can't start until the driver calls register */ 356 refcount_set(&device->refcount, 1); 357 358 vfio_device_group_register(device); 359 vfio_device_debugfs_init(device); 360 361 return 0; 362 err_out: 363 vfio_device_remove_group(device); 364 return ret; 365 } 366 367 int vfio_register_group_dev(struct vfio_device *device) 368 { 369 return __vfio_register_dev(device, VFIO_IOMMU); 370 } 371 EXPORT_SYMBOL_GPL(vfio_register_group_dev); 372 373 /* 374 * Register a virtual device without IOMMU backing. The user of this 375 * device must not be able to directly trigger unmediated DMA. 376 */ 377 int vfio_register_emulated_iommu_dev(struct vfio_device *device) 378 { 379 return __vfio_register_dev(device, VFIO_EMULATED_IOMMU); 380 } 381 EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev); 382 383 /* 384 * Decrement the device reference count and wait for the device to be 385 * removed. Open file descriptors for the device... */ 386 void vfio_unregister_group_dev(struct vfio_device *device) 387 { 388 unsigned int i = 0; 389 bool interrupted = false; 390 long rc; 391 392 /* 393 * Prevent new device opened by userspace via the 394 * VFIO_GROUP_GET_DEVICE_FD in the group path. 395 */ 396 vfio_device_group_unregister(device); 397 398 /* 399 * Balances vfio_device_add() in register path, also prevents 400 * new device opened by userspace in the cdev path. 401 */ 402 vfio_device_del(device); 403 404 vfio_device_put_registration(device); 405 rc = try_wait_for_completion(&device->comp); 406 while (rc <= 0) { 407 if (device->ops->request) 408 device->ops->request(device, i++); 409 410 if (interrupted) { 411 rc = wait_for_completion_timeout(&device->comp, 412 HZ * 10); 413 } else { 414 rc = wait_for_completion_interruptible_timeout( 415 &device->comp, HZ * 10); 416 if (rc < 0) { 417 interrupted = true; 418 dev_warn(device->dev, 419 "Device is currently in use, task" 420 " \"%s\" (%d) " 421 "blocked until device is released", 422 current->comm, task_pid_nr(current)); 423 } 424 } 425 } 426 427 vfio_device_debugfs_exit(device); 428 /* Balances vfio_device_set_group in register path */ 429 vfio_device_remove_group(device); 430 } 431 EXPORT_SYMBOL_GPL(vfio_unregister_group_dev); 432 433 #if IS_ENABLED(CONFIG_KVM) 434 void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm) 435 { 436 void (*pfn)(struct kvm *kvm); 437 bool (*fn)(struct kvm *kvm); 438 bool ret; 439 440 lockdep_assert_held(&device->dev_set->lock); 441 442 if (!kvm) 443 return; 444 445 pfn = symbol_get(kvm_put_kvm); 446 if (WARN_ON(!pfn)) 447 return; 448 449 fn = symbol_get(kvm_get_kvm_safe); 450 if (WARN_ON(!fn)) { 451 symbol_put(kvm_put_kvm); 452 return; 453 } 454 455 ret = fn(kvm); 456 symbol_put(kvm_get_kvm_safe); 457 if (!ret) { 458 symbol_put(kvm_put_kvm); 459 return; 460 } 461 462 device->put_kvm = pfn; 463 device->kvm = kvm; 464 } 465 466 void vfio_device_put_kvm(struct vfio_device *device) 467 { 468 lockdep_assert_held(&device->dev_set->lock); 469 470 if (!device->kvm) 471 return; 472 473 if (WARN_ON(!device->put_kvm)) 474 goto clear; 475 476 device->put_kvm(device->kvm); 477 device->put_kvm = NULL; 478 symbol_put(kvm_put_kvm); 479 480 clear: 481 device->kvm = NULL; 482 } 483 #endif 484 485 /* true if the vfio_device has open_device() called but not close_device() */ 486 static bool vfio_assert_device_open(struct vfio_device *device) 487 { 488 return !WARN_ON_ONCE(!READ_ONCE(device->open_count)); 489 } 490 491 struct vfio_device_file * 492 vfio_allocate_device_file(struct vfio_device *device) 493 { 494 struct vfio_device_file *df; 495 496 df = kzalloc(sizeof(*df), GFP_KERNEL_ACCOUNT); 497 if (!df) 498 return ERR_PTR(-ENOMEM); 499 500 df->device = device; 501 spin_lock_init(&df->kvm_ref_lock); 502 503 return df; 504 } 505 506 static int vfio_df_device_first_open(struct vfio_device_file *df) 507 { 508 struct vfio_device *device = df->device; 509 struct iommufd_ctx *iommufd = df->iommufd; 510 int ret; 511 512 lockdep_assert_held(&device->dev_set->lock); 513 514 if (!try_module_get(device->dev->driver->owner)) 515 return -ENODEV; 516 517 if (iommufd) 518 ret = vfio_df_iommufd_bind(df); 519 else 520 ret = vfio_device_group_use_iommu(device); 521 if (ret) 522 goto err_module_put; 523 524 if (device->ops->open_device) { 525 ret = device->ops->open_device(device); 526 if (ret) 527 goto err_unuse_iommu; 528 } 529 return 0; 530 531 err_unuse_iommu: 532 if (iommufd) 533 vfio_df_iommufd_unbind(df); 534 else 535 vfio_device_group_unuse_iommu(device); 536 err_module_put: 537 module_put(device->dev->driver->owner); 538 return ret; 539 } 540 541 static void vfio_df_device_last_close(struct vfio_device_file *df) 542 { 543 struct vfio_device *device = df->device; 544 struct iommufd_ctx *iommufd = df->iommufd; 545 546 lockdep_assert_held(&device->dev_set->lock); 547 548 if (device->ops->close_device) 549 device->ops->close_device(device); 550 if (iommufd) 551 vfio_df_iommufd_unbind(df); 552 else 553 vfio_device_group_unuse_iommu(device); 554 module_put(device->dev->driver->owner); 555 } 556 557 int vfio_df_open(struct vfio_device_file *df) 558 { 559 struct vfio_device *device = df->device; 560 int ret = 0; 561 562 lockdep_assert_held(&device->dev_set->lock); 563 564 /* 565 * Only the group path allows the device to be opened multiple 566 * times. The device cdev path doesn't have a secure way for it. 567 */ 568 if (device->open_count != 0 && !df->group) 569 return -EINVAL; 570 571 device->open_count++; 572 if (device->open_count == 1) { 573 ret = vfio_df_device_first_open(df); 574 if (ret) 575 device->open_count--; 576 } 577 578 return ret; 579 } 580 581 void vfio_df_close(struct vfio_device_file *df) 582 { 583 struct vfio_device *device = df->device; 584 585 lockdep_assert_held(&device->dev_set->lock); 586 587 if (!vfio_assert_device_open(device)) 588 return; 589 if (device->open_count == 1) 590 vfio_df_device_last_close(df); 591 device->open_count--; 592 } 593 594 /* 595 * Wrapper around pm_runtime_resume_and_get(). 596 * Return error code on failure or 0 on success. 597 */ 598 static inline int vfio_device_pm_runtime_get(struct vfio_device *device) 599 { 600 struct device *dev = device->dev; 601 602 if (dev->driver && dev->driver->pm) { 603 int ret; 604 605 ret = pm_runtime_resume_and_get(dev); 606 if (ret) { 607 dev_info_ratelimited(dev, 608 "vfio: runtime resume failed %d\n", ret); 609 return -EIO; 610 } 611 } 612 613 return 0; 614 } 615 616 /* 617 * Wrapper around pm_runtime_put(). 618 */ 619 static inline void vfio_device_pm_runtime_put(struct vfio_device *device) 620 { 621 struct device *dev = device->dev; 622 623 if (dev->driver && dev->driver->pm) 624 pm_runtime_put(dev); 625 } 626 627 /* 628 * VFIO Device fd 629 */ 630 static int vfio_device_fops_release(struct inode *inode, struct file *filep) 631 { 632 struct vfio_device_file *df = filep->private_data; 633 struct vfio_device *device = df->device; 634 635 if (df->group) 636 vfio_df_group_close(df); 637 else 638 vfio_df_unbind_iommufd(df); 639 640 vfio_device_put_registration(device); 641 642 kfree(df); 643 644 return 0; 645 } 646 647 /* 648 * vfio_mig_get_next_state - Compute the next step in the FSM 649 * @cur_fsm - The current state the device is in 650 * @new_fsm - The target state to reach 651 * @next_fsm - Pointer to the next step to get to new_fsm 652 * 653 * Return 0 upon success, otherwise -errno 654 * Upon success the next step in the state progression between cur_fsm and 655 * new_fsm will be set in next_fsm. 656 * 657 * This breaks down requests for combination transitions into smaller steps and 658 * returns the next step to get to new_fsm. The function may need to be called 659 * multiple times before reaching new_fsm. 660 * 661 */ 662 int vfio_mig_get_next_state(struct vfio_device *device, 663 enum vfio_device_mig_state cur_fsm, 664 enum vfio_device_mig_state new_fsm, 665 enum vfio_device_mig_state *next_fsm) 666 { 667 enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 }; 668 /* 669 * The coding in this table requires the driver to implement the 670 * following FSM arcs: 671 * RESUMING -> STOP 672 * STOP -> RESUMING 673 * STOP -> STOP_COPY 674 * STOP_COPY -> STOP 675 * 676 * If P2P is supported then the driver must also implement these FSM 677 * arcs: 678 * RUNNING -> RUNNING_P2P 679 * RUNNING_P2P -> RUNNING 680 * RUNNING_P2P -> STOP 681 * STOP -> RUNNING_P2P 682 * 683 * If precopy is supported then the driver must support these additional 684 * FSM arcs: 685 * RUNNING -> PRE_COPY 686 * PRE_COPY -> RUNNING 687 * PRE_COPY -> STOP_COPY 688 * However, if precopy and P2P are supported together then the driver 689 * must support these additional arcs beyond the P2P arcs above: 690 * PRE_COPY -> RUNNING 691 * PRE_COPY -> PRE_COPY_P2P 692 * PRE_COPY_P2P -> PRE_COPY 693 * PRE_COPY_P2P -> RUNNING_P2P 694 * PRE_COPY_P2P -> STOP_COPY 695 * RUNNING -> PRE_COPY 696 * RUNNING_P2P -> PRE_COPY_P2P 697 * 698 * Without P2P and precopy the driver must implement: 699 * RUNNING -> STOP 700 * STOP -> RUNNING 701 * 702 * The coding will step through multiple states for some combination 703 * transitions; if all optional features are supported, this means the 704 * following ones: 705 * PRE_COPY -> PRE_COPY_P2P -> STOP_COPY 706 * PRE_COPY -> RUNNING -> RUNNING_P2P 707 * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP 708 * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING 709 * PRE_COPY_P2P -> RUNNING_P2P -> RUNNING 710 * PRE_COPY_P2P -> RUNNING_P2P -> STOP 711 * PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING 712 * RESUMING -> STOP -> RUNNING_P2P 713 * RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P 714 * RESUMING -> STOP -> RUNNING_P2P -> RUNNING 715 * RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 716 * RESUMING -> STOP -> STOP_COPY 717 * RUNNING -> RUNNING_P2P -> PRE_COPY_P2P 718 * RUNNING -> RUNNING_P2P -> STOP 719 * RUNNING -> RUNNING_P2P -> STOP -> RESUMING 720 * RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY 721 * RUNNING_P2P -> RUNNING -> PRE_COPY 722 * RUNNING_P2P -> STOP -> RESUMING 723 * RUNNING_P2P -> STOP -> STOP_COPY 724 * STOP -> RUNNING_P2P -> PRE_COPY_P2P 725 * STOP -> RUNNING_P2P -> RUNNING 726 * STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 727 * STOP_COPY -> STOP -> RESUMING 728 * STOP_COPY -> STOP -> RUNNING_P2P 729 * STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING 730 * 731 * The following transitions are blocked: 732 * STOP_COPY -> PRE_COPY 733 * STOP_COPY -> PRE_COPY_P2P 734 */ 735 static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = { 736 [VFIO_DEVICE_STATE_STOP] = { 737 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 738 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 739 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 740 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 741 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 742 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 743 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 744 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 745 }, 746 [VFIO_DEVICE_STATE_RUNNING] = { 747 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 748 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 749 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 750 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 751 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 752 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 753 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 754 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 755 }, 756 [VFIO_DEVICE_STATE_PRE_COPY] = { 757 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING, 758 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 759 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 760 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 761 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 762 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING, 763 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING, 764 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 765 }, 766 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = { 767 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 768 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 769 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 770 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 771 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 772 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 773 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 774 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 775 }, 776 [VFIO_DEVICE_STATE_STOP_COPY] = { 777 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 778 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 779 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 780 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 781 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 782 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 783 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 784 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 785 }, 786 [VFIO_DEVICE_STATE_RESUMING] = { 787 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 788 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 789 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP, 790 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP, 791 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 792 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 793 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 794 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 795 }, 796 [VFIO_DEVICE_STATE_RUNNING_P2P] = { 797 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 798 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 799 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING, 800 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 801 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 802 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 803 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 804 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 805 }, 806 [VFIO_DEVICE_STATE_ERROR] = { 807 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR, 808 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR, 809 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 810 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 811 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR, 812 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR, 813 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR, 814 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 815 }, 816 }; 817 818 static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = { 819 [VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY, 820 [VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY, 821 [VFIO_DEVICE_STATE_PRE_COPY] = 822 VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY, 823 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY | 824 VFIO_MIGRATION_P2P | 825 VFIO_MIGRATION_PRE_COPY, 826 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY, 827 [VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY, 828 [VFIO_DEVICE_STATE_RUNNING_P2P] = 829 VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P, 830 [VFIO_DEVICE_STATE_ERROR] = ~0U, 831 }; 832 833 if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 834 (state_flags_table[cur_fsm] & device->migration_flags) != 835 state_flags_table[cur_fsm])) 836 return -EINVAL; 837 838 if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 839 (state_flags_table[new_fsm] & device->migration_flags) != 840 state_flags_table[new_fsm]) 841 return -EINVAL; 842 843 /* 844 * Arcs touching optional and unsupported states are skipped over. The 845 * driver will instead see an arc from the original state to the next 846 * logical state, as per the above comment. 847 */ 848 *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm]; 849 while ((state_flags_table[*next_fsm] & device->migration_flags) != 850 state_flags_table[*next_fsm]) 851 *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm]; 852 853 return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL; 854 } 855 EXPORT_SYMBOL_GPL(vfio_mig_get_next_state); 856 857 /* 858 * Convert the drivers's struct file into a FD number and return it to userspace 859 */ 860 static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg, 861 struct vfio_device_feature_mig_state *mig) 862 { 863 int ret; 864 int fd; 865 866 fd = get_unused_fd_flags(O_CLOEXEC); 867 if (fd < 0) { 868 ret = fd; 869 goto out_fput; 870 } 871 872 mig->data_fd = fd; 873 if (copy_to_user(arg, mig, sizeof(*mig))) { 874 ret = -EFAULT; 875 goto out_put_unused; 876 } 877 fd_install(fd, filp); 878 return 0; 879 880 out_put_unused: 881 put_unused_fd(fd); 882 out_fput: 883 fput(filp); 884 return ret; 885 } 886 887 static int 888 vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device, 889 u32 flags, void __user *arg, 890 size_t argsz) 891 { 892 size_t minsz = 893 offsetofend(struct vfio_device_feature_mig_state, data_fd); 894 struct vfio_device_feature_mig_state mig; 895 struct file *filp = NULL; 896 int ret; 897 898 if (!device->mig_ops) 899 return -ENOTTY; 900 901 ret = vfio_check_feature(flags, argsz, 902 VFIO_DEVICE_FEATURE_SET | 903 VFIO_DEVICE_FEATURE_GET, 904 sizeof(mig)); 905 if (ret != 1) 906 return ret; 907 908 if (copy_from_user(&mig, arg, minsz)) 909 return -EFAULT; 910 911 if (flags & VFIO_DEVICE_FEATURE_GET) { 912 enum vfio_device_mig_state curr_state; 913 914 ret = device->mig_ops->migration_get_state(device, 915 &curr_state); 916 if (ret) 917 return ret; 918 mig.device_state = curr_state; 919 goto out_copy; 920 } 921 922 /* Handle the VFIO_DEVICE_FEATURE_SET */ 923 filp = device->mig_ops->migration_set_state(device, mig.device_state); 924 if (IS_ERR(filp) || !filp) 925 goto out_copy; 926 927 return vfio_ioct_mig_return_fd(filp, arg, &mig); 928 out_copy: 929 mig.data_fd = -1; 930 if (copy_to_user(arg, &mig, sizeof(mig))) 931 return -EFAULT; 932 if (IS_ERR(filp)) 933 return PTR_ERR(filp); 934 return 0; 935 } 936 937 static int 938 vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device, 939 u32 flags, void __user *arg, 940 size_t argsz) 941 { 942 struct vfio_device_feature_mig_data_size data_size = {}; 943 unsigned long stop_copy_length; 944 int ret; 945 946 if (!device->mig_ops) 947 return -ENOTTY; 948 949 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 950 sizeof(data_size)); 951 if (ret != 1) 952 return ret; 953 954 ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length); 955 if (ret) 956 return ret; 957 958 data_size.stop_copy_length = stop_copy_length; 959 if (copy_to_user(arg, &data_size, sizeof(data_size))) 960 return -EFAULT; 961 962 return 0; 963 } 964 965 static int vfio_ioctl_device_feature_migration(struct vfio_device *device, 966 u32 flags, void __user *arg, 967 size_t argsz) 968 { 969 struct vfio_device_feature_migration mig = { 970 .flags = device->migration_flags, 971 }; 972 int ret; 973 974 if (!device->mig_ops) 975 return -ENOTTY; 976 977 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 978 sizeof(mig)); 979 if (ret != 1) 980 return ret; 981 if (copy_to_user(arg, &mig, sizeof(mig))) 982 return -EFAULT; 983 return 0; 984 } 985 986 void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes, 987 u32 req_nodes) 988 { 989 struct interval_tree_node *prev, *curr, *comb_start, *comb_end; 990 unsigned long min_gap, curr_gap; 991 992 /* Special shortcut when a single range is required */ 993 if (req_nodes == 1) { 994 unsigned long last; 995 996 comb_start = interval_tree_iter_first(root, 0, ULONG_MAX); 997 998 /* Empty list */ 999 if (WARN_ON_ONCE(!comb_start)) 1000 return; 1001 1002 curr = comb_start; 1003 while (curr) { 1004 last = curr->last; 1005 prev = curr; 1006 curr = interval_tree_iter_next(curr, 0, ULONG_MAX); 1007 if (prev != comb_start) 1008 interval_tree_remove(prev, root); 1009 } 1010 comb_start->last = last; 1011 return; 1012 } 1013 1014 /* Combine ranges which have the smallest gap */ 1015 while (cur_nodes > req_nodes) { 1016 prev = NULL; 1017 min_gap = ULONG_MAX; 1018 curr = interval_tree_iter_first(root, 0, ULONG_MAX); 1019 while (curr) { 1020 if (prev) { 1021 curr_gap = curr->start - prev->last; 1022 if (curr_gap < min_gap) { 1023 min_gap = curr_gap; 1024 comb_start = prev; 1025 comb_end = curr; 1026 } 1027 } 1028 prev = curr; 1029 curr = interval_tree_iter_next(curr, 0, ULONG_MAX); 1030 } 1031 1032 /* Empty list or no nodes to combine */ 1033 if (WARN_ON_ONCE(min_gap == ULONG_MAX)) 1034 break; 1035 1036 comb_start->last = comb_end->last; 1037 interval_tree_remove(comb_end, root); 1038 cur_nodes--; 1039 } 1040 } 1041 EXPORT_SYMBOL_GPL(vfio_combine_iova_ranges); 1042 1043 /* Ranges should fit into a single kernel page */ 1044 #define LOG_MAX_RANGES \ 1045 (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range)) 1046 1047 static int 1048 vfio_ioctl_device_feature_logging_start(struct vfio_device *device, 1049 u32 flags, void __user *arg, 1050 size_t argsz) 1051 { 1052 size_t minsz = 1053 offsetofend(struct vfio_device_feature_dma_logging_control, 1054 ranges); 1055 struct vfio_device_feature_dma_logging_range __user *ranges; 1056 struct vfio_device_feature_dma_logging_control control; 1057 struct vfio_device_feature_dma_logging_range range; 1058 struct rb_root_cached root = RB_ROOT_CACHED; 1059 struct interval_tree_node *nodes; 1060 u64 iova_end; 1061 u32 nnodes; 1062 int i, ret; 1063 1064 if (!device->log_ops) 1065 return -ENOTTY; 1066 1067 ret = vfio_check_feature(flags, argsz, 1068 VFIO_DEVICE_FEATURE_SET, 1069 sizeof(control)); 1070 if (ret != 1) 1071 return ret; 1072 1073 if (copy_from_user(&control, arg, minsz)) 1074 return -EFAULT; 1075 1076 nnodes = control.num_ranges; 1077 if (!nnodes) 1078 return -EINVAL; 1079 1080 if (nnodes > LOG_MAX_RANGES) 1081 return -E2BIG; 1082 1083 ranges = u64_to_user_ptr(control.ranges); 1084 nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node), 1085 GFP_KERNEL); 1086 if (!nodes) 1087 return -ENOMEM; 1088 1089 for (i = 0; i < nnodes; i++) { 1090 if (copy_from_user(&range, &ranges[i], sizeof(range))) { 1091 ret = -EFAULT; 1092 goto end; 1093 } 1094 if (!IS_ALIGNED(range.iova, control.page_size) || 1095 !IS_ALIGNED(range.length, control.page_size)) { 1096 ret = -EINVAL; 1097 goto end; 1098 } 1099 1100 if (check_add_overflow(range.iova, range.length, &iova_end) || 1101 iova_end > ULONG_MAX) { 1102 ret = -EOVERFLOW; 1103 goto end; 1104 } 1105 1106 nodes[i].start = range.iova; 1107 nodes[i].last = range.iova + range.length - 1; 1108 if (interval_tree_iter_first(&root, nodes[i].start, 1109 nodes[i].last)) { 1110 /* Range overlapping */ 1111 ret = -EINVAL; 1112 goto end; 1113 } 1114 interval_tree_insert(nodes + i, &root); 1115 } 1116 1117 ret = device->log_ops->log_start(device, &root, nnodes, 1118 &control.page_size); 1119 if (ret) 1120 goto end; 1121 1122 if (copy_to_user(arg, &control, sizeof(control))) { 1123 ret = -EFAULT; 1124 device->log_ops->log_stop(device); 1125 } 1126 1127 end: 1128 kfree(nodes); 1129 return ret; 1130 } 1131 1132 static int 1133 vfio_ioctl_device_feature_logging_stop(struct vfio_device *device, 1134 u32 flags, void __user *arg, 1135 size_t argsz) 1136 { 1137 int ret; 1138 1139 if (!device->log_ops) 1140 return -ENOTTY; 1141 1142 ret = vfio_check_feature(flags, argsz, 1143 VFIO_DEVICE_FEATURE_SET, 0); 1144 if (ret != 1) 1145 return ret; 1146 1147 return device->log_ops->log_stop(device); 1148 } 1149 1150 static int vfio_device_log_read_and_clear(struct iova_bitmap *iter, 1151 unsigned long iova, size_t length, 1152 void *opaque) 1153 { 1154 struct vfio_device *device = opaque; 1155 1156 return device->log_ops->log_read_and_clear(device, iova, length, iter); 1157 } 1158 1159 static int 1160 vfio_ioctl_device_feature_logging_report(struct vfio_device *device, 1161 u32 flags, void __user *arg, 1162 size_t argsz) 1163 { 1164 size_t minsz = 1165 offsetofend(struct vfio_device_feature_dma_logging_report, 1166 bitmap); 1167 struct vfio_device_feature_dma_logging_report report; 1168 struct iova_bitmap *iter; 1169 u64 iova_end; 1170 int ret; 1171 1172 if (!device->log_ops) 1173 return -ENOTTY; 1174 1175 ret = vfio_check_feature(flags, argsz, 1176 VFIO_DEVICE_FEATURE_GET, 1177 sizeof(report)); 1178 if (ret != 1) 1179 return ret; 1180 1181 if (copy_from_user(&report, arg, minsz)) 1182 return -EFAULT; 1183 1184 if (report.page_size < SZ_4K || !is_power_of_2(report.page_size)) 1185 return -EINVAL; 1186 1187 if (check_add_overflow(report.iova, report.length, &iova_end) || 1188 iova_end > ULONG_MAX) 1189 return -EOVERFLOW; 1190 1191 iter = iova_bitmap_alloc(report.iova, report.length, 1192 report.page_size, 1193 u64_to_user_ptr(report.bitmap)); 1194 if (IS_ERR(iter)) 1195 return PTR_ERR(iter); 1196 1197 ret = iova_bitmap_for_each(iter, device, 1198 vfio_device_log_read_and_clear); 1199 1200 iova_bitmap_free(iter); 1201 return ret; 1202 } 1203 1204 static int vfio_ioctl_device_feature(struct vfio_device *device, 1205 struct vfio_device_feature __user *arg) 1206 { 1207 size_t minsz = offsetofend(struct vfio_device_feature, flags); 1208 struct vfio_device_feature feature; 1209 1210 if (copy_from_user(&feature, arg, minsz)) 1211 return -EFAULT; 1212 1213 if (feature.argsz < minsz) 1214 return -EINVAL; 1215 1216 /* Check unknown flags */ 1217 if (feature.flags & 1218 ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET | 1219 VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE)) 1220 return -EINVAL; 1221 1222 /* GET & SET are mutually exclusive except with PROBE */ 1223 if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) && 1224 (feature.flags & VFIO_DEVICE_FEATURE_SET) && 1225 (feature.flags & VFIO_DEVICE_FEATURE_GET)) 1226 return -EINVAL; 1227 1228 switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) { 1229 case VFIO_DEVICE_FEATURE_MIGRATION: 1230 return vfio_ioctl_device_feature_migration( 1231 device, feature.flags, arg->data, 1232 feature.argsz - minsz); 1233 case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE: 1234 return vfio_ioctl_device_feature_mig_device_state( 1235 device, feature.flags, arg->data, 1236 feature.argsz - minsz); 1237 case VFIO_DEVICE_FEATURE_DMA_LOGGING_START: 1238 return vfio_ioctl_device_feature_logging_start( 1239 device, feature.flags, arg->data, 1240 feature.argsz - minsz); 1241 case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP: 1242 return vfio_ioctl_device_feature_logging_stop( 1243 device, feature.flags, arg->data, 1244 feature.argsz - minsz); 1245 case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT: 1246 return vfio_ioctl_device_feature_logging_report( 1247 device, feature.flags, arg->data, 1248 feature.argsz - minsz); 1249 case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE: 1250 return vfio_ioctl_device_feature_migration_data_size( 1251 device, feature.flags, arg->data, 1252 feature.argsz - minsz); 1253 default: 1254 if (unlikely(!device->ops->device_feature)) 1255 return -ENOTTY; 1256 return device->ops->device_feature(device, feature.flags, 1257 arg->data, 1258 feature.argsz - minsz); 1259 } 1260 } 1261 1262 static long vfio_get_region_info(struct vfio_device *device, 1263 struct vfio_region_info __user *arg) 1264 { 1265 unsigned long minsz = offsetofend(struct vfio_region_info, offset); 1266 struct vfio_region_info info = {}; 1267 struct vfio_info_cap caps = {}; 1268 int ret; 1269 1270 if (copy_from_user(&info, arg, minsz)) 1271 return -EFAULT; 1272 if (info.argsz < minsz) 1273 return -EINVAL; 1274 1275 if (device->ops->get_region_info_caps) { 1276 ret = device->ops->get_region_info_caps(device, &info, &caps); 1277 if (ret) 1278 goto out_free; 1279 1280 if (caps.size) { 1281 info.flags |= VFIO_REGION_INFO_FLAG_CAPS; 1282 if (info.argsz < sizeof(info) + caps.size) { 1283 info.argsz = sizeof(info) + caps.size; 1284 info.cap_offset = 0; 1285 } else { 1286 vfio_info_cap_shift(&caps, sizeof(info)); 1287 if (copy_to_user(arg + 1, caps.buf, 1288 caps.size)) { 1289 ret = -EFAULT; 1290 goto out_free; 1291 } 1292 info.cap_offset = sizeof(info); 1293 } 1294 } 1295 1296 if (copy_to_user(arg, &info, minsz)) { 1297 ret = -EFAULT; 1298 goto out_free; 1299 } 1300 } else if (device->ops->get_region_info) { 1301 ret = device->ops->get_region_info(device, arg); 1302 if (ret) 1303 return ret; 1304 } else { 1305 return -EINVAL; 1306 } 1307 1308 out_free: 1309 kfree(caps.buf); 1310 return ret; 1311 } 1312 1313 static long vfio_device_fops_unl_ioctl(struct file *filep, 1314 unsigned int cmd, unsigned long arg) 1315 { 1316 struct vfio_device_file *df = filep->private_data; 1317 struct vfio_device *device = df->device; 1318 void __user *uptr = (void __user *)arg; 1319 int ret; 1320 1321 if (cmd == VFIO_DEVICE_BIND_IOMMUFD) 1322 return vfio_df_ioctl_bind_iommufd(df, uptr); 1323 1324 /* Paired with smp_store_release() following vfio_df_open() */ 1325 if (!smp_load_acquire(&df->access_granted)) 1326 return -EINVAL; 1327 1328 ret = vfio_device_pm_runtime_get(device); 1329 if (ret) 1330 return ret; 1331 1332 /* cdev only ioctls */ 1333 if (IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) && !df->group) { 1334 switch (cmd) { 1335 case VFIO_DEVICE_ATTACH_IOMMUFD_PT: 1336 ret = vfio_df_ioctl_attach_pt(df, uptr); 1337 goto out; 1338 1339 case VFIO_DEVICE_DETACH_IOMMUFD_PT: 1340 ret = vfio_df_ioctl_detach_pt(df, uptr); 1341 goto out; 1342 } 1343 } 1344 1345 switch (cmd) { 1346 case VFIO_DEVICE_FEATURE: 1347 ret = vfio_ioctl_device_feature(device, uptr); 1348 break; 1349 1350 case VFIO_DEVICE_GET_REGION_INFO: 1351 ret = vfio_get_region_info(device, uptr); 1352 break; 1353 1354 default: 1355 if (unlikely(!device->ops->ioctl)) 1356 ret = -EINVAL; 1357 else 1358 ret = device->ops->ioctl(device, cmd, arg); 1359 break; 1360 } 1361 out: 1362 vfio_device_pm_runtime_put(device); 1363 return ret; 1364 } 1365 1366 static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf, 1367 size_t count, loff_t *ppos) 1368 { 1369 struct vfio_device_file *df = filep->private_data; 1370 struct vfio_device *device = df->device; 1371 1372 /* Paired with smp_store_release() following vfio_df_open() */ 1373 if (!smp_load_acquire(&df->access_granted)) 1374 return -EINVAL; 1375 1376 if (unlikely(!device->ops->read)) 1377 return -EINVAL; 1378 1379 return device->ops->read(device, buf, count, ppos); 1380 } 1381 1382 static ssize_t vfio_device_fops_write(struct file *filep, 1383 const char __user *buf, 1384 size_t count, loff_t *ppos) 1385 { 1386 struct vfio_device_file *df = filep->private_data; 1387 struct vfio_device *device = df->device; 1388 1389 /* Paired with smp_store_release() following vfio_df_open() */ 1390 if (!smp_load_acquire(&df->access_granted)) 1391 return -EINVAL; 1392 1393 if (unlikely(!device->ops->write)) 1394 return -EINVAL; 1395 1396 return device->ops->write(device, buf, count, ppos); 1397 } 1398 1399 static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma) 1400 { 1401 struct vfio_device_file *df = filep->private_data; 1402 struct vfio_device *device = df->device; 1403 1404 /* Paired with smp_store_release() following vfio_df_open() */ 1405 if (!smp_load_acquire(&df->access_granted)) 1406 return -EINVAL; 1407 1408 if (unlikely(!device->ops->mmap)) 1409 return -EINVAL; 1410 1411 return device->ops->mmap(device, vma); 1412 } 1413 1414 #ifdef CONFIG_PROC_FS 1415 static void vfio_device_show_fdinfo(struct seq_file *m, struct file *filep) 1416 { 1417 char *path; 1418 struct vfio_device_file *df = filep->private_data; 1419 struct vfio_device *device = df->device; 1420 1421 path = kobject_get_path(&device->dev->kobj, GFP_KERNEL); 1422 if (!path) 1423 return; 1424 1425 seq_printf(m, "vfio-device-syspath: /sys%s\n", path); 1426 kfree(path); 1427 } 1428 #endif 1429 1430 const struct file_operations vfio_device_fops = { 1431 .owner = THIS_MODULE, 1432 .open = vfio_device_fops_cdev_open, 1433 .release = vfio_device_fops_release, 1434 .read = vfio_device_fops_read, 1435 .write = vfio_device_fops_write, 1436 .unlocked_ioctl = vfio_device_fops_unl_ioctl, 1437 .compat_ioctl = compat_ptr_ioctl, 1438 .mmap = vfio_device_fops_mmap, 1439 #ifdef CONFIG_PROC_FS 1440 .show_fdinfo = vfio_device_show_fdinfo, 1441 #endif 1442 }; 1443 1444 static struct vfio_device *vfio_device_from_file(struct file *file) 1445 { 1446 struct vfio_device_file *df = file->private_data; 1447 1448 if (file->f_op != &vfio_device_fops) 1449 return NULL; 1450 return df->device; 1451 } 1452 1453 /** 1454 * vfio_file_is_valid - True if the file is valid vfio file 1455 * @file: VFIO group file or VFIO device file 1456 */ 1457 bool vfio_file_is_valid(struct file *file) 1458 { 1459 return vfio_group_from_file(file) || 1460 vfio_device_from_file(file); 1461 } 1462 EXPORT_SYMBOL_GPL(vfio_file_is_valid); 1463 1464 /** 1465 * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file 1466 * is always CPU cache coherent 1467 * @file: VFIO group file or VFIO device file 1468 * 1469 * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop 1470 * bit in DMA transactions. A return of false indicates that the user has 1471 * rights to access additional instructions such as wbinvd on x86. 1472 */ 1473 bool vfio_file_enforced_coherent(struct file *file) 1474 { 1475 struct vfio_device *device; 1476 struct vfio_group *group; 1477 1478 group = vfio_group_from_file(file); 1479 if (group) 1480 return vfio_group_enforced_coherent(group); 1481 1482 device = vfio_device_from_file(file); 1483 if (device) 1484 return device_iommu_capable(device->dev, 1485 IOMMU_CAP_ENFORCE_CACHE_COHERENCY); 1486 1487 return true; 1488 } 1489 EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent); 1490 1491 static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm) 1492 { 1493 struct vfio_device_file *df = file->private_data; 1494 1495 /* 1496 * The kvm is first recorded in the vfio_device_file, and will 1497 * be propagated to vfio_device::kvm when the file is bound to 1498 * iommufd successfully in the vfio device cdev path. 1499 */ 1500 spin_lock(&df->kvm_ref_lock); 1501 df->kvm = kvm; 1502 spin_unlock(&df->kvm_ref_lock); 1503 } 1504 1505 /** 1506 * vfio_file_set_kvm - Link a kvm with VFIO drivers 1507 * @file: VFIO group file or VFIO device file 1508 * @kvm: KVM to link 1509 * 1510 * When a VFIO device is first opened the KVM will be available in 1511 * device->kvm if one was associated with the file. 1512 */ 1513 void vfio_file_set_kvm(struct file *file, struct kvm *kvm) 1514 { 1515 struct vfio_group *group; 1516 1517 group = vfio_group_from_file(file); 1518 if (group) 1519 vfio_group_set_kvm(group, kvm); 1520 1521 if (vfio_device_from_file(file)) 1522 vfio_device_file_set_kvm(file, kvm); 1523 } 1524 EXPORT_SYMBOL_GPL(vfio_file_set_kvm); 1525 1526 /* 1527 * Sub-module support 1528 */ 1529 /* 1530 * Helper for managing a buffer of info chain capabilities, allocate or 1531 * reallocate a buffer with additional @size, filling in @id and @version 1532 * of the capability. A pointer to the new capability is returned. 1533 * 1534 * NB. The chain is based at the head of the buffer, so new entries are 1535 * added to the tail, vfio_info_cap_shift() should be called to fixup the 1536 * next offsets prior to copying to the user buffer. 1537 */ 1538 struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps, 1539 size_t size, u16 id, u16 version) 1540 { 1541 void *buf; 1542 struct vfio_info_cap_header *header, *tmp; 1543 1544 /* Ensure that the next capability struct will be aligned */ 1545 size = ALIGN(size, sizeof(u64)); 1546 1547 buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL); 1548 if (!buf) { 1549 kfree(caps->buf); 1550 caps->buf = NULL; 1551 caps->size = 0; 1552 return ERR_PTR(-ENOMEM); 1553 } 1554 1555 caps->buf = buf; 1556 header = buf + caps->size; 1557 1558 /* Eventually copied to user buffer, zero */ 1559 memset(header, 0, size); 1560 1561 header->id = id; 1562 header->version = version; 1563 1564 /* Add to the end of the capability chain */ 1565 for (tmp = buf; tmp->next; tmp = buf + tmp->next) 1566 ; /* nothing */ 1567 1568 tmp->next = caps->size; 1569 caps->size += size; 1570 1571 return header; 1572 } 1573 EXPORT_SYMBOL_GPL(vfio_info_cap_add); 1574 1575 void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset) 1576 { 1577 struct vfio_info_cap_header *tmp; 1578 void *buf = (void *)caps->buf; 1579 1580 /* Capability structs should start with proper alignment */ 1581 WARN_ON(!IS_ALIGNED(offset, sizeof(u64))); 1582 1583 for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset) 1584 tmp->next += offset; 1585 } 1586 EXPORT_SYMBOL(vfio_info_cap_shift); 1587 1588 int vfio_info_add_capability(struct vfio_info_cap *caps, 1589 struct vfio_info_cap_header *cap, size_t size) 1590 { 1591 struct vfio_info_cap_header *header; 1592 1593 header = vfio_info_cap_add(caps, size, cap->id, cap->version); 1594 if (IS_ERR(header)) 1595 return PTR_ERR(header); 1596 1597 memcpy(header + 1, cap + 1, size - sizeof(*header)); 1598 1599 return 0; 1600 } 1601 EXPORT_SYMBOL(vfio_info_add_capability); 1602 1603 int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs, 1604 int max_irq_type, size_t *data_size) 1605 { 1606 unsigned long minsz; 1607 size_t size; 1608 1609 minsz = offsetofend(struct vfio_irq_set, count); 1610 1611 if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) || 1612 (hdr->count >= (U32_MAX - hdr->start)) || 1613 (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK | 1614 VFIO_IRQ_SET_ACTION_TYPE_MASK))) 1615 return -EINVAL; 1616 1617 if (data_size) 1618 *data_size = 0; 1619 1620 if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs) 1621 return -EINVAL; 1622 1623 switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 1624 case VFIO_IRQ_SET_DATA_NONE: 1625 size = 0; 1626 break; 1627 case VFIO_IRQ_SET_DATA_BOOL: 1628 size = sizeof(uint8_t); 1629 break; 1630 case VFIO_IRQ_SET_DATA_EVENTFD: 1631 size = sizeof(int32_t); 1632 break; 1633 default: 1634 return -EINVAL; 1635 } 1636 1637 if (size) { 1638 if (hdr->argsz - minsz < hdr->count * size) 1639 return -EINVAL; 1640 1641 if (!data_size) 1642 return -EINVAL; 1643 1644 *data_size = hdr->count * size; 1645 } 1646 1647 return 0; 1648 } 1649 EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare); 1650 1651 /* 1652 * Pin contiguous user pages and return their associated host pages for local 1653 * domain only. 1654 * @device [in] : device 1655 * @iova [in] : starting IOVA of user pages to be pinned. 1656 * @npage [in] : count of pages to be pinned. This count should not 1657 * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 1658 * @prot [in] : protection flags 1659 * @pages[out] : array of host pages 1660 * Return error or number of pages pinned. 1661 * 1662 * A driver may only call this function if the vfio_device was created 1663 * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages(). 1664 */ 1665 int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, 1666 int npage, int prot, struct page **pages) 1667 { 1668 /* group->container cannot change while a vfio device is open */ 1669 if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device))) 1670 return -EINVAL; 1671 if (!device->ops->dma_unmap) 1672 return -EINVAL; 1673 if (vfio_device_has_container(device)) 1674 return vfio_device_container_pin_pages(device, iova, 1675 npage, prot, pages); 1676 if (device->iommufd_access) { 1677 int ret; 1678 1679 if (iova > ULONG_MAX) 1680 return -EINVAL; 1681 /* 1682 * VFIO ignores the sub page offset, npages is from the start of 1683 * a PAGE_SIZE chunk of IOVA. The caller is expected to recover 1684 * the sub page offset by doing: 1685 * pages[0] + (iova % PAGE_SIZE) 1686 */ 1687 ret = iommufd_access_pin_pages( 1688 device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE), 1689 npage * PAGE_SIZE, pages, 1690 (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0); 1691 if (ret) 1692 return ret; 1693 return npage; 1694 } 1695 return -EINVAL; 1696 } 1697 EXPORT_SYMBOL(vfio_pin_pages); 1698 1699 /* 1700 * Unpin contiguous host pages for local domain only. 1701 * @device [in] : device 1702 * @iova [in] : starting address of user pages to be unpinned. 1703 * @npage [in] : count of pages to be unpinned. This count should not 1704 * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 1705 */ 1706 void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage) 1707 { 1708 if (WARN_ON(!vfio_assert_device_open(device))) 1709 return; 1710 if (WARN_ON(!device->ops->dma_unmap)) 1711 return; 1712 1713 if (vfio_device_has_container(device)) { 1714 vfio_device_container_unpin_pages(device, iova, npage); 1715 return; 1716 } 1717 if (device->iommufd_access) { 1718 if (WARN_ON(iova > ULONG_MAX)) 1719 return; 1720 iommufd_access_unpin_pages(device->iommufd_access, 1721 ALIGN_DOWN(iova, PAGE_SIZE), 1722 npage * PAGE_SIZE); 1723 return; 1724 } 1725 } 1726 EXPORT_SYMBOL(vfio_unpin_pages); 1727 1728 /* 1729 * This interface allows the CPUs to perform some sort of virtual DMA on 1730 * behalf of the device. 1731 * 1732 * CPUs read/write from/into a range of IOVAs pointing to user space memory 1733 * into/from a kernel buffer. 1734 * 1735 * As the read/write of user space memory is conducted via the CPUs and is 1736 * not a real device DMA, it is not necessary to pin the user space memory. 1737 * 1738 * @device [in] : VFIO device 1739 * @iova [in] : base IOVA of a user space buffer 1740 * @data [in] : pointer to kernel buffer 1741 * @len [in] : kernel buffer length 1742 * @write : indicate read or write 1743 * Return error code on failure or 0 on success. 1744 */ 1745 int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, 1746 size_t len, bool write) 1747 { 1748 if (!data || len <= 0 || !vfio_assert_device_open(device)) 1749 return -EINVAL; 1750 1751 if (vfio_device_has_container(device)) 1752 return vfio_device_container_dma_rw(device, iova, 1753 data, len, write); 1754 1755 if (device->iommufd_access) { 1756 unsigned int flags = 0; 1757 1758 if (iova > ULONG_MAX) 1759 return -EINVAL; 1760 1761 /* VFIO historically tries to auto-detect a kthread */ 1762 if (!current->mm) 1763 flags |= IOMMUFD_ACCESS_RW_KTHREAD; 1764 if (write) 1765 flags |= IOMMUFD_ACCESS_RW_WRITE; 1766 return iommufd_access_rw(device->iommufd_access, iova, data, 1767 len, flags); 1768 } 1769 return -EINVAL; 1770 } 1771 EXPORT_SYMBOL(vfio_dma_rw); 1772 1773 /* 1774 * Module/class support 1775 */ 1776 static int __init vfio_init(void) 1777 { 1778 int ret; 1779 1780 ida_init(&vfio.device_ida); 1781 1782 ret = vfio_group_init(); 1783 if (ret) 1784 return ret; 1785 1786 ret = vfio_virqfd_init(); 1787 if (ret) 1788 goto err_virqfd; 1789 1790 /* /sys/class/vfio-dev/vfioX */ 1791 vfio.device_class = class_create("vfio-dev"); 1792 if (IS_ERR(vfio.device_class)) { 1793 ret = PTR_ERR(vfio.device_class); 1794 goto err_dev_class; 1795 } 1796 1797 ret = vfio_cdev_init(vfio.device_class); 1798 if (ret) 1799 goto err_alloc_dev_chrdev; 1800 1801 vfio_debugfs_create_root(); 1802 pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n"); 1803 return 0; 1804 1805 err_alloc_dev_chrdev: 1806 class_destroy(vfio.device_class); 1807 vfio.device_class = NULL; 1808 err_dev_class: 1809 vfio_virqfd_exit(); 1810 err_virqfd: 1811 vfio_group_cleanup(); 1812 return ret; 1813 } 1814 1815 static void __exit vfio_cleanup(void) 1816 { 1817 vfio_debugfs_remove_root(); 1818 ida_destroy(&vfio.device_ida); 1819 vfio_cdev_cleanup(); 1820 class_destroy(vfio.device_class); 1821 vfio.device_class = NULL; 1822 vfio_virqfd_exit(); 1823 vfio_group_cleanup(); 1824 xa_destroy(&vfio_device_set_xa); 1825 } 1826 1827 module_init(vfio_init); 1828 module_exit(vfio_cleanup); 1829 1830 MODULE_IMPORT_NS("IOMMUFD"); 1831 MODULE_VERSION(DRIVER_VERSION); 1832 MODULE_LICENSE("GPL v2"); 1833 MODULE_AUTHOR(DRIVER_AUTHOR); 1834 MODULE_DESCRIPTION(DRIVER_DESC); 1835 MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce"); 1836