1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO core 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 * 8 * Derived from original vfio: 9 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 10 * Author: Tom Lyon, pugs@cisco.com 11 */ 12 13 #include <linux/cdev.h> 14 #include <linux/compat.h> 15 #include <linux/device.h> 16 #include <linux/fs.h> 17 #include <linux/idr.h> 18 #include <linux/iommu.h> 19 #if IS_ENABLED(CONFIG_KVM) 20 #include <linux/kvm_host.h> 21 #endif 22 #include <linux/list.h> 23 #include <linux/miscdevice.h> 24 #include <linux/module.h> 25 #include <linux/mutex.h> 26 #include <linux/pci.h> 27 #include <linux/rwsem.h> 28 #include <linux/sched.h> 29 #include <linux/slab.h> 30 #include <linux/stat.h> 31 #include <linux/string.h> 32 #include <linux/uaccess.h> 33 #include <linux/vfio.h> 34 #include <linux/wait.h> 35 #include <linux/sched/signal.h> 36 #include <linux/pm_runtime.h> 37 #include <linux/interval_tree.h> 38 #include <linux/iova_bitmap.h> 39 #include <linux/iommufd.h> 40 #include "vfio.h" 41 42 #define DRIVER_VERSION "0.3" 43 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 44 #define DRIVER_DESC "VFIO - User Level meta-driver" 45 46 static struct vfio { 47 struct class *device_class; 48 struct ida device_ida; 49 } vfio; 50 51 #ifdef CONFIG_VFIO_NOIOMMU 52 bool vfio_noiommu __read_mostly; 53 module_param_named(enable_unsafe_noiommu_mode, 54 vfio_noiommu, bool, S_IRUGO | S_IWUSR); 55 MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)"); 56 #endif 57 58 static DEFINE_XARRAY(vfio_device_set_xa); 59 60 int vfio_assign_device_set(struct vfio_device *device, void *set_id) 61 { 62 unsigned long idx = (unsigned long)set_id; 63 struct vfio_device_set *new_dev_set; 64 struct vfio_device_set *dev_set; 65 66 if (WARN_ON(!set_id)) 67 return -EINVAL; 68 69 /* 70 * Atomically acquire a singleton object in the xarray for this set_id 71 */ 72 xa_lock(&vfio_device_set_xa); 73 dev_set = xa_load(&vfio_device_set_xa, idx); 74 if (dev_set) 75 goto found_get_ref; 76 xa_unlock(&vfio_device_set_xa); 77 78 new_dev_set = kzalloc(sizeof(*new_dev_set), GFP_KERNEL); 79 if (!new_dev_set) 80 return -ENOMEM; 81 mutex_init(&new_dev_set->lock); 82 INIT_LIST_HEAD(&new_dev_set->device_list); 83 new_dev_set->set_id = set_id; 84 85 xa_lock(&vfio_device_set_xa); 86 dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set, 87 GFP_KERNEL); 88 if (!dev_set) { 89 dev_set = new_dev_set; 90 goto found_get_ref; 91 } 92 93 kfree(new_dev_set); 94 if (xa_is_err(dev_set)) { 95 xa_unlock(&vfio_device_set_xa); 96 return xa_err(dev_set); 97 } 98 99 found_get_ref: 100 dev_set->device_count++; 101 xa_unlock(&vfio_device_set_xa); 102 mutex_lock(&dev_set->lock); 103 device->dev_set = dev_set; 104 list_add_tail(&device->dev_set_list, &dev_set->device_list); 105 mutex_unlock(&dev_set->lock); 106 return 0; 107 } 108 EXPORT_SYMBOL_GPL(vfio_assign_device_set); 109 110 static void vfio_release_device_set(struct vfio_device *device) 111 { 112 struct vfio_device_set *dev_set = device->dev_set; 113 114 if (!dev_set) 115 return; 116 117 mutex_lock(&dev_set->lock); 118 list_del(&device->dev_set_list); 119 mutex_unlock(&dev_set->lock); 120 121 xa_lock(&vfio_device_set_xa); 122 if (!--dev_set->device_count) { 123 __xa_erase(&vfio_device_set_xa, 124 (unsigned long)dev_set->set_id); 125 mutex_destroy(&dev_set->lock); 126 kfree(dev_set); 127 } 128 xa_unlock(&vfio_device_set_xa); 129 } 130 131 unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set) 132 { 133 struct vfio_device *cur; 134 unsigned int open_count = 0; 135 136 lockdep_assert_held(&dev_set->lock); 137 138 list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 139 open_count += cur->open_count; 140 return open_count; 141 } 142 EXPORT_SYMBOL_GPL(vfio_device_set_open_count); 143 144 struct vfio_device * 145 vfio_find_device_in_devset(struct vfio_device_set *dev_set, 146 struct device *dev) 147 { 148 struct vfio_device *cur; 149 150 lockdep_assert_held(&dev_set->lock); 151 152 list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 153 if (cur->dev == dev) 154 return cur; 155 return NULL; 156 } 157 EXPORT_SYMBOL_GPL(vfio_find_device_in_devset); 158 159 /* 160 * Device objects - create, release, get, put, search 161 */ 162 /* Device reference always implies a group reference */ 163 void vfio_device_put_registration(struct vfio_device *device) 164 { 165 if (refcount_dec_and_test(&device->refcount)) 166 complete(&device->comp); 167 } 168 169 bool vfio_device_try_get_registration(struct vfio_device *device) 170 { 171 return refcount_inc_not_zero(&device->refcount); 172 } 173 174 /* 175 * VFIO driver API 176 */ 177 /* Release helper called by vfio_put_device() */ 178 static void vfio_device_release(struct device *dev) 179 { 180 struct vfio_device *device = 181 container_of(dev, struct vfio_device, device); 182 183 vfio_release_device_set(device); 184 ida_free(&vfio.device_ida, device->index); 185 186 if (device->ops->release) 187 device->ops->release(device); 188 189 kvfree(device); 190 } 191 192 static int vfio_init_device(struct vfio_device *device, struct device *dev, 193 const struct vfio_device_ops *ops); 194 195 /* 196 * Allocate and initialize vfio_device so it can be registered to vfio 197 * core. 198 * 199 * Drivers should use the wrapper vfio_alloc_device() for allocation. 200 * @size is the size of the structure to be allocated, including any 201 * private data used by the driver. 202 * 203 * Driver may provide an @init callback to cover device private data. 204 * 205 * Use vfio_put_device() to release the structure after success return. 206 */ 207 struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, 208 const struct vfio_device_ops *ops) 209 { 210 struct vfio_device *device; 211 int ret; 212 213 if (WARN_ON(size < sizeof(struct vfio_device))) 214 return ERR_PTR(-EINVAL); 215 216 device = kvzalloc(size, GFP_KERNEL); 217 if (!device) 218 return ERR_PTR(-ENOMEM); 219 220 ret = vfio_init_device(device, dev, ops); 221 if (ret) 222 goto out_free; 223 return device; 224 225 out_free: 226 kvfree(device); 227 return ERR_PTR(ret); 228 } 229 EXPORT_SYMBOL_GPL(_vfio_alloc_device); 230 231 /* 232 * Initialize a vfio_device so it can be registered to vfio core. 233 */ 234 static int vfio_init_device(struct vfio_device *device, struct device *dev, 235 const struct vfio_device_ops *ops) 236 { 237 int ret; 238 239 ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL); 240 if (ret < 0) { 241 dev_dbg(dev, "Error to alloc index\n"); 242 return ret; 243 } 244 245 device->index = ret; 246 init_completion(&device->comp); 247 device->dev = dev; 248 device->ops = ops; 249 250 if (ops->init) { 251 ret = ops->init(device); 252 if (ret) 253 goto out_uninit; 254 } 255 256 device_initialize(&device->device); 257 device->device.release = vfio_device_release; 258 device->device.class = vfio.device_class; 259 device->device.parent = device->dev; 260 return 0; 261 262 out_uninit: 263 vfio_release_device_set(device); 264 ida_free(&vfio.device_ida, device->index); 265 return ret; 266 } 267 268 static int __vfio_register_dev(struct vfio_device *device, 269 enum vfio_group_type type) 270 { 271 int ret; 272 273 if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) && 274 (!device->ops->bind_iommufd || 275 !device->ops->unbind_iommufd || 276 !device->ops->attach_ioas || 277 !device->ops->detach_ioas))) 278 return -EINVAL; 279 280 /* 281 * If the driver doesn't specify a set then the device is added to a 282 * singleton set just for itself. 283 */ 284 if (!device->dev_set) 285 vfio_assign_device_set(device, device); 286 287 ret = dev_set_name(&device->device, "vfio%d", device->index); 288 if (ret) 289 return ret; 290 291 ret = vfio_device_set_group(device, type); 292 if (ret) 293 return ret; 294 295 /* 296 * VFIO always sets IOMMU_CACHE because we offer no way for userspace to 297 * restore cache coherency. It has to be checked here because it is only 298 * valid for cases where we are using iommu groups. 299 */ 300 if (type == VFIO_IOMMU && !vfio_device_is_noiommu(device) && 301 !device_iommu_capable(device->dev, IOMMU_CAP_CACHE_COHERENCY)) { 302 ret = -EINVAL; 303 goto err_out; 304 } 305 306 ret = vfio_device_add(device); 307 if (ret) 308 goto err_out; 309 310 /* Refcounting can't start until the driver calls register */ 311 refcount_set(&device->refcount, 1); 312 313 vfio_device_group_register(device); 314 vfio_device_debugfs_init(device); 315 316 return 0; 317 err_out: 318 vfio_device_remove_group(device); 319 return ret; 320 } 321 322 int vfio_register_group_dev(struct vfio_device *device) 323 { 324 return __vfio_register_dev(device, VFIO_IOMMU); 325 } 326 EXPORT_SYMBOL_GPL(vfio_register_group_dev); 327 328 /* 329 * Register a virtual device without IOMMU backing. The user of this 330 * device must not be able to directly trigger unmediated DMA. 331 */ 332 int vfio_register_emulated_iommu_dev(struct vfio_device *device) 333 { 334 return __vfio_register_dev(device, VFIO_EMULATED_IOMMU); 335 } 336 EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev); 337 338 /* 339 * Decrement the device reference count and wait for the device to be 340 * removed. Open file descriptors for the device... */ 341 void vfio_unregister_group_dev(struct vfio_device *device) 342 { 343 unsigned int i = 0; 344 bool interrupted = false; 345 long rc; 346 347 /* 348 * Prevent new device opened by userspace via the 349 * VFIO_GROUP_GET_DEVICE_FD in the group path. 350 */ 351 vfio_device_group_unregister(device); 352 353 /* 354 * Balances vfio_device_add() in register path, also prevents 355 * new device opened by userspace in the cdev path. 356 */ 357 vfio_device_del(device); 358 359 vfio_device_put_registration(device); 360 rc = try_wait_for_completion(&device->comp); 361 while (rc <= 0) { 362 if (device->ops->request) 363 device->ops->request(device, i++); 364 365 if (interrupted) { 366 rc = wait_for_completion_timeout(&device->comp, 367 HZ * 10); 368 } else { 369 rc = wait_for_completion_interruptible_timeout( 370 &device->comp, HZ * 10); 371 if (rc < 0) { 372 interrupted = true; 373 dev_warn(device->dev, 374 "Device is currently in use, task" 375 " \"%s\" (%d) " 376 "blocked until device is released", 377 current->comm, task_pid_nr(current)); 378 } 379 } 380 } 381 382 vfio_device_debugfs_exit(device); 383 /* Balances vfio_device_set_group in register path */ 384 vfio_device_remove_group(device); 385 } 386 EXPORT_SYMBOL_GPL(vfio_unregister_group_dev); 387 388 #if IS_ENABLED(CONFIG_KVM) 389 void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm) 390 { 391 void (*pfn)(struct kvm *kvm); 392 bool (*fn)(struct kvm *kvm); 393 bool ret; 394 395 lockdep_assert_held(&device->dev_set->lock); 396 397 if (!kvm) 398 return; 399 400 pfn = symbol_get(kvm_put_kvm); 401 if (WARN_ON(!pfn)) 402 return; 403 404 fn = symbol_get(kvm_get_kvm_safe); 405 if (WARN_ON(!fn)) { 406 symbol_put(kvm_put_kvm); 407 return; 408 } 409 410 ret = fn(kvm); 411 symbol_put(kvm_get_kvm_safe); 412 if (!ret) { 413 symbol_put(kvm_put_kvm); 414 return; 415 } 416 417 device->put_kvm = pfn; 418 device->kvm = kvm; 419 } 420 421 void vfio_device_put_kvm(struct vfio_device *device) 422 { 423 lockdep_assert_held(&device->dev_set->lock); 424 425 if (!device->kvm) 426 return; 427 428 if (WARN_ON(!device->put_kvm)) 429 goto clear; 430 431 device->put_kvm(device->kvm); 432 device->put_kvm = NULL; 433 symbol_put(kvm_put_kvm); 434 435 clear: 436 device->kvm = NULL; 437 } 438 #endif 439 440 /* true if the vfio_device has open_device() called but not close_device() */ 441 static bool vfio_assert_device_open(struct vfio_device *device) 442 { 443 return !WARN_ON_ONCE(!READ_ONCE(device->open_count)); 444 } 445 446 struct vfio_device_file * 447 vfio_allocate_device_file(struct vfio_device *device) 448 { 449 struct vfio_device_file *df; 450 451 df = kzalloc(sizeof(*df), GFP_KERNEL_ACCOUNT); 452 if (!df) 453 return ERR_PTR(-ENOMEM); 454 455 df->device = device; 456 spin_lock_init(&df->kvm_ref_lock); 457 458 return df; 459 } 460 461 static int vfio_df_device_first_open(struct vfio_device_file *df) 462 { 463 struct vfio_device *device = df->device; 464 struct iommufd_ctx *iommufd = df->iommufd; 465 int ret; 466 467 lockdep_assert_held(&device->dev_set->lock); 468 469 if (!try_module_get(device->dev->driver->owner)) 470 return -ENODEV; 471 472 if (iommufd) 473 ret = vfio_df_iommufd_bind(df); 474 else 475 ret = vfio_device_group_use_iommu(device); 476 if (ret) 477 goto err_module_put; 478 479 if (device->ops->open_device) { 480 ret = device->ops->open_device(device); 481 if (ret) 482 goto err_unuse_iommu; 483 } 484 return 0; 485 486 err_unuse_iommu: 487 if (iommufd) 488 vfio_df_iommufd_unbind(df); 489 else 490 vfio_device_group_unuse_iommu(device); 491 err_module_put: 492 module_put(device->dev->driver->owner); 493 return ret; 494 } 495 496 static void vfio_df_device_last_close(struct vfio_device_file *df) 497 { 498 struct vfio_device *device = df->device; 499 struct iommufd_ctx *iommufd = df->iommufd; 500 501 lockdep_assert_held(&device->dev_set->lock); 502 503 if (device->ops->close_device) 504 device->ops->close_device(device); 505 if (iommufd) 506 vfio_df_iommufd_unbind(df); 507 else 508 vfio_device_group_unuse_iommu(device); 509 module_put(device->dev->driver->owner); 510 } 511 512 int vfio_df_open(struct vfio_device_file *df) 513 { 514 struct vfio_device *device = df->device; 515 int ret = 0; 516 517 lockdep_assert_held(&device->dev_set->lock); 518 519 /* 520 * Only the group path allows the device to be opened multiple 521 * times. The device cdev path doesn't have a secure way for it. 522 */ 523 if (device->open_count != 0 && !df->group) 524 return -EINVAL; 525 526 device->open_count++; 527 if (device->open_count == 1) { 528 ret = vfio_df_device_first_open(df); 529 if (ret) 530 device->open_count--; 531 } 532 533 return ret; 534 } 535 536 void vfio_df_close(struct vfio_device_file *df) 537 { 538 struct vfio_device *device = df->device; 539 540 lockdep_assert_held(&device->dev_set->lock); 541 542 vfio_assert_device_open(device); 543 if (device->open_count == 1) 544 vfio_df_device_last_close(df); 545 device->open_count--; 546 } 547 548 /* 549 * Wrapper around pm_runtime_resume_and_get(). 550 * Return error code on failure or 0 on success. 551 */ 552 static inline int vfio_device_pm_runtime_get(struct vfio_device *device) 553 { 554 struct device *dev = device->dev; 555 556 if (dev->driver && dev->driver->pm) { 557 int ret; 558 559 ret = pm_runtime_resume_and_get(dev); 560 if (ret) { 561 dev_info_ratelimited(dev, 562 "vfio: runtime resume failed %d\n", ret); 563 return -EIO; 564 } 565 } 566 567 return 0; 568 } 569 570 /* 571 * Wrapper around pm_runtime_put(). 572 */ 573 static inline void vfio_device_pm_runtime_put(struct vfio_device *device) 574 { 575 struct device *dev = device->dev; 576 577 if (dev->driver && dev->driver->pm) 578 pm_runtime_put(dev); 579 } 580 581 /* 582 * VFIO Device fd 583 */ 584 static int vfio_device_fops_release(struct inode *inode, struct file *filep) 585 { 586 struct vfio_device_file *df = filep->private_data; 587 struct vfio_device *device = df->device; 588 589 if (df->group) 590 vfio_df_group_close(df); 591 else 592 vfio_df_unbind_iommufd(df); 593 594 vfio_device_put_registration(device); 595 596 kfree(df); 597 598 return 0; 599 } 600 601 /* 602 * vfio_mig_get_next_state - Compute the next step in the FSM 603 * @cur_fsm - The current state the device is in 604 * @new_fsm - The target state to reach 605 * @next_fsm - Pointer to the next step to get to new_fsm 606 * 607 * Return 0 upon success, otherwise -errno 608 * Upon success the next step in the state progression between cur_fsm and 609 * new_fsm will be set in next_fsm. 610 * 611 * This breaks down requests for combination transitions into smaller steps and 612 * returns the next step to get to new_fsm. The function may need to be called 613 * multiple times before reaching new_fsm. 614 * 615 */ 616 int vfio_mig_get_next_state(struct vfio_device *device, 617 enum vfio_device_mig_state cur_fsm, 618 enum vfio_device_mig_state new_fsm, 619 enum vfio_device_mig_state *next_fsm) 620 { 621 enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 }; 622 /* 623 * The coding in this table requires the driver to implement the 624 * following FSM arcs: 625 * RESUMING -> STOP 626 * STOP -> RESUMING 627 * STOP -> STOP_COPY 628 * STOP_COPY -> STOP 629 * 630 * If P2P is supported then the driver must also implement these FSM 631 * arcs: 632 * RUNNING -> RUNNING_P2P 633 * RUNNING_P2P -> RUNNING 634 * RUNNING_P2P -> STOP 635 * STOP -> RUNNING_P2P 636 * 637 * If precopy is supported then the driver must support these additional 638 * FSM arcs: 639 * RUNNING -> PRE_COPY 640 * PRE_COPY -> RUNNING 641 * PRE_COPY -> STOP_COPY 642 * However, if precopy and P2P are supported together then the driver 643 * must support these additional arcs beyond the P2P arcs above: 644 * PRE_COPY -> RUNNING 645 * PRE_COPY -> PRE_COPY_P2P 646 * PRE_COPY_P2P -> PRE_COPY 647 * PRE_COPY_P2P -> RUNNING_P2P 648 * PRE_COPY_P2P -> STOP_COPY 649 * RUNNING -> PRE_COPY 650 * RUNNING_P2P -> PRE_COPY_P2P 651 * 652 * Without P2P and precopy the driver must implement: 653 * RUNNING -> STOP 654 * STOP -> RUNNING 655 * 656 * The coding will step through multiple states for some combination 657 * transitions; if all optional features are supported, this means the 658 * following ones: 659 * PRE_COPY -> PRE_COPY_P2P -> STOP_COPY 660 * PRE_COPY -> RUNNING -> RUNNING_P2P 661 * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP 662 * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING 663 * PRE_COPY_P2P -> RUNNING_P2P -> RUNNING 664 * PRE_COPY_P2P -> RUNNING_P2P -> STOP 665 * PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING 666 * RESUMING -> STOP -> RUNNING_P2P 667 * RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P 668 * RESUMING -> STOP -> RUNNING_P2P -> RUNNING 669 * RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 670 * RESUMING -> STOP -> STOP_COPY 671 * RUNNING -> RUNNING_P2P -> PRE_COPY_P2P 672 * RUNNING -> RUNNING_P2P -> STOP 673 * RUNNING -> RUNNING_P2P -> STOP -> RESUMING 674 * RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY 675 * RUNNING_P2P -> RUNNING -> PRE_COPY 676 * RUNNING_P2P -> STOP -> RESUMING 677 * RUNNING_P2P -> STOP -> STOP_COPY 678 * STOP -> RUNNING_P2P -> PRE_COPY_P2P 679 * STOP -> RUNNING_P2P -> RUNNING 680 * STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 681 * STOP_COPY -> STOP -> RESUMING 682 * STOP_COPY -> STOP -> RUNNING_P2P 683 * STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING 684 * 685 * The following transitions are blocked: 686 * STOP_COPY -> PRE_COPY 687 * STOP_COPY -> PRE_COPY_P2P 688 */ 689 static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = { 690 [VFIO_DEVICE_STATE_STOP] = { 691 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 692 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 693 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 694 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 695 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 696 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 697 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 698 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 699 }, 700 [VFIO_DEVICE_STATE_RUNNING] = { 701 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 702 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 703 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 704 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 705 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 706 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 707 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 708 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 709 }, 710 [VFIO_DEVICE_STATE_PRE_COPY] = { 711 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING, 712 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 713 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 714 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 715 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 716 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING, 717 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING, 718 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 719 }, 720 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = { 721 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 722 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 723 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 724 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 725 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 726 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 727 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 728 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 729 }, 730 [VFIO_DEVICE_STATE_STOP_COPY] = { 731 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 732 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 733 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 734 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 735 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 736 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 737 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 738 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 739 }, 740 [VFIO_DEVICE_STATE_RESUMING] = { 741 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 742 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 743 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP, 744 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP, 745 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 746 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 747 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 748 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 749 }, 750 [VFIO_DEVICE_STATE_RUNNING_P2P] = { 751 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 752 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 753 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING, 754 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 755 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 756 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 757 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 758 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 759 }, 760 [VFIO_DEVICE_STATE_ERROR] = { 761 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR, 762 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR, 763 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 764 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 765 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR, 766 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR, 767 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR, 768 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 769 }, 770 }; 771 772 static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = { 773 [VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY, 774 [VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY, 775 [VFIO_DEVICE_STATE_PRE_COPY] = 776 VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY, 777 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY | 778 VFIO_MIGRATION_P2P | 779 VFIO_MIGRATION_PRE_COPY, 780 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY, 781 [VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY, 782 [VFIO_DEVICE_STATE_RUNNING_P2P] = 783 VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P, 784 [VFIO_DEVICE_STATE_ERROR] = ~0U, 785 }; 786 787 if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 788 (state_flags_table[cur_fsm] & device->migration_flags) != 789 state_flags_table[cur_fsm])) 790 return -EINVAL; 791 792 if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 793 (state_flags_table[new_fsm] & device->migration_flags) != 794 state_flags_table[new_fsm]) 795 return -EINVAL; 796 797 /* 798 * Arcs touching optional and unsupported states are skipped over. The 799 * driver will instead see an arc from the original state to the next 800 * logical state, as per the above comment. 801 */ 802 *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm]; 803 while ((state_flags_table[*next_fsm] & device->migration_flags) != 804 state_flags_table[*next_fsm]) 805 *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm]; 806 807 return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL; 808 } 809 EXPORT_SYMBOL_GPL(vfio_mig_get_next_state); 810 811 /* 812 * Convert the drivers's struct file into a FD number and return it to userspace 813 */ 814 static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg, 815 struct vfio_device_feature_mig_state *mig) 816 { 817 int ret; 818 int fd; 819 820 fd = get_unused_fd_flags(O_CLOEXEC); 821 if (fd < 0) { 822 ret = fd; 823 goto out_fput; 824 } 825 826 mig->data_fd = fd; 827 if (copy_to_user(arg, mig, sizeof(*mig))) { 828 ret = -EFAULT; 829 goto out_put_unused; 830 } 831 fd_install(fd, filp); 832 return 0; 833 834 out_put_unused: 835 put_unused_fd(fd); 836 out_fput: 837 fput(filp); 838 return ret; 839 } 840 841 static int 842 vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device, 843 u32 flags, void __user *arg, 844 size_t argsz) 845 { 846 size_t minsz = 847 offsetofend(struct vfio_device_feature_mig_state, data_fd); 848 struct vfio_device_feature_mig_state mig; 849 struct file *filp = NULL; 850 int ret; 851 852 if (!device->mig_ops) 853 return -ENOTTY; 854 855 ret = vfio_check_feature(flags, argsz, 856 VFIO_DEVICE_FEATURE_SET | 857 VFIO_DEVICE_FEATURE_GET, 858 sizeof(mig)); 859 if (ret != 1) 860 return ret; 861 862 if (copy_from_user(&mig, arg, minsz)) 863 return -EFAULT; 864 865 if (flags & VFIO_DEVICE_FEATURE_GET) { 866 enum vfio_device_mig_state curr_state; 867 868 ret = device->mig_ops->migration_get_state(device, 869 &curr_state); 870 if (ret) 871 return ret; 872 mig.device_state = curr_state; 873 goto out_copy; 874 } 875 876 /* Handle the VFIO_DEVICE_FEATURE_SET */ 877 filp = device->mig_ops->migration_set_state(device, mig.device_state); 878 if (IS_ERR(filp) || !filp) 879 goto out_copy; 880 881 return vfio_ioct_mig_return_fd(filp, arg, &mig); 882 out_copy: 883 mig.data_fd = -1; 884 if (copy_to_user(arg, &mig, sizeof(mig))) 885 return -EFAULT; 886 if (IS_ERR(filp)) 887 return PTR_ERR(filp); 888 return 0; 889 } 890 891 static int 892 vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device, 893 u32 flags, void __user *arg, 894 size_t argsz) 895 { 896 struct vfio_device_feature_mig_data_size data_size = {}; 897 unsigned long stop_copy_length; 898 int ret; 899 900 if (!device->mig_ops) 901 return -ENOTTY; 902 903 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 904 sizeof(data_size)); 905 if (ret != 1) 906 return ret; 907 908 ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length); 909 if (ret) 910 return ret; 911 912 data_size.stop_copy_length = stop_copy_length; 913 if (copy_to_user(arg, &data_size, sizeof(data_size))) 914 return -EFAULT; 915 916 return 0; 917 } 918 919 static int vfio_ioctl_device_feature_migration(struct vfio_device *device, 920 u32 flags, void __user *arg, 921 size_t argsz) 922 { 923 struct vfio_device_feature_migration mig = { 924 .flags = device->migration_flags, 925 }; 926 int ret; 927 928 if (!device->mig_ops) 929 return -ENOTTY; 930 931 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 932 sizeof(mig)); 933 if (ret != 1) 934 return ret; 935 if (copy_to_user(arg, &mig, sizeof(mig))) 936 return -EFAULT; 937 return 0; 938 } 939 940 void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes, 941 u32 req_nodes) 942 { 943 struct interval_tree_node *prev, *curr, *comb_start, *comb_end; 944 unsigned long min_gap, curr_gap; 945 946 /* Special shortcut when a single range is required */ 947 if (req_nodes == 1) { 948 unsigned long last; 949 950 comb_start = interval_tree_iter_first(root, 0, ULONG_MAX); 951 952 /* Empty list */ 953 if (WARN_ON_ONCE(!comb_start)) 954 return; 955 956 curr = comb_start; 957 while (curr) { 958 last = curr->last; 959 prev = curr; 960 curr = interval_tree_iter_next(curr, 0, ULONG_MAX); 961 if (prev != comb_start) 962 interval_tree_remove(prev, root); 963 } 964 comb_start->last = last; 965 return; 966 } 967 968 /* Combine ranges which have the smallest gap */ 969 while (cur_nodes > req_nodes) { 970 prev = NULL; 971 min_gap = ULONG_MAX; 972 curr = interval_tree_iter_first(root, 0, ULONG_MAX); 973 while (curr) { 974 if (prev) { 975 curr_gap = curr->start - prev->last; 976 if (curr_gap < min_gap) { 977 min_gap = curr_gap; 978 comb_start = prev; 979 comb_end = curr; 980 } 981 } 982 prev = curr; 983 curr = interval_tree_iter_next(curr, 0, ULONG_MAX); 984 } 985 986 /* Empty list or no nodes to combine */ 987 if (WARN_ON_ONCE(min_gap == ULONG_MAX)) 988 break; 989 990 comb_start->last = comb_end->last; 991 interval_tree_remove(comb_end, root); 992 cur_nodes--; 993 } 994 } 995 EXPORT_SYMBOL_GPL(vfio_combine_iova_ranges); 996 997 /* Ranges should fit into a single kernel page */ 998 #define LOG_MAX_RANGES \ 999 (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range)) 1000 1001 static int 1002 vfio_ioctl_device_feature_logging_start(struct vfio_device *device, 1003 u32 flags, void __user *arg, 1004 size_t argsz) 1005 { 1006 size_t minsz = 1007 offsetofend(struct vfio_device_feature_dma_logging_control, 1008 ranges); 1009 struct vfio_device_feature_dma_logging_range __user *ranges; 1010 struct vfio_device_feature_dma_logging_control control; 1011 struct vfio_device_feature_dma_logging_range range; 1012 struct rb_root_cached root = RB_ROOT_CACHED; 1013 struct interval_tree_node *nodes; 1014 u64 iova_end; 1015 u32 nnodes; 1016 int i, ret; 1017 1018 if (!device->log_ops) 1019 return -ENOTTY; 1020 1021 ret = vfio_check_feature(flags, argsz, 1022 VFIO_DEVICE_FEATURE_SET, 1023 sizeof(control)); 1024 if (ret != 1) 1025 return ret; 1026 1027 if (copy_from_user(&control, arg, minsz)) 1028 return -EFAULT; 1029 1030 nnodes = control.num_ranges; 1031 if (!nnodes) 1032 return -EINVAL; 1033 1034 if (nnodes > LOG_MAX_RANGES) 1035 return -E2BIG; 1036 1037 ranges = u64_to_user_ptr(control.ranges); 1038 nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node), 1039 GFP_KERNEL); 1040 if (!nodes) 1041 return -ENOMEM; 1042 1043 for (i = 0; i < nnodes; i++) { 1044 if (copy_from_user(&range, &ranges[i], sizeof(range))) { 1045 ret = -EFAULT; 1046 goto end; 1047 } 1048 if (!IS_ALIGNED(range.iova, control.page_size) || 1049 !IS_ALIGNED(range.length, control.page_size)) { 1050 ret = -EINVAL; 1051 goto end; 1052 } 1053 1054 if (check_add_overflow(range.iova, range.length, &iova_end) || 1055 iova_end > ULONG_MAX) { 1056 ret = -EOVERFLOW; 1057 goto end; 1058 } 1059 1060 nodes[i].start = range.iova; 1061 nodes[i].last = range.iova + range.length - 1; 1062 if (interval_tree_iter_first(&root, nodes[i].start, 1063 nodes[i].last)) { 1064 /* Range overlapping */ 1065 ret = -EINVAL; 1066 goto end; 1067 } 1068 interval_tree_insert(nodes + i, &root); 1069 } 1070 1071 ret = device->log_ops->log_start(device, &root, nnodes, 1072 &control.page_size); 1073 if (ret) 1074 goto end; 1075 1076 if (copy_to_user(arg, &control, sizeof(control))) { 1077 ret = -EFAULT; 1078 device->log_ops->log_stop(device); 1079 } 1080 1081 end: 1082 kfree(nodes); 1083 return ret; 1084 } 1085 1086 static int 1087 vfio_ioctl_device_feature_logging_stop(struct vfio_device *device, 1088 u32 flags, void __user *arg, 1089 size_t argsz) 1090 { 1091 int ret; 1092 1093 if (!device->log_ops) 1094 return -ENOTTY; 1095 1096 ret = vfio_check_feature(flags, argsz, 1097 VFIO_DEVICE_FEATURE_SET, 0); 1098 if (ret != 1) 1099 return ret; 1100 1101 return device->log_ops->log_stop(device); 1102 } 1103 1104 static int vfio_device_log_read_and_clear(struct iova_bitmap *iter, 1105 unsigned long iova, size_t length, 1106 void *opaque) 1107 { 1108 struct vfio_device *device = opaque; 1109 1110 return device->log_ops->log_read_and_clear(device, iova, length, iter); 1111 } 1112 1113 static int 1114 vfio_ioctl_device_feature_logging_report(struct vfio_device *device, 1115 u32 flags, void __user *arg, 1116 size_t argsz) 1117 { 1118 size_t minsz = 1119 offsetofend(struct vfio_device_feature_dma_logging_report, 1120 bitmap); 1121 struct vfio_device_feature_dma_logging_report report; 1122 struct iova_bitmap *iter; 1123 u64 iova_end; 1124 int ret; 1125 1126 if (!device->log_ops) 1127 return -ENOTTY; 1128 1129 ret = vfio_check_feature(flags, argsz, 1130 VFIO_DEVICE_FEATURE_GET, 1131 sizeof(report)); 1132 if (ret != 1) 1133 return ret; 1134 1135 if (copy_from_user(&report, arg, minsz)) 1136 return -EFAULT; 1137 1138 if (report.page_size < SZ_4K || !is_power_of_2(report.page_size)) 1139 return -EINVAL; 1140 1141 if (check_add_overflow(report.iova, report.length, &iova_end) || 1142 iova_end > ULONG_MAX) 1143 return -EOVERFLOW; 1144 1145 iter = iova_bitmap_alloc(report.iova, report.length, 1146 report.page_size, 1147 u64_to_user_ptr(report.bitmap)); 1148 if (IS_ERR(iter)) 1149 return PTR_ERR(iter); 1150 1151 ret = iova_bitmap_for_each(iter, device, 1152 vfio_device_log_read_and_clear); 1153 1154 iova_bitmap_free(iter); 1155 return ret; 1156 } 1157 1158 static int vfio_ioctl_device_feature(struct vfio_device *device, 1159 struct vfio_device_feature __user *arg) 1160 { 1161 size_t minsz = offsetofend(struct vfio_device_feature, flags); 1162 struct vfio_device_feature feature; 1163 1164 if (copy_from_user(&feature, arg, minsz)) 1165 return -EFAULT; 1166 1167 if (feature.argsz < minsz) 1168 return -EINVAL; 1169 1170 /* Check unknown flags */ 1171 if (feature.flags & 1172 ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET | 1173 VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE)) 1174 return -EINVAL; 1175 1176 /* GET & SET are mutually exclusive except with PROBE */ 1177 if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) && 1178 (feature.flags & VFIO_DEVICE_FEATURE_SET) && 1179 (feature.flags & VFIO_DEVICE_FEATURE_GET)) 1180 return -EINVAL; 1181 1182 switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) { 1183 case VFIO_DEVICE_FEATURE_MIGRATION: 1184 return vfio_ioctl_device_feature_migration( 1185 device, feature.flags, arg->data, 1186 feature.argsz - minsz); 1187 case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE: 1188 return vfio_ioctl_device_feature_mig_device_state( 1189 device, feature.flags, arg->data, 1190 feature.argsz - minsz); 1191 case VFIO_DEVICE_FEATURE_DMA_LOGGING_START: 1192 return vfio_ioctl_device_feature_logging_start( 1193 device, feature.flags, arg->data, 1194 feature.argsz - minsz); 1195 case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP: 1196 return vfio_ioctl_device_feature_logging_stop( 1197 device, feature.flags, arg->data, 1198 feature.argsz - minsz); 1199 case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT: 1200 return vfio_ioctl_device_feature_logging_report( 1201 device, feature.flags, arg->data, 1202 feature.argsz - minsz); 1203 case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE: 1204 return vfio_ioctl_device_feature_migration_data_size( 1205 device, feature.flags, arg->data, 1206 feature.argsz - minsz); 1207 default: 1208 if (unlikely(!device->ops->device_feature)) 1209 return -EINVAL; 1210 return device->ops->device_feature(device, feature.flags, 1211 arg->data, 1212 feature.argsz - minsz); 1213 } 1214 } 1215 1216 static long vfio_device_fops_unl_ioctl(struct file *filep, 1217 unsigned int cmd, unsigned long arg) 1218 { 1219 struct vfio_device_file *df = filep->private_data; 1220 struct vfio_device *device = df->device; 1221 void __user *uptr = (void __user *)arg; 1222 int ret; 1223 1224 if (cmd == VFIO_DEVICE_BIND_IOMMUFD) 1225 return vfio_df_ioctl_bind_iommufd(df, uptr); 1226 1227 /* Paired with smp_store_release() following vfio_df_open() */ 1228 if (!smp_load_acquire(&df->access_granted)) 1229 return -EINVAL; 1230 1231 ret = vfio_device_pm_runtime_get(device); 1232 if (ret) 1233 return ret; 1234 1235 /* cdev only ioctls */ 1236 if (IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) && !df->group) { 1237 switch (cmd) { 1238 case VFIO_DEVICE_ATTACH_IOMMUFD_PT: 1239 ret = vfio_df_ioctl_attach_pt(df, uptr); 1240 goto out; 1241 1242 case VFIO_DEVICE_DETACH_IOMMUFD_PT: 1243 ret = vfio_df_ioctl_detach_pt(df, uptr); 1244 goto out; 1245 } 1246 } 1247 1248 switch (cmd) { 1249 case VFIO_DEVICE_FEATURE: 1250 ret = vfio_ioctl_device_feature(device, uptr); 1251 break; 1252 1253 default: 1254 if (unlikely(!device->ops->ioctl)) 1255 ret = -EINVAL; 1256 else 1257 ret = device->ops->ioctl(device, cmd, arg); 1258 break; 1259 } 1260 out: 1261 vfio_device_pm_runtime_put(device); 1262 return ret; 1263 } 1264 1265 static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf, 1266 size_t count, loff_t *ppos) 1267 { 1268 struct vfio_device_file *df = filep->private_data; 1269 struct vfio_device *device = df->device; 1270 1271 /* Paired with smp_store_release() following vfio_df_open() */ 1272 if (!smp_load_acquire(&df->access_granted)) 1273 return -EINVAL; 1274 1275 if (unlikely(!device->ops->read)) 1276 return -EINVAL; 1277 1278 return device->ops->read(device, buf, count, ppos); 1279 } 1280 1281 static ssize_t vfio_device_fops_write(struct file *filep, 1282 const char __user *buf, 1283 size_t count, loff_t *ppos) 1284 { 1285 struct vfio_device_file *df = filep->private_data; 1286 struct vfio_device *device = df->device; 1287 1288 /* Paired with smp_store_release() following vfio_df_open() */ 1289 if (!smp_load_acquire(&df->access_granted)) 1290 return -EINVAL; 1291 1292 if (unlikely(!device->ops->write)) 1293 return -EINVAL; 1294 1295 return device->ops->write(device, buf, count, ppos); 1296 } 1297 1298 static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma) 1299 { 1300 struct vfio_device_file *df = filep->private_data; 1301 struct vfio_device *device = df->device; 1302 1303 /* Paired with smp_store_release() following vfio_df_open() */ 1304 if (!smp_load_acquire(&df->access_granted)) 1305 return -EINVAL; 1306 1307 if (unlikely(!device->ops->mmap)) 1308 return -EINVAL; 1309 1310 return device->ops->mmap(device, vma); 1311 } 1312 1313 const struct file_operations vfio_device_fops = { 1314 .owner = THIS_MODULE, 1315 .open = vfio_device_fops_cdev_open, 1316 .release = vfio_device_fops_release, 1317 .read = vfio_device_fops_read, 1318 .write = vfio_device_fops_write, 1319 .unlocked_ioctl = vfio_device_fops_unl_ioctl, 1320 .compat_ioctl = compat_ptr_ioctl, 1321 .mmap = vfio_device_fops_mmap, 1322 }; 1323 1324 static struct vfio_device *vfio_device_from_file(struct file *file) 1325 { 1326 struct vfio_device_file *df = file->private_data; 1327 1328 if (file->f_op != &vfio_device_fops) 1329 return NULL; 1330 return df->device; 1331 } 1332 1333 /** 1334 * vfio_file_is_valid - True if the file is valid vfio file 1335 * @file: VFIO group file or VFIO device file 1336 */ 1337 bool vfio_file_is_valid(struct file *file) 1338 { 1339 return vfio_group_from_file(file) || 1340 vfio_device_from_file(file); 1341 } 1342 EXPORT_SYMBOL_GPL(vfio_file_is_valid); 1343 1344 /** 1345 * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file 1346 * is always CPU cache coherent 1347 * @file: VFIO group file or VFIO device file 1348 * 1349 * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop 1350 * bit in DMA transactions. A return of false indicates that the user has 1351 * rights to access additional instructions such as wbinvd on x86. 1352 */ 1353 bool vfio_file_enforced_coherent(struct file *file) 1354 { 1355 struct vfio_device *device; 1356 struct vfio_group *group; 1357 1358 group = vfio_group_from_file(file); 1359 if (group) 1360 return vfio_group_enforced_coherent(group); 1361 1362 device = vfio_device_from_file(file); 1363 if (device) 1364 return device_iommu_capable(device->dev, 1365 IOMMU_CAP_ENFORCE_CACHE_COHERENCY); 1366 1367 return true; 1368 } 1369 EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent); 1370 1371 static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm) 1372 { 1373 struct vfio_device_file *df = file->private_data; 1374 1375 /* 1376 * The kvm is first recorded in the vfio_device_file, and will 1377 * be propagated to vfio_device::kvm when the file is bound to 1378 * iommufd successfully in the vfio device cdev path. 1379 */ 1380 spin_lock(&df->kvm_ref_lock); 1381 df->kvm = kvm; 1382 spin_unlock(&df->kvm_ref_lock); 1383 } 1384 1385 /** 1386 * vfio_file_set_kvm - Link a kvm with VFIO drivers 1387 * @file: VFIO group file or VFIO device file 1388 * @kvm: KVM to link 1389 * 1390 * When a VFIO device is first opened the KVM will be available in 1391 * device->kvm if one was associated with the file. 1392 */ 1393 void vfio_file_set_kvm(struct file *file, struct kvm *kvm) 1394 { 1395 struct vfio_group *group; 1396 1397 group = vfio_group_from_file(file); 1398 if (group) 1399 vfio_group_set_kvm(group, kvm); 1400 1401 if (vfio_device_from_file(file)) 1402 vfio_device_file_set_kvm(file, kvm); 1403 } 1404 EXPORT_SYMBOL_GPL(vfio_file_set_kvm); 1405 1406 /* 1407 * Sub-module support 1408 */ 1409 /* 1410 * Helper for managing a buffer of info chain capabilities, allocate or 1411 * reallocate a buffer with additional @size, filling in @id and @version 1412 * of the capability. A pointer to the new capability is returned. 1413 * 1414 * NB. The chain is based at the head of the buffer, so new entries are 1415 * added to the tail, vfio_info_cap_shift() should be called to fixup the 1416 * next offsets prior to copying to the user buffer. 1417 */ 1418 struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps, 1419 size_t size, u16 id, u16 version) 1420 { 1421 void *buf; 1422 struct vfio_info_cap_header *header, *tmp; 1423 1424 /* Ensure that the next capability struct will be aligned */ 1425 size = ALIGN(size, sizeof(u64)); 1426 1427 buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL); 1428 if (!buf) { 1429 kfree(caps->buf); 1430 caps->buf = NULL; 1431 caps->size = 0; 1432 return ERR_PTR(-ENOMEM); 1433 } 1434 1435 caps->buf = buf; 1436 header = buf + caps->size; 1437 1438 /* Eventually copied to user buffer, zero */ 1439 memset(header, 0, size); 1440 1441 header->id = id; 1442 header->version = version; 1443 1444 /* Add to the end of the capability chain */ 1445 for (tmp = buf; tmp->next; tmp = buf + tmp->next) 1446 ; /* nothing */ 1447 1448 tmp->next = caps->size; 1449 caps->size += size; 1450 1451 return header; 1452 } 1453 EXPORT_SYMBOL_GPL(vfio_info_cap_add); 1454 1455 void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset) 1456 { 1457 struct vfio_info_cap_header *tmp; 1458 void *buf = (void *)caps->buf; 1459 1460 /* Capability structs should start with proper alignment */ 1461 WARN_ON(!IS_ALIGNED(offset, sizeof(u64))); 1462 1463 for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset) 1464 tmp->next += offset; 1465 } 1466 EXPORT_SYMBOL(vfio_info_cap_shift); 1467 1468 int vfio_info_add_capability(struct vfio_info_cap *caps, 1469 struct vfio_info_cap_header *cap, size_t size) 1470 { 1471 struct vfio_info_cap_header *header; 1472 1473 header = vfio_info_cap_add(caps, size, cap->id, cap->version); 1474 if (IS_ERR(header)) 1475 return PTR_ERR(header); 1476 1477 memcpy(header + 1, cap + 1, size - sizeof(*header)); 1478 1479 return 0; 1480 } 1481 EXPORT_SYMBOL(vfio_info_add_capability); 1482 1483 int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs, 1484 int max_irq_type, size_t *data_size) 1485 { 1486 unsigned long minsz; 1487 size_t size; 1488 1489 minsz = offsetofend(struct vfio_irq_set, count); 1490 1491 if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) || 1492 (hdr->count >= (U32_MAX - hdr->start)) || 1493 (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK | 1494 VFIO_IRQ_SET_ACTION_TYPE_MASK))) 1495 return -EINVAL; 1496 1497 if (data_size) 1498 *data_size = 0; 1499 1500 if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs) 1501 return -EINVAL; 1502 1503 switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 1504 case VFIO_IRQ_SET_DATA_NONE: 1505 size = 0; 1506 break; 1507 case VFIO_IRQ_SET_DATA_BOOL: 1508 size = sizeof(uint8_t); 1509 break; 1510 case VFIO_IRQ_SET_DATA_EVENTFD: 1511 size = sizeof(int32_t); 1512 break; 1513 default: 1514 return -EINVAL; 1515 } 1516 1517 if (size) { 1518 if (hdr->argsz - minsz < hdr->count * size) 1519 return -EINVAL; 1520 1521 if (!data_size) 1522 return -EINVAL; 1523 1524 *data_size = hdr->count * size; 1525 } 1526 1527 return 0; 1528 } 1529 EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare); 1530 1531 /* 1532 * Pin contiguous user pages and return their associated host pages for local 1533 * domain only. 1534 * @device [in] : device 1535 * @iova [in] : starting IOVA of user pages to be pinned. 1536 * @npage [in] : count of pages to be pinned. This count should not 1537 * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 1538 * @prot [in] : protection flags 1539 * @pages[out] : array of host pages 1540 * Return error or number of pages pinned. 1541 * 1542 * A driver may only call this function if the vfio_device was created 1543 * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages(). 1544 */ 1545 int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, 1546 int npage, int prot, struct page **pages) 1547 { 1548 /* group->container cannot change while a vfio device is open */ 1549 if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device))) 1550 return -EINVAL; 1551 if (!device->ops->dma_unmap) 1552 return -EINVAL; 1553 if (vfio_device_has_container(device)) 1554 return vfio_device_container_pin_pages(device, iova, 1555 npage, prot, pages); 1556 if (device->iommufd_access) { 1557 int ret; 1558 1559 if (iova > ULONG_MAX) 1560 return -EINVAL; 1561 /* 1562 * VFIO ignores the sub page offset, npages is from the start of 1563 * a PAGE_SIZE chunk of IOVA. The caller is expected to recover 1564 * the sub page offset by doing: 1565 * pages[0] + (iova % PAGE_SIZE) 1566 */ 1567 ret = iommufd_access_pin_pages( 1568 device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE), 1569 npage * PAGE_SIZE, pages, 1570 (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0); 1571 if (ret) 1572 return ret; 1573 return npage; 1574 } 1575 return -EINVAL; 1576 } 1577 EXPORT_SYMBOL(vfio_pin_pages); 1578 1579 /* 1580 * Unpin contiguous host pages for local domain only. 1581 * @device [in] : device 1582 * @iova [in] : starting address of user pages to be unpinned. 1583 * @npage [in] : count of pages to be unpinned. This count should not 1584 * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 1585 */ 1586 void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage) 1587 { 1588 if (WARN_ON(!vfio_assert_device_open(device))) 1589 return; 1590 if (WARN_ON(!device->ops->dma_unmap)) 1591 return; 1592 1593 if (vfio_device_has_container(device)) { 1594 vfio_device_container_unpin_pages(device, iova, npage); 1595 return; 1596 } 1597 if (device->iommufd_access) { 1598 if (WARN_ON(iova > ULONG_MAX)) 1599 return; 1600 iommufd_access_unpin_pages(device->iommufd_access, 1601 ALIGN_DOWN(iova, PAGE_SIZE), 1602 npage * PAGE_SIZE); 1603 return; 1604 } 1605 } 1606 EXPORT_SYMBOL(vfio_unpin_pages); 1607 1608 /* 1609 * This interface allows the CPUs to perform some sort of virtual DMA on 1610 * behalf of the device. 1611 * 1612 * CPUs read/write from/into a range of IOVAs pointing to user space memory 1613 * into/from a kernel buffer. 1614 * 1615 * As the read/write of user space memory is conducted via the CPUs and is 1616 * not a real device DMA, it is not necessary to pin the user space memory. 1617 * 1618 * @device [in] : VFIO device 1619 * @iova [in] : base IOVA of a user space buffer 1620 * @data [in] : pointer to kernel buffer 1621 * @len [in] : kernel buffer length 1622 * @write : indicate read or write 1623 * Return error code on failure or 0 on success. 1624 */ 1625 int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, 1626 size_t len, bool write) 1627 { 1628 if (!data || len <= 0 || !vfio_assert_device_open(device)) 1629 return -EINVAL; 1630 1631 if (vfio_device_has_container(device)) 1632 return vfio_device_container_dma_rw(device, iova, 1633 data, len, write); 1634 1635 if (device->iommufd_access) { 1636 unsigned int flags = 0; 1637 1638 if (iova > ULONG_MAX) 1639 return -EINVAL; 1640 1641 /* VFIO historically tries to auto-detect a kthread */ 1642 if (!current->mm) 1643 flags |= IOMMUFD_ACCESS_RW_KTHREAD; 1644 if (write) 1645 flags |= IOMMUFD_ACCESS_RW_WRITE; 1646 return iommufd_access_rw(device->iommufd_access, iova, data, 1647 len, flags); 1648 } 1649 return -EINVAL; 1650 } 1651 EXPORT_SYMBOL(vfio_dma_rw); 1652 1653 /* 1654 * Module/class support 1655 */ 1656 static int __init vfio_init(void) 1657 { 1658 int ret; 1659 1660 ida_init(&vfio.device_ida); 1661 1662 ret = vfio_group_init(); 1663 if (ret) 1664 return ret; 1665 1666 ret = vfio_virqfd_init(); 1667 if (ret) 1668 goto err_virqfd; 1669 1670 /* /sys/class/vfio-dev/vfioX */ 1671 vfio.device_class = class_create("vfio-dev"); 1672 if (IS_ERR(vfio.device_class)) { 1673 ret = PTR_ERR(vfio.device_class); 1674 goto err_dev_class; 1675 } 1676 1677 ret = vfio_cdev_init(vfio.device_class); 1678 if (ret) 1679 goto err_alloc_dev_chrdev; 1680 1681 vfio_debugfs_create_root(); 1682 pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n"); 1683 return 0; 1684 1685 err_alloc_dev_chrdev: 1686 class_destroy(vfio.device_class); 1687 vfio.device_class = NULL; 1688 err_dev_class: 1689 vfio_virqfd_exit(); 1690 err_virqfd: 1691 vfio_group_cleanup(); 1692 return ret; 1693 } 1694 1695 static void __exit vfio_cleanup(void) 1696 { 1697 vfio_debugfs_remove_root(); 1698 ida_destroy(&vfio.device_ida); 1699 vfio_cdev_cleanup(); 1700 class_destroy(vfio.device_class); 1701 vfio.device_class = NULL; 1702 vfio_virqfd_exit(); 1703 vfio_group_cleanup(); 1704 xa_destroy(&vfio_device_set_xa); 1705 } 1706 1707 module_init(vfio_init); 1708 module_exit(vfio_cleanup); 1709 1710 MODULE_IMPORT_NS(IOMMUFD); 1711 MODULE_VERSION(DRIVER_VERSION); 1712 MODULE_LICENSE("GPL v2"); 1713 MODULE_AUTHOR(DRIVER_AUTHOR); 1714 MODULE_DESCRIPTION(DRIVER_DESC); 1715 MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce"); 1716