1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * VFIO core
4 *
5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
6 * Author: Alex Williamson <alex.williamson@redhat.com>
7 *
8 * Derived from original vfio:
9 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
10 * Author: Tom Lyon, pugs@cisco.com
11 */
12
13 #include <linux/cdev.h>
14 #include <linux/compat.h>
15 #include <linux/device.h>
16 #include <linux/fs.h>
17 #include <linux/idr.h>
18 #include <linux/iommu.h>
19 #if IS_ENABLED(CONFIG_KVM)
20 #include <linux/kvm_host.h>
21 #endif
22 #include <linux/list.h>
23 #include <linux/miscdevice.h>
24 #include <linux/module.h>
25 #include <linux/mount.h>
26 #include <linux/mutex.h>
27 #include <linux/pci.h>
28 #include <linux/pseudo_fs.h>
29 #include <linux/rwsem.h>
30 #include <linux/sched.h>
31 #include <linux/seq_file.h>
32 #include <linux/slab.h>
33 #include <linux/stat.h>
34 #include <linux/string.h>
35 #include <linux/uaccess.h>
36 #include <linux/vfio.h>
37 #include <linux/wait.h>
38 #include <linux/sched/signal.h>
39 #include <linux/pm_runtime.h>
40 #include <linux/interval_tree.h>
41 #include <linux/iova_bitmap.h>
42 #include <linux/iommufd.h>
43 #include "vfio.h"
44
45 #define DRIVER_VERSION "0.3"
46 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
47 #define DRIVER_DESC "VFIO - User Level meta-driver"
48
49 #define VFIO_MAGIC 0x5646494f /* "VFIO" */
50
51 static struct vfio {
52 struct ida device_ida;
53 struct vfsmount *vfs_mount;
54 int fs_count;
55 } vfio;
56
57 #ifdef CONFIG_VFIO_NOIOMMU
58 bool vfio_noiommu __read_mostly;
59 module_param_named(enable_unsafe_noiommu_mode,
60 vfio_noiommu, bool, S_IRUGO | S_IWUSR);
61 MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)");
62 #endif
63
64 static DEFINE_XARRAY(vfio_device_set_xa);
65
vfio_device_devnode(const struct device * dev,umode_t * mode)66 static char *vfio_device_devnode(const struct device *dev, umode_t *mode)
67 {
68 return kasprintf(GFP_KERNEL, "vfio/devices/%s", dev_name(dev));
69 }
70
71 static const struct class vfio_device_class = {
72 .name = "vfio-dev",
73 .devnode = vfio_device_devnode
74 };
75
vfio_assign_device_set(struct vfio_device * device,void * set_id)76 int vfio_assign_device_set(struct vfio_device *device, void *set_id)
77 {
78 unsigned long idx = (unsigned long)set_id;
79 struct vfio_device_set *new_dev_set;
80 struct vfio_device_set *dev_set;
81
82 if (WARN_ON(!set_id))
83 return -EINVAL;
84
85 /*
86 * Atomically acquire a singleton object in the xarray for this set_id
87 */
88 xa_lock(&vfio_device_set_xa);
89 dev_set = xa_load(&vfio_device_set_xa, idx);
90 if (dev_set)
91 goto found_get_ref;
92 xa_unlock(&vfio_device_set_xa);
93
94 new_dev_set = kzalloc_obj(*new_dev_set);
95 if (!new_dev_set)
96 return -ENOMEM;
97 mutex_init(&new_dev_set->lock);
98 INIT_LIST_HEAD(&new_dev_set->device_list);
99 new_dev_set->set_id = set_id;
100
101 xa_lock(&vfio_device_set_xa);
102 dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set,
103 GFP_KERNEL);
104 if (!dev_set) {
105 dev_set = new_dev_set;
106 goto found_get_ref;
107 }
108
109 kfree(new_dev_set);
110 if (xa_is_err(dev_set)) {
111 xa_unlock(&vfio_device_set_xa);
112 return xa_err(dev_set);
113 }
114
115 found_get_ref:
116 dev_set->device_count++;
117 xa_unlock(&vfio_device_set_xa);
118 mutex_lock(&dev_set->lock);
119 device->dev_set = dev_set;
120 list_add_tail(&device->dev_set_list, &dev_set->device_list);
121 mutex_unlock(&dev_set->lock);
122 return 0;
123 }
124 EXPORT_SYMBOL_GPL(vfio_assign_device_set);
125
vfio_release_device_set(struct vfio_device * device)126 static void vfio_release_device_set(struct vfio_device *device)
127 {
128 struct vfio_device_set *dev_set = device->dev_set;
129
130 if (!dev_set)
131 return;
132
133 mutex_lock(&dev_set->lock);
134 list_del(&device->dev_set_list);
135 mutex_unlock(&dev_set->lock);
136
137 xa_lock(&vfio_device_set_xa);
138 if (!--dev_set->device_count) {
139 __xa_erase(&vfio_device_set_xa,
140 (unsigned long)dev_set->set_id);
141 mutex_destroy(&dev_set->lock);
142 kfree(dev_set);
143 }
144 xa_unlock(&vfio_device_set_xa);
145 }
146
vfio_device_set_open_count(struct vfio_device_set * dev_set)147 unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set)
148 {
149 struct vfio_device *cur;
150 unsigned int open_count = 0;
151
152 lockdep_assert_held(&dev_set->lock);
153
154 list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
155 open_count += cur->open_count;
156 return open_count;
157 }
158 EXPORT_SYMBOL_GPL(vfio_device_set_open_count);
159
160 struct vfio_device *
vfio_find_device_in_devset(struct vfio_device_set * dev_set,struct device * dev)161 vfio_find_device_in_devset(struct vfio_device_set *dev_set,
162 struct device *dev)
163 {
164 struct vfio_device *cur;
165
166 lockdep_assert_held(&dev_set->lock);
167
168 list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
169 if (cur->dev == dev)
170 return cur;
171 return NULL;
172 }
173 EXPORT_SYMBOL_GPL(vfio_find_device_in_devset);
174
175 /*
176 * Device objects - create, release, get, put, search
177 */
178 /* Device reference always implies a group reference */
vfio_device_put_registration(struct vfio_device * device)179 void vfio_device_put_registration(struct vfio_device *device)
180 {
181 if (refcount_dec_and_test(&device->refcount))
182 complete(&device->comp);
183 }
184 EXPORT_SYMBOL_GPL(vfio_device_put_registration);
185
vfio_device_try_get_registration(struct vfio_device * device)186 bool vfio_device_try_get_registration(struct vfio_device *device)
187 {
188 return refcount_inc_not_zero(&device->refcount);
189 }
190 EXPORT_SYMBOL_GPL(vfio_device_try_get_registration);
191
192 /*
193 * VFIO driver API
194 */
195 /* Release helper called by vfio_put_device() */
vfio_device_release(struct device * dev)196 static void vfio_device_release(struct device *dev)
197 {
198 struct vfio_device *device =
199 container_of(dev, struct vfio_device, device);
200
201 vfio_release_device_set(device);
202 ida_free(&vfio.device_ida, device->index);
203
204 if (device->ops->release)
205 device->ops->release(device);
206
207 iput(device->inode);
208 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count);
209 kvfree(device);
210 }
211
212 static int vfio_init_device(struct vfio_device *device, struct device *dev,
213 const struct vfio_device_ops *ops);
214
215 /*
216 * Allocate and initialize vfio_device so it can be registered to vfio
217 * core.
218 *
219 * Drivers should use the wrapper vfio_alloc_device() for allocation.
220 * @size is the size of the structure to be allocated, including any
221 * private data used by the driver.
222 *
223 * Driver may provide an @init callback to cover device private data.
224 *
225 * Use vfio_put_device() to release the structure after success return.
226 */
_vfio_alloc_device(size_t size,struct device * dev,const struct vfio_device_ops * ops)227 struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev,
228 const struct vfio_device_ops *ops)
229 {
230 struct vfio_device *device;
231 int ret;
232
233 if (WARN_ON(size < sizeof(struct vfio_device)))
234 return ERR_PTR(-EINVAL);
235
236 device = kvzalloc(size, GFP_KERNEL);
237 if (!device)
238 return ERR_PTR(-ENOMEM);
239
240 ret = vfio_init_device(device, dev, ops);
241 if (ret)
242 goto out_free;
243 return device;
244
245 out_free:
246 kvfree(device);
247 return ERR_PTR(ret);
248 }
249 EXPORT_SYMBOL_GPL(_vfio_alloc_device);
250
vfio_fs_init_fs_context(struct fs_context * fc)251 static int vfio_fs_init_fs_context(struct fs_context *fc)
252 {
253 return init_pseudo(fc, VFIO_MAGIC) ? 0 : -ENOMEM;
254 }
255
256 static struct file_system_type vfio_fs_type = {
257 .name = "vfio",
258 .owner = THIS_MODULE,
259 .init_fs_context = vfio_fs_init_fs_context,
260 .kill_sb = kill_anon_super,
261 };
262
vfio_fs_inode_new(void)263 static struct inode *vfio_fs_inode_new(void)
264 {
265 struct inode *inode;
266 int ret;
267
268 ret = simple_pin_fs(&vfio_fs_type, &vfio.vfs_mount, &vfio.fs_count);
269 if (ret)
270 return ERR_PTR(ret);
271
272 inode = alloc_anon_inode(vfio.vfs_mount->mnt_sb);
273 if (IS_ERR(inode))
274 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count);
275
276 return inode;
277 }
278
279 /*
280 * Initialize a vfio_device so it can be registered to vfio core.
281 */
vfio_init_device(struct vfio_device * device,struct device * dev,const struct vfio_device_ops * ops)282 static int vfio_init_device(struct vfio_device *device, struct device *dev,
283 const struct vfio_device_ops *ops)
284 {
285 int ret;
286
287 ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL);
288 if (ret < 0) {
289 dev_dbg(dev, "Error to alloc index\n");
290 return ret;
291 }
292
293 device->index = ret;
294 init_completion(&device->comp);
295 device->dev = dev;
296 device->ops = ops;
297 device->inode = vfio_fs_inode_new();
298 if (IS_ERR(device->inode)) {
299 ret = PTR_ERR(device->inode);
300 goto out_inode;
301 }
302
303 if (ops->init) {
304 ret = ops->init(device);
305 if (ret)
306 goto out_uninit;
307 }
308
309 device_initialize(&device->device);
310 device->device.release = vfio_device_release;
311 device->device.class = &vfio_device_class;
312 device->device.parent = device->dev;
313 return 0;
314
315 out_uninit:
316 iput(device->inode);
317 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count);
318 out_inode:
319 vfio_release_device_set(device);
320 ida_free(&vfio.device_ida, device->index);
321 return ret;
322 }
323
vfio_device_set_noiommu_and_name(struct vfio_device * device,enum vfio_group_type type)324 static int vfio_device_set_noiommu_and_name(struct vfio_device *device, enum vfio_group_type type)
325 {
326 if (IS_ENABLED(CONFIG_IOMMUFD_NOIOMMU) && vfio_noiommu &&
327 !device->dev->iommu && type == VFIO_IOMMU)
328 device->noiommu = true;
329
330 /*
331 * device->noiommu records no-IOMMU support for the standalone cdev
332 * interface. VFIO_NOIOMMU enables both group and cdev no-IOMMU; when
333 * cdev no-IOMMU is available, device->noiommu is set before
334 * vfio_device_set_group(), so the cdev is named noiommu-vfio%d up
335 * front. If IOMMUFD_NOIOMMU is unavailable, no-IOMMU devices are
336 * limited to the group interface and do not receive a device cdev.
337 */
338 return dev_set_name(&device->device, "%svfio%d",
339 device->noiommu ? "noiommu-" : "", device->index);
340 }
341
__vfio_register_dev(struct vfio_device * device,enum vfio_group_type type)342 static int __vfio_register_dev(struct vfio_device *device,
343 enum vfio_group_type type)
344 {
345 int ret;
346
347 if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) &&
348 (!device->ops->bind_iommufd ||
349 !device->ops->unbind_iommufd ||
350 !device->ops->attach_ioas ||
351 !device->ops->detach_ioas)))
352 return -EINVAL;
353
354 /*
355 * If the driver doesn't specify a set then the device is added to a
356 * singleton set just for itself.
357 */
358 if (!device->dev_set)
359 vfio_assign_device_set(device, device);
360
361 ret = vfio_device_set_noiommu_and_name(device, type);
362 if (ret)
363 return ret;
364
365 ret = vfio_device_set_group(device, type);
366 if (ret)
367 return ret;
368
369 if (vfio_device_is_noiommu(device) && IS_ENABLED(CONFIG_IOMMUFD_NOIOMMU)) {
370 add_taint(TAINT_USER, LOCKDEP_STILL_OK);
371 dev_warn(device->dev,
372 "Adding kernel taint for vfio-noiommu cdev\n");
373 }
374
375 /*
376 * VFIO always sets IOMMU_CACHE because we offer no way for userspace to
377 * restore cache coherency. It has to be checked here because it is only
378 * valid for cases where we are using iommu groups.
379 */
380 if (type == VFIO_IOMMU && !vfio_device_is_noiommu(device) &&
381 !device_iommu_capable(device->dev, IOMMU_CAP_CACHE_COHERENCY)) {
382 ret = -EINVAL;
383 goto err_out;
384 }
385
386 ret = vfio_device_add(device);
387 if (ret)
388 goto err_out;
389
390 /* Refcounting can't start until the driver calls register */
391 refcount_set(&device->refcount, 1);
392
393 vfio_device_group_register(device);
394 vfio_device_debugfs_init(device);
395
396 return 0;
397 err_out:
398 vfio_device_remove_group(device);
399 return ret;
400 }
401
vfio_register_group_dev(struct vfio_device * device)402 int vfio_register_group_dev(struct vfio_device *device)
403 {
404 return __vfio_register_dev(device, VFIO_IOMMU);
405 }
406 EXPORT_SYMBOL_GPL(vfio_register_group_dev);
407
408 /*
409 * Register a virtual device without IOMMU backing. The user of this
410 * device must not be able to directly trigger unmediated DMA.
411 */
vfio_register_emulated_iommu_dev(struct vfio_device * device)412 int vfio_register_emulated_iommu_dev(struct vfio_device *device)
413 {
414 return __vfio_register_dev(device, VFIO_EMULATED_IOMMU);
415 }
416 EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev);
417
418 /*
419 * Decrement the device reference count and wait for the device to be
420 * removed. Open file descriptors for the device... */
vfio_unregister_group_dev(struct vfio_device * device)421 void vfio_unregister_group_dev(struct vfio_device *device)
422 {
423 unsigned int i = 0;
424 bool interrupted = false;
425 long rc;
426
427 /*
428 * Prevent new device opened by userspace via the
429 * VFIO_GROUP_GET_DEVICE_FD in the group path.
430 */
431 vfio_device_group_unregister(device);
432
433 /*
434 * Remove debugfs before device_del(), which releases devres. Some
435 * debugfs entries are created with debugfs_create_devm_seqfile() and
436 * therefore rely on devres-managed inode private data.
437 */
438 vfio_device_debugfs_exit(device);
439
440 /*
441 * Balances vfio_device_add() in register path, also prevents
442 * new device opened by userspace in the cdev path.
443 */
444 vfio_device_del(device);
445
446 vfio_device_put_registration(device);
447 rc = try_wait_for_completion(&device->comp);
448 while (rc <= 0) {
449 if (device->ops->request)
450 device->ops->request(device, i++);
451
452 if (interrupted) {
453 rc = wait_for_completion_timeout(&device->comp,
454 HZ * 10);
455 } else {
456 rc = wait_for_completion_interruptible_timeout(
457 &device->comp, HZ * 10);
458 if (rc < 0) {
459 interrupted = true;
460 dev_warn(device->dev,
461 "Device is currently in use, task"
462 " \"%s\" (%d) "
463 "blocked until device is released",
464 current->comm, task_pid_nr(current));
465 }
466 }
467 }
468
469 /* Balances vfio_device_set_group in register path */
470 vfio_device_remove_group(device);
471 }
472 EXPORT_SYMBOL_GPL(vfio_unregister_group_dev);
473
474 #if IS_ENABLED(CONFIG_KVM)
vfio_device_get_kvm_safe(struct vfio_device * device,struct kvm * kvm)475 void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm)
476 {
477 void (*pfn)(struct kvm *kvm);
478 bool (*fn)(struct kvm *kvm);
479 bool ret;
480
481 lockdep_assert_held(&device->dev_set->lock);
482
483 if (!kvm)
484 return;
485
486 pfn = symbol_get(kvm_put_kvm);
487 if (WARN_ON(!pfn))
488 return;
489
490 fn = symbol_get(kvm_get_kvm_safe);
491 if (WARN_ON(!fn)) {
492 symbol_put(kvm_put_kvm);
493 return;
494 }
495
496 ret = fn(kvm);
497 symbol_put(kvm_get_kvm_safe);
498 if (!ret) {
499 symbol_put(kvm_put_kvm);
500 return;
501 }
502
503 device->put_kvm = pfn;
504 device->kvm = kvm;
505 }
506
vfio_device_put_kvm(struct vfio_device * device)507 void vfio_device_put_kvm(struct vfio_device *device)
508 {
509 lockdep_assert_held(&device->dev_set->lock);
510
511 if (!device->kvm)
512 return;
513
514 if (WARN_ON(!device->put_kvm))
515 goto clear;
516
517 device->put_kvm(device->kvm);
518 device->put_kvm = NULL;
519 symbol_put(kvm_put_kvm);
520
521 clear:
522 device->kvm = NULL;
523 }
524 #endif
525
526 /* true if the vfio_device has open_device() called but not close_device() */
vfio_assert_device_open(struct vfio_device * device)527 static bool vfio_assert_device_open(struct vfio_device *device)
528 {
529 return !WARN_ON_ONCE(!READ_ONCE(device->open_count));
530 }
531
532 struct vfio_device_file *
vfio_allocate_device_file(struct vfio_device * device)533 vfio_allocate_device_file(struct vfio_device *device)
534 {
535 struct vfio_device_file *df;
536
537 df = kzalloc_obj(*df, GFP_KERNEL_ACCOUNT);
538 if (!df)
539 return ERR_PTR(-ENOMEM);
540
541 df->device = device;
542 spin_lock_init(&df->kvm_ref_lock);
543
544 return df;
545 }
546
vfio_df_device_first_open(struct vfio_device_file * df)547 static int vfio_df_device_first_open(struct vfio_device_file *df)
548 {
549 struct vfio_device *device = df->device;
550 struct iommufd_ctx *iommufd = df->iommufd;
551 int ret;
552
553 lockdep_assert_held(&device->dev_set->lock);
554
555 if (!try_module_get(device->dev->driver->owner))
556 return -ENODEV;
557
558 if (iommufd)
559 ret = vfio_df_iommufd_bind(df);
560 else
561 ret = vfio_device_group_use_iommu(device);
562 if (ret)
563 goto err_module_put;
564
565 if (device->ops->open_device) {
566 ret = device->ops->open_device(device);
567 if (ret)
568 goto err_unuse_iommu;
569 }
570 return 0;
571
572 err_unuse_iommu:
573 if (iommufd)
574 vfio_df_iommufd_unbind(df);
575 else
576 vfio_device_group_unuse_iommu(device);
577 err_module_put:
578 module_put(device->dev->driver->owner);
579 return ret;
580 }
581
vfio_df_device_last_close(struct vfio_device_file * df)582 static void vfio_df_device_last_close(struct vfio_device_file *df)
583 {
584 struct vfio_device *device = df->device;
585 struct iommufd_ctx *iommufd = df->iommufd;
586
587 lockdep_assert_held(&device->dev_set->lock);
588
589 if (device->ops->close_device)
590 device->ops->close_device(device);
591 if (iommufd)
592 vfio_df_iommufd_unbind(df);
593 else
594 vfio_device_group_unuse_iommu(device);
595 device->precopy_info_v2 = 0;
596 module_put(device->dev->driver->owner);
597 }
598
vfio_df_open(struct vfio_device_file * df)599 int vfio_df_open(struct vfio_device_file *df)
600 {
601 struct vfio_device *device = df->device;
602 int ret = 0;
603
604 lockdep_assert_held(&device->dev_set->lock);
605
606 /*
607 * Only the group path allows the device to be opened multiple
608 * times. The device cdev path doesn't have a secure way for it.
609 */
610 if (device->open_count != 0 && !df->group)
611 return -EINVAL;
612
613 device->open_count++;
614 if (device->open_count == 1) {
615 ret = vfio_df_device_first_open(df);
616 if (ret)
617 device->open_count--;
618 }
619
620 return ret;
621 }
622
vfio_df_close(struct vfio_device_file * df)623 void vfio_df_close(struct vfio_device_file *df)
624 {
625 struct vfio_device *device = df->device;
626
627 lockdep_assert_held(&device->dev_set->lock);
628
629 if (!vfio_assert_device_open(device))
630 return;
631 if (device->open_count == 1)
632 vfio_df_device_last_close(df);
633 device->open_count--;
634 }
635
636 /*
637 * Wrapper around pm_runtime_resume_and_get().
638 * Return error code on failure or 0 on success.
639 */
vfio_device_pm_runtime_get(struct vfio_device * device)640 static inline int vfio_device_pm_runtime_get(struct vfio_device *device)
641 {
642 struct device *dev = device->dev;
643
644 if (dev->driver && dev->driver->pm) {
645 int ret;
646
647 ret = pm_runtime_resume_and_get(dev);
648 if (ret) {
649 dev_info_ratelimited(dev,
650 "vfio: runtime resume failed %d\n", ret);
651 return -EIO;
652 }
653 }
654
655 return 0;
656 }
657
658 /*
659 * Wrapper around pm_runtime_put().
660 */
vfio_device_pm_runtime_put(struct vfio_device * device)661 static inline void vfio_device_pm_runtime_put(struct vfio_device *device)
662 {
663 struct device *dev = device->dev;
664
665 if (dev->driver && dev->driver->pm)
666 pm_runtime_put(dev);
667 }
668
669 /*
670 * VFIO Device fd
671 */
vfio_device_fops_release(struct inode * inode,struct file * filep)672 static int vfio_device_fops_release(struct inode *inode, struct file *filep)
673 {
674 struct vfio_device_file *df = filep->private_data;
675 struct vfio_device *device = df->device;
676
677 if (df->group)
678 vfio_df_group_close(df);
679 else
680 vfio_df_unbind_iommufd(df);
681
682 vfio_device_put_registration(device);
683
684 kfree(df);
685
686 return 0;
687 }
688
689 /*
690 * vfio_mig_get_next_state - Compute the next step in the FSM
691 * @cur_fsm - The current state the device is in
692 * @new_fsm - The target state to reach
693 * @next_fsm - Pointer to the next step to get to new_fsm
694 *
695 * Return 0 upon success, otherwise -errno
696 * Upon success the next step in the state progression between cur_fsm and
697 * new_fsm will be set in next_fsm.
698 *
699 * This breaks down requests for combination transitions into smaller steps and
700 * returns the next step to get to new_fsm. The function may need to be called
701 * multiple times before reaching new_fsm.
702 *
703 */
vfio_mig_get_next_state(struct vfio_device * device,enum vfio_device_mig_state cur_fsm,enum vfio_device_mig_state new_fsm,enum vfio_device_mig_state * next_fsm)704 int vfio_mig_get_next_state(struct vfio_device *device,
705 enum vfio_device_mig_state cur_fsm,
706 enum vfio_device_mig_state new_fsm,
707 enum vfio_device_mig_state *next_fsm)
708 {
709 enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 };
710 /*
711 * The coding in this table requires the driver to implement the
712 * following FSM arcs:
713 * RESUMING -> STOP
714 * STOP -> RESUMING
715 * STOP -> STOP_COPY
716 * STOP_COPY -> STOP
717 *
718 * If P2P is supported then the driver must also implement these FSM
719 * arcs:
720 * RUNNING -> RUNNING_P2P
721 * RUNNING_P2P -> RUNNING
722 * RUNNING_P2P -> STOP
723 * STOP -> RUNNING_P2P
724 *
725 * If precopy is supported then the driver must support these additional
726 * FSM arcs:
727 * RUNNING -> PRE_COPY
728 * PRE_COPY -> RUNNING
729 * PRE_COPY -> STOP_COPY
730 * However, if precopy and P2P are supported together then the driver
731 * must support these additional arcs beyond the P2P arcs above:
732 * PRE_COPY -> RUNNING
733 * PRE_COPY -> PRE_COPY_P2P
734 * PRE_COPY_P2P -> PRE_COPY
735 * PRE_COPY_P2P -> RUNNING_P2P
736 * PRE_COPY_P2P -> STOP_COPY
737 * RUNNING -> PRE_COPY
738 * RUNNING_P2P -> PRE_COPY_P2P
739 *
740 * Without P2P and precopy the driver must implement:
741 * RUNNING -> STOP
742 * STOP -> RUNNING
743 *
744 * The coding will step through multiple states for some combination
745 * transitions; if all optional features are supported, this means the
746 * following ones:
747 * PRE_COPY -> PRE_COPY_P2P -> STOP_COPY
748 * PRE_COPY -> RUNNING -> RUNNING_P2P
749 * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP
750 * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING
751 * PRE_COPY_P2P -> RUNNING_P2P -> RUNNING
752 * PRE_COPY_P2P -> RUNNING_P2P -> STOP
753 * PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING
754 * RESUMING -> STOP -> RUNNING_P2P
755 * RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P
756 * RESUMING -> STOP -> RUNNING_P2P -> RUNNING
757 * RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY
758 * RESUMING -> STOP -> STOP_COPY
759 * RUNNING -> RUNNING_P2P -> PRE_COPY_P2P
760 * RUNNING -> RUNNING_P2P -> STOP
761 * RUNNING -> RUNNING_P2P -> STOP -> RESUMING
762 * RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY
763 * RUNNING_P2P -> RUNNING -> PRE_COPY
764 * RUNNING_P2P -> STOP -> RESUMING
765 * RUNNING_P2P -> STOP -> STOP_COPY
766 * STOP -> RUNNING_P2P -> PRE_COPY_P2P
767 * STOP -> RUNNING_P2P -> RUNNING
768 * STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY
769 * STOP_COPY -> STOP -> RESUMING
770 * STOP_COPY -> STOP -> RUNNING_P2P
771 * STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING
772 *
773 * The following transitions are blocked:
774 * STOP_COPY -> PRE_COPY
775 * STOP_COPY -> PRE_COPY_P2P
776 */
777 static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = {
778 [VFIO_DEVICE_STATE_STOP] = {
779 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
780 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
781 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
782 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
783 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
784 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
785 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
786 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
787 },
788 [VFIO_DEVICE_STATE_RUNNING] = {
789 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
790 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
791 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
792 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
793 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
794 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
795 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
796 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
797 },
798 [VFIO_DEVICE_STATE_PRE_COPY] = {
799 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING,
800 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
801 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
802 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
803 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
804 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING,
805 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING,
806 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
807 },
808 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = {
809 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
810 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
811 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
812 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
813 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
814 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
815 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
816 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
817 },
818 [VFIO_DEVICE_STATE_STOP_COPY] = {
819 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
820 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
821 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR,
822 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR,
823 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
824 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
825 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
826 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
827 },
828 [VFIO_DEVICE_STATE_RESUMING] = {
829 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
830 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
831 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP,
832 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP,
833 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
834 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
835 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
836 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
837 },
838 [VFIO_DEVICE_STATE_RUNNING_P2P] = {
839 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
840 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
841 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING,
842 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
843 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
844 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
845 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
846 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
847 },
848 [VFIO_DEVICE_STATE_ERROR] = {
849 [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR,
850 [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR,
851 [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR,
852 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR,
853 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR,
854 [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR,
855 [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR,
856 [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
857 },
858 };
859
860 static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = {
861 [VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY,
862 [VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY,
863 [VFIO_DEVICE_STATE_PRE_COPY] =
864 VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY,
865 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY |
866 VFIO_MIGRATION_P2P |
867 VFIO_MIGRATION_PRE_COPY,
868 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY,
869 [VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY,
870 [VFIO_DEVICE_STATE_RUNNING_P2P] =
871 VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P,
872 [VFIO_DEVICE_STATE_ERROR] = ~0U,
873 };
874
875 if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
876 (state_flags_table[cur_fsm] & device->migration_flags) !=
877 state_flags_table[cur_fsm]))
878 return -EINVAL;
879
880 if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
881 (state_flags_table[new_fsm] & device->migration_flags) !=
882 state_flags_table[new_fsm])
883 return -EINVAL;
884
885 /*
886 * Arcs touching optional and unsupported states are skipped over. The
887 * driver will instead see an arc from the original state to the next
888 * logical state, as per the above comment.
889 */
890 *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm];
891 while (*next_fsm != VFIO_DEVICE_STATE_ERROR &&
892 (state_flags_table[*next_fsm] & device->migration_flags) !=
893 state_flags_table[*next_fsm])
894 *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm];
895
896 return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL;
897 }
898 EXPORT_SYMBOL_GPL(vfio_mig_get_next_state);
899
900 /*
901 * Convert the drivers's struct file into a FD number and return it to userspace
902 */
vfio_ioct_mig_return_fd(struct file * filp,void __user * arg,struct vfio_device_feature_mig_state * mig)903 static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg,
904 struct vfio_device_feature_mig_state *mig)
905 {
906 int ret;
907 int fd;
908
909 fd = get_unused_fd_flags(O_CLOEXEC);
910 if (fd < 0) {
911 ret = fd;
912 goto out_fput;
913 }
914
915 mig->data_fd = fd;
916 if (copy_to_user(arg, mig, sizeof(*mig))) {
917 ret = -EFAULT;
918 goto out_put_unused;
919 }
920 fd_install(fd, filp);
921 return 0;
922
923 out_put_unused:
924 put_unused_fd(fd);
925 out_fput:
926 fput(filp);
927 return ret;
928 }
929
930 static int
vfio_ioctl_device_feature_mig_device_state(struct vfio_device * device,u32 flags,void __user * arg,size_t argsz)931 vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device,
932 u32 flags, void __user *arg,
933 size_t argsz)
934 {
935 size_t minsz =
936 offsetofend(struct vfio_device_feature_mig_state, data_fd);
937 struct vfio_device_feature_mig_state mig;
938 struct file *filp = NULL;
939 int ret;
940
941 if (!device->mig_ops)
942 return -ENOTTY;
943
944 ret = vfio_check_feature(flags, argsz,
945 VFIO_DEVICE_FEATURE_SET |
946 VFIO_DEVICE_FEATURE_GET,
947 sizeof(mig));
948 if (ret != 1)
949 return ret;
950
951 if (copy_from_user(&mig, arg, minsz))
952 return -EFAULT;
953
954 if (flags & VFIO_DEVICE_FEATURE_GET) {
955 enum vfio_device_mig_state curr_state;
956
957 ret = device->mig_ops->migration_get_state(device,
958 &curr_state);
959 if (ret)
960 return ret;
961 mig.device_state = curr_state;
962 goto out_copy;
963 }
964
965 /* Handle the VFIO_DEVICE_FEATURE_SET */
966 filp = device->mig_ops->migration_set_state(device, mig.device_state);
967 if (IS_ERR(filp) || !filp)
968 goto out_copy;
969
970 return vfio_ioct_mig_return_fd(filp, arg, &mig);
971 out_copy:
972 mig.data_fd = -1;
973 if (copy_to_user(arg, &mig, sizeof(mig)))
974 return -EFAULT;
975 if (IS_ERR(filp))
976 return PTR_ERR(filp);
977 return 0;
978 }
979
980 static int
vfio_ioctl_device_feature_migration_data_size(struct vfio_device * device,u32 flags,void __user * arg,size_t argsz)981 vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device,
982 u32 flags, void __user *arg,
983 size_t argsz)
984 {
985 struct vfio_device_feature_mig_data_size data_size = {};
986 unsigned long stop_copy_length;
987 int ret;
988
989 if (!device->mig_ops)
990 return -ENOTTY;
991
992 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
993 sizeof(data_size));
994 if (ret != 1)
995 return ret;
996
997 ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length);
998 if (ret)
999 return ret;
1000
1001 data_size.stop_copy_length = stop_copy_length;
1002 if (copy_to_user(arg, &data_size, sizeof(data_size)))
1003 return -EFAULT;
1004
1005 return 0;
1006 }
1007
1008 static int
vfio_ioctl_device_feature_migration_precopy_info_v2(struct vfio_device * device,u32 flags,size_t argsz)1009 vfio_ioctl_device_feature_migration_precopy_info_v2(struct vfio_device *device,
1010 u32 flags, size_t argsz)
1011 {
1012 int ret;
1013
1014 if (!(device->migration_flags & VFIO_MIGRATION_PRE_COPY))
1015 return -EINVAL;
1016
1017 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0);
1018 if (ret != 1)
1019 return ret;
1020
1021 device->precopy_info_v2 = 1;
1022 return 0;
1023 }
1024
vfio_ioctl_device_feature_migration(struct vfio_device * device,u32 flags,void __user * arg,size_t argsz)1025 static int vfio_ioctl_device_feature_migration(struct vfio_device *device,
1026 u32 flags, void __user *arg,
1027 size_t argsz)
1028 {
1029 struct vfio_device_feature_migration mig = {
1030 .flags = device->migration_flags,
1031 };
1032 int ret;
1033
1034 if (!device->mig_ops)
1035 return -ENOTTY;
1036
1037 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
1038 sizeof(mig));
1039 if (ret != 1)
1040 return ret;
1041 if (copy_to_user(arg, &mig, sizeof(mig)))
1042 return -EFAULT;
1043 return 0;
1044 }
1045
vfio_combine_iova_ranges(struct rb_root_cached * root,u32 cur_nodes,u32 req_nodes)1046 void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes,
1047 u32 req_nodes)
1048 {
1049 struct interval_tree_node *prev, *curr, *comb_start, *comb_end;
1050 unsigned long min_gap, curr_gap;
1051
1052 /* Special shortcut when a single range is required */
1053 if (req_nodes == 1) {
1054 unsigned long last;
1055
1056 comb_start = interval_tree_iter_first(root, 0, ULONG_MAX);
1057
1058 /* Empty list */
1059 if (WARN_ON_ONCE(!comb_start))
1060 return;
1061
1062 curr = comb_start;
1063 while (curr) {
1064 last = curr->last;
1065 prev = curr;
1066 curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
1067 if (prev != comb_start)
1068 interval_tree_remove(prev, root);
1069 }
1070 comb_start->last = last;
1071 return;
1072 }
1073
1074 /* Combine ranges which have the smallest gap */
1075 while (cur_nodes > req_nodes) {
1076 prev = NULL;
1077 min_gap = ULONG_MAX;
1078 curr = interval_tree_iter_first(root, 0, ULONG_MAX);
1079 while (curr) {
1080 if (prev) {
1081 curr_gap = curr->start - prev->last;
1082 if (curr_gap < min_gap) {
1083 min_gap = curr_gap;
1084 comb_start = prev;
1085 comb_end = curr;
1086 }
1087 }
1088 prev = curr;
1089 curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
1090 }
1091
1092 /* Empty list or no nodes to combine */
1093 if (WARN_ON_ONCE(min_gap == ULONG_MAX))
1094 break;
1095
1096 comb_start->last = comb_end->last;
1097 interval_tree_remove(comb_end, root);
1098 cur_nodes--;
1099 }
1100 }
1101 EXPORT_SYMBOL_GPL(vfio_combine_iova_ranges);
1102
1103 /* Ranges should fit into a single kernel page */
1104 #define LOG_MAX_RANGES \
1105 (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range))
1106
1107 static int
vfio_ioctl_device_feature_logging_start(struct vfio_device * device,u32 flags,void __user * arg,size_t argsz)1108 vfio_ioctl_device_feature_logging_start(struct vfio_device *device,
1109 u32 flags, void __user *arg,
1110 size_t argsz)
1111 {
1112 size_t minsz =
1113 offsetofend(struct vfio_device_feature_dma_logging_control,
1114 ranges);
1115 struct vfio_device_feature_dma_logging_range __user *ranges;
1116 struct vfio_device_feature_dma_logging_control control;
1117 struct vfio_device_feature_dma_logging_range range;
1118 struct rb_root_cached root = RB_ROOT_CACHED;
1119 struct interval_tree_node *nodes;
1120 u64 iova_end;
1121 u32 nnodes;
1122 int i, ret;
1123
1124 if (!device->log_ops)
1125 return -ENOTTY;
1126
1127 ret = vfio_check_feature(flags, argsz,
1128 VFIO_DEVICE_FEATURE_SET,
1129 sizeof(control));
1130 if (ret != 1)
1131 return ret;
1132
1133 if (copy_from_user(&control, arg, minsz))
1134 return -EFAULT;
1135
1136 nnodes = control.num_ranges;
1137 if (!nnodes)
1138 return -EINVAL;
1139
1140 if (nnodes > LOG_MAX_RANGES)
1141 return -E2BIG;
1142
1143 ranges = u64_to_user_ptr(control.ranges);
1144 nodes = kmalloc_objs(struct interval_tree_node, nnodes);
1145 if (!nodes)
1146 return -ENOMEM;
1147
1148 for (i = 0; i < nnodes; i++) {
1149 if (copy_from_user(&range, &ranges[i], sizeof(range))) {
1150 ret = -EFAULT;
1151 goto end;
1152 }
1153 if (!IS_ALIGNED(range.iova, control.page_size) ||
1154 !IS_ALIGNED(range.length, control.page_size)) {
1155 ret = -EINVAL;
1156 goto end;
1157 }
1158
1159 if (check_add_overflow(range.iova, range.length, &iova_end) ||
1160 iova_end > ULONG_MAX) {
1161 ret = -EOVERFLOW;
1162 goto end;
1163 }
1164
1165 nodes[i].start = range.iova;
1166 nodes[i].last = range.iova + range.length - 1;
1167 if (interval_tree_iter_first(&root, nodes[i].start,
1168 nodes[i].last)) {
1169 /* Range overlapping */
1170 ret = -EINVAL;
1171 goto end;
1172 }
1173 interval_tree_insert(nodes + i, &root);
1174 }
1175
1176 ret = device->log_ops->log_start(device, &root, nnodes,
1177 &control.page_size);
1178 if (ret)
1179 goto end;
1180
1181 if (copy_to_user(arg, &control, sizeof(control))) {
1182 ret = -EFAULT;
1183 device->log_ops->log_stop(device);
1184 }
1185
1186 end:
1187 kfree(nodes);
1188 return ret;
1189 }
1190
1191 static int
vfio_ioctl_device_feature_logging_stop(struct vfio_device * device,u32 flags,void __user * arg,size_t argsz)1192 vfio_ioctl_device_feature_logging_stop(struct vfio_device *device,
1193 u32 flags, void __user *arg,
1194 size_t argsz)
1195 {
1196 int ret;
1197
1198 if (!device->log_ops)
1199 return -ENOTTY;
1200
1201 ret = vfio_check_feature(flags, argsz,
1202 VFIO_DEVICE_FEATURE_SET, 0);
1203 if (ret != 1)
1204 return ret;
1205
1206 return device->log_ops->log_stop(device);
1207 }
1208
vfio_device_log_read_and_clear(struct iova_bitmap * iter,unsigned long iova,size_t length,void * opaque)1209 static int vfio_device_log_read_and_clear(struct iova_bitmap *iter,
1210 unsigned long iova, size_t length,
1211 void *opaque)
1212 {
1213 struct vfio_device *device = opaque;
1214
1215 return device->log_ops->log_read_and_clear(device, iova, length, iter);
1216 }
1217
1218 static int
vfio_ioctl_device_feature_logging_report(struct vfio_device * device,u32 flags,void __user * arg,size_t argsz)1219 vfio_ioctl_device_feature_logging_report(struct vfio_device *device,
1220 u32 flags, void __user *arg,
1221 size_t argsz)
1222 {
1223 size_t minsz =
1224 offsetofend(struct vfio_device_feature_dma_logging_report,
1225 bitmap);
1226 struct vfio_device_feature_dma_logging_report report;
1227 struct iova_bitmap *iter;
1228 u64 iova_end;
1229 int ret;
1230
1231 if (!device->log_ops)
1232 return -ENOTTY;
1233
1234 ret = vfio_check_feature(flags, argsz,
1235 VFIO_DEVICE_FEATURE_GET,
1236 sizeof(report));
1237 if (ret != 1)
1238 return ret;
1239
1240 if (copy_from_user(&report, arg, minsz))
1241 return -EFAULT;
1242
1243 if (report.page_size < SZ_4K || !is_power_of_2(report.page_size))
1244 return -EINVAL;
1245
1246 if (check_add_overflow(report.iova, report.length, &iova_end) ||
1247 iova_end > ULONG_MAX)
1248 return -EOVERFLOW;
1249
1250 iter = iova_bitmap_alloc(report.iova, report.length,
1251 report.page_size,
1252 u64_to_user_ptr(report.bitmap));
1253 if (IS_ERR(iter))
1254 return PTR_ERR(iter);
1255
1256 ret = iova_bitmap_for_each(iter, device,
1257 vfio_device_log_read_and_clear);
1258
1259 iova_bitmap_free(iter);
1260 return ret;
1261 }
1262
vfio_ioctl_device_feature(struct vfio_device * device,struct vfio_device_feature __user * arg)1263 static int vfio_ioctl_device_feature(struct vfio_device *device,
1264 struct vfio_device_feature __user *arg)
1265 {
1266 size_t minsz = offsetofend(struct vfio_device_feature, flags);
1267 struct vfio_device_feature feature;
1268
1269 if (copy_from_user(&feature, arg, minsz))
1270 return -EFAULT;
1271
1272 if (feature.argsz < minsz)
1273 return -EINVAL;
1274
1275 /* Check unknown flags */
1276 if (feature.flags &
1277 ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET |
1278 VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE))
1279 return -EINVAL;
1280
1281 /* GET & SET are mutually exclusive except with PROBE */
1282 if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) &&
1283 (feature.flags & VFIO_DEVICE_FEATURE_SET) &&
1284 (feature.flags & VFIO_DEVICE_FEATURE_GET))
1285 return -EINVAL;
1286
1287 switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) {
1288 case VFIO_DEVICE_FEATURE_MIGRATION:
1289 return vfio_ioctl_device_feature_migration(
1290 device, feature.flags, arg->data,
1291 feature.argsz - minsz);
1292 case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE:
1293 return vfio_ioctl_device_feature_mig_device_state(
1294 device, feature.flags, arg->data,
1295 feature.argsz - minsz);
1296 case VFIO_DEVICE_FEATURE_DMA_LOGGING_START:
1297 return vfio_ioctl_device_feature_logging_start(
1298 device, feature.flags, arg->data,
1299 feature.argsz - minsz);
1300 case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP:
1301 return vfio_ioctl_device_feature_logging_stop(
1302 device, feature.flags, arg->data,
1303 feature.argsz - minsz);
1304 case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT:
1305 return vfio_ioctl_device_feature_logging_report(
1306 device, feature.flags, arg->data,
1307 feature.argsz - minsz);
1308 case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE:
1309 return vfio_ioctl_device_feature_migration_data_size(
1310 device, feature.flags, arg->data,
1311 feature.argsz - minsz);
1312 case VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2:
1313 return vfio_ioctl_device_feature_migration_precopy_info_v2(
1314 device, feature.flags, feature.argsz - minsz);
1315 default:
1316 if (unlikely(!device->ops->device_feature))
1317 return -ENOTTY;
1318 return device->ops->device_feature(device, feature.flags,
1319 arg->data,
1320 feature.argsz - minsz);
1321 }
1322 }
1323
vfio_get_region_info(struct vfio_device * device,struct vfio_region_info __user * arg)1324 static long vfio_get_region_info(struct vfio_device *device,
1325 struct vfio_region_info __user *arg)
1326 {
1327 unsigned long minsz = offsetofend(struct vfio_region_info, offset);
1328 struct vfio_region_info info = {};
1329 struct vfio_info_cap caps = {};
1330 int ret;
1331
1332 if (unlikely(!device->ops->get_region_info_caps))
1333 return -EINVAL;
1334
1335 if (copy_from_user(&info, arg, minsz))
1336 return -EFAULT;
1337 if (info.argsz < minsz)
1338 return -EINVAL;
1339
1340 ret = device->ops->get_region_info_caps(device, &info, &caps);
1341 if (ret)
1342 goto out_free;
1343
1344 if (caps.size) {
1345 info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
1346 if (info.argsz < sizeof(info) + caps.size) {
1347 info.argsz = sizeof(info) + caps.size;
1348 info.cap_offset = 0;
1349 } else {
1350 vfio_info_cap_shift(&caps, sizeof(info));
1351 if (copy_to_user(arg + 1, caps.buf, caps.size)) {
1352 ret = -EFAULT;
1353 goto out_free;
1354 }
1355 info.cap_offset = sizeof(info);
1356 }
1357 }
1358
1359 if (copy_to_user(arg, &info, minsz)){
1360 ret = -EFAULT;
1361 goto out_free;
1362 }
1363
1364 out_free:
1365 kfree(caps.buf);
1366 return ret;
1367 }
1368
vfio_device_fops_unl_ioctl(struct file * filep,unsigned int cmd,unsigned long arg)1369 static long vfio_device_fops_unl_ioctl(struct file *filep,
1370 unsigned int cmd, unsigned long arg)
1371 {
1372 struct vfio_device_file *df = filep->private_data;
1373 struct vfio_device *device = df->device;
1374 void __user *uptr = (void __user *)arg;
1375 int ret;
1376
1377 if (cmd == VFIO_DEVICE_BIND_IOMMUFD)
1378 return vfio_df_ioctl_bind_iommufd(df, uptr);
1379
1380 /* Paired with smp_store_release() following vfio_df_open() */
1381 if (!smp_load_acquire(&df->access_granted))
1382 return -EINVAL;
1383
1384 ret = vfio_device_pm_runtime_get(device);
1385 if (ret)
1386 return ret;
1387
1388 /* cdev only ioctls */
1389 if (IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) && !df->group) {
1390 switch (cmd) {
1391 case VFIO_DEVICE_ATTACH_IOMMUFD_PT:
1392 ret = vfio_df_ioctl_attach_pt(df, uptr);
1393 goto out;
1394
1395 case VFIO_DEVICE_DETACH_IOMMUFD_PT:
1396 ret = vfio_df_ioctl_detach_pt(df, uptr);
1397 goto out;
1398 }
1399 }
1400
1401 switch (cmd) {
1402 case VFIO_DEVICE_FEATURE:
1403 ret = vfio_ioctl_device_feature(device, uptr);
1404 break;
1405
1406 case VFIO_DEVICE_GET_REGION_INFO:
1407 ret = vfio_get_region_info(device, uptr);
1408 break;
1409
1410 default:
1411 if (unlikely(!device->ops->ioctl))
1412 ret = -EINVAL;
1413 else
1414 ret = device->ops->ioctl(device, cmd, arg);
1415 break;
1416 }
1417 out:
1418 vfio_device_pm_runtime_put(device);
1419 return ret;
1420 }
1421
vfio_device_fops_read(struct file * filep,char __user * buf,size_t count,loff_t * ppos)1422 static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf,
1423 size_t count, loff_t *ppos)
1424 {
1425 struct vfio_device_file *df = filep->private_data;
1426 struct vfio_device *device = df->device;
1427
1428 /* Paired with smp_store_release() following vfio_df_open() */
1429 if (!smp_load_acquire(&df->access_granted))
1430 return -EINVAL;
1431
1432 if (unlikely(!device->ops->read))
1433 return -EINVAL;
1434
1435 return device->ops->read(device, buf, count, ppos);
1436 }
1437
vfio_device_fops_write(struct file * filep,const char __user * buf,size_t count,loff_t * ppos)1438 static ssize_t vfio_device_fops_write(struct file *filep,
1439 const char __user *buf,
1440 size_t count, loff_t *ppos)
1441 {
1442 struct vfio_device_file *df = filep->private_data;
1443 struct vfio_device *device = df->device;
1444
1445 /* Paired with smp_store_release() following vfio_df_open() */
1446 if (!smp_load_acquire(&df->access_granted))
1447 return -EINVAL;
1448
1449 if (unlikely(!device->ops->write))
1450 return -EINVAL;
1451
1452 return device->ops->write(device, buf, count, ppos);
1453 }
1454
vfio_device_fops_mmap(struct file * filep,struct vm_area_struct * vma)1455 static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma)
1456 {
1457 struct vfio_device_file *df = filep->private_data;
1458 struct vfio_device *device = df->device;
1459
1460 /* Paired with smp_store_release() following vfio_df_open() */
1461 if (!smp_load_acquire(&df->access_granted))
1462 return -EINVAL;
1463
1464 if (unlikely(!device->ops->mmap))
1465 return -EINVAL;
1466
1467 return device->ops->mmap(device, vma);
1468 }
1469
1470 #ifdef CONFIG_PROC_FS
vfio_device_show_fdinfo(struct seq_file * m,struct file * filep)1471 static void vfio_device_show_fdinfo(struct seq_file *m, struct file *filep)
1472 {
1473 char *path;
1474 struct vfio_device_file *df = filep->private_data;
1475 struct vfio_device *device = df->device;
1476
1477 path = kobject_get_path(&device->dev->kobj, GFP_KERNEL);
1478 if (!path)
1479 return;
1480
1481 seq_printf(m, "vfio-device-syspath: /sys%s\n", path);
1482 kfree(path);
1483 }
1484 #endif
1485
1486 const struct file_operations vfio_device_fops = {
1487 .owner = THIS_MODULE,
1488 .open = vfio_device_fops_cdev_open,
1489 .release = vfio_device_fops_release,
1490 .read = vfio_device_fops_read,
1491 .write = vfio_device_fops_write,
1492 .unlocked_ioctl = vfio_device_fops_unl_ioctl,
1493 .compat_ioctl = compat_ptr_ioctl,
1494 .mmap = vfio_device_fops_mmap,
1495 #ifdef CONFIG_PROC_FS
1496 .show_fdinfo = vfio_device_show_fdinfo,
1497 #endif
1498 };
1499
vfio_device_from_file(struct file * file)1500 static struct vfio_device *vfio_device_from_file(struct file *file)
1501 {
1502 struct vfio_device_file *df = file->private_data;
1503
1504 if (file->f_op != &vfio_device_fops)
1505 return NULL;
1506 return df->device;
1507 }
1508
1509 /**
1510 * vfio_file_is_valid - True if the file is valid vfio file
1511 * @file: VFIO group file or VFIO device file
1512 */
vfio_file_is_valid(struct file * file)1513 bool vfio_file_is_valid(struct file *file)
1514 {
1515 return vfio_group_from_file(file) ||
1516 vfio_device_from_file(file);
1517 }
1518 EXPORT_SYMBOL_GPL(vfio_file_is_valid);
1519
1520 /**
1521 * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file
1522 * is always CPU cache coherent
1523 * @file: VFIO group file or VFIO device file
1524 *
1525 * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop
1526 * bit in DMA transactions. A return of false indicates that the user has
1527 * rights to access additional instructions such as wbinvd on x86.
1528 */
vfio_file_enforced_coherent(struct file * file)1529 bool vfio_file_enforced_coherent(struct file *file)
1530 {
1531 struct vfio_device *device;
1532 struct vfio_group *group;
1533
1534 group = vfio_group_from_file(file);
1535 if (group)
1536 return vfio_group_enforced_coherent(group);
1537
1538 device = vfio_device_from_file(file);
1539 if (device)
1540 return device_iommu_capable(device->dev,
1541 IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
1542
1543 return true;
1544 }
1545 EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent);
1546
vfio_device_file_set_kvm(struct file * file,struct kvm * kvm)1547 static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm)
1548 {
1549 struct vfio_device_file *df = file->private_data;
1550
1551 /*
1552 * The kvm is first recorded in the vfio_device_file, and will
1553 * be propagated to vfio_device::kvm when the file is bound to
1554 * iommufd successfully in the vfio device cdev path.
1555 */
1556 spin_lock(&df->kvm_ref_lock);
1557 df->kvm = kvm;
1558 spin_unlock(&df->kvm_ref_lock);
1559 }
1560
1561 /**
1562 * vfio_file_set_kvm - Link a kvm with VFIO drivers
1563 * @file: VFIO group file or VFIO device file
1564 * @kvm: KVM to link
1565 *
1566 * When a VFIO device is first opened the KVM will be available in
1567 * device->kvm if one was associated with the file.
1568 */
vfio_file_set_kvm(struct file * file,struct kvm * kvm)1569 void vfio_file_set_kvm(struct file *file, struct kvm *kvm)
1570 {
1571 struct vfio_group *group;
1572
1573 group = vfio_group_from_file(file);
1574 if (group)
1575 vfio_group_set_kvm(group, kvm);
1576
1577 if (vfio_device_from_file(file))
1578 vfio_device_file_set_kvm(file, kvm);
1579 }
1580 EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
1581
1582 /*
1583 * Sub-module support
1584 */
1585 /*
1586 * Helper for managing a buffer of info chain capabilities, allocate or
1587 * reallocate a buffer with additional @size, filling in @id and @version
1588 * of the capability. A pointer to the new capability is returned.
1589 *
1590 * NB. The chain is based at the head of the buffer, so new entries are
1591 * added to the tail, vfio_info_cap_shift() should be called to fixup the
1592 * next offsets prior to copying to the user buffer.
1593 */
vfio_info_cap_add(struct vfio_info_cap * caps,size_t size,u16 id,u16 version)1594 struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps,
1595 size_t size, u16 id, u16 version)
1596 {
1597 void *buf;
1598 struct vfio_info_cap_header *header, *tmp;
1599
1600 /* Ensure that the next capability struct will be aligned */
1601 size = ALIGN(size, sizeof(u64));
1602
1603 buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL);
1604 if (!buf) {
1605 kfree(caps->buf);
1606 caps->buf = NULL;
1607 caps->size = 0;
1608 return ERR_PTR(-ENOMEM);
1609 }
1610
1611 caps->buf = buf;
1612 header = buf + caps->size;
1613
1614 /* Eventually copied to user buffer, zero */
1615 memset(header, 0, size);
1616
1617 header->id = id;
1618 header->version = version;
1619
1620 /* Add to the end of the capability chain */
1621 for (tmp = buf; tmp->next; tmp = buf + tmp->next)
1622 ; /* nothing */
1623
1624 tmp->next = caps->size;
1625 caps->size += size;
1626
1627 return header;
1628 }
1629 EXPORT_SYMBOL_GPL(vfio_info_cap_add);
1630
vfio_info_cap_shift(struct vfio_info_cap * caps,size_t offset)1631 void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset)
1632 {
1633 struct vfio_info_cap_header *tmp;
1634 void *buf = (void *)caps->buf;
1635
1636 /* Capability structs should start with proper alignment */
1637 WARN_ON(!IS_ALIGNED(offset, sizeof(u64)));
1638
1639 for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset)
1640 tmp->next += offset;
1641 }
1642 EXPORT_SYMBOL(vfio_info_cap_shift);
1643
vfio_info_add_capability(struct vfio_info_cap * caps,struct vfio_info_cap_header * cap,size_t size)1644 int vfio_info_add_capability(struct vfio_info_cap *caps,
1645 struct vfio_info_cap_header *cap, size_t size)
1646 {
1647 struct vfio_info_cap_header *header;
1648
1649 header = vfio_info_cap_add(caps, size, cap->id, cap->version);
1650 if (IS_ERR(header))
1651 return PTR_ERR(header);
1652
1653 memcpy(header + 1, cap + 1, size - sizeof(*header));
1654
1655 return 0;
1656 }
1657 EXPORT_SYMBOL(vfio_info_add_capability);
1658
vfio_set_irqs_validate_and_prepare(struct vfio_irq_set * hdr,int num_irqs,int max_irq_type,size_t * data_size)1659 int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs,
1660 int max_irq_type, size_t *data_size)
1661 {
1662 unsigned long minsz;
1663 size_t size;
1664
1665 minsz = offsetofend(struct vfio_irq_set, count);
1666
1667 if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) ||
1668 (hdr->count >= (U32_MAX - hdr->start)) ||
1669 (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
1670 VFIO_IRQ_SET_ACTION_TYPE_MASK)))
1671 return -EINVAL;
1672
1673 if (data_size)
1674 *data_size = 0;
1675
1676 if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs)
1677 return -EINVAL;
1678
1679 switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
1680 case VFIO_IRQ_SET_DATA_NONE:
1681 size = 0;
1682 break;
1683 case VFIO_IRQ_SET_DATA_BOOL:
1684 size = sizeof(uint8_t);
1685 break;
1686 case VFIO_IRQ_SET_DATA_EVENTFD:
1687 size = sizeof(int32_t);
1688 break;
1689 default:
1690 return -EINVAL;
1691 }
1692
1693 if (size) {
1694 if (hdr->argsz - minsz < hdr->count * size)
1695 return -EINVAL;
1696
1697 if (!data_size)
1698 return -EINVAL;
1699
1700 *data_size = hdr->count * size;
1701 }
1702
1703 return 0;
1704 }
1705 EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);
1706
1707 /*
1708 * Pin contiguous user pages and return their associated host pages for local
1709 * domain only.
1710 * @device [in] : device
1711 * @iova [in] : starting IOVA of user pages to be pinned.
1712 * @npage [in] : count of pages to be pinned. This count should not
1713 * be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1714 * @prot [in] : protection flags
1715 * @pages[out] : array of host pages
1716 * Return error or number of pages pinned.
1717 *
1718 * A driver may only call this function if the vfio_device was created
1719 * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages().
1720 */
vfio_pin_pages(struct vfio_device * device,dma_addr_t iova,int npage,int prot,struct page ** pages)1721 int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova,
1722 int npage, int prot, struct page **pages)
1723 {
1724 /* group->container cannot change while a vfio device is open */
1725 if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device)))
1726 return -EINVAL;
1727 if (!device->ops->dma_unmap)
1728 return -EINVAL;
1729 if (vfio_device_has_container(device))
1730 return vfio_device_container_pin_pages(device, iova,
1731 npage, prot, pages);
1732 if (device->iommufd_access) {
1733 int ret;
1734
1735 if (iova > ULONG_MAX)
1736 return -EINVAL;
1737 /*
1738 * VFIO ignores the sub page offset, npages is from the start of
1739 * a PAGE_SIZE chunk of IOVA. The caller is expected to recover
1740 * the sub page offset by doing:
1741 * pages[0] + (iova % PAGE_SIZE)
1742 */
1743 ret = iommufd_access_pin_pages(
1744 device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE),
1745 npage * PAGE_SIZE, pages,
1746 (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
1747 if (ret)
1748 return ret;
1749 return npage;
1750 }
1751 return -EINVAL;
1752 }
1753 EXPORT_SYMBOL(vfio_pin_pages);
1754
1755 /*
1756 * Unpin contiguous host pages for local domain only.
1757 * @device [in] : device
1758 * @iova [in] : starting address of user pages to be unpinned.
1759 * @npage [in] : count of pages to be unpinned. This count should not
1760 * be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1761 */
vfio_unpin_pages(struct vfio_device * device,dma_addr_t iova,int npage)1762 void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage)
1763 {
1764 if (WARN_ON(!vfio_assert_device_open(device)))
1765 return;
1766 if (WARN_ON(!device->ops->dma_unmap))
1767 return;
1768
1769 if (vfio_device_has_container(device)) {
1770 vfio_device_container_unpin_pages(device, iova, npage);
1771 return;
1772 }
1773 if (device->iommufd_access) {
1774 if (WARN_ON(iova > ULONG_MAX))
1775 return;
1776 iommufd_access_unpin_pages(device->iommufd_access,
1777 ALIGN_DOWN(iova, PAGE_SIZE),
1778 npage * PAGE_SIZE);
1779 return;
1780 }
1781 }
1782 EXPORT_SYMBOL(vfio_unpin_pages);
1783
1784 /*
1785 * This interface allows the CPUs to perform some sort of virtual DMA on
1786 * behalf of the device.
1787 *
1788 * CPUs read/write from/into a range of IOVAs pointing to user space memory
1789 * into/from a kernel buffer.
1790 *
1791 * As the read/write of user space memory is conducted via the CPUs and is
1792 * not a real device DMA, it is not necessary to pin the user space memory.
1793 *
1794 * @device [in] : VFIO device
1795 * @iova [in] : base IOVA of a user space buffer
1796 * @data [in] : pointer to kernel buffer
1797 * @len [in] : kernel buffer length
1798 * @write : indicate read or write
1799 * Return error code on failure or 0 on success.
1800 */
vfio_dma_rw(struct vfio_device * device,dma_addr_t iova,void * data,size_t len,bool write)1801 int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data,
1802 size_t len, bool write)
1803 {
1804 if (!data || len <= 0 || !vfio_assert_device_open(device))
1805 return -EINVAL;
1806
1807 if (vfio_device_has_container(device))
1808 return vfio_device_container_dma_rw(device, iova,
1809 data, len, write);
1810
1811 if (device->iommufd_access) {
1812 unsigned int flags = 0;
1813
1814 if (iova > ULONG_MAX)
1815 return -EINVAL;
1816
1817 /* VFIO historically tries to auto-detect a kthread */
1818 if (!current->mm)
1819 flags |= IOMMUFD_ACCESS_RW_KTHREAD;
1820 if (write)
1821 flags |= IOMMUFD_ACCESS_RW_WRITE;
1822 return iommufd_access_rw(device->iommufd_access, iova, data,
1823 len, flags);
1824 }
1825 return -EINVAL;
1826 }
1827 EXPORT_SYMBOL(vfio_dma_rw);
1828
1829 /*
1830 * Module/class support
1831 */
vfio_init(void)1832 static int __init vfio_init(void)
1833 {
1834 int ret;
1835
1836 ida_init(&vfio.device_ida);
1837
1838 ret = vfio_group_init();
1839 if (ret)
1840 return ret;
1841
1842 ret = vfio_virqfd_init();
1843 if (ret)
1844 goto err_virqfd;
1845
1846 /* /sys/class/vfio-dev/vfioX */
1847 ret = class_register(&vfio_device_class);
1848 if (ret)
1849 goto err_dev_class;
1850
1851 ret = vfio_cdev_init();
1852 if (ret)
1853 goto err_alloc_dev_chrdev;
1854
1855 vfio_debugfs_create_root();
1856 pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
1857 return 0;
1858
1859 err_alloc_dev_chrdev:
1860 class_unregister(&vfio_device_class);
1861 err_dev_class:
1862 vfio_virqfd_exit();
1863 err_virqfd:
1864 vfio_group_cleanup();
1865 return ret;
1866 }
1867
vfio_cleanup(void)1868 static void __exit vfio_cleanup(void)
1869 {
1870 vfio_debugfs_remove_root();
1871 ida_destroy(&vfio.device_ida);
1872 vfio_cdev_cleanup();
1873 class_unregister(&vfio_device_class);
1874 vfio_virqfd_exit();
1875 vfio_group_cleanup();
1876 xa_destroy(&vfio_device_set_xa);
1877 }
1878
1879 module_init(vfio_init);
1880 module_exit(vfio_cleanup);
1881
1882 MODULE_IMPORT_NS("IOMMUFD");
1883 MODULE_VERSION(DRIVER_VERSION);
1884 MODULE_LICENSE("GPL v2");
1885 MODULE_AUTHOR(DRIVER_AUTHOR);
1886 MODULE_DESCRIPTION(DRIVER_DESC);
1887 MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce");
1888