1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2009 Red Hat, Inc. 3 * Copyright (C) 2006 Rusty Russell IBM Corporation 4 * 5 * Author: Michael S. Tsirkin <mst@redhat.com> 6 * 7 * Inspiration, some code, and most witty comments come from 8 * Documentation/virtual/lguest/lguest.c, by Rusty Russell 9 * 10 * Generic code for virtio server in host kernel. 11 */ 12 13 #include <linux/eventfd.h> 14 #include <linux/vhost.h> 15 #include <linux/uio.h> 16 #include <linux/mm.h> 17 #include <linux/miscdevice.h> 18 #include <linux/mutex.h> 19 #include <linux/poll.h> 20 #include <linux/file.h> 21 #include <linux/highmem.h> 22 #include <linux/slab.h> 23 #include <linux/vmalloc.h> 24 #include <linux/kthread.h> 25 #include <linux/cgroup.h> 26 #include <linux/module.h> 27 #include <linux/sort.h> 28 #include <linux/sched/mm.h> 29 #include <linux/sched/signal.h> 30 #include <linux/interval_tree_generic.h> 31 #include <linux/nospec.h> 32 #include <linux/kcov.h> 33 34 #include "vhost.h" 35 36 static ushort max_mem_regions = 64; 37 module_param(max_mem_regions, ushort, 0444); 38 MODULE_PARM_DESC(max_mem_regions, 39 "Maximum number of memory regions in memory map. (default: 64)"); 40 static int max_iotlb_entries = 2048; 41 module_param(max_iotlb_entries, int, 0444); 42 MODULE_PARM_DESC(max_iotlb_entries, 43 "Maximum number of iotlb entries. (default: 2048)"); 44 45 enum { 46 VHOST_MEMORY_F_LOG = 0x1, 47 }; 48 49 #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num]) 50 #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num]) 51 52 #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY 53 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) 54 { 55 vq->user_be = !virtio_legacy_is_little_endian(); 56 } 57 58 static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq) 59 { 60 vq->user_be = true; 61 } 62 63 static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq) 64 { 65 vq->user_be = false; 66 } 67 68 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp) 69 { 70 struct vhost_vring_state s; 71 72 if (vq->private_data) 73 return -EBUSY; 74 75 if (copy_from_user(&s, argp, sizeof(s))) 76 return -EFAULT; 77 78 if (s.num != VHOST_VRING_LITTLE_ENDIAN && 79 s.num != VHOST_VRING_BIG_ENDIAN) 80 return -EINVAL; 81 82 if (s.num == VHOST_VRING_BIG_ENDIAN) 83 vhost_enable_cross_endian_big(vq); 84 else 85 vhost_enable_cross_endian_little(vq); 86 87 return 0; 88 } 89 90 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, 91 int __user *argp) 92 { 93 struct vhost_vring_state s = { 94 .index = idx, 95 .num = vq->user_be 96 }; 97 98 if (copy_to_user(argp, &s, sizeof(s))) 99 return -EFAULT; 100 101 return 0; 102 } 103 104 static void vhost_init_is_le(struct vhost_virtqueue *vq) 105 { 106 /* Note for legacy virtio: user_be is initialized at reset time 107 * according to the host endianness. If userspace does not set an 108 * explicit endianness, the default behavior is native endian, as 109 * expected by legacy virtio. 110 */ 111 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be; 112 } 113 #else 114 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) 115 { 116 } 117 118 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp) 119 { 120 return -ENOIOCTLCMD; 121 } 122 123 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, 124 int __user *argp) 125 { 126 return -ENOIOCTLCMD; 127 } 128 129 static void vhost_init_is_le(struct vhost_virtqueue *vq) 130 { 131 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) 132 || virtio_legacy_is_little_endian(); 133 } 134 #endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */ 135 136 static void vhost_reset_is_le(struct vhost_virtqueue *vq) 137 { 138 vhost_init_is_le(vq); 139 } 140 141 struct vhost_flush_struct { 142 struct vhost_work work; 143 struct completion wait_event; 144 }; 145 146 static void vhost_flush_work(struct vhost_work *work) 147 { 148 struct vhost_flush_struct *s; 149 150 s = container_of(work, struct vhost_flush_struct, work); 151 complete(&s->wait_event); 152 } 153 154 static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh, 155 poll_table *pt) 156 { 157 struct vhost_poll *poll; 158 159 poll = container_of(pt, struct vhost_poll, table); 160 poll->wqh = wqh; 161 add_wait_queue(wqh, &poll->wait); 162 } 163 164 static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, 165 void *key) 166 { 167 struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait); 168 struct vhost_work *work = &poll->work; 169 170 if (!(key_to_poll(key) & poll->mask)) 171 return 0; 172 173 if (!poll->dev->use_worker) 174 work->fn(work); 175 else 176 vhost_poll_queue(poll); 177 178 return 0; 179 } 180 181 void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn) 182 { 183 clear_bit(VHOST_WORK_QUEUED, &work->flags); 184 work->fn = fn; 185 } 186 EXPORT_SYMBOL_GPL(vhost_work_init); 187 188 /* Init poll structure */ 189 void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, 190 __poll_t mask, struct vhost_dev *dev) 191 { 192 init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup); 193 init_poll_funcptr(&poll->table, vhost_poll_func); 194 poll->mask = mask; 195 poll->dev = dev; 196 poll->wqh = NULL; 197 198 vhost_work_init(&poll->work, fn); 199 } 200 EXPORT_SYMBOL_GPL(vhost_poll_init); 201 202 /* Start polling a file. We add ourselves to file's wait queue. The caller must 203 * keep a reference to a file until after vhost_poll_stop is called. */ 204 int vhost_poll_start(struct vhost_poll *poll, struct file *file) 205 { 206 __poll_t mask; 207 208 if (poll->wqh) 209 return 0; 210 211 mask = vfs_poll(file, &poll->table); 212 if (mask) 213 vhost_poll_wakeup(&poll->wait, 0, 0, poll_to_key(mask)); 214 if (mask & EPOLLERR) { 215 vhost_poll_stop(poll); 216 return -EINVAL; 217 } 218 219 return 0; 220 } 221 EXPORT_SYMBOL_GPL(vhost_poll_start); 222 223 /* Stop polling a file. After this function returns, it becomes safe to drop the 224 * file reference. You must also flush afterwards. */ 225 void vhost_poll_stop(struct vhost_poll *poll) 226 { 227 if (poll->wqh) { 228 remove_wait_queue(poll->wqh, &poll->wait); 229 poll->wqh = NULL; 230 } 231 } 232 EXPORT_SYMBOL_GPL(vhost_poll_stop); 233 234 void vhost_work_dev_flush(struct vhost_dev *dev) 235 { 236 struct vhost_flush_struct flush; 237 238 if (dev->worker) { 239 init_completion(&flush.wait_event); 240 vhost_work_init(&flush.work, vhost_flush_work); 241 242 vhost_work_queue(dev, &flush.work); 243 wait_for_completion(&flush.wait_event); 244 } 245 } 246 EXPORT_SYMBOL_GPL(vhost_work_dev_flush); 247 248 /* Flush any work that has been scheduled. When calling this, don't hold any 249 * locks that are also used by the callback. */ 250 void vhost_poll_flush(struct vhost_poll *poll) 251 { 252 vhost_work_dev_flush(poll->dev); 253 } 254 EXPORT_SYMBOL_GPL(vhost_poll_flush); 255 256 void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work) 257 { 258 if (!dev->worker) 259 return; 260 261 if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) { 262 /* We can only add the work to the list after we're 263 * sure it was not in the list. 264 * test_and_set_bit() implies a memory barrier. 265 */ 266 llist_add(&work->node, &dev->work_list); 267 wake_up_process(dev->worker); 268 } 269 } 270 EXPORT_SYMBOL_GPL(vhost_work_queue); 271 272 /* A lockless hint for busy polling code to exit the loop */ 273 bool vhost_has_work(struct vhost_dev *dev) 274 { 275 return !llist_empty(&dev->work_list); 276 } 277 EXPORT_SYMBOL_GPL(vhost_has_work); 278 279 void vhost_poll_queue(struct vhost_poll *poll) 280 { 281 vhost_work_queue(poll->dev, &poll->work); 282 } 283 EXPORT_SYMBOL_GPL(vhost_poll_queue); 284 285 static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq) 286 { 287 int j; 288 289 for (j = 0; j < VHOST_NUM_ADDRS; j++) 290 vq->meta_iotlb[j] = NULL; 291 } 292 293 static void vhost_vq_meta_reset(struct vhost_dev *d) 294 { 295 int i; 296 297 for (i = 0; i < d->nvqs; ++i) 298 __vhost_vq_meta_reset(d->vqs[i]); 299 } 300 301 static void vhost_vring_call_reset(struct vhost_vring_call *call_ctx) 302 { 303 call_ctx->ctx = NULL; 304 memset(&call_ctx->producer, 0x0, sizeof(struct irq_bypass_producer)); 305 } 306 307 bool vhost_vq_is_setup(struct vhost_virtqueue *vq) 308 { 309 return vq->avail && vq->desc && vq->used && vhost_vq_access_ok(vq); 310 } 311 EXPORT_SYMBOL_GPL(vhost_vq_is_setup); 312 313 static void vhost_vq_reset(struct vhost_dev *dev, 314 struct vhost_virtqueue *vq) 315 { 316 vq->num = 1; 317 vq->desc = NULL; 318 vq->avail = NULL; 319 vq->used = NULL; 320 vq->last_avail_idx = 0; 321 vq->avail_idx = 0; 322 vq->last_used_idx = 0; 323 vq->signalled_used = 0; 324 vq->signalled_used_valid = false; 325 vq->used_flags = 0; 326 vq->log_used = false; 327 vq->log_addr = -1ull; 328 vq->private_data = NULL; 329 vq->acked_features = 0; 330 vq->acked_backend_features = 0; 331 vq->log_base = NULL; 332 vq->error_ctx = NULL; 333 vq->kick = NULL; 334 vq->log_ctx = NULL; 335 vhost_disable_cross_endian(vq); 336 vhost_reset_is_le(vq); 337 vq->busyloop_timeout = 0; 338 vq->umem = NULL; 339 vq->iotlb = NULL; 340 vhost_vring_call_reset(&vq->call_ctx); 341 __vhost_vq_meta_reset(vq); 342 } 343 344 static int vhost_worker(void *data) 345 { 346 struct vhost_dev *dev = data; 347 struct vhost_work *work, *work_next; 348 struct llist_node *node; 349 350 kthread_use_mm(dev->mm); 351 352 for (;;) { 353 /* mb paired w/ kthread_stop */ 354 set_current_state(TASK_INTERRUPTIBLE); 355 356 if (kthread_should_stop()) { 357 __set_current_state(TASK_RUNNING); 358 break; 359 } 360 361 node = llist_del_all(&dev->work_list); 362 if (!node) 363 schedule(); 364 365 node = llist_reverse_order(node); 366 /* make sure flag is seen after deletion */ 367 smp_wmb(); 368 llist_for_each_entry_safe(work, work_next, node, node) { 369 clear_bit(VHOST_WORK_QUEUED, &work->flags); 370 __set_current_state(TASK_RUNNING); 371 kcov_remote_start_common(dev->kcov_handle); 372 work->fn(work); 373 kcov_remote_stop(); 374 if (need_resched()) 375 schedule(); 376 } 377 } 378 kthread_unuse_mm(dev->mm); 379 return 0; 380 } 381 382 static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq) 383 { 384 kfree(vq->indirect); 385 vq->indirect = NULL; 386 kfree(vq->log); 387 vq->log = NULL; 388 kfree(vq->heads); 389 vq->heads = NULL; 390 } 391 392 /* Helper to allocate iovec buffers for all vqs. */ 393 static long vhost_dev_alloc_iovecs(struct vhost_dev *dev) 394 { 395 struct vhost_virtqueue *vq; 396 int i; 397 398 for (i = 0; i < dev->nvqs; ++i) { 399 vq = dev->vqs[i]; 400 vq->indirect = kmalloc_array(UIO_MAXIOV, 401 sizeof(*vq->indirect), 402 GFP_KERNEL); 403 vq->log = kmalloc_array(dev->iov_limit, sizeof(*vq->log), 404 GFP_KERNEL); 405 vq->heads = kmalloc_array(dev->iov_limit, sizeof(*vq->heads), 406 GFP_KERNEL); 407 if (!vq->indirect || !vq->log || !vq->heads) 408 goto err_nomem; 409 } 410 return 0; 411 412 err_nomem: 413 for (; i >= 0; --i) 414 vhost_vq_free_iovecs(dev->vqs[i]); 415 return -ENOMEM; 416 } 417 418 static void vhost_dev_free_iovecs(struct vhost_dev *dev) 419 { 420 int i; 421 422 for (i = 0; i < dev->nvqs; ++i) 423 vhost_vq_free_iovecs(dev->vqs[i]); 424 } 425 426 bool vhost_exceeds_weight(struct vhost_virtqueue *vq, 427 int pkts, int total_len) 428 { 429 struct vhost_dev *dev = vq->dev; 430 431 if ((dev->byte_weight && total_len >= dev->byte_weight) || 432 pkts >= dev->weight) { 433 vhost_poll_queue(&vq->poll); 434 return true; 435 } 436 437 return false; 438 } 439 EXPORT_SYMBOL_GPL(vhost_exceeds_weight); 440 441 static size_t vhost_get_avail_size(struct vhost_virtqueue *vq, 442 unsigned int num) 443 { 444 size_t event __maybe_unused = 445 vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 446 447 return sizeof(*vq->avail) + 448 sizeof(*vq->avail->ring) * num + event; 449 } 450 451 static size_t vhost_get_used_size(struct vhost_virtqueue *vq, 452 unsigned int num) 453 { 454 size_t event __maybe_unused = 455 vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 456 457 return sizeof(*vq->used) + 458 sizeof(*vq->used->ring) * num + event; 459 } 460 461 static size_t vhost_get_desc_size(struct vhost_virtqueue *vq, 462 unsigned int num) 463 { 464 return sizeof(*vq->desc) * num; 465 } 466 467 void vhost_dev_init(struct vhost_dev *dev, 468 struct vhost_virtqueue **vqs, int nvqs, 469 int iov_limit, int weight, int byte_weight, 470 bool use_worker, 471 int (*msg_handler)(struct vhost_dev *dev, u32 asid, 472 struct vhost_iotlb_msg *msg)) 473 { 474 struct vhost_virtqueue *vq; 475 int i; 476 477 dev->vqs = vqs; 478 dev->nvqs = nvqs; 479 mutex_init(&dev->mutex); 480 dev->log_ctx = NULL; 481 dev->umem = NULL; 482 dev->iotlb = NULL; 483 dev->mm = NULL; 484 dev->worker = NULL; 485 dev->iov_limit = iov_limit; 486 dev->weight = weight; 487 dev->byte_weight = byte_weight; 488 dev->use_worker = use_worker; 489 dev->msg_handler = msg_handler; 490 init_llist_head(&dev->work_list); 491 init_waitqueue_head(&dev->wait); 492 INIT_LIST_HEAD(&dev->read_list); 493 INIT_LIST_HEAD(&dev->pending_list); 494 spin_lock_init(&dev->iotlb_lock); 495 496 497 for (i = 0; i < dev->nvqs; ++i) { 498 vq = dev->vqs[i]; 499 vq->log = NULL; 500 vq->indirect = NULL; 501 vq->heads = NULL; 502 vq->dev = dev; 503 mutex_init(&vq->mutex); 504 vhost_vq_reset(dev, vq); 505 if (vq->handle_kick) 506 vhost_poll_init(&vq->poll, vq->handle_kick, 507 EPOLLIN, dev); 508 } 509 } 510 EXPORT_SYMBOL_GPL(vhost_dev_init); 511 512 /* Caller should have device mutex */ 513 long vhost_dev_check_owner(struct vhost_dev *dev) 514 { 515 /* Are you the owner? If not, I don't think you mean to do that */ 516 return dev->mm == current->mm ? 0 : -EPERM; 517 } 518 EXPORT_SYMBOL_GPL(vhost_dev_check_owner); 519 520 struct vhost_attach_cgroups_struct { 521 struct vhost_work work; 522 struct task_struct *owner; 523 int ret; 524 }; 525 526 static void vhost_attach_cgroups_work(struct vhost_work *work) 527 { 528 struct vhost_attach_cgroups_struct *s; 529 530 s = container_of(work, struct vhost_attach_cgroups_struct, work); 531 s->ret = cgroup_attach_task_all(s->owner, current); 532 } 533 534 static int vhost_attach_cgroups(struct vhost_dev *dev) 535 { 536 struct vhost_attach_cgroups_struct attach; 537 538 attach.owner = current; 539 vhost_work_init(&attach.work, vhost_attach_cgroups_work); 540 vhost_work_queue(dev, &attach.work); 541 vhost_work_dev_flush(dev); 542 return attach.ret; 543 } 544 545 /* Caller should have device mutex */ 546 bool vhost_dev_has_owner(struct vhost_dev *dev) 547 { 548 return dev->mm; 549 } 550 EXPORT_SYMBOL_GPL(vhost_dev_has_owner); 551 552 static void vhost_attach_mm(struct vhost_dev *dev) 553 { 554 /* No owner, become one */ 555 if (dev->use_worker) { 556 dev->mm = get_task_mm(current); 557 } else { 558 /* vDPA device does not use worker thead, so there's 559 * no need to hold the address space for mm. This help 560 * to avoid deadlock in the case of mmap() which may 561 * held the refcnt of the file and depends on release 562 * method to remove vma. 563 */ 564 dev->mm = current->mm; 565 mmgrab(dev->mm); 566 } 567 } 568 569 static void vhost_detach_mm(struct vhost_dev *dev) 570 { 571 if (!dev->mm) 572 return; 573 574 if (dev->use_worker) 575 mmput(dev->mm); 576 else 577 mmdrop(dev->mm); 578 579 dev->mm = NULL; 580 } 581 582 /* Caller should have device mutex */ 583 long vhost_dev_set_owner(struct vhost_dev *dev) 584 { 585 struct task_struct *worker; 586 int err; 587 588 /* Is there an owner already? */ 589 if (vhost_dev_has_owner(dev)) { 590 err = -EBUSY; 591 goto err_mm; 592 } 593 594 vhost_attach_mm(dev); 595 596 dev->kcov_handle = kcov_common_handle(); 597 if (dev->use_worker) { 598 worker = kthread_create(vhost_worker, dev, 599 "vhost-%d", current->pid); 600 if (IS_ERR(worker)) { 601 err = PTR_ERR(worker); 602 goto err_worker; 603 } 604 605 dev->worker = worker; 606 wake_up_process(worker); /* avoid contributing to loadavg */ 607 608 err = vhost_attach_cgroups(dev); 609 if (err) 610 goto err_cgroup; 611 } 612 613 err = vhost_dev_alloc_iovecs(dev); 614 if (err) 615 goto err_cgroup; 616 617 return 0; 618 err_cgroup: 619 if (dev->worker) { 620 kthread_stop(dev->worker); 621 dev->worker = NULL; 622 } 623 err_worker: 624 vhost_detach_mm(dev); 625 dev->kcov_handle = 0; 626 err_mm: 627 return err; 628 } 629 EXPORT_SYMBOL_GPL(vhost_dev_set_owner); 630 631 static struct vhost_iotlb *iotlb_alloc(void) 632 { 633 return vhost_iotlb_alloc(max_iotlb_entries, 634 VHOST_IOTLB_FLAG_RETIRE); 635 } 636 637 struct vhost_iotlb *vhost_dev_reset_owner_prepare(void) 638 { 639 return iotlb_alloc(); 640 } 641 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare); 642 643 /* Caller should have device mutex */ 644 void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem) 645 { 646 int i; 647 648 vhost_dev_cleanup(dev); 649 650 dev->umem = umem; 651 /* We don't need VQ locks below since vhost_dev_cleanup makes sure 652 * VQs aren't running. 653 */ 654 for (i = 0; i < dev->nvqs; ++i) 655 dev->vqs[i]->umem = umem; 656 } 657 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner); 658 659 void vhost_dev_stop(struct vhost_dev *dev) 660 { 661 int i; 662 663 for (i = 0; i < dev->nvqs; ++i) { 664 if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) { 665 vhost_poll_stop(&dev->vqs[i]->poll); 666 vhost_poll_flush(&dev->vqs[i]->poll); 667 } 668 } 669 } 670 EXPORT_SYMBOL_GPL(vhost_dev_stop); 671 672 static void vhost_clear_msg(struct vhost_dev *dev) 673 { 674 struct vhost_msg_node *node, *n; 675 676 spin_lock(&dev->iotlb_lock); 677 678 list_for_each_entry_safe(node, n, &dev->read_list, node) { 679 list_del(&node->node); 680 kfree(node); 681 } 682 683 list_for_each_entry_safe(node, n, &dev->pending_list, node) { 684 list_del(&node->node); 685 kfree(node); 686 } 687 688 spin_unlock(&dev->iotlb_lock); 689 } 690 691 void vhost_dev_cleanup(struct vhost_dev *dev) 692 { 693 int i; 694 695 for (i = 0; i < dev->nvqs; ++i) { 696 if (dev->vqs[i]->error_ctx) 697 eventfd_ctx_put(dev->vqs[i]->error_ctx); 698 if (dev->vqs[i]->kick) 699 fput(dev->vqs[i]->kick); 700 if (dev->vqs[i]->call_ctx.ctx) 701 eventfd_ctx_put(dev->vqs[i]->call_ctx.ctx); 702 vhost_vq_reset(dev, dev->vqs[i]); 703 } 704 vhost_dev_free_iovecs(dev); 705 if (dev->log_ctx) 706 eventfd_ctx_put(dev->log_ctx); 707 dev->log_ctx = NULL; 708 /* No one will access memory at this point */ 709 vhost_iotlb_free(dev->umem); 710 dev->umem = NULL; 711 vhost_iotlb_free(dev->iotlb); 712 dev->iotlb = NULL; 713 vhost_clear_msg(dev); 714 wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM); 715 WARN_ON(!llist_empty(&dev->work_list)); 716 if (dev->worker) { 717 kthread_stop(dev->worker); 718 dev->worker = NULL; 719 dev->kcov_handle = 0; 720 } 721 vhost_detach_mm(dev); 722 } 723 EXPORT_SYMBOL_GPL(vhost_dev_cleanup); 724 725 static bool log_access_ok(void __user *log_base, u64 addr, unsigned long sz) 726 { 727 u64 a = addr / VHOST_PAGE_SIZE / 8; 728 729 /* Make sure 64 bit math will not overflow. */ 730 if (a > ULONG_MAX - (unsigned long)log_base || 731 a + (unsigned long)log_base > ULONG_MAX) 732 return false; 733 734 return access_ok(log_base + a, 735 (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8); 736 } 737 738 /* Make sure 64 bit math will not overflow. */ 739 static bool vhost_overflow(u64 uaddr, u64 size) 740 { 741 if (uaddr > ULONG_MAX || size > ULONG_MAX) 742 return true; 743 744 if (!size) 745 return false; 746 747 return uaddr > ULONG_MAX - size + 1; 748 } 749 750 /* Caller should have vq mutex and device mutex. */ 751 static bool vq_memory_access_ok(void __user *log_base, struct vhost_iotlb *umem, 752 int log_all) 753 { 754 struct vhost_iotlb_map *map; 755 756 if (!umem) 757 return false; 758 759 list_for_each_entry(map, &umem->list, link) { 760 unsigned long a = map->addr; 761 762 if (vhost_overflow(map->addr, map->size)) 763 return false; 764 765 766 if (!access_ok((void __user *)a, map->size)) 767 return false; 768 else if (log_all && !log_access_ok(log_base, 769 map->start, 770 map->size)) 771 return false; 772 } 773 return true; 774 } 775 776 static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq, 777 u64 addr, unsigned int size, 778 int type) 779 { 780 const struct vhost_iotlb_map *map = vq->meta_iotlb[type]; 781 782 if (!map) 783 return NULL; 784 785 return (void __user *)(uintptr_t)(map->addr + addr - map->start); 786 } 787 788 /* Can we switch to this memory table? */ 789 /* Caller should have device mutex but not vq mutex */ 790 static bool memory_access_ok(struct vhost_dev *d, struct vhost_iotlb *umem, 791 int log_all) 792 { 793 int i; 794 795 for (i = 0; i < d->nvqs; ++i) { 796 bool ok; 797 bool log; 798 799 mutex_lock(&d->vqs[i]->mutex); 800 log = log_all || vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL); 801 /* If ring is inactive, will check when it's enabled. */ 802 if (d->vqs[i]->private_data) 803 ok = vq_memory_access_ok(d->vqs[i]->log_base, 804 umem, log); 805 else 806 ok = true; 807 mutex_unlock(&d->vqs[i]->mutex); 808 if (!ok) 809 return false; 810 } 811 return true; 812 } 813 814 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, 815 struct iovec iov[], int iov_size, int access); 816 817 static int vhost_copy_to_user(struct vhost_virtqueue *vq, void __user *to, 818 const void *from, unsigned size) 819 { 820 int ret; 821 822 if (!vq->iotlb) 823 return __copy_to_user(to, from, size); 824 else { 825 /* This function should be called after iotlb 826 * prefetch, which means we're sure that all vq 827 * could be access through iotlb. So -EAGAIN should 828 * not happen in this case. 829 */ 830 struct iov_iter t; 831 void __user *uaddr = vhost_vq_meta_fetch(vq, 832 (u64)(uintptr_t)to, size, 833 VHOST_ADDR_USED); 834 835 if (uaddr) 836 return __copy_to_user(uaddr, from, size); 837 838 ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov, 839 ARRAY_SIZE(vq->iotlb_iov), 840 VHOST_ACCESS_WO); 841 if (ret < 0) 842 goto out; 843 iov_iter_init(&t, WRITE, vq->iotlb_iov, ret, size); 844 ret = copy_to_iter(from, size, &t); 845 if (ret == size) 846 ret = 0; 847 } 848 out: 849 return ret; 850 } 851 852 static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to, 853 void __user *from, unsigned size) 854 { 855 int ret; 856 857 if (!vq->iotlb) 858 return __copy_from_user(to, from, size); 859 else { 860 /* This function should be called after iotlb 861 * prefetch, which means we're sure that vq 862 * could be access through iotlb. So -EAGAIN should 863 * not happen in this case. 864 */ 865 void __user *uaddr = vhost_vq_meta_fetch(vq, 866 (u64)(uintptr_t)from, size, 867 VHOST_ADDR_DESC); 868 struct iov_iter f; 869 870 if (uaddr) 871 return __copy_from_user(to, uaddr, size); 872 873 ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov, 874 ARRAY_SIZE(vq->iotlb_iov), 875 VHOST_ACCESS_RO); 876 if (ret < 0) { 877 vq_err(vq, "IOTLB translation failure: uaddr " 878 "%p size 0x%llx\n", from, 879 (unsigned long long) size); 880 goto out; 881 } 882 iov_iter_init(&f, READ, vq->iotlb_iov, ret, size); 883 ret = copy_from_iter(to, size, &f); 884 if (ret == size) 885 ret = 0; 886 } 887 888 out: 889 return ret; 890 } 891 892 static void __user *__vhost_get_user_slow(struct vhost_virtqueue *vq, 893 void __user *addr, unsigned int size, 894 int type) 895 { 896 int ret; 897 898 ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov, 899 ARRAY_SIZE(vq->iotlb_iov), 900 VHOST_ACCESS_RO); 901 if (ret < 0) { 902 vq_err(vq, "IOTLB translation failure: uaddr " 903 "%p size 0x%llx\n", addr, 904 (unsigned long long) size); 905 return NULL; 906 } 907 908 if (ret != 1 || vq->iotlb_iov[0].iov_len != size) { 909 vq_err(vq, "Non atomic userspace memory access: uaddr " 910 "%p size 0x%llx\n", addr, 911 (unsigned long long) size); 912 return NULL; 913 } 914 915 return vq->iotlb_iov[0].iov_base; 916 } 917 918 /* This function should be called after iotlb 919 * prefetch, which means we're sure that vq 920 * could be access through iotlb. So -EAGAIN should 921 * not happen in this case. 922 */ 923 static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq, 924 void __user *addr, unsigned int size, 925 int type) 926 { 927 void __user *uaddr = vhost_vq_meta_fetch(vq, 928 (u64)(uintptr_t)addr, size, type); 929 if (uaddr) 930 return uaddr; 931 932 return __vhost_get_user_slow(vq, addr, size, type); 933 } 934 935 #define vhost_put_user(vq, x, ptr) \ 936 ({ \ 937 int ret; \ 938 if (!vq->iotlb) { \ 939 ret = __put_user(x, ptr); \ 940 } else { \ 941 __typeof__(ptr) to = \ 942 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \ 943 sizeof(*ptr), VHOST_ADDR_USED); \ 944 if (to != NULL) \ 945 ret = __put_user(x, to); \ 946 else \ 947 ret = -EFAULT; \ 948 } \ 949 ret; \ 950 }) 951 952 static inline int vhost_put_avail_event(struct vhost_virtqueue *vq) 953 { 954 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx), 955 vhost_avail_event(vq)); 956 } 957 958 static inline int vhost_put_used(struct vhost_virtqueue *vq, 959 struct vring_used_elem *head, int idx, 960 int count) 961 { 962 return vhost_copy_to_user(vq, vq->used->ring + idx, head, 963 count * sizeof(*head)); 964 } 965 966 static inline int vhost_put_used_flags(struct vhost_virtqueue *vq) 967 968 { 969 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags), 970 &vq->used->flags); 971 } 972 973 static inline int vhost_put_used_idx(struct vhost_virtqueue *vq) 974 975 { 976 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx), 977 &vq->used->idx); 978 } 979 980 #define vhost_get_user(vq, x, ptr, type) \ 981 ({ \ 982 int ret; \ 983 if (!vq->iotlb) { \ 984 ret = __get_user(x, ptr); \ 985 } else { \ 986 __typeof__(ptr) from = \ 987 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \ 988 sizeof(*ptr), \ 989 type); \ 990 if (from != NULL) \ 991 ret = __get_user(x, from); \ 992 else \ 993 ret = -EFAULT; \ 994 } \ 995 ret; \ 996 }) 997 998 #define vhost_get_avail(vq, x, ptr) \ 999 vhost_get_user(vq, x, ptr, VHOST_ADDR_AVAIL) 1000 1001 #define vhost_get_used(vq, x, ptr) \ 1002 vhost_get_user(vq, x, ptr, VHOST_ADDR_USED) 1003 1004 static void vhost_dev_lock_vqs(struct vhost_dev *d) 1005 { 1006 int i = 0; 1007 for (i = 0; i < d->nvqs; ++i) 1008 mutex_lock_nested(&d->vqs[i]->mutex, i); 1009 } 1010 1011 static void vhost_dev_unlock_vqs(struct vhost_dev *d) 1012 { 1013 int i = 0; 1014 for (i = 0; i < d->nvqs; ++i) 1015 mutex_unlock(&d->vqs[i]->mutex); 1016 } 1017 1018 static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, 1019 __virtio16 *idx) 1020 { 1021 return vhost_get_avail(vq, *idx, &vq->avail->idx); 1022 } 1023 1024 static inline int vhost_get_avail_head(struct vhost_virtqueue *vq, 1025 __virtio16 *head, int idx) 1026 { 1027 return vhost_get_avail(vq, *head, 1028 &vq->avail->ring[idx & (vq->num - 1)]); 1029 } 1030 1031 static inline int vhost_get_avail_flags(struct vhost_virtqueue *vq, 1032 __virtio16 *flags) 1033 { 1034 return vhost_get_avail(vq, *flags, &vq->avail->flags); 1035 } 1036 1037 static inline int vhost_get_used_event(struct vhost_virtqueue *vq, 1038 __virtio16 *event) 1039 { 1040 return vhost_get_avail(vq, *event, vhost_used_event(vq)); 1041 } 1042 1043 static inline int vhost_get_used_idx(struct vhost_virtqueue *vq, 1044 __virtio16 *idx) 1045 { 1046 return vhost_get_used(vq, *idx, &vq->used->idx); 1047 } 1048 1049 static inline int vhost_get_desc(struct vhost_virtqueue *vq, 1050 struct vring_desc *desc, int idx) 1051 { 1052 return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc)); 1053 } 1054 1055 static void vhost_iotlb_notify_vq(struct vhost_dev *d, 1056 struct vhost_iotlb_msg *msg) 1057 { 1058 struct vhost_msg_node *node, *n; 1059 1060 spin_lock(&d->iotlb_lock); 1061 1062 list_for_each_entry_safe(node, n, &d->pending_list, node) { 1063 struct vhost_iotlb_msg *vq_msg = &node->msg.iotlb; 1064 if (msg->iova <= vq_msg->iova && 1065 msg->iova + msg->size - 1 >= vq_msg->iova && 1066 vq_msg->type == VHOST_IOTLB_MISS) { 1067 vhost_poll_queue(&node->vq->poll); 1068 list_del(&node->node); 1069 kfree(node); 1070 } 1071 } 1072 1073 spin_unlock(&d->iotlb_lock); 1074 } 1075 1076 static bool umem_access_ok(u64 uaddr, u64 size, int access) 1077 { 1078 unsigned long a = uaddr; 1079 1080 /* Make sure 64 bit math will not overflow. */ 1081 if (vhost_overflow(uaddr, size)) 1082 return false; 1083 1084 if ((access & VHOST_ACCESS_RO) && 1085 !access_ok((void __user *)a, size)) 1086 return false; 1087 if ((access & VHOST_ACCESS_WO) && 1088 !access_ok((void __user *)a, size)) 1089 return false; 1090 return true; 1091 } 1092 1093 static int vhost_process_iotlb_msg(struct vhost_dev *dev, u32 asid, 1094 struct vhost_iotlb_msg *msg) 1095 { 1096 int ret = 0; 1097 1098 if (asid != 0) 1099 return -EINVAL; 1100 1101 mutex_lock(&dev->mutex); 1102 vhost_dev_lock_vqs(dev); 1103 switch (msg->type) { 1104 case VHOST_IOTLB_UPDATE: 1105 if (!dev->iotlb) { 1106 ret = -EFAULT; 1107 break; 1108 } 1109 if (!umem_access_ok(msg->uaddr, msg->size, msg->perm)) { 1110 ret = -EFAULT; 1111 break; 1112 } 1113 vhost_vq_meta_reset(dev); 1114 if (vhost_iotlb_add_range(dev->iotlb, msg->iova, 1115 msg->iova + msg->size - 1, 1116 msg->uaddr, msg->perm)) { 1117 ret = -ENOMEM; 1118 break; 1119 } 1120 vhost_iotlb_notify_vq(dev, msg); 1121 break; 1122 case VHOST_IOTLB_INVALIDATE: 1123 if (!dev->iotlb) { 1124 ret = -EFAULT; 1125 break; 1126 } 1127 vhost_vq_meta_reset(dev); 1128 vhost_iotlb_del_range(dev->iotlb, msg->iova, 1129 msg->iova + msg->size - 1); 1130 break; 1131 default: 1132 ret = -EINVAL; 1133 break; 1134 } 1135 1136 vhost_dev_unlock_vqs(dev); 1137 mutex_unlock(&dev->mutex); 1138 1139 return ret; 1140 } 1141 ssize_t vhost_chr_write_iter(struct vhost_dev *dev, 1142 struct iov_iter *from) 1143 { 1144 struct vhost_iotlb_msg msg; 1145 size_t offset; 1146 int type, ret; 1147 u32 asid = 0; 1148 1149 ret = copy_from_iter(&type, sizeof(type), from); 1150 if (ret != sizeof(type)) { 1151 ret = -EINVAL; 1152 goto done; 1153 } 1154 1155 switch (type) { 1156 case VHOST_IOTLB_MSG: 1157 /* There maybe a hole after type for V1 message type, 1158 * so skip it here. 1159 */ 1160 offset = offsetof(struct vhost_msg, iotlb) - sizeof(int); 1161 break; 1162 case VHOST_IOTLB_MSG_V2: 1163 if (vhost_backend_has_feature(dev->vqs[0], 1164 VHOST_BACKEND_F_IOTLB_ASID)) { 1165 ret = copy_from_iter(&asid, sizeof(asid), from); 1166 if (ret != sizeof(asid)) { 1167 ret = -EINVAL; 1168 goto done; 1169 } 1170 offset = 0; 1171 } else 1172 offset = sizeof(__u32); 1173 break; 1174 default: 1175 ret = -EINVAL; 1176 goto done; 1177 } 1178 1179 iov_iter_advance(from, offset); 1180 ret = copy_from_iter(&msg, sizeof(msg), from); 1181 if (ret != sizeof(msg)) { 1182 ret = -EINVAL; 1183 goto done; 1184 } 1185 1186 if ((msg.type == VHOST_IOTLB_UPDATE || 1187 msg.type == VHOST_IOTLB_INVALIDATE) && 1188 msg.size == 0) { 1189 ret = -EINVAL; 1190 goto done; 1191 } 1192 1193 if (dev->msg_handler) 1194 ret = dev->msg_handler(dev, asid, &msg); 1195 else 1196 ret = vhost_process_iotlb_msg(dev, asid, &msg); 1197 if (ret) { 1198 ret = -EFAULT; 1199 goto done; 1200 } 1201 1202 ret = (type == VHOST_IOTLB_MSG) ? sizeof(struct vhost_msg) : 1203 sizeof(struct vhost_msg_v2); 1204 done: 1205 return ret; 1206 } 1207 EXPORT_SYMBOL(vhost_chr_write_iter); 1208 1209 __poll_t vhost_chr_poll(struct file *file, struct vhost_dev *dev, 1210 poll_table *wait) 1211 { 1212 __poll_t mask = 0; 1213 1214 poll_wait(file, &dev->wait, wait); 1215 1216 if (!list_empty(&dev->read_list)) 1217 mask |= EPOLLIN | EPOLLRDNORM; 1218 1219 return mask; 1220 } 1221 EXPORT_SYMBOL(vhost_chr_poll); 1222 1223 ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to, 1224 int noblock) 1225 { 1226 DEFINE_WAIT(wait); 1227 struct vhost_msg_node *node; 1228 ssize_t ret = 0; 1229 unsigned size = sizeof(struct vhost_msg); 1230 1231 if (iov_iter_count(to) < size) 1232 return 0; 1233 1234 while (1) { 1235 if (!noblock) 1236 prepare_to_wait(&dev->wait, &wait, 1237 TASK_INTERRUPTIBLE); 1238 1239 node = vhost_dequeue_msg(dev, &dev->read_list); 1240 if (node) 1241 break; 1242 if (noblock) { 1243 ret = -EAGAIN; 1244 break; 1245 } 1246 if (signal_pending(current)) { 1247 ret = -ERESTARTSYS; 1248 break; 1249 } 1250 if (!dev->iotlb) { 1251 ret = -EBADFD; 1252 break; 1253 } 1254 1255 schedule(); 1256 } 1257 1258 if (!noblock) 1259 finish_wait(&dev->wait, &wait); 1260 1261 if (node) { 1262 struct vhost_iotlb_msg *msg; 1263 void *start = &node->msg; 1264 1265 switch (node->msg.type) { 1266 case VHOST_IOTLB_MSG: 1267 size = sizeof(node->msg); 1268 msg = &node->msg.iotlb; 1269 break; 1270 case VHOST_IOTLB_MSG_V2: 1271 size = sizeof(node->msg_v2); 1272 msg = &node->msg_v2.iotlb; 1273 break; 1274 default: 1275 BUG(); 1276 break; 1277 } 1278 1279 ret = copy_to_iter(start, size, to); 1280 if (ret != size || msg->type != VHOST_IOTLB_MISS) { 1281 kfree(node); 1282 return ret; 1283 } 1284 vhost_enqueue_msg(dev, &dev->pending_list, node); 1285 } 1286 1287 return ret; 1288 } 1289 EXPORT_SYMBOL_GPL(vhost_chr_read_iter); 1290 1291 static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access) 1292 { 1293 struct vhost_dev *dev = vq->dev; 1294 struct vhost_msg_node *node; 1295 struct vhost_iotlb_msg *msg; 1296 bool v2 = vhost_backend_has_feature(vq, VHOST_BACKEND_F_IOTLB_MSG_V2); 1297 1298 node = vhost_new_msg(vq, v2 ? VHOST_IOTLB_MSG_V2 : VHOST_IOTLB_MSG); 1299 if (!node) 1300 return -ENOMEM; 1301 1302 if (v2) { 1303 node->msg_v2.type = VHOST_IOTLB_MSG_V2; 1304 msg = &node->msg_v2.iotlb; 1305 } else { 1306 msg = &node->msg.iotlb; 1307 } 1308 1309 msg->type = VHOST_IOTLB_MISS; 1310 msg->iova = iova; 1311 msg->perm = access; 1312 1313 vhost_enqueue_msg(dev, &dev->read_list, node); 1314 1315 return 0; 1316 } 1317 1318 static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, 1319 vring_desc_t __user *desc, 1320 vring_avail_t __user *avail, 1321 vring_used_t __user *used) 1322 1323 { 1324 /* If an IOTLB device is present, the vring addresses are 1325 * GIOVAs. Access validation occurs at prefetch time. */ 1326 if (vq->iotlb) 1327 return true; 1328 1329 return access_ok(desc, vhost_get_desc_size(vq, num)) && 1330 access_ok(avail, vhost_get_avail_size(vq, num)) && 1331 access_ok(used, vhost_get_used_size(vq, num)); 1332 } 1333 1334 static void vhost_vq_meta_update(struct vhost_virtqueue *vq, 1335 const struct vhost_iotlb_map *map, 1336 int type) 1337 { 1338 int access = (type == VHOST_ADDR_USED) ? 1339 VHOST_ACCESS_WO : VHOST_ACCESS_RO; 1340 1341 if (likely(map->perm & access)) 1342 vq->meta_iotlb[type] = map; 1343 } 1344 1345 static bool iotlb_access_ok(struct vhost_virtqueue *vq, 1346 int access, u64 addr, u64 len, int type) 1347 { 1348 const struct vhost_iotlb_map *map; 1349 struct vhost_iotlb *umem = vq->iotlb; 1350 u64 s = 0, size, orig_addr = addr, last = addr + len - 1; 1351 1352 if (vhost_vq_meta_fetch(vq, addr, len, type)) 1353 return true; 1354 1355 while (len > s) { 1356 map = vhost_iotlb_itree_first(umem, addr, last); 1357 if (map == NULL || map->start > addr) { 1358 vhost_iotlb_miss(vq, addr, access); 1359 return false; 1360 } else if (!(map->perm & access)) { 1361 /* Report the possible access violation by 1362 * request another translation from userspace. 1363 */ 1364 return false; 1365 } 1366 1367 size = map->size - addr + map->start; 1368 1369 if (orig_addr == addr && size >= len) 1370 vhost_vq_meta_update(vq, map, type); 1371 1372 s += size; 1373 addr += size; 1374 } 1375 1376 return true; 1377 } 1378 1379 int vq_meta_prefetch(struct vhost_virtqueue *vq) 1380 { 1381 unsigned int num = vq->num; 1382 1383 if (!vq->iotlb) 1384 return 1; 1385 1386 return iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->desc, 1387 vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) && 1388 iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->avail, 1389 vhost_get_avail_size(vq, num), 1390 VHOST_ADDR_AVAIL) && 1391 iotlb_access_ok(vq, VHOST_MAP_WO, (u64)(uintptr_t)vq->used, 1392 vhost_get_used_size(vq, num), VHOST_ADDR_USED); 1393 } 1394 EXPORT_SYMBOL_GPL(vq_meta_prefetch); 1395 1396 /* Can we log writes? */ 1397 /* Caller should have device mutex but not vq mutex */ 1398 bool vhost_log_access_ok(struct vhost_dev *dev) 1399 { 1400 return memory_access_ok(dev, dev->umem, 1); 1401 } 1402 EXPORT_SYMBOL_GPL(vhost_log_access_ok); 1403 1404 static bool vq_log_used_access_ok(struct vhost_virtqueue *vq, 1405 void __user *log_base, 1406 bool log_used, 1407 u64 log_addr) 1408 { 1409 /* If an IOTLB device is present, log_addr is a GIOVA that 1410 * will never be logged by log_used(). */ 1411 if (vq->iotlb) 1412 return true; 1413 1414 return !log_used || log_access_ok(log_base, log_addr, 1415 vhost_get_used_size(vq, vq->num)); 1416 } 1417 1418 /* Verify access for write logging. */ 1419 /* Caller should have vq mutex and device mutex */ 1420 static bool vq_log_access_ok(struct vhost_virtqueue *vq, 1421 void __user *log_base) 1422 { 1423 return vq_memory_access_ok(log_base, vq->umem, 1424 vhost_has_feature(vq, VHOST_F_LOG_ALL)) && 1425 vq_log_used_access_ok(vq, log_base, vq->log_used, vq->log_addr); 1426 } 1427 1428 /* Can we start vq? */ 1429 /* Caller should have vq mutex and device mutex */ 1430 bool vhost_vq_access_ok(struct vhost_virtqueue *vq) 1431 { 1432 if (!vq_log_access_ok(vq, vq->log_base)) 1433 return false; 1434 1435 return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used); 1436 } 1437 EXPORT_SYMBOL_GPL(vhost_vq_access_ok); 1438 1439 static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) 1440 { 1441 struct vhost_memory mem, *newmem; 1442 struct vhost_memory_region *region; 1443 struct vhost_iotlb *newumem, *oldumem; 1444 unsigned long size = offsetof(struct vhost_memory, regions); 1445 int i; 1446 1447 if (copy_from_user(&mem, m, size)) 1448 return -EFAULT; 1449 if (mem.padding) 1450 return -EOPNOTSUPP; 1451 if (mem.nregions > max_mem_regions) 1452 return -E2BIG; 1453 newmem = kvzalloc(struct_size(newmem, regions, mem.nregions), 1454 GFP_KERNEL); 1455 if (!newmem) 1456 return -ENOMEM; 1457 1458 memcpy(newmem, &mem, size); 1459 if (copy_from_user(newmem->regions, m->regions, 1460 flex_array_size(newmem, regions, mem.nregions))) { 1461 kvfree(newmem); 1462 return -EFAULT; 1463 } 1464 1465 newumem = iotlb_alloc(); 1466 if (!newumem) { 1467 kvfree(newmem); 1468 return -ENOMEM; 1469 } 1470 1471 for (region = newmem->regions; 1472 region < newmem->regions + mem.nregions; 1473 region++) { 1474 if (vhost_iotlb_add_range(newumem, 1475 region->guest_phys_addr, 1476 region->guest_phys_addr + 1477 region->memory_size - 1, 1478 region->userspace_addr, 1479 VHOST_MAP_RW)) 1480 goto err; 1481 } 1482 1483 if (!memory_access_ok(d, newumem, 0)) 1484 goto err; 1485 1486 oldumem = d->umem; 1487 d->umem = newumem; 1488 1489 /* All memory accesses are done under some VQ mutex. */ 1490 for (i = 0; i < d->nvqs; ++i) { 1491 mutex_lock(&d->vqs[i]->mutex); 1492 d->vqs[i]->umem = newumem; 1493 mutex_unlock(&d->vqs[i]->mutex); 1494 } 1495 1496 kvfree(newmem); 1497 vhost_iotlb_free(oldumem); 1498 return 0; 1499 1500 err: 1501 vhost_iotlb_free(newumem); 1502 kvfree(newmem); 1503 return -EFAULT; 1504 } 1505 1506 static long vhost_vring_set_num(struct vhost_dev *d, 1507 struct vhost_virtqueue *vq, 1508 void __user *argp) 1509 { 1510 struct vhost_vring_state s; 1511 1512 /* Resizing ring with an active backend? 1513 * You don't want to do that. */ 1514 if (vq->private_data) 1515 return -EBUSY; 1516 1517 if (copy_from_user(&s, argp, sizeof s)) 1518 return -EFAULT; 1519 1520 if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) 1521 return -EINVAL; 1522 vq->num = s.num; 1523 1524 return 0; 1525 } 1526 1527 static long vhost_vring_set_addr(struct vhost_dev *d, 1528 struct vhost_virtqueue *vq, 1529 void __user *argp) 1530 { 1531 struct vhost_vring_addr a; 1532 1533 if (copy_from_user(&a, argp, sizeof a)) 1534 return -EFAULT; 1535 if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) 1536 return -EOPNOTSUPP; 1537 1538 /* For 32bit, verify that the top 32bits of the user 1539 data are set to zero. */ 1540 if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr || 1541 (u64)(unsigned long)a.used_user_addr != a.used_user_addr || 1542 (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) 1543 return -EFAULT; 1544 1545 /* Make sure it's safe to cast pointers to vring types. */ 1546 BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE); 1547 BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE); 1548 if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) || 1549 (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) || 1550 (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1))) 1551 return -EINVAL; 1552 1553 /* We only verify access here if backend is configured. 1554 * If it is not, we don't as size might not have been setup. 1555 * We will verify when backend is configured. */ 1556 if (vq->private_data) { 1557 if (!vq_access_ok(vq, vq->num, 1558 (void __user *)(unsigned long)a.desc_user_addr, 1559 (void __user *)(unsigned long)a.avail_user_addr, 1560 (void __user *)(unsigned long)a.used_user_addr)) 1561 return -EINVAL; 1562 1563 /* Also validate log access for used ring if enabled. */ 1564 if (!vq_log_used_access_ok(vq, vq->log_base, 1565 a.flags & (0x1 << VHOST_VRING_F_LOG), 1566 a.log_guest_addr)) 1567 return -EINVAL; 1568 } 1569 1570 vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG)); 1571 vq->desc = (void __user *)(unsigned long)a.desc_user_addr; 1572 vq->avail = (void __user *)(unsigned long)a.avail_user_addr; 1573 vq->log_addr = a.log_guest_addr; 1574 vq->used = (void __user *)(unsigned long)a.used_user_addr; 1575 1576 return 0; 1577 } 1578 1579 static long vhost_vring_set_num_addr(struct vhost_dev *d, 1580 struct vhost_virtqueue *vq, 1581 unsigned int ioctl, 1582 void __user *argp) 1583 { 1584 long r; 1585 1586 mutex_lock(&vq->mutex); 1587 1588 switch (ioctl) { 1589 case VHOST_SET_VRING_NUM: 1590 r = vhost_vring_set_num(d, vq, argp); 1591 break; 1592 case VHOST_SET_VRING_ADDR: 1593 r = vhost_vring_set_addr(d, vq, argp); 1594 break; 1595 default: 1596 BUG(); 1597 } 1598 1599 mutex_unlock(&vq->mutex); 1600 1601 return r; 1602 } 1603 long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) 1604 { 1605 struct file *eventfp, *filep = NULL; 1606 bool pollstart = false, pollstop = false; 1607 struct eventfd_ctx *ctx = NULL; 1608 u32 __user *idxp = argp; 1609 struct vhost_virtqueue *vq; 1610 struct vhost_vring_state s; 1611 struct vhost_vring_file f; 1612 u32 idx; 1613 long r; 1614 1615 r = get_user(idx, idxp); 1616 if (r < 0) 1617 return r; 1618 if (idx >= d->nvqs) 1619 return -ENOBUFS; 1620 1621 idx = array_index_nospec(idx, d->nvqs); 1622 vq = d->vqs[idx]; 1623 1624 if (ioctl == VHOST_SET_VRING_NUM || 1625 ioctl == VHOST_SET_VRING_ADDR) { 1626 return vhost_vring_set_num_addr(d, vq, ioctl, argp); 1627 } 1628 1629 mutex_lock(&vq->mutex); 1630 1631 switch (ioctl) { 1632 case VHOST_SET_VRING_BASE: 1633 /* Moving base with an active backend? 1634 * You don't want to do that. */ 1635 if (vq->private_data) { 1636 r = -EBUSY; 1637 break; 1638 } 1639 if (copy_from_user(&s, argp, sizeof s)) { 1640 r = -EFAULT; 1641 break; 1642 } 1643 if (s.num > 0xffff) { 1644 r = -EINVAL; 1645 break; 1646 } 1647 vq->last_avail_idx = s.num; 1648 /* Forget the cached index value. */ 1649 vq->avail_idx = vq->last_avail_idx; 1650 break; 1651 case VHOST_GET_VRING_BASE: 1652 s.index = idx; 1653 s.num = vq->last_avail_idx; 1654 if (copy_to_user(argp, &s, sizeof s)) 1655 r = -EFAULT; 1656 break; 1657 case VHOST_SET_VRING_KICK: 1658 if (copy_from_user(&f, argp, sizeof f)) { 1659 r = -EFAULT; 1660 break; 1661 } 1662 eventfp = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_fget(f.fd); 1663 if (IS_ERR(eventfp)) { 1664 r = PTR_ERR(eventfp); 1665 break; 1666 } 1667 if (eventfp != vq->kick) { 1668 pollstop = (filep = vq->kick) != NULL; 1669 pollstart = (vq->kick = eventfp) != NULL; 1670 } else 1671 filep = eventfp; 1672 break; 1673 case VHOST_SET_VRING_CALL: 1674 if (copy_from_user(&f, argp, sizeof f)) { 1675 r = -EFAULT; 1676 break; 1677 } 1678 ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd); 1679 if (IS_ERR(ctx)) { 1680 r = PTR_ERR(ctx); 1681 break; 1682 } 1683 1684 swap(ctx, vq->call_ctx.ctx); 1685 break; 1686 case VHOST_SET_VRING_ERR: 1687 if (copy_from_user(&f, argp, sizeof f)) { 1688 r = -EFAULT; 1689 break; 1690 } 1691 ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd); 1692 if (IS_ERR(ctx)) { 1693 r = PTR_ERR(ctx); 1694 break; 1695 } 1696 swap(ctx, vq->error_ctx); 1697 break; 1698 case VHOST_SET_VRING_ENDIAN: 1699 r = vhost_set_vring_endian(vq, argp); 1700 break; 1701 case VHOST_GET_VRING_ENDIAN: 1702 r = vhost_get_vring_endian(vq, idx, argp); 1703 break; 1704 case VHOST_SET_VRING_BUSYLOOP_TIMEOUT: 1705 if (copy_from_user(&s, argp, sizeof(s))) { 1706 r = -EFAULT; 1707 break; 1708 } 1709 vq->busyloop_timeout = s.num; 1710 break; 1711 case VHOST_GET_VRING_BUSYLOOP_TIMEOUT: 1712 s.index = idx; 1713 s.num = vq->busyloop_timeout; 1714 if (copy_to_user(argp, &s, sizeof(s))) 1715 r = -EFAULT; 1716 break; 1717 default: 1718 r = -ENOIOCTLCMD; 1719 } 1720 1721 if (pollstop && vq->handle_kick) 1722 vhost_poll_stop(&vq->poll); 1723 1724 if (!IS_ERR_OR_NULL(ctx)) 1725 eventfd_ctx_put(ctx); 1726 if (filep) 1727 fput(filep); 1728 1729 if (pollstart && vq->handle_kick) 1730 r = vhost_poll_start(&vq->poll, vq->kick); 1731 1732 mutex_unlock(&vq->mutex); 1733 1734 if (pollstop && vq->handle_kick) 1735 vhost_poll_flush(&vq->poll); 1736 return r; 1737 } 1738 EXPORT_SYMBOL_GPL(vhost_vring_ioctl); 1739 1740 int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled) 1741 { 1742 struct vhost_iotlb *niotlb, *oiotlb; 1743 int i; 1744 1745 niotlb = iotlb_alloc(); 1746 if (!niotlb) 1747 return -ENOMEM; 1748 1749 oiotlb = d->iotlb; 1750 d->iotlb = niotlb; 1751 1752 for (i = 0; i < d->nvqs; ++i) { 1753 struct vhost_virtqueue *vq = d->vqs[i]; 1754 1755 mutex_lock(&vq->mutex); 1756 vq->iotlb = niotlb; 1757 __vhost_vq_meta_reset(vq); 1758 mutex_unlock(&vq->mutex); 1759 } 1760 1761 vhost_iotlb_free(oiotlb); 1762 1763 return 0; 1764 } 1765 EXPORT_SYMBOL_GPL(vhost_init_device_iotlb); 1766 1767 /* Caller must have device mutex */ 1768 long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) 1769 { 1770 struct eventfd_ctx *ctx; 1771 u64 p; 1772 long r; 1773 int i, fd; 1774 1775 /* If you are not the owner, you can become one */ 1776 if (ioctl == VHOST_SET_OWNER) { 1777 r = vhost_dev_set_owner(d); 1778 goto done; 1779 } 1780 1781 /* You must be the owner to do anything else */ 1782 r = vhost_dev_check_owner(d); 1783 if (r) 1784 goto done; 1785 1786 switch (ioctl) { 1787 case VHOST_SET_MEM_TABLE: 1788 r = vhost_set_memory(d, argp); 1789 break; 1790 case VHOST_SET_LOG_BASE: 1791 if (copy_from_user(&p, argp, sizeof p)) { 1792 r = -EFAULT; 1793 break; 1794 } 1795 if ((u64)(unsigned long)p != p) { 1796 r = -EFAULT; 1797 break; 1798 } 1799 for (i = 0; i < d->nvqs; ++i) { 1800 struct vhost_virtqueue *vq; 1801 void __user *base = (void __user *)(unsigned long)p; 1802 vq = d->vqs[i]; 1803 mutex_lock(&vq->mutex); 1804 /* If ring is inactive, will check when it's enabled. */ 1805 if (vq->private_data && !vq_log_access_ok(vq, base)) 1806 r = -EFAULT; 1807 else 1808 vq->log_base = base; 1809 mutex_unlock(&vq->mutex); 1810 } 1811 break; 1812 case VHOST_SET_LOG_FD: 1813 r = get_user(fd, (int __user *)argp); 1814 if (r < 0) 1815 break; 1816 ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd); 1817 if (IS_ERR(ctx)) { 1818 r = PTR_ERR(ctx); 1819 break; 1820 } 1821 swap(ctx, d->log_ctx); 1822 for (i = 0; i < d->nvqs; ++i) { 1823 mutex_lock(&d->vqs[i]->mutex); 1824 d->vqs[i]->log_ctx = d->log_ctx; 1825 mutex_unlock(&d->vqs[i]->mutex); 1826 } 1827 if (ctx) 1828 eventfd_ctx_put(ctx); 1829 break; 1830 default: 1831 r = -ENOIOCTLCMD; 1832 break; 1833 } 1834 done: 1835 return r; 1836 } 1837 EXPORT_SYMBOL_GPL(vhost_dev_ioctl); 1838 1839 /* TODO: This is really inefficient. We need something like get_user() 1840 * (instruction directly accesses the data, with an exception table entry 1841 * returning -EFAULT). See Documentation/x86/exception-tables.rst. 1842 */ 1843 static int set_bit_to_user(int nr, void __user *addr) 1844 { 1845 unsigned long log = (unsigned long)addr; 1846 struct page *page; 1847 void *base; 1848 int bit = nr + (log % PAGE_SIZE) * 8; 1849 int r; 1850 1851 r = pin_user_pages_fast(log, 1, FOLL_WRITE, &page); 1852 if (r < 0) 1853 return r; 1854 BUG_ON(r != 1); 1855 base = kmap_atomic(page); 1856 set_bit(bit, base); 1857 kunmap_atomic(base); 1858 unpin_user_pages_dirty_lock(&page, 1, true); 1859 return 0; 1860 } 1861 1862 static int log_write(void __user *log_base, 1863 u64 write_address, u64 write_length) 1864 { 1865 u64 write_page = write_address / VHOST_PAGE_SIZE; 1866 int r; 1867 1868 if (!write_length) 1869 return 0; 1870 write_length += write_address % VHOST_PAGE_SIZE; 1871 for (;;) { 1872 u64 base = (u64)(unsigned long)log_base; 1873 u64 log = base + write_page / 8; 1874 int bit = write_page % 8; 1875 if ((u64)(unsigned long)log != log) 1876 return -EFAULT; 1877 r = set_bit_to_user(bit, (void __user *)(unsigned long)log); 1878 if (r < 0) 1879 return r; 1880 if (write_length <= VHOST_PAGE_SIZE) 1881 break; 1882 write_length -= VHOST_PAGE_SIZE; 1883 write_page += 1; 1884 } 1885 return r; 1886 } 1887 1888 static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len) 1889 { 1890 struct vhost_iotlb *umem = vq->umem; 1891 struct vhost_iotlb_map *u; 1892 u64 start, end, l, min; 1893 int r; 1894 bool hit = false; 1895 1896 while (len) { 1897 min = len; 1898 /* More than one GPAs can be mapped into a single HVA. So 1899 * iterate all possible umems here to be safe. 1900 */ 1901 list_for_each_entry(u, &umem->list, link) { 1902 if (u->addr > hva - 1 + len || 1903 u->addr - 1 + u->size < hva) 1904 continue; 1905 start = max(u->addr, hva); 1906 end = min(u->addr - 1 + u->size, hva - 1 + len); 1907 l = end - start + 1; 1908 r = log_write(vq->log_base, 1909 u->start + start - u->addr, 1910 l); 1911 if (r < 0) 1912 return r; 1913 hit = true; 1914 min = min(l, min); 1915 } 1916 1917 if (!hit) 1918 return -EFAULT; 1919 1920 len -= min; 1921 hva += min; 1922 } 1923 1924 return 0; 1925 } 1926 1927 static int log_used(struct vhost_virtqueue *vq, u64 used_offset, u64 len) 1928 { 1929 struct iovec *iov = vq->log_iov; 1930 int i, ret; 1931 1932 if (!vq->iotlb) 1933 return log_write(vq->log_base, vq->log_addr + used_offset, len); 1934 1935 ret = translate_desc(vq, (uintptr_t)vq->used + used_offset, 1936 len, iov, 64, VHOST_ACCESS_WO); 1937 if (ret < 0) 1938 return ret; 1939 1940 for (i = 0; i < ret; i++) { 1941 ret = log_write_hva(vq, (uintptr_t)iov[i].iov_base, 1942 iov[i].iov_len); 1943 if (ret) 1944 return ret; 1945 } 1946 1947 return 0; 1948 } 1949 1950 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, 1951 unsigned int log_num, u64 len, struct iovec *iov, int count) 1952 { 1953 int i, r; 1954 1955 /* Make sure data written is seen before log. */ 1956 smp_wmb(); 1957 1958 if (vq->iotlb) { 1959 for (i = 0; i < count; i++) { 1960 r = log_write_hva(vq, (uintptr_t)iov[i].iov_base, 1961 iov[i].iov_len); 1962 if (r < 0) 1963 return r; 1964 } 1965 return 0; 1966 } 1967 1968 for (i = 0; i < log_num; ++i) { 1969 u64 l = min(log[i].len, len); 1970 r = log_write(vq->log_base, log[i].addr, l); 1971 if (r < 0) 1972 return r; 1973 len -= l; 1974 if (!len) { 1975 if (vq->log_ctx) 1976 eventfd_signal(vq->log_ctx, 1); 1977 return 0; 1978 } 1979 } 1980 /* Length written exceeds what we have stored. This is a bug. */ 1981 BUG(); 1982 return 0; 1983 } 1984 EXPORT_SYMBOL_GPL(vhost_log_write); 1985 1986 static int vhost_update_used_flags(struct vhost_virtqueue *vq) 1987 { 1988 void __user *used; 1989 if (vhost_put_used_flags(vq)) 1990 return -EFAULT; 1991 if (unlikely(vq->log_used)) { 1992 /* Make sure the flag is seen before log. */ 1993 smp_wmb(); 1994 /* Log used flag write. */ 1995 used = &vq->used->flags; 1996 log_used(vq, (used - (void __user *)vq->used), 1997 sizeof vq->used->flags); 1998 if (vq->log_ctx) 1999 eventfd_signal(vq->log_ctx, 1); 2000 } 2001 return 0; 2002 } 2003 2004 static int vhost_update_avail_event(struct vhost_virtqueue *vq) 2005 { 2006 if (vhost_put_avail_event(vq)) 2007 return -EFAULT; 2008 if (unlikely(vq->log_used)) { 2009 void __user *used; 2010 /* Make sure the event is seen before log. */ 2011 smp_wmb(); 2012 /* Log avail event write */ 2013 used = vhost_avail_event(vq); 2014 log_used(vq, (used - (void __user *)vq->used), 2015 sizeof *vhost_avail_event(vq)); 2016 if (vq->log_ctx) 2017 eventfd_signal(vq->log_ctx, 1); 2018 } 2019 return 0; 2020 } 2021 2022 int vhost_vq_init_access(struct vhost_virtqueue *vq) 2023 { 2024 __virtio16 last_used_idx; 2025 int r; 2026 bool is_le = vq->is_le; 2027 2028 if (!vq->private_data) 2029 return 0; 2030 2031 vhost_init_is_le(vq); 2032 2033 r = vhost_update_used_flags(vq); 2034 if (r) 2035 goto err; 2036 vq->signalled_used_valid = false; 2037 if (!vq->iotlb && 2038 !access_ok(&vq->used->idx, sizeof vq->used->idx)) { 2039 r = -EFAULT; 2040 goto err; 2041 } 2042 r = vhost_get_used_idx(vq, &last_used_idx); 2043 if (r) { 2044 vq_err(vq, "Can't access used idx at %p\n", 2045 &vq->used->idx); 2046 goto err; 2047 } 2048 vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx); 2049 return 0; 2050 2051 err: 2052 vq->is_le = is_le; 2053 return r; 2054 } 2055 EXPORT_SYMBOL_GPL(vhost_vq_init_access); 2056 2057 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, 2058 struct iovec iov[], int iov_size, int access) 2059 { 2060 const struct vhost_iotlb_map *map; 2061 struct vhost_dev *dev = vq->dev; 2062 struct vhost_iotlb *umem = dev->iotlb ? dev->iotlb : dev->umem; 2063 struct iovec *_iov; 2064 u64 s = 0; 2065 int ret = 0; 2066 2067 while ((u64)len > s) { 2068 u64 size; 2069 if (unlikely(ret >= iov_size)) { 2070 ret = -ENOBUFS; 2071 break; 2072 } 2073 2074 map = vhost_iotlb_itree_first(umem, addr, addr + len - 1); 2075 if (map == NULL || map->start > addr) { 2076 if (umem != dev->iotlb) { 2077 ret = -EFAULT; 2078 break; 2079 } 2080 ret = -EAGAIN; 2081 break; 2082 } else if (!(map->perm & access)) { 2083 ret = -EPERM; 2084 break; 2085 } 2086 2087 _iov = iov + ret; 2088 size = map->size - addr + map->start; 2089 _iov->iov_len = min((u64)len - s, size); 2090 _iov->iov_base = (void __user *)(unsigned long) 2091 (map->addr + addr - map->start); 2092 s += size; 2093 addr += size; 2094 ++ret; 2095 } 2096 2097 if (ret == -EAGAIN) 2098 vhost_iotlb_miss(vq, addr, access); 2099 return ret; 2100 } 2101 2102 /* Each buffer in the virtqueues is actually a chain of descriptors. This 2103 * function returns the next descriptor in the chain, 2104 * or -1U if we're at the end. */ 2105 static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc) 2106 { 2107 unsigned int next; 2108 2109 /* If this descriptor says it doesn't chain, we're done. */ 2110 if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT))) 2111 return -1U; 2112 2113 /* Check they're not leading us off end of descriptors. */ 2114 next = vhost16_to_cpu(vq, READ_ONCE(desc->next)); 2115 return next; 2116 } 2117 2118 static int get_indirect(struct vhost_virtqueue *vq, 2119 struct iovec iov[], unsigned int iov_size, 2120 unsigned int *out_num, unsigned int *in_num, 2121 struct vhost_log *log, unsigned int *log_num, 2122 struct vring_desc *indirect) 2123 { 2124 struct vring_desc desc; 2125 unsigned int i = 0, count, found = 0; 2126 u32 len = vhost32_to_cpu(vq, indirect->len); 2127 struct iov_iter from; 2128 int ret, access; 2129 2130 /* Sanity check */ 2131 if (unlikely(len % sizeof desc)) { 2132 vq_err(vq, "Invalid length in indirect descriptor: " 2133 "len 0x%llx not multiple of 0x%zx\n", 2134 (unsigned long long)len, 2135 sizeof desc); 2136 return -EINVAL; 2137 } 2138 2139 ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect, 2140 UIO_MAXIOV, VHOST_ACCESS_RO); 2141 if (unlikely(ret < 0)) { 2142 if (ret != -EAGAIN) 2143 vq_err(vq, "Translation failure %d in indirect.\n", ret); 2144 return ret; 2145 } 2146 iov_iter_init(&from, READ, vq->indirect, ret, len); 2147 count = len / sizeof desc; 2148 /* Buffers are chained via a 16 bit next field, so 2149 * we can have at most 2^16 of these. */ 2150 if (unlikely(count > USHRT_MAX + 1)) { 2151 vq_err(vq, "Indirect buffer length too big: %d\n", 2152 indirect->len); 2153 return -E2BIG; 2154 } 2155 2156 do { 2157 unsigned iov_count = *in_num + *out_num; 2158 if (unlikely(++found > count)) { 2159 vq_err(vq, "Loop detected: last one at %u " 2160 "indirect size %u\n", 2161 i, count); 2162 return -EINVAL; 2163 } 2164 if (unlikely(!copy_from_iter_full(&desc, sizeof(desc), &from))) { 2165 vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n", 2166 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); 2167 return -EINVAL; 2168 } 2169 if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) { 2170 vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n", 2171 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); 2172 return -EINVAL; 2173 } 2174 2175 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) 2176 access = VHOST_ACCESS_WO; 2177 else 2178 access = VHOST_ACCESS_RO; 2179 2180 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), 2181 vhost32_to_cpu(vq, desc.len), iov + iov_count, 2182 iov_size - iov_count, access); 2183 if (unlikely(ret < 0)) { 2184 if (ret != -EAGAIN) 2185 vq_err(vq, "Translation failure %d indirect idx %d\n", 2186 ret, i); 2187 return ret; 2188 } 2189 /* If this is an input descriptor, increment that count. */ 2190 if (access == VHOST_ACCESS_WO) { 2191 *in_num += ret; 2192 if (unlikely(log && ret)) { 2193 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); 2194 log[*log_num].len = vhost32_to_cpu(vq, desc.len); 2195 ++*log_num; 2196 } 2197 } else { 2198 /* If it's an output descriptor, they're all supposed 2199 * to come before any input descriptors. */ 2200 if (unlikely(*in_num)) { 2201 vq_err(vq, "Indirect descriptor " 2202 "has out after in: idx %d\n", i); 2203 return -EINVAL; 2204 } 2205 *out_num += ret; 2206 } 2207 } while ((i = next_desc(vq, &desc)) != -1); 2208 return 0; 2209 } 2210 2211 /* This looks in the virtqueue and for the first available buffer, and converts 2212 * it to an iovec for convenient access. Since descriptors consist of some 2213 * number of output then some number of input descriptors, it's actually two 2214 * iovecs, but we pack them into one and note how many of each there were. 2215 * 2216 * This function returns the descriptor number found, or vq->num (which is 2217 * never a valid descriptor number) if none was found. A negative code is 2218 * returned on error. */ 2219 int vhost_get_vq_desc(struct vhost_virtqueue *vq, 2220 struct iovec iov[], unsigned int iov_size, 2221 unsigned int *out_num, unsigned int *in_num, 2222 struct vhost_log *log, unsigned int *log_num) 2223 { 2224 struct vring_desc desc; 2225 unsigned int i, head, found = 0; 2226 u16 last_avail_idx; 2227 __virtio16 avail_idx; 2228 __virtio16 ring_head; 2229 int ret, access; 2230 2231 /* Check it isn't doing very strange things with descriptor numbers. */ 2232 last_avail_idx = vq->last_avail_idx; 2233 2234 if (vq->avail_idx == vq->last_avail_idx) { 2235 if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) { 2236 vq_err(vq, "Failed to access avail idx at %p\n", 2237 &vq->avail->idx); 2238 return -EFAULT; 2239 } 2240 vq->avail_idx = vhost16_to_cpu(vq, avail_idx); 2241 2242 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) { 2243 vq_err(vq, "Guest moved used index from %u to %u", 2244 last_avail_idx, vq->avail_idx); 2245 return -EFAULT; 2246 } 2247 2248 /* If there's nothing new since last we looked, return 2249 * invalid. 2250 */ 2251 if (vq->avail_idx == last_avail_idx) 2252 return vq->num; 2253 2254 /* Only get avail ring entries after they have been 2255 * exposed by guest. 2256 */ 2257 smp_rmb(); 2258 } 2259 2260 /* Grab the next descriptor number they're advertising, and increment 2261 * the index we've seen. */ 2262 if (unlikely(vhost_get_avail_head(vq, &ring_head, last_avail_idx))) { 2263 vq_err(vq, "Failed to read head: idx %d address %p\n", 2264 last_avail_idx, 2265 &vq->avail->ring[last_avail_idx % vq->num]); 2266 return -EFAULT; 2267 } 2268 2269 head = vhost16_to_cpu(vq, ring_head); 2270 2271 /* If their number is silly, that's an error. */ 2272 if (unlikely(head >= vq->num)) { 2273 vq_err(vq, "Guest says index %u > %u is available", 2274 head, vq->num); 2275 return -EINVAL; 2276 } 2277 2278 /* When we start there are none of either input nor output. */ 2279 *out_num = *in_num = 0; 2280 if (unlikely(log)) 2281 *log_num = 0; 2282 2283 i = head; 2284 do { 2285 unsigned iov_count = *in_num + *out_num; 2286 if (unlikely(i >= vq->num)) { 2287 vq_err(vq, "Desc index is %u > %u, head = %u", 2288 i, vq->num, head); 2289 return -EINVAL; 2290 } 2291 if (unlikely(++found > vq->num)) { 2292 vq_err(vq, "Loop detected: last one at %u " 2293 "vq size %u head %u\n", 2294 i, vq->num, head); 2295 return -EINVAL; 2296 } 2297 ret = vhost_get_desc(vq, &desc, i); 2298 if (unlikely(ret)) { 2299 vq_err(vq, "Failed to get descriptor: idx %d addr %p\n", 2300 i, vq->desc + i); 2301 return -EFAULT; 2302 } 2303 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) { 2304 ret = get_indirect(vq, iov, iov_size, 2305 out_num, in_num, 2306 log, log_num, &desc); 2307 if (unlikely(ret < 0)) { 2308 if (ret != -EAGAIN) 2309 vq_err(vq, "Failure detected " 2310 "in indirect descriptor at idx %d\n", i); 2311 return ret; 2312 } 2313 continue; 2314 } 2315 2316 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) 2317 access = VHOST_ACCESS_WO; 2318 else 2319 access = VHOST_ACCESS_RO; 2320 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), 2321 vhost32_to_cpu(vq, desc.len), iov + iov_count, 2322 iov_size - iov_count, access); 2323 if (unlikely(ret < 0)) { 2324 if (ret != -EAGAIN) 2325 vq_err(vq, "Translation failure %d descriptor idx %d\n", 2326 ret, i); 2327 return ret; 2328 } 2329 if (access == VHOST_ACCESS_WO) { 2330 /* If this is an input descriptor, 2331 * increment that count. */ 2332 *in_num += ret; 2333 if (unlikely(log && ret)) { 2334 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); 2335 log[*log_num].len = vhost32_to_cpu(vq, desc.len); 2336 ++*log_num; 2337 } 2338 } else { 2339 /* If it's an output descriptor, they're all supposed 2340 * to come before any input descriptors. */ 2341 if (unlikely(*in_num)) { 2342 vq_err(vq, "Descriptor has out after in: " 2343 "idx %d\n", i); 2344 return -EINVAL; 2345 } 2346 *out_num += ret; 2347 } 2348 } while ((i = next_desc(vq, &desc)) != -1); 2349 2350 /* On success, increment avail index. */ 2351 vq->last_avail_idx++; 2352 2353 /* Assume notifications from guest are disabled at this point, 2354 * if they aren't we would need to update avail_event index. */ 2355 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY)); 2356 return head; 2357 } 2358 EXPORT_SYMBOL_GPL(vhost_get_vq_desc); 2359 2360 /* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */ 2361 void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n) 2362 { 2363 vq->last_avail_idx -= n; 2364 } 2365 EXPORT_SYMBOL_GPL(vhost_discard_vq_desc); 2366 2367 /* After we've used one of their buffers, we tell them about it. We'll then 2368 * want to notify the guest, using eventfd. */ 2369 int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len) 2370 { 2371 struct vring_used_elem heads = { 2372 cpu_to_vhost32(vq, head), 2373 cpu_to_vhost32(vq, len) 2374 }; 2375 2376 return vhost_add_used_n(vq, &heads, 1); 2377 } 2378 EXPORT_SYMBOL_GPL(vhost_add_used); 2379 2380 static int __vhost_add_used_n(struct vhost_virtqueue *vq, 2381 struct vring_used_elem *heads, 2382 unsigned count) 2383 { 2384 vring_used_elem_t __user *used; 2385 u16 old, new; 2386 int start; 2387 2388 start = vq->last_used_idx & (vq->num - 1); 2389 used = vq->used->ring + start; 2390 if (vhost_put_used(vq, heads, start, count)) { 2391 vq_err(vq, "Failed to write used"); 2392 return -EFAULT; 2393 } 2394 if (unlikely(vq->log_used)) { 2395 /* Make sure data is seen before log. */ 2396 smp_wmb(); 2397 /* Log used ring entry write. */ 2398 log_used(vq, ((void __user *)used - (void __user *)vq->used), 2399 count * sizeof *used); 2400 } 2401 old = vq->last_used_idx; 2402 new = (vq->last_used_idx += count); 2403 /* If the driver never bothers to signal in a very long while, 2404 * used index might wrap around. If that happens, invalidate 2405 * signalled_used index we stored. TODO: make sure driver 2406 * signals at least once in 2^16 and remove this. */ 2407 if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old))) 2408 vq->signalled_used_valid = false; 2409 return 0; 2410 } 2411 2412 /* After we've used one of their buffers, we tell them about it. We'll then 2413 * want to notify the guest, using eventfd. */ 2414 int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, 2415 unsigned count) 2416 { 2417 int start, n, r; 2418 2419 start = vq->last_used_idx & (vq->num - 1); 2420 n = vq->num - start; 2421 if (n < count) { 2422 r = __vhost_add_used_n(vq, heads, n); 2423 if (r < 0) 2424 return r; 2425 heads += n; 2426 count -= n; 2427 } 2428 r = __vhost_add_used_n(vq, heads, count); 2429 2430 /* Make sure buffer is written before we update index. */ 2431 smp_wmb(); 2432 if (vhost_put_used_idx(vq)) { 2433 vq_err(vq, "Failed to increment used idx"); 2434 return -EFAULT; 2435 } 2436 if (unlikely(vq->log_used)) { 2437 /* Make sure used idx is seen before log. */ 2438 smp_wmb(); 2439 /* Log used index update. */ 2440 log_used(vq, offsetof(struct vring_used, idx), 2441 sizeof vq->used->idx); 2442 if (vq->log_ctx) 2443 eventfd_signal(vq->log_ctx, 1); 2444 } 2445 return r; 2446 } 2447 EXPORT_SYMBOL_GPL(vhost_add_used_n); 2448 2449 static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2450 { 2451 __u16 old, new; 2452 __virtio16 event; 2453 bool v; 2454 /* Flush out used index updates. This is paired 2455 * with the barrier that the Guest executes when enabling 2456 * interrupts. */ 2457 smp_mb(); 2458 2459 if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) && 2460 unlikely(vq->avail_idx == vq->last_avail_idx)) 2461 return true; 2462 2463 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { 2464 __virtio16 flags; 2465 if (vhost_get_avail_flags(vq, &flags)) { 2466 vq_err(vq, "Failed to get flags"); 2467 return true; 2468 } 2469 return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT)); 2470 } 2471 old = vq->signalled_used; 2472 v = vq->signalled_used_valid; 2473 new = vq->signalled_used = vq->last_used_idx; 2474 vq->signalled_used_valid = true; 2475 2476 if (unlikely(!v)) 2477 return true; 2478 2479 if (vhost_get_used_event(vq, &event)) { 2480 vq_err(vq, "Failed to get used event idx"); 2481 return true; 2482 } 2483 return vring_need_event(vhost16_to_cpu(vq, event), new, old); 2484 } 2485 2486 /* This actually signals the guest, using eventfd. */ 2487 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2488 { 2489 /* Signal the Guest tell them we used something up. */ 2490 if (vq->call_ctx.ctx && vhost_notify(dev, vq)) 2491 eventfd_signal(vq->call_ctx.ctx, 1); 2492 } 2493 EXPORT_SYMBOL_GPL(vhost_signal); 2494 2495 /* And here's the combo meal deal. Supersize me! */ 2496 void vhost_add_used_and_signal(struct vhost_dev *dev, 2497 struct vhost_virtqueue *vq, 2498 unsigned int head, int len) 2499 { 2500 vhost_add_used(vq, head, len); 2501 vhost_signal(dev, vq); 2502 } 2503 EXPORT_SYMBOL_GPL(vhost_add_used_and_signal); 2504 2505 /* multi-buffer version of vhost_add_used_and_signal */ 2506 void vhost_add_used_and_signal_n(struct vhost_dev *dev, 2507 struct vhost_virtqueue *vq, 2508 struct vring_used_elem *heads, unsigned count) 2509 { 2510 vhost_add_used_n(vq, heads, count); 2511 vhost_signal(dev, vq); 2512 } 2513 EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n); 2514 2515 /* return true if we're sure that avaiable ring is empty */ 2516 bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2517 { 2518 __virtio16 avail_idx; 2519 int r; 2520 2521 if (vq->avail_idx != vq->last_avail_idx) 2522 return false; 2523 2524 r = vhost_get_avail_idx(vq, &avail_idx); 2525 if (unlikely(r)) 2526 return false; 2527 vq->avail_idx = vhost16_to_cpu(vq, avail_idx); 2528 2529 return vq->avail_idx == vq->last_avail_idx; 2530 } 2531 EXPORT_SYMBOL_GPL(vhost_vq_avail_empty); 2532 2533 /* OK, now we need to know about added descriptors. */ 2534 bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2535 { 2536 __virtio16 avail_idx; 2537 int r; 2538 2539 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) 2540 return false; 2541 vq->used_flags &= ~VRING_USED_F_NO_NOTIFY; 2542 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { 2543 r = vhost_update_used_flags(vq); 2544 if (r) { 2545 vq_err(vq, "Failed to enable notification at %p: %d\n", 2546 &vq->used->flags, r); 2547 return false; 2548 } 2549 } else { 2550 r = vhost_update_avail_event(vq); 2551 if (r) { 2552 vq_err(vq, "Failed to update avail event index at %p: %d\n", 2553 vhost_avail_event(vq), r); 2554 return false; 2555 } 2556 } 2557 /* They could have slipped one in as we were doing that: make 2558 * sure it's written, then check again. */ 2559 smp_mb(); 2560 r = vhost_get_avail_idx(vq, &avail_idx); 2561 if (r) { 2562 vq_err(vq, "Failed to check avail idx at %p: %d\n", 2563 &vq->avail->idx, r); 2564 return false; 2565 } 2566 vq->avail_idx = vhost16_to_cpu(vq, avail_idx); 2567 2568 return vq->avail_idx != vq->last_avail_idx; 2569 } 2570 EXPORT_SYMBOL_GPL(vhost_enable_notify); 2571 2572 /* We don't need to be notified again. */ 2573 void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2574 { 2575 int r; 2576 2577 if (vq->used_flags & VRING_USED_F_NO_NOTIFY) 2578 return; 2579 vq->used_flags |= VRING_USED_F_NO_NOTIFY; 2580 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { 2581 r = vhost_update_used_flags(vq); 2582 if (r) 2583 vq_err(vq, "Failed to disable notification at %p: %d\n", 2584 &vq->used->flags, r); 2585 } 2586 } 2587 EXPORT_SYMBOL_GPL(vhost_disable_notify); 2588 2589 /* Create a new message. */ 2590 struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type) 2591 { 2592 struct vhost_msg_node *node = kmalloc(sizeof *node, GFP_KERNEL); 2593 if (!node) 2594 return NULL; 2595 2596 /* Make sure all padding within the structure is initialized. */ 2597 memset(&node->msg, 0, sizeof node->msg); 2598 node->vq = vq; 2599 node->msg.type = type; 2600 return node; 2601 } 2602 EXPORT_SYMBOL_GPL(vhost_new_msg); 2603 2604 void vhost_enqueue_msg(struct vhost_dev *dev, struct list_head *head, 2605 struct vhost_msg_node *node) 2606 { 2607 spin_lock(&dev->iotlb_lock); 2608 list_add_tail(&node->node, head); 2609 spin_unlock(&dev->iotlb_lock); 2610 2611 wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM); 2612 } 2613 EXPORT_SYMBOL_GPL(vhost_enqueue_msg); 2614 2615 struct vhost_msg_node *vhost_dequeue_msg(struct vhost_dev *dev, 2616 struct list_head *head) 2617 { 2618 struct vhost_msg_node *node = NULL; 2619 2620 spin_lock(&dev->iotlb_lock); 2621 if (!list_empty(head)) { 2622 node = list_first_entry(head, struct vhost_msg_node, 2623 node); 2624 list_del(&node->node); 2625 } 2626 spin_unlock(&dev->iotlb_lock); 2627 2628 return node; 2629 } 2630 EXPORT_SYMBOL_GPL(vhost_dequeue_msg); 2631 2632 void vhost_set_backend_features(struct vhost_dev *dev, u64 features) 2633 { 2634 struct vhost_virtqueue *vq; 2635 int i; 2636 2637 mutex_lock(&dev->mutex); 2638 for (i = 0; i < dev->nvqs; ++i) { 2639 vq = dev->vqs[i]; 2640 mutex_lock(&vq->mutex); 2641 vq->acked_backend_features = features; 2642 mutex_unlock(&vq->mutex); 2643 } 2644 mutex_unlock(&dev->mutex); 2645 } 2646 EXPORT_SYMBOL_GPL(vhost_set_backend_features); 2647 2648 static int __init vhost_init(void) 2649 { 2650 return 0; 2651 } 2652 2653 static void __exit vhost_exit(void) 2654 { 2655 } 2656 2657 module_init(vhost_init); 2658 module_exit(vhost_exit); 2659 2660 MODULE_VERSION("0.0.1"); 2661 MODULE_LICENSE("GPL v2"); 2662 MODULE_AUTHOR("Michael S. Tsirkin"); 2663 MODULE_DESCRIPTION("Host kernel accelerator for virtio"); 2664