1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2009 Red Hat, Inc. 3 * Copyright (C) 2006 Rusty Russell IBM Corporation 4 * 5 * Author: Michael S. Tsirkin <mst@redhat.com> 6 * 7 * Inspiration, some code, and most witty comments come from 8 * Documentation/virtual/lguest/lguest.c, by Rusty Russell 9 * 10 * Generic code for virtio server in host kernel. 11 */ 12 13 #include <linux/eventfd.h> 14 #include <linux/vhost.h> 15 #include <linux/uio.h> 16 #include <linux/mm.h> 17 #include <linux/miscdevice.h> 18 #include <linux/mutex.h> 19 #include <linux/poll.h> 20 #include <linux/file.h> 21 #include <linux/highmem.h> 22 #include <linux/slab.h> 23 #include <linux/vmalloc.h> 24 #include <linux/kthread.h> 25 #include <linux/cgroup.h> 26 #include <linux/module.h> 27 #include <linux/sort.h> 28 #include <linux/sched/mm.h> 29 #include <linux/sched/signal.h> 30 #include <linux/interval_tree_generic.h> 31 #include <linux/nospec.h> 32 #include <linux/kcov.h> 33 34 #include "vhost.h" 35 36 static ushort max_mem_regions = 64; 37 module_param(max_mem_regions, ushort, 0444); 38 MODULE_PARM_DESC(max_mem_regions, 39 "Maximum number of memory regions in memory map. (default: 64)"); 40 static int max_iotlb_entries = 2048; 41 module_param(max_iotlb_entries, int, 0444); 42 MODULE_PARM_DESC(max_iotlb_entries, 43 "Maximum number of iotlb entries. (default: 2048)"); 44 45 enum { 46 VHOST_MEMORY_F_LOG = 0x1, 47 }; 48 49 #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num]) 50 #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num]) 51 52 #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY 53 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) 54 { 55 vq->user_be = !virtio_legacy_is_little_endian(); 56 } 57 58 static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq) 59 { 60 vq->user_be = true; 61 } 62 63 static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq) 64 { 65 vq->user_be = false; 66 } 67 68 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp) 69 { 70 struct vhost_vring_state s; 71 72 if (vq->private_data) 73 return -EBUSY; 74 75 if (copy_from_user(&s, argp, sizeof(s))) 76 return -EFAULT; 77 78 if (s.num != VHOST_VRING_LITTLE_ENDIAN && 79 s.num != VHOST_VRING_BIG_ENDIAN) 80 return -EINVAL; 81 82 if (s.num == VHOST_VRING_BIG_ENDIAN) 83 vhost_enable_cross_endian_big(vq); 84 else 85 vhost_enable_cross_endian_little(vq); 86 87 return 0; 88 } 89 90 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, 91 int __user *argp) 92 { 93 struct vhost_vring_state s = { 94 .index = idx, 95 .num = vq->user_be 96 }; 97 98 if (copy_to_user(argp, &s, sizeof(s))) 99 return -EFAULT; 100 101 return 0; 102 } 103 104 static void vhost_init_is_le(struct vhost_virtqueue *vq) 105 { 106 /* Note for legacy virtio: user_be is initialized at reset time 107 * according to the host endianness. If userspace does not set an 108 * explicit endianness, the default behavior is native endian, as 109 * expected by legacy virtio. 110 */ 111 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be; 112 } 113 #else 114 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) 115 { 116 } 117 118 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp) 119 { 120 return -ENOIOCTLCMD; 121 } 122 123 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, 124 int __user *argp) 125 { 126 return -ENOIOCTLCMD; 127 } 128 129 static void vhost_init_is_le(struct vhost_virtqueue *vq) 130 { 131 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) 132 || virtio_legacy_is_little_endian(); 133 } 134 #endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */ 135 136 static void vhost_reset_is_le(struct vhost_virtqueue *vq) 137 { 138 vhost_init_is_le(vq); 139 } 140 141 struct vhost_flush_struct { 142 struct vhost_work work; 143 struct completion wait_event; 144 }; 145 146 static void vhost_flush_work(struct vhost_work *work) 147 { 148 struct vhost_flush_struct *s; 149 150 s = container_of(work, struct vhost_flush_struct, work); 151 complete(&s->wait_event); 152 } 153 154 static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh, 155 poll_table *pt) 156 { 157 struct vhost_poll *poll; 158 159 poll = container_of(pt, struct vhost_poll, table); 160 poll->wqh = wqh; 161 add_wait_queue(wqh, &poll->wait); 162 } 163 164 static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, 165 void *key) 166 { 167 struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait); 168 struct vhost_work *work = &poll->work; 169 170 if (!(key_to_poll(key) & poll->mask)) 171 return 0; 172 173 if (!poll->dev->use_worker) 174 work->fn(work); 175 else 176 vhost_poll_queue(poll); 177 178 return 0; 179 } 180 181 void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn) 182 { 183 clear_bit(VHOST_WORK_QUEUED, &work->flags); 184 work->fn = fn; 185 } 186 EXPORT_SYMBOL_GPL(vhost_work_init); 187 188 /* Init poll structure */ 189 void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, 190 __poll_t mask, struct vhost_dev *dev) 191 { 192 init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup); 193 init_poll_funcptr(&poll->table, vhost_poll_func); 194 poll->mask = mask; 195 poll->dev = dev; 196 poll->wqh = NULL; 197 198 vhost_work_init(&poll->work, fn); 199 } 200 EXPORT_SYMBOL_GPL(vhost_poll_init); 201 202 /* Start polling a file. We add ourselves to file's wait queue. The caller must 203 * keep a reference to a file until after vhost_poll_stop is called. */ 204 int vhost_poll_start(struct vhost_poll *poll, struct file *file) 205 { 206 __poll_t mask; 207 208 if (poll->wqh) 209 return 0; 210 211 mask = vfs_poll(file, &poll->table); 212 if (mask) 213 vhost_poll_wakeup(&poll->wait, 0, 0, poll_to_key(mask)); 214 if (mask & EPOLLERR) { 215 vhost_poll_stop(poll); 216 return -EINVAL; 217 } 218 219 return 0; 220 } 221 EXPORT_SYMBOL_GPL(vhost_poll_start); 222 223 /* Stop polling a file. After this function returns, it becomes safe to drop the 224 * file reference. You must also flush afterwards. */ 225 void vhost_poll_stop(struct vhost_poll *poll) 226 { 227 if (poll->wqh) { 228 remove_wait_queue(poll->wqh, &poll->wait); 229 poll->wqh = NULL; 230 } 231 } 232 EXPORT_SYMBOL_GPL(vhost_poll_stop); 233 234 void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work) 235 { 236 struct vhost_flush_struct flush; 237 238 if (dev->worker) { 239 init_completion(&flush.wait_event); 240 vhost_work_init(&flush.work, vhost_flush_work); 241 242 vhost_work_queue(dev, &flush.work); 243 wait_for_completion(&flush.wait_event); 244 } 245 } 246 EXPORT_SYMBOL_GPL(vhost_work_flush); 247 248 /* Flush any work that has been scheduled. When calling this, don't hold any 249 * locks that are also used by the callback. */ 250 void vhost_poll_flush(struct vhost_poll *poll) 251 { 252 vhost_work_flush(poll->dev, &poll->work); 253 } 254 EXPORT_SYMBOL_GPL(vhost_poll_flush); 255 256 void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work) 257 { 258 if (!dev->worker) 259 return; 260 261 if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) { 262 /* We can only add the work to the list after we're 263 * sure it was not in the list. 264 * test_and_set_bit() implies a memory barrier. 265 */ 266 llist_add(&work->node, &dev->work_list); 267 wake_up_process(dev->worker); 268 } 269 } 270 EXPORT_SYMBOL_GPL(vhost_work_queue); 271 272 /* A lockless hint for busy polling code to exit the loop */ 273 bool vhost_has_work(struct vhost_dev *dev) 274 { 275 return !llist_empty(&dev->work_list); 276 } 277 EXPORT_SYMBOL_GPL(vhost_has_work); 278 279 void vhost_poll_queue(struct vhost_poll *poll) 280 { 281 vhost_work_queue(poll->dev, &poll->work); 282 } 283 EXPORT_SYMBOL_GPL(vhost_poll_queue); 284 285 static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq) 286 { 287 int j; 288 289 for (j = 0; j < VHOST_NUM_ADDRS; j++) 290 vq->meta_iotlb[j] = NULL; 291 } 292 293 static void vhost_vq_meta_reset(struct vhost_dev *d) 294 { 295 int i; 296 297 for (i = 0; i < d->nvqs; ++i) 298 __vhost_vq_meta_reset(d->vqs[i]); 299 } 300 301 static void vhost_vring_call_reset(struct vhost_vring_call *call_ctx) 302 { 303 call_ctx->ctx = NULL; 304 memset(&call_ctx->producer, 0x0, sizeof(struct irq_bypass_producer)); 305 } 306 307 bool vhost_vq_is_setup(struct vhost_virtqueue *vq) 308 { 309 return vq->avail && vq->desc && vq->used && vhost_vq_access_ok(vq); 310 } 311 EXPORT_SYMBOL_GPL(vhost_vq_is_setup); 312 313 static void vhost_vq_reset(struct vhost_dev *dev, 314 struct vhost_virtqueue *vq) 315 { 316 vq->num = 1; 317 vq->desc = NULL; 318 vq->avail = NULL; 319 vq->used = NULL; 320 vq->last_avail_idx = 0; 321 vq->avail_idx = 0; 322 vq->last_used_idx = 0; 323 vq->signalled_used = 0; 324 vq->signalled_used_valid = false; 325 vq->used_flags = 0; 326 vq->log_used = false; 327 vq->log_addr = -1ull; 328 vq->private_data = NULL; 329 vq->acked_features = 0; 330 vq->acked_backend_features = 0; 331 vq->log_base = NULL; 332 vq->error_ctx = NULL; 333 vq->kick = NULL; 334 vq->log_ctx = NULL; 335 vhost_reset_is_le(vq); 336 vhost_disable_cross_endian(vq); 337 vq->busyloop_timeout = 0; 338 vq->umem = NULL; 339 vq->iotlb = NULL; 340 vhost_vring_call_reset(&vq->call_ctx); 341 __vhost_vq_meta_reset(vq); 342 } 343 344 static int vhost_worker(void *data) 345 { 346 struct vhost_dev *dev = data; 347 struct vhost_work *work, *work_next; 348 struct llist_node *node; 349 350 kthread_use_mm(dev->mm); 351 352 for (;;) { 353 /* mb paired w/ kthread_stop */ 354 set_current_state(TASK_INTERRUPTIBLE); 355 356 if (kthread_should_stop()) { 357 __set_current_state(TASK_RUNNING); 358 break; 359 } 360 361 node = llist_del_all(&dev->work_list); 362 if (!node) 363 schedule(); 364 365 node = llist_reverse_order(node); 366 /* make sure flag is seen after deletion */ 367 smp_wmb(); 368 llist_for_each_entry_safe(work, work_next, node, node) { 369 clear_bit(VHOST_WORK_QUEUED, &work->flags); 370 __set_current_state(TASK_RUNNING); 371 kcov_remote_start_common(dev->kcov_handle); 372 work->fn(work); 373 kcov_remote_stop(); 374 if (need_resched()) 375 schedule(); 376 } 377 } 378 kthread_unuse_mm(dev->mm); 379 return 0; 380 } 381 382 static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq) 383 { 384 kfree(vq->indirect); 385 vq->indirect = NULL; 386 kfree(vq->log); 387 vq->log = NULL; 388 kfree(vq->heads); 389 vq->heads = NULL; 390 } 391 392 /* Helper to allocate iovec buffers for all vqs. */ 393 static long vhost_dev_alloc_iovecs(struct vhost_dev *dev) 394 { 395 struct vhost_virtqueue *vq; 396 int i; 397 398 for (i = 0; i < dev->nvqs; ++i) { 399 vq = dev->vqs[i]; 400 vq->indirect = kmalloc_array(UIO_MAXIOV, 401 sizeof(*vq->indirect), 402 GFP_KERNEL); 403 vq->log = kmalloc_array(dev->iov_limit, sizeof(*vq->log), 404 GFP_KERNEL); 405 vq->heads = kmalloc_array(dev->iov_limit, sizeof(*vq->heads), 406 GFP_KERNEL); 407 if (!vq->indirect || !vq->log || !vq->heads) 408 goto err_nomem; 409 } 410 return 0; 411 412 err_nomem: 413 for (; i >= 0; --i) 414 vhost_vq_free_iovecs(dev->vqs[i]); 415 return -ENOMEM; 416 } 417 418 static void vhost_dev_free_iovecs(struct vhost_dev *dev) 419 { 420 int i; 421 422 for (i = 0; i < dev->nvqs; ++i) 423 vhost_vq_free_iovecs(dev->vqs[i]); 424 } 425 426 bool vhost_exceeds_weight(struct vhost_virtqueue *vq, 427 int pkts, int total_len) 428 { 429 struct vhost_dev *dev = vq->dev; 430 431 if ((dev->byte_weight && total_len >= dev->byte_weight) || 432 pkts >= dev->weight) { 433 vhost_poll_queue(&vq->poll); 434 return true; 435 } 436 437 return false; 438 } 439 EXPORT_SYMBOL_GPL(vhost_exceeds_weight); 440 441 static size_t vhost_get_avail_size(struct vhost_virtqueue *vq, 442 unsigned int num) 443 { 444 size_t event __maybe_unused = 445 vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 446 447 return sizeof(*vq->avail) + 448 sizeof(*vq->avail->ring) * num + event; 449 } 450 451 static size_t vhost_get_used_size(struct vhost_virtqueue *vq, 452 unsigned int num) 453 { 454 size_t event __maybe_unused = 455 vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 456 457 return sizeof(*vq->used) + 458 sizeof(*vq->used->ring) * num + event; 459 } 460 461 static size_t vhost_get_desc_size(struct vhost_virtqueue *vq, 462 unsigned int num) 463 { 464 return sizeof(*vq->desc) * num; 465 } 466 467 void vhost_dev_init(struct vhost_dev *dev, 468 struct vhost_virtqueue **vqs, int nvqs, 469 int iov_limit, int weight, int byte_weight, 470 bool use_worker, 471 int (*msg_handler)(struct vhost_dev *dev, 472 struct vhost_iotlb_msg *msg)) 473 { 474 struct vhost_virtqueue *vq; 475 int i; 476 477 dev->vqs = vqs; 478 dev->nvqs = nvqs; 479 mutex_init(&dev->mutex); 480 dev->log_ctx = NULL; 481 dev->umem = NULL; 482 dev->iotlb = NULL; 483 dev->mm = NULL; 484 dev->worker = NULL; 485 dev->iov_limit = iov_limit; 486 dev->weight = weight; 487 dev->byte_weight = byte_weight; 488 dev->use_worker = use_worker; 489 dev->msg_handler = msg_handler; 490 init_llist_head(&dev->work_list); 491 init_waitqueue_head(&dev->wait); 492 INIT_LIST_HEAD(&dev->read_list); 493 INIT_LIST_HEAD(&dev->pending_list); 494 spin_lock_init(&dev->iotlb_lock); 495 496 497 for (i = 0; i < dev->nvqs; ++i) { 498 vq = dev->vqs[i]; 499 vq->log = NULL; 500 vq->indirect = NULL; 501 vq->heads = NULL; 502 vq->dev = dev; 503 mutex_init(&vq->mutex); 504 vhost_vq_reset(dev, vq); 505 if (vq->handle_kick) 506 vhost_poll_init(&vq->poll, vq->handle_kick, 507 EPOLLIN, dev); 508 } 509 } 510 EXPORT_SYMBOL_GPL(vhost_dev_init); 511 512 /* Caller should have device mutex */ 513 long vhost_dev_check_owner(struct vhost_dev *dev) 514 { 515 /* Are you the owner? If not, I don't think you mean to do that */ 516 return dev->mm == current->mm ? 0 : -EPERM; 517 } 518 EXPORT_SYMBOL_GPL(vhost_dev_check_owner); 519 520 struct vhost_attach_cgroups_struct { 521 struct vhost_work work; 522 struct task_struct *owner; 523 int ret; 524 }; 525 526 static void vhost_attach_cgroups_work(struct vhost_work *work) 527 { 528 struct vhost_attach_cgroups_struct *s; 529 530 s = container_of(work, struct vhost_attach_cgroups_struct, work); 531 s->ret = cgroup_attach_task_all(s->owner, current); 532 } 533 534 static int vhost_attach_cgroups(struct vhost_dev *dev) 535 { 536 struct vhost_attach_cgroups_struct attach; 537 538 attach.owner = current; 539 vhost_work_init(&attach.work, vhost_attach_cgroups_work); 540 vhost_work_queue(dev, &attach.work); 541 vhost_work_flush(dev, &attach.work); 542 return attach.ret; 543 } 544 545 /* Caller should have device mutex */ 546 bool vhost_dev_has_owner(struct vhost_dev *dev) 547 { 548 return dev->mm; 549 } 550 EXPORT_SYMBOL_GPL(vhost_dev_has_owner); 551 552 static void vhost_attach_mm(struct vhost_dev *dev) 553 { 554 /* No owner, become one */ 555 if (dev->use_worker) { 556 dev->mm = get_task_mm(current); 557 } else { 558 /* vDPA device does not use worker thead, so there's 559 * no need to hold the address space for mm. This help 560 * to avoid deadlock in the case of mmap() which may 561 * held the refcnt of the file and depends on release 562 * method to remove vma. 563 */ 564 dev->mm = current->mm; 565 mmgrab(dev->mm); 566 } 567 } 568 569 static void vhost_detach_mm(struct vhost_dev *dev) 570 { 571 if (!dev->mm) 572 return; 573 574 if (dev->use_worker) 575 mmput(dev->mm); 576 else 577 mmdrop(dev->mm); 578 579 dev->mm = NULL; 580 } 581 582 /* Caller should have device mutex */ 583 long vhost_dev_set_owner(struct vhost_dev *dev) 584 { 585 struct task_struct *worker; 586 int err; 587 588 /* Is there an owner already? */ 589 if (vhost_dev_has_owner(dev)) { 590 err = -EBUSY; 591 goto err_mm; 592 } 593 594 vhost_attach_mm(dev); 595 596 dev->kcov_handle = kcov_common_handle(); 597 if (dev->use_worker) { 598 worker = kthread_create(vhost_worker, dev, 599 "vhost-%d", current->pid); 600 if (IS_ERR(worker)) { 601 err = PTR_ERR(worker); 602 goto err_worker; 603 } 604 605 dev->worker = worker; 606 wake_up_process(worker); /* avoid contributing to loadavg */ 607 608 err = vhost_attach_cgroups(dev); 609 if (err) 610 goto err_cgroup; 611 } 612 613 err = vhost_dev_alloc_iovecs(dev); 614 if (err) 615 goto err_cgroup; 616 617 return 0; 618 err_cgroup: 619 if (dev->worker) { 620 kthread_stop(dev->worker); 621 dev->worker = NULL; 622 } 623 err_worker: 624 vhost_detach_mm(dev); 625 dev->kcov_handle = 0; 626 err_mm: 627 return err; 628 } 629 EXPORT_SYMBOL_GPL(vhost_dev_set_owner); 630 631 static struct vhost_iotlb *iotlb_alloc(void) 632 { 633 return vhost_iotlb_alloc(max_iotlb_entries, 634 VHOST_IOTLB_FLAG_RETIRE); 635 } 636 637 struct vhost_iotlb *vhost_dev_reset_owner_prepare(void) 638 { 639 return iotlb_alloc(); 640 } 641 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare); 642 643 /* Caller should have device mutex */ 644 void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem) 645 { 646 int i; 647 648 vhost_dev_cleanup(dev); 649 650 dev->umem = umem; 651 /* We don't need VQ locks below since vhost_dev_cleanup makes sure 652 * VQs aren't running. 653 */ 654 for (i = 0; i < dev->nvqs; ++i) 655 dev->vqs[i]->umem = umem; 656 } 657 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner); 658 659 void vhost_dev_stop(struct vhost_dev *dev) 660 { 661 int i; 662 663 for (i = 0; i < dev->nvqs; ++i) { 664 if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) { 665 vhost_poll_stop(&dev->vqs[i]->poll); 666 vhost_poll_flush(&dev->vqs[i]->poll); 667 } 668 } 669 } 670 EXPORT_SYMBOL_GPL(vhost_dev_stop); 671 672 static void vhost_clear_msg(struct vhost_dev *dev) 673 { 674 struct vhost_msg_node *node, *n; 675 676 spin_lock(&dev->iotlb_lock); 677 678 list_for_each_entry_safe(node, n, &dev->read_list, node) { 679 list_del(&node->node); 680 kfree(node); 681 } 682 683 list_for_each_entry_safe(node, n, &dev->pending_list, node) { 684 list_del(&node->node); 685 kfree(node); 686 } 687 688 spin_unlock(&dev->iotlb_lock); 689 } 690 691 void vhost_dev_cleanup(struct vhost_dev *dev) 692 { 693 int i; 694 695 for (i = 0; i < dev->nvqs; ++i) { 696 if (dev->vqs[i]->error_ctx) 697 eventfd_ctx_put(dev->vqs[i]->error_ctx); 698 if (dev->vqs[i]->kick) 699 fput(dev->vqs[i]->kick); 700 if (dev->vqs[i]->call_ctx.ctx) 701 eventfd_ctx_put(dev->vqs[i]->call_ctx.ctx); 702 vhost_vq_reset(dev, dev->vqs[i]); 703 } 704 vhost_dev_free_iovecs(dev); 705 if (dev->log_ctx) 706 eventfd_ctx_put(dev->log_ctx); 707 dev->log_ctx = NULL; 708 /* No one will access memory at this point */ 709 vhost_iotlb_free(dev->umem); 710 dev->umem = NULL; 711 vhost_iotlb_free(dev->iotlb); 712 dev->iotlb = NULL; 713 vhost_clear_msg(dev); 714 wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM); 715 WARN_ON(!llist_empty(&dev->work_list)); 716 if (dev->worker) { 717 kthread_stop(dev->worker); 718 dev->worker = NULL; 719 dev->kcov_handle = 0; 720 } 721 vhost_detach_mm(dev); 722 } 723 EXPORT_SYMBOL_GPL(vhost_dev_cleanup); 724 725 static bool log_access_ok(void __user *log_base, u64 addr, unsigned long sz) 726 { 727 u64 a = addr / VHOST_PAGE_SIZE / 8; 728 729 /* Make sure 64 bit math will not overflow. */ 730 if (a > ULONG_MAX - (unsigned long)log_base || 731 a + (unsigned long)log_base > ULONG_MAX) 732 return false; 733 734 return access_ok(log_base + a, 735 (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8); 736 } 737 738 static bool vhost_overflow(u64 uaddr, u64 size) 739 { 740 /* Make sure 64 bit math will not overflow. */ 741 return uaddr > ULONG_MAX || size > ULONG_MAX || uaddr > ULONG_MAX - size; 742 } 743 744 /* Caller should have vq mutex and device mutex. */ 745 static bool vq_memory_access_ok(void __user *log_base, struct vhost_iotlb *umem, 746 int log_all) 747 { 748 struct vhost_iotlb_map *map; 749 750 if (!umem) 751 return false; 752 753 list_for_each_entry(map, &umem->list, link) { 754 unsigned long a = map->addr; 755 756 if (vhost_overflow(map->addr, map->size)) 757 return false; 758 759 760 if (!access_ok((void __user *)a, map->size)) 761 return false; 762 else if (log_all && !log_access_ok(log_base, 763 map->start, 764 map->size)) 765 return false; 766 } 767 return true; 768 } 769 770 static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq, 771 u64 addr, unsigned int size, 772 int type) 773 { 774 const struct vhost_iotlb_map *map = vq->meta_iotlb[type]; 775 776 if (!map) 777 return NULL; 778 779 return (void __user *)(uintptr_t)(map->addr + addr - map->start); 780 } 781 782 /* Can we switch to this memory table? */ 783 /* Caller should have device mutex but not vq mutex */ 784 static bool memory_access_ok(struct vhost_dev *d, struct vhost_iotlb *umem, 785 int log_all) 786 { 787 int i; 788 789 for (i = 0; i < d->nvqs; ++i) { 790 bool ok; 791 bool log; 792 793 mutex_lock(&d->vqs[i]->mutex); 794 log = log_all || vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL); 795 /* If ring is inactive, will check when it's enabled. */ 796 if (d->vqs[i]->private_data) 797 ok = vq_memory_access_ok(d->vqs[i]->log_base, 798 umem, log); 799 else 800 ok = true; 801 mutex_unlock(&d->vqs[i]->mutex); 802 if (!ok) 803 return false; 804 } 805 return true; 806 } 807 808 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, 809 struct iovec iov[], int iov_size, int access); 810 811 static int vhost_copy_to_user(struct vhost_virtqueue *vq, void __user *to, 812 const void *from, unsigned size) 813 { 814 int ret; 815 816 if (!vq->iotlb) 817 return __copy_to_user(to, from, size); 818 else { 819 /* This function should be called after iotlb 820 * prefetch, which means we're sure that all vq 821 * could be access through iotlb. So -EAGAIN should 822 * not happen in this case. 823 */ 824 struct iov_iter t; 825 void __user *uaddr = vhost_vq_meta_fetch(vq, 826 (u64)(uintptr_t)to, size, 827 VHOST_ADDR_USED); 828 829 if (uaddr) 830 return __copy_to_user(uaddr, from, size); 831 832 ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov, 833 ARRAY_SIZE(vq->iotlb_iov), 834 VHOST_ACCESS_WO); 835 if (ret < 0) 836 goto out; 837 iov_iter_init(&t, WRITE, vq->iotlb_iov, ret, size); 838 ret = copy_to_iter(from, size, &t); 839 if (ret == size) 840 ret = 0; 841 } 842 out: 843 return ret; 844 } 845 846 static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to, 847 void __user *from, unsigned size) 848 { 849 int ret; 850 851 if (!vq->iotlb) 852 return __copy_from_user(to, from, size); 853 else { 854 /* This function should be called after iotlb 855 * prefetch, which means we're sure that vq 856 * could be access through iotlb. So -EAGAIN should 857 * not happen in this case. 858 */ 859 void __user *uaddr = vhost_vq_meta_fetch(vq, 860 (u64)(uintptr_t)from, size, 861 VHOST_ADDR_DESC); 862 struct iov_iter f; 863 864 if (uaddr) 865 return __copy_from_user(to, uaddr, size); 866 867 ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov, 868 ARRAY_SIZE(vq->iotlb_iov), 869 VHOST_ACCESS_RO); 870 if (ret < 0) { 871 vq_err(vq, "IOTLB translation failure: uaddr " 872 "%p size 0x%llx\n", from, 873 (unsigned long long) size); 874 goto out; 875 } 876 iov_iter_init(&f, READ, vq->iotlb_iov, ret, size); 877 ret = copy_from_iter(to, size, &f); 878 if (ret == size) 879 ret = 0; 880 } 881 882 out: 883 return ret; 884 } 885 886 static void __user *__vhost_get_user_slow(struct vhost_virtqueue *vq, 887 void __user *addr, unsigned int size, 888 int type) 889 { 890 int ret; 891 892 ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov, 893 ARRAY_SIZE(vq->iotlb_iov), 894 VHOST_ACCESS_RO); 895 if (ret < 0) { 896 vq_err(vq, "IOTLB translation failure: uaddr " 897 "%p size 0x%llx\n", addr, 898 (unsigned long long) size); 899 return NULL; 900 } 901 902 if (ret != 1 || vq->iotlb_iov[0].iov_len != size) { 903 vq_err(vq, "Non atomic userspace memory access: uaddr " 904 "%p size 0x%llx\n", addr, 905 (unsigned long long) size); 906 return NULL; 907 } 908 909 return vq->iotlb_iov[0].iov_base; 910 } 911 912 /* This function should be called after iotlb 913 * prefetch, which means we're sure that vq 914 * could be access through iotlb. So -EAGAIN should 915 * not happen in this case. 916 */ 917 static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq, 918 void __user *addr, unsigned int size, 919 int type) 920 { 921 void __user *uaddr = vhost_vq_meta_fetch(vq, 922 (u64)(uintptr_t)addr, size, type); 923 if (uaddr) 924 return uaddr; 925 926 return __vhost_get_user_slow(vq, addr, size, type); 927 } 928 929 #define vhost_put_user(vq, x, ptr) \ 930 ({ \ 931 int ret; \ 932 if (!vq->iotlb) { \ 933 ret = __put_user(x, ptr); \ 934 } else { \ 935 __typeof__(ptr) to = \ 936 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \ 937 sizeof(*ptr), VHOST_ADDR_USED); \ 938 if (to != NULL) \ 939 ret = __put_user(x, to); \ 940 else \ 941 ret = -EFAULT; \ 942 } \ 943 ret; \ 944 }) 945 946 static inline int vhost_put_avail_event(struct vhost_virtqueue *vq) 947 { 948 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx), 949 vhost_avail_event(vq)); 950 } 951 952 static inline int vhost_put_used(struct vhost_virtqueue *vq, 953 struct vring_used_elem *head, int idx, 954 int count) 955 { 956 return vhost_copy_to_user(vq, vq->used->ring + idx, head, 957 count * sizeof(*head)); 958 } 959 960 static inline int vhost_put_used_flags(struct vhost_virtqueue *vq) 961 962 { 963 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags), 964 &vq->used->flags); 965 } 966 967 static inline int vhost_put_used_idx(struct vhost_virtqueue *vq) 968 969 { 970 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx), 971 &vq->used->idx); 972 } 973 974 #define vhost_get_user(vq, x, ptr, type) \ 975 ({ \ 976 int ret; \ 977 if (!vq->iotlb) { \ 978 ret = __get_user(x, ptr); \ 979 } else { \ 980 __typeof__(ptr) from = \ 981 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \ 982 sizeof(*ptr), \ 983 type); \ 984 if (from != NULL) \ 985 ret = __get_user(x, from); \ 986 else \ 987 ret = -EFAULT; \ 988 } \ 989 ret; \ 990 }) 991 992 #define vhost_get_avail(vq, x, ptr) \ 993 vhost_get_user(vq, x, ptr, VHOST_ADDR_AVAIL) 994 995 #define vhost_get_used(vq, x, ptr) \ 996 vhost_get_user(vq, x, ptr, VHOST_ADDR_USED) 997 998 static void vhost_dev_lock_vqs(struct vhost_dev *d) 999 { 1000 int i = 0; 1001 for (i = 0; i < d->nvqs; ++i) 1002 mutex_lock_nested(&d->vqs[i]->mutex, i); 1003 } 1004 1005 static void vhost_dev_unlock_vqs(struct vhost_dev *d) 1006 { 1007 int i = 0; 1008 for (i = 0; i < d->nvqs; ++i) 1009 mutex_unlock(&d->vqs[i]->mutex); 1010 } 1011 1012 static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, 1013 __virtio16 *idx) 1014 { 1015 return vhost_get_avail(vq, *idx, &vq->avail->idx); 1016 } 1017 1018 static inline int vhost_get_avail_head(struct vhost_virtqueue *vq, 1019 __virtio16 *head, int idx) 1020 { 1021 return vhost_get_avail(vq, *head, 1022 &vq->avail->ring[idx & (vq->num - 1)]); 1023 } 1024 1025 static inline int vhost_get_avail_flags(struct vhost_virtqueue *vq, 1026 __virtio16 *flags) 1027 { 1028 return vhost_get_avail(vq, *flags, &vq->avail->flags); 1029 } 1030 1031 static inline int vhost_get_used_event(struct vhost_virtqueue *vq, 1032 __virtio16 *event) 1033 { 1034 return vhost_get_avail(vq, *event, vhost_used_event(vq)); 1035 } 1036 1037 static inline int vhost_get_used_idx(struct vhost_virtqueue *vq, 1038 __virtio16 *idx) 1039 { 1040 return vhost_get_used(vq, *idx, &vq->used->idx); 1041 } 1042 1043 static inline int vhost_get_desc(struct vhost_virtqueue *vq, 1044 struct vring_desc *desc, int idx) 1045 { 1046 return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc)); 1047 } 1048 1049 static void vhost_iotlb_notify_vq(struct vhost_dev *d, 1050 struct vhost_iotlb_msg *msg) 1051 { 1052 struct vhost_msg_node *node, *n; 1053 1054 spin_lock(&d->iotlb_lock); 1055 1056 list_for_each_entry_safe(node, n, &d->pending_list, node) { 1057 struct vhost_iotlb_msg *vq_msg = &node->msg.iotlb; 1058 if (msg->iova <= vq_msg->iova && 1059 msg->iova + msg->size - 1 >= vq_msg->iova && 1060 vq_msg->type == VHOST_IOTLB_MISS) { 1061 vhost_poll_queue(&node->vq->poll); 1062 list_del(&node->node); 1063 kfree(node); 1064 } 1065 } 1066 1067 spin_unlock(&d->iotlb_lock); 1068 } 1069 1070 static bool umem_access_ok(u64 uaddr, u64 size, int access) 1071 { 1072 unsigned long a = uaddr; 1073 1074 /* Make sure 64 bit math will not overflow. */ 1075 if (vhost_overflow(uaddr, size)) 1076 return false; 1077 1078 if ((access & VHOST_ACCESS_RO) && 1079 !access_ok((void __user *)a, size)) 1080 return false; 1081 if ((access & VHOST_ACCESS_WO) && 1082 !access_ok((void __user *)a, size)) 1083 return false; 1084 return true; 1085 } 1086 1087 static int vhost_process_iotlb_msg(struct vhost_dev *dev, 1088 struct vhost_iotlb_msg *msg) 1089 { 1090 int ret = 0; 1091 1092 mutex_lock(&dev->mutex); 1093 vhost_dev_lock_vqs(dev); 1094 switch (msg->type) { 1095 case VHOST_IOTLB_UPDATE: 1096 if (!dev->iotlb) { 1097 ret = -EFAULT; 1098 break; 1099 } 1100 if (!umem_access_ok(msg->uaddr, msg->size, msg->perm)) { 1101 ret = -EFAULT; 1102 break; 1103 } 1104 vhost_vq_meta_reset(dev); 1105 if (vhost_iotlb_add_range(dev->iotlb, msg->iova, 1106 msg->iova + msg->size - 1, 1107 msg->uaddr, msg->perm)) { 1108 ret = -ENOMEM; 1109 break; 1110 } 1111 vhost_iotlb_notify_vq(dev, msg); 1112 break; 1113 case VHOST_IOTLB_INVALIDATE: 1114 if (!dev->iotlb) { 1115 ret = -EFAULT; 1116 break; 1117 } 1118 vhost_vq_meta_reset(dev); 1119 vhost_iotlb_del_range(dev->iotlb, msg->iova, 1120 msg->iova + msg->size - 1); 1121 break; 1122 default: 1123 ret = -EINVAL; 1124 break; 1125 } 1126 1127 vhost_dev_unlock_vqs(dev); 1128 mutex_unlock(&dev->mutex); 1129 1130 return ret; 1131 } 1132 ssize_t vhost_chr_write_iter(struct vhost_dev *dev, 1133 struct iov_iter *from) 1134 { 1135 struct vhost_iotlb_msg msg; 1136 size_t offset; 1137 int type, ret; 1138 1139 ret = copy_from_iter(&type, sizeof(type), from); 1140 if (ret != sizeof(type)) { 1141 ret = -EINVAL; 1142 goto done; 1143 } 1144 1145 switch (type) { 1146 case VHOST_IOTLB_MSG: 1147 /* There maybe a hole after type for V1 message type, 1148 * so skip it here. 1149 */ 1150 offset = offsetof(struct vhost_msg, iotlb) - sizeof(int); 1151 break; 1152 case VHOST_IOTLB_MSG_V2: 1153 offset = sizeof(__u32); 1154 break; 1155 default: 1156 ret = -EINVAL; 1157 goto done; 1158 } 1159 1160 iov_iter_advance(from, offset); 1161 ret = copy_from_iter(&msg, sizeof(msg), from); 1162 if (ret != sizeof(msg)) { 1163 ret = -EINVAL; 1164 goto done; 1165 } 1166 1167 if (dev->msg_handler) 1168 ret = dev->msg_handler(dev, &msg); 1169 else 1170 ret = vhost_process_iotlb_msg(dev, &msg); 1171 if (ret) { 1172 ret = -EFAULT; 1173 goto done; 1174 } 1175 1176 ret = (type == VHOST_IOTLB_MSG) ? sizeof(struct vhost_msg) : 1177 sizeof(struct vhost_msg_v2); 1178 done: 1179 return ret; 1180 } 1181 EXPORT_SYMBOL(vhost_chr_write_iter); 1182 1183 __poll_t vhost_chr_poll(struct file *file, struct vhost_dev *dev, 1184 poll_table *wait) 1185 { 1186 __poll_t mask = 0; 1187 1188 poll_wait(file, &dev->wait, wait); 1189 1190 if (!list_empty(&dev->read_list)) 1191 mask |= EPOLLIN | EPOLLRDNORM; 1192 1193 return mask; 1194 } 1195 EXPORT_SYMBOL(vhost_chr_poll); 1196 1197 ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to, 1198 int noblock) 1199 { 1200 DEFINE_WAIT(wait); 1201 struct vhost_msg_node *node; 1202 ssize_t ret = 0; 1203 unsigned size = sizeof(struct vhost_msg); 1204 1205 if (iov_iter_count(to) < size) 1206 return 0; 1207 1208 while (1) { 1209 if (!noblock) 1210 prepare_to_wait(&dev->wait, &wait, 1211 TASK_INTERRUPTIBLE); 1212 1213 node = vhost_dequeue_msg(dev, &dev->read_list); 1214 if (node) 1215 break; 1216 if (noblock) { 1217 ret = -EAGAIN; 1218 break; 1219 } 1220 if (signal_pending(current)) { 1221 ret = -ERESTARTSYS; 1222 break; 1223 } 1224 if (!dev->iotlb) { 1225 ret = -EBADFD; 1226 break; 1227 } 1228 1229 schedule(); 1230 } 1231 1232 if (!noblock) 1233 finish_wait(&dev->wait, &wait); 1234 1235 if (node) { 1236 struct vhost_iotlb_msg *msg; 1237 void *start = &node->msg; 1238 1239 switch (node->msg.type) { 1240 case VHOST_IOTLB_MSG: 1241 size = sizeof(node->msg); 1242 msg = &node->msg.iotlb; 1243 break; 1244 case VHOST_IOTLB_MSG_V2: 1245 size = sizeof(node->msg_v2); 1246 msg = &node->msg_v2.iotlb; 1247 break; 1248 default: 1249 BUG(); 1250 break; 1251 } 1252 1253 ret = copy_to_iter(start, size, to); 1254 if (ret != size || msg->type != VHOST_IOTLB_MISS) { 1255 kfree(node); 1256 return ret; 1257 } 1258 vhost_enqueue_msg(dev, &dev->pending_list, node); 1259 } 1260 1261 return ret; 1262 } 1263 EXPORT_SYMBOL_GPL(vhost_chr_read_iter); 1264 1265 static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access) 1266 { 1267 struct vhost_dev *dev = vq->dev; 1268 struct vhost_msg_node *node; 1269 struct vhost_iotlb_msg *msg; 1270 bool v2 = vhost_backend_has_feature(vq, VHOST_BACKEND_F_IOTLB_MSG_V2); 1271 1272 node = vhost_new_msg(vq, v2 ? VHOST_IOTLB_MSG_V2 : VHOST_IOTLB_MSG); 1273 if (!node) 1274 return -ENOMEM; 1275 1276 if (v2) { 1277 node->msg_v2.type = VHOST_IOTLB_MSG_V2; 1278 msg = &node->msg_v2.iotlb; 1279 } else { 1280 msg = &node->msg.iotlb; 1281 } 1282 1283 msg->type = VHOST_IOTLB_MISS; 1284 msg->iova = iova; 1285 msg->perm = access; 1286 1287 vhost_enqueue_msg(dev, &dev->read_list, node); 1288 1289 return 0; 1290 } 1291 1292 static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, 1293 vring_desc_t __user *desc, 1294 vring_avail_t __user *avail, 1295 vring_used_t __user *used) 1296 1297 { 1298 /* If an IOTLB device is present, the vring addresses are 1299 * GIOVAs. Access validation occurs at prefetch time. */ 1300 if (vq->iotlb) 1301 return true; 1302 1303 return access_ok(desc, vhost_get_desc_size(vq, num)) && 1304 access_ok(avail, vhost_get_avail_size(vq, num)) && 1305 access_ok(used, vhost_get_used_size(vq, num)); 1306 } 1307 1308 static void vhost_vq_meta_update(struct vhost_virtqueue *vq, 1309 const struct vhost_iotlb_map *map, 1310 int type) 1311 { 1312 int access = (type == VHOST_ADDR_USED) ? 1313 VHOST_ACCESS_WO : VHOST_ACCESS_RO; 1314 1315 if (likely(map->perm & access)) 1316 vq->meta_iotlb[type] = map; 1317 } 1318 1319 static bool iotlb_access_ok(struct vhost_virtqueue *vq, 1320 int access, u64 addr, u64 len, int type) 1321 { 1322 const struct vhost_iotlb_map *map; 1323 struct vhost_iotlb *umem = vq->iotlb; 1324 u64 s = 0, size, orig_addr = addr, last = addr + len - 1; 1325 1326 if (vhost_vq_meta_fetch(vq, addr, len, type)) 1327 return true; 1328 1329 while (len > s) { 1330 map = vhost_iotlb_itree_first(umem, addr, last); 1331 if (map == NULL || map->start > addr) { 1332 vhost_iotlb_miss(vq, addr, access); 1333 return false; 1334 } else if (!(map->perm & access)) { 1335 /* Report the possible access violation by 1336 * request another translation from userspace. 1337 */ 1338 return false; 1339 } 1340 1341 size = map->size - addr + map->start; 1342 1343 if (orig_addr == addr && size >= len) 1344 vhost_vq_meta_update(vq, map, type); 1345 1346 s += size; 1347 addr += size; 1348 } 1349 1350 return true; 1351 } 1352 1353 int vq_meta_prefetch(struct vhost_virtqueue *vq) 1354 { 1355 unsigned int num = vq->num; 1356 1357 if (!vq->iotlb) 1358 return 1; 1359 1360 return iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->desc, 1361 vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) && 1362 iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->avail, 1363 vhost_get_avail_size(vq, num), 1364 VHOST_ADDR_AVAIL) && 1365 iotlb_access_ok(vq, VHOST_MAP_WO, (u64)(uintptr_t)vq->used, 1366 vhost_get_used_size(vq, num), VHOST_ADDR_USED); 1367 } 1368 EXPORT_SYMBOL_GPL(vq_meta_prefetch); 1369 1370 /* Can we log writes? */ 1371 /* Caller should have device mutex but not vq mutex */ 1372 bool vhost_log_access_ok(struct vhost_dev *dev) 1373 { 1374 return memory_access_ok(dev, dev->umem, 1); 1375 } 1376 EXPORT_SYMBOL_GPL(vhost_log_access_ok); 1377 1378 static bool vq_log_used_access_ok(struct vhost_virtqueue *vq, 1379 void __user *log_base, 1380 bool log_used, 1381 u64 log_addr) 1382 { 1383 /* If an IOTLB device is present, log_addr is a GIOVA that 1384 * will never be logged by log_used(). */ 1385 if (vq->iotlb) 1386 return true; 1387 1388 return !log_used || log_access_ok(log_base, log_addr, 1389 vhost_get_used_size(vq, vq->num)); 1390 } 1391 1392 /* Verify access for write logging. */ 1393 /* Caller should have vq mutex and device mutex */ 1394 static bool vq_log_access_ok(struct vhost_virtqueue *vq, 1395 void __user *log_base) 1396 { 1397 return vq_memory_access_ok(log_base, vq->umem, 1398 vhost_has_feature(vq, VHOST_F_LOG_ALL)) && 1399 vq_log_used_access_ok(vq, log_base, vq->log_used, vq->log_addr); 1400 } 1401 1402 /* Can we start vq? */ 1403 /* Caller should have vq mutex and device mutex */ 1404 bool vhost_vq_access_ok(struct vhost_virtqueue *vq) 1405 { 1406 if (!vq_log_access_ok(vq, vq->log_base)) 1407 return false; 1408 1409 return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used); 1410 } 1411 EXPORT_SYMBOL_GPL(vhost_vq_access_ok); 1412 1413 static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) 1414 { 1415 struct vhost_memory mem, *newmem; 1416 struct vhost_memory_region *region; 1417 struct vhost_iotlb *newumem, *oldumem; 1418 unsigned long size = offsetof(struct vhost_memory, regions); 1419 int i; 1420 1421 if (copy_from_user(&mem, m, size)) 1422 return -EFAULT; 1423 if (mem.padding) 1424 return -EOPNOTSUPP; 1425 if (mem.nregions > max_mem_regions) 1426 return -E2BIG; 1427 newmem = kvzalloc(struct_size(newmem, regions, mem.nregions), 1428 GFP_KERNEL); 1429 if (!newmem) 1430 return -ENOMEM; 1431 1432 memcpy(newmem, &mem, size); 1433 if (copy_from_user(newmem->regions, m->regions, 1434 flex_array_size(newmem, regions, mem.nregions))) { 1435 kvfree(newmem); 1436 return -EFAULT; 1437 } 1438 1439 newumem = iotlb_alloc(); 1440 if (!newumem) { 1441 kvfree(newmem); 1442 return -ENOMEM; 1443 } 1444 1445 for (region = newmem->regions; 1446 region < newmem->regions + mem.nregions; 1447 region++) { 1448 if (vhost_iotlb_add_range(newumem, 1449 region->guest_phys_addr, 1450 region->guest_phys_addr + 1451 region->memory_size - 1, 1452 region->userspace_addr, 1453 VHOST_MAP_RW)) 1454 goto err; 1455 } 1456 1457 if (!memory_access_ok(d, newumem, 0)) 1458 goto err; 1459 1460 oldumem = d->umem; 1461 d->umem = newumem; 1462 1463 /* All memory accesses are done under some VQ mutex. */ 1464 for (i = 0; i < d->nvqs; ++i) { 1465 mutex_lock(&d->vqs[i]->mutex); 1466 d->vqs[i]->umem = newumem; 1467 mutex_unlock(&d->vqs[i]->mutex); 1468 } 1469 1470 kvfree(newmem); 1471 vhost_iotlb_free(oldumem); 1472 return 0; 1473 1474 err: 1475 vhost_iotlb_free(newumem); 1476 kvfree(newmem); 1477 return -EFAULT; 1478 } 1479 1480 static long vhost_vring_set_num(struct vhost_dev *d, 1481 struct vhost_virtqueue *vq, 1482 void __user *argp) 1483 { 1484 struct vhost_vring_state s; 1485 1486 /* Resizing ring with an active backend? 1487 * You don't want to do that. */ 1488 if (vq->private_data) 1489 return -EBUSY; 1490 1491 if (copy_from_user(&s, argp, sizeof s)) 1492 return -EFAULT; 1493 1494 if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) 1495 return -EINVAL; 1496 vq->num = s.num; 1497 1498 return 0; 1499 } 1500 1501 static long vhost_vring_set_addr(struct vhost_dev *d, 1502 struct vhost_virtqueue *vq, 1503 void __user *argp) 1504 { 1505 struct vhost_vring_addr a; 1506 1507 if (copy_from_user(&a, argp, sizeof a)) 1508 return -EFAULT; 1509 if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) 1510 return -EOPNOTSUPP; 1511 1512 /* For 32bit, verify that the top 32bits of the user 1513 data are set to zero. */ 1514 if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr || 1515 (u64)(unsigned long)a.used_user_addr != a.used_user_addr || 1516 (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) 1517 return -EFAULT; 1518 1519 /* Make sure it's safe to cast pointers to vring types. */ 1520 BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE); 1521 BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE); 1522 if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) || 1523 (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) || 1524 (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1))) 1525 return -EINVAL; 1526 1527 /* We only verify access here if backend is configured. 1528 * If it is not, we don't as size might not have been setup. 1529 * We will verify when backend is configured. */ 1530 if (vq->private_data) { 1531 if (!vq_access_ok(vq, vq->num, 1532 (void __user *)(unsigned long)a.desc_user_addr, 1533 (void __user *)(unsigned long)a.avail_user_addr, 1534 (void __user *)(unsigned long)a.used_user_addr)) 1535 return -EINVAL; 1536 1537 /* Also validate log access for used ring if enabled. */ 1538 if (!vq_log_used_access_ok(vq, vq->log_base, 1539 a.flags & (0x1 << VHOST_VRING_F_LOG), 1540 a.log_guest_addr)) 1541 return -EINVAL; 1542 } 1543 1544 vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG)); 1545 vq->desc = (void __user *)(unsigned long)a.desc_user_addr; 1546 vq->avail = (void __user *)(unsigned long)a.avail_user_addr; 1547 vq->log_addr = a.log_guest_addr; 1548 vq->used = (void __user *)(unsigned long)a.used_user_addr; 1549 1550 return 0; 1551 } 1552 1553 static long vhost_vring_set_num_addr(struct vhost_dev *d, 1554 struct vhost_virtqueue *vq, 1555 unsigned int ioctl, 1556 void __user *argp) 1557 { 1558 long r; 1559 1560 mutex_lock(&vq->mutex); 1561 1562 switch (ioctl) { 1563 case VHOST_SET_VRING_NUM: 1564 r = vhost_vring_set_num(d, vq, argp); 1565 break; 1566 case VHOST_SET_VRING_ADDR: 1567 r = vhost_vring_set_addr(d, vq, argp); 1568 break; 1569 default: 1570 BUG(); 1571 } 1572 1573 mutex_unlock(&vq->mutex); 1574 1575 return r; 1576 } 1577 long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) 1578 { 1579 struct file *eventfp, *filep = NULL; 1580 bool pollstart = false, pollstop = false; 1581 struct eventfd_ctx *ctx = NULL; 1582 u32 __user *idxp = argp; 1583 struct vhost_virtqueue *vq; 1584 struct vhost_vring_state s; 1585 struct vhost_vring_file f; 1586 u32 idx; 1587 long r; 1588 1589 r = get_user(idx, idxp); 1590 if (r < 0) 1591 return r; 1592 if (idx >= d->nvqs) 1593 return -ENOBUFS; 1594 1595 idx = array_index_nospec(idx, d->nvqs); 1596 vq = d->vqs[idx]; 1597 1598 if (ioctl == VHOST_SET_VRING_NUM || 1599 ioctl == VHOST_SET_VRING_ADDR) { 1600 return vhost_vring_set_num_addr(d, vq, ioctl, argp); 1601 } 1602 1603 mutex_lock(&vq->mutex); 1604 1605 switch (ioctl) { 1606 case VHOST_SET_VRING_BASE: 1607 /* Moving base with an active backend? 1608 * You don't want to do that. */ 1609 if (vq->private_data) { 1610 r = -EBUSY; 1611 break; 1612 } 1613 if (copy_from_user(&s, argp, sizeof s)) { 1614 r = -EFAULT; 1615 break; 1616 } 1617 if (s.num > 0xffff) { 1618 r = -EINVAL; 1619 break; 1620 } 1621 vq->last_avail_idx = s.num; 1622 /* Forget the cached index value. */ 1623 vq->avail_idx = vq->last_avail_idx; 1624 break; 1625 case VHOST_GET_VRING_BASE: 1626 s.index = idx; 1627 s.num = vq->last_avail_idx; 1628 if (copy_to_user(argp, &s, sizeof s)) 1629 r = -EFAULT; 1630 break; 1631 case VHOST_SET_VRING_KICK: 1632 if (copy_from_user(&f, argp, sizeof f)) { 1633 r = -EFAULT; 1634 break; 1635 } 1636 eventfp = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_fget(f.fd); 1637 if (IS_ERR(eventfp)) { 1638 r = PTR_ERR(eventfp); 1639 break; 1640 } 1641 if (eventfp != vq->kick) { 1642 pollstop = (filep = vq->kick) != NULL; 1643 pollstart = (vq->kick = eventfp) != NULL; 1644 } else 1645 filep = eventfp; 1646 break; 1647 case VHOST_SET_VRING_CALL: 1648 if (copy_from_user(&f, argp, sizeof f)) { 1649 r = -EFAULT; 1650 break; 1651 } 1652 ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd); 1653 if (IS_ERR(ctx)) { 1654 r = PTR_ERR(ctx); 1655 break; 1656 } 1657 1658 swap(ctx, vq->call_ctx.ctx); 1659 break; 1660 case VHOST_SET_VRING_ERR: 1661 if (copy_from_user(&f, argp, sizeof f)) { 1662 r = -EFAULT; 1663 break; 1664 } 1665 ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd); 1666 if (IS_ERR(ctx)) { 1667 r = PTR_ERR(ctx); 1668 break; 1669 } 1670 swap(ctx, vq->error_ctx); 1671 break; 1672 case VHOST_SET_VRING_ENDIAN: 1673 r = vhost_set_vring_endian(vq, argp); 1674 break; 1675 case VHOST_GET_VRING_ENDIAN: 1676 r = vhost_get_vring_endian(vq, idx, argp); 1677 break; 1678 case VHOST_SET_VRING_BUSYLOOP_TIMEOUT: 1679 if (copy_from_user(&s, argp, sizeof(s))) { 1680 r = -EFAULT; 1681 break; 1682 } 1683 vq->busyloop_timeout = s.num; 1684 break; 1685 case VHOST_GET_VRING_BUSYLOOP_TIMEOUT: 1686 s.index = idx; 1687 s.num = vq->busyloop_timeout; 1688 if (copy_to_user(argp, &s, sizeof(s))) 1689 r = -EFAULT; 1690 break; 1691 default: 1692 r = -ENOIOCTLCMD; 1693 } 1694 1695 if (pollstop && vq->handle_kick) 1696 vhost_poll_stop(&vq->poll); 1697 1698 if (!IS_ERR_OR_NULL(ctx)) 1699 eventfd_ctx_put(ctx); 1700 if (filep) 1701 fput(filep); 1702 1703 if (pollstart && vq->handle_kick) 1704 r = vhost_poll_start(&vq->poll, vq->kick); 1705 1706 mutex_unlock(&vq->mutex); 1707 1708 if (pollstop && vq->handle_kick) 1709 vhost_poll_flush(&vq->poll); 1710 return r; 1711 } 1712 EXPORT_SYMBOL_GPL(vhost_vring_ioctl); 1713 1714 int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled) 1715 { 1716 struct vhost_iotlb *niotlb, *oiotlb; 1717 int i; 1718 1719 niotlb = iotlb_alloc(); 1720 if (!niotlb) 1721 return -ENOMEM; 1722 1723 oiotlb = d->iotlb; 1724 d->iotlb = niotlb; 1725 1726 for (i = 0; i < d->nvqs; ++i) { 1727 struct vhost_virtqueue *vq = d->vqs[i]; 1728 1729 mutex_lock(&vq->mutex); 1730 vq->iotlb = niotlb; 1731 __vhost_vq_meta_reset(vq); 1732 mutex_unlock(&vq->mutex); 1733 } 1734 1735 vhost_iotlb_free(oiotlb); 1736 1737 return 0; 1738 } 1739 EXPORT_SYMBOL_GPL(vhost_init_device_iotlb); 1740 1741 /* Caller must have device mutex */ 1742 long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) 1743 { 1744 struct eventfd_ctx *ctx; 1745 u64 p; 1746 long r; 1747 int i, fd; 1748 1749 /* If you are not the owner, you can become one */ 1750 if (ioctl == VHOST_SET_OWNER) { 1751 r = vhost_dev_set_owner(d); 1752 goto done; 1753 } 1754 1755 /* You must be the owner to do anything else */ 1756 r = vhost_dev_check_owner(d); 1757 if (r) 1758 goto done; 1759 1760 switch (ioctl) { 1761 case VHOST_SET_MEM_TABLE: 1762 r = vhost_set_memory(d, argp); 1763 break; 1764 case VHOST_SET_LOG_BASE: 1765 if (copy_from_user(&p, argp, sizeof p)) { 1766 r = -EFAULT; 1767 break; 1768 } 1769 if ((u64)(unsigned long)p != p) { 1770 r = -EFAULT; 1771 break; 1772 } 1773 for (i = 0; i < d->nvqs; ++i) { 1774 struct vhost_virtqueue *vq; 1775 void __user *base = (void __user *)(unsigned long)p; 1776 vq = d->vqs[i]; 1777 mutex_lock(&vq->mutex); 1778 /* If ring is inactive, will check when it's enabled. */ 1779 if (vq->private_data && !vq_log_access_ok(vq, base)) 1780 r = -EFAULT; 1781 else 1782 vq->log_base = base; 1783 mutex_unlock(&vq->mutex); 1784 } 1785 break; 1786 case VHOST_SET_LOG_FD: 1787 r = get_user(fd, (int __user *)argp); 1788 if (r < 0) 1789 break; 1790 ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd); 1791 if (IS_ERR(ctx)) { 1792 r = PTR_ERR(ctx); 1793 break; 1794 } 1795 swap(ctx, d->log_ctx); 1796 for (i = 0; i < d->nvqs; ++i) { 1797 mutex_lock(&d->vqs[i]->mutex); 1798 d->vqs[i]->log_ctx = d->log_ctx; 1799 mutex_unlock(&d->vqs[i]->mutex); 1800 } 1801 if (ctx) 1802 eventfd_ctx_put(ctx); 1803 break; 1804 default: 1805 r = -ENOIOCTLCMD; 1806 break; 1807 } 1808 done: 1809 return r; 1810 } 1811 EXPORT_SYMBOL_GPL(vhost_dev_ioctl); 1812 1813 /* TODO: This is really inefficient. We need something like get_user() 1814 * (instruction directly accesses the data, with an exception table entry 1815 * returning -EFAULT). See Documentation/x86/exception-tables.rst. 1816 */ 1817 static int set_bit_to_user(int nr, void __user *addr) 1818 { 1819 unsigned long log = (unsigned long)addr; 1820 struct page *page; 1821 void *base; 1822 int bit = nr + (log % PAGE_SIZE) * 8; 1823 int r; 1824 1825 r = pin_user_pages_fast(log, 1, FOLL_WRITE, &page); 1826 if (r < 0) 1827 return r; 1828 BUG_ON(r != 1); 1829 base = kmap_atomic(page); 1830 set_bit(bit, base); 1831 kunmap_atomic(base); 1832 unpin_user_pages_dirty_lock(&page, 1, true); 1833 return 0; 1834 } 1835 1836 static int log_write(void __user *log_base, 1837 u64 write_address, u64 write_length) 1838 { 1839 u64 write_page = write_address / VHOST_PAGE_SIZE; 1840 int r; 1841 1842 if (!write_length) 1843 return 0; 1844 write_length += write_address % VHOST_PAGE_SIZE; 1845 for (;;) { 1846 u64 base = (u64)(unsigned long)log_base; 1847 u64 log = base + write_page / 8; 1848 int bit = write_page % 8; 1849 if ((u64)(unsigned long)log != log) 1850 return -EFAULT; 1851 r = set_bit_to_user(bit, (void __user *)(unsigned long)log); 1852 if (r < 0) 1853 return r; 1854 if (write_length <= VHOST_PAGE_SIZE) 1855 break; 1856 write_length -= VHOST_PAGE_SIZE; 1857 write_page += 1; 1858 } 1859 return r; 1860 } 1861 1862 static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len) 1863 { 1864 struct vhost_iotlb *umem = vq->umem; 1865 struct vhost_iotlb_map *u; 1866 u64 start, end, l, min; 1867 int r; 1868 bool hit = false; 1869 1870 while (len) { 1871 min = len; 1872 /* More than one GPAs can be mapped into a single HVA. So 1873 * iterate all possible umems here to be safe. 1874 */ 1875 list_for_each_entry(u, &umem->list, link) { 1876 if (u->addr > hva - 1 + len || 1877 u->addr - 1 + u->size < hva) 1878 continue; 1879 start = max(u->addr, hva); 1880 end = min(u->addr - 1 + u->size, hva - 1 + len); 1881 l = end - start + 1; 1882 r = log_write(vq->log_base, 1883 u->start + start - u->addr, 1884 l); 1885 if (r < 0) 1886 return r; 1887 hit = true; 1888 min = min(l, min); 1889 } 1890 1891 if (!hit) 1892 return -EFAULT; 1893 1894 len -= min; 1895 hva += min; 1896 } 1897 1898 return 0; 1899 } 1900 1901 static int log_used(struct vhost_virtqueue *vq, u64 used_offset, u64 len) 1902 { 1903 struct iovec *iov = vq->log_iov; 1904 int i, ret; 1905 1906 if (!vq->iotlb) 1907 return log_write(vq->log_base, vq->log_addr + used_offset, len); 1908 1909 ret = translate_desc(vq, (uintptr_t)vq->used + used_offset, 1910 len, iov, 64, VHOST_ACCESS_WO); 1911 if (ret < 0) 1912 return ret; 1913 1914 for (i = 0; i < ret; i++) { 1915 ret = log_write_hva(vq, (uintptr_t)iov[i].iov_base, 1916 iov[i].iov_len); 1917 if (ret) 1918 return ret; 1919 } 1920 1921 return 0; 1922 } 1923 1924 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, 1925 unsigned int log_num, u64 len, struct iovec *iov, int count) 1926 { 1927 int i, r; 1928 1929 /* Make sure data written is seen before log. */ 1930 smp_wmb(); 1931 1932 if (vq->iotlb) { 1933 for (i = 0; i < count; i++) { 1934 r = log_write_hva(vq, (uintptr_t)iov[i].iov_base, 1935 iov[i].iov_len); 1936 if (r < 0) 1937 return r; 1938 } 1939 return 0; 1940 } 1941 1942 for (i = 0; i < log_num; ++i) { 1943 u64 l = min(log[i].len, len); 1944 r = log_write(vq->log_base, log[i].addr, l); 1945 if (r < 0) 1946 return r; 1947 len -= l; 1948 if (!len) { 1949 if (vq->log_ctx) 1950 eventfd_signal(vq->log_ctx, 1); 1951 return 0; 1952 } 1953 } 1954 /* Length written exceeds what we have stored. This is a bug. */ 1955 BUG(); 1956 return 0; 1957 } 1958 EXPORT_SYMBOL_GPL(vhost_log_write); 1959 1960 static int vhost_update_used_flags(struct vhost_virtqueue *vq) 1961 { 1962 void __user *used; 1963 if (vhost_put_used_flags(vq)) 1964 return -EFAULT; 1965 if (unlikely(vq->log_used)) { 1966 /* Make sure the flag is seen before log. */ 1967 smp_wmb(); 1968 /* Log used flag write. */ 1969 used = &vq->used->flags; 1970 log_used(vq, (used - (void __user *)vq->used), 1971 sizeof vq->used->flags); 1972 if (vq->log_ctx) 1973 eventfd_signal(vq->log_ctx, 1); 1974 } 1975 return 0; 1976 } 1977 1978 static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event) 1979 { 1980 if (vhost_put_avail_event(vq)) 1981 return -EFAULT; 1982 if (unlikely(vq->log_used)) { 1983 void __user *used; 1984 /* Make sure the event is seen before log. */ 1985 smp_wmb(); 1986 /* Log avail event write */ 1987 used = vhost_avail_event(vq); 1988 log_used(vq, (used - (void __user *)vq->used), 1989 sizeof *vhost_avail_event(vq)); 1990 if (vq->log_ctx) 1991 eventfd_signal(vq->log_ctx, 1); 1992 } 1993 return 0; 1994 } 1995 1996 int vhost_vq_init_access(struct vhost_virtqueue *vq) 1997 { 1998 __virtio16 last_used_idx; 1999 int r; 2000 bool is_le = vq->is_le; 2001 2002 if (!vq->private_data) 2003 return 0; 2004 2005 vhost_init_is_le(vq); 2006 2007 r = vhost_update_used_flags(vq); 2008 if (r) 2009 goto err; 2010 vq->signalled_used_valid = false; 2011 if (!vq->iotlb && 2012 !access_ok(&vq->used->idx, sizeof vq->used->idx)) { 2013 r = -EFAULT; 2014 goto err; 2015 } 2016 r = vhost_get_used_idx(vq, &last_used_idx); 2017 if (r) { 2018 vq_err(vq, "Can't access used idx at %p\n", 2019 &vq->used->idx); 2020 goto err; 2021 } 2022 vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx); 2023 return 0; 2024 2025 err: 2026 vq->is_le = is_le; 2027 return r; 2028 } 2029 EXPORT_SYMBOL_GPL(vhost_vq_init_access); 2030 2031 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, 2032 struct iovec iov[], int iov_size, int access) 2033 { 2034 const struct vhost_iotlb_map *map; 2035 struct vhost_dev *dev = vq->dev; 2036 struct vhost_iotlb *umem = dev->iotlb ? dev->iotlb : dev->umem; 2037 struct iovec *_iov; 2038 u64 s = 0; 2039 int ret = 0; 2040 2041 while ((u64)len > s) { 2042 u64 size; 2043 if (unlikely(ret >= iov_size)) { 2044 ret = -ENOBUFS; 2045 break; 2046 } 2047 2048 map = vhost_iotlb_itree_first(umem, addr, addr + len - 1); 2049 if (map == NULL || map->start > addr) { 2050 if (umem != dev->iotlb) { 2051 ret = -EFAULT; 2052 break; 2053 } 2054 ret = -EAGAIN; 2055 break; 2056 } else if (!(map->perm & access)) { 2057 ret = -EPERM; 2058 break; 2059 } 2060 2061 _iov = iov + ret; 2062 size = map->size - addr + map->start; 2063 _iov->iov_len = min((u64)len - s, size); 2064 _iov->iov_base = (void __user *)(unsigned long) 2065 (map->addr + addr - map->start); 2066 s += size; 2067 addr += size; 2068 ++ret; 2069 } 2070 2071 if (ret == -EAGAIN) 2072 vhost_iotlb_miss(vq, addr, access); 2073 return ret; 2074 } 2075 2076 /* Each buffer in the virtqueues is actually a chain of descriptors. This 2077 * function returns the next descriptor in the chain, 2078 * or -1U if we're at the end. */ 2079 static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc) 2080 { 2081 unsigned int next; 2082 2083 /* If this descriptor says it doesn't chain, we're done. */ 2084 if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT))) 2085 return -1U; 2086 2087 /* Check they're not leading us off end of descriptors. */ 2088 next = vhost16_to_cpu(vq, READ_ONCE(desc->next)); 2089 return next; 2090 } 2091 2092 static int get_indirect(struct vhost_virtqueue *vq, 2093 struct iovec iov[], unsigned int iov_size, 2094 unsigned int *out_num, unsigned int *in_num, 2095 struct vhost_log *log, unsigned int *log_num, 2096 struct vring_desc *indirect) 2097 { 2098 struct vring_desc desc; 2099 unsigned int i = 0, count, found = 0; 2100 u32 len = vhost32_to_cpu(vq, indirect->len); 2101 struct iov_iter from; 2102 int ret, access; 2103 2104 /* Sanity check */ 2105 if (unlikely(len % sizeof desc)) { 2106 vq_err(vq, "Invalid length in indirect descriptor: " 2107 "len 0x%llx not multiple of 0x%zx\n", 2108 (unsigned long long)len, 2109 sizeof desc); 2110 return -EINVAL; 2111 } 2112 2113 ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect, 2114 UIO_MAXIOV, VHOST_ACCESS_RO); 2115 if (unlikely(ret < 0)) { 2116 if (ret != -EAGAIN) 2117 vq_err(vq, "Translation failure %d in indirect.\n", ret); 2118 return ret; 2119 } 2120 iov_iter_init(&from, READ, vq->indirect, ret, len); 2121 count = len / sizeof desc; 2122 /* Buffers are chained via a 16 bit next field, so 2123 * we can have at most 2^16 of these. */ 2124 if (unlikely(count > USHRT_MAX + 1)) { 2125 vq_err(vq, "Indirect buffer length too big: %d\n", 2126 indirect->len); 2127 return -E2BIG; 2128 } 2129 2130 do { 2131 unsigned iov_count = *in_num + *out_num; 2132 if (unlikely(++found > count)) { 2133 vq_err(vq, "Loop detected: last one at %u " 2134 "indirect size %u\n", 2135 i, count); 2136 return -EINVAL; 2137 } 2138 if (unlikely(!copy_from_iter_full(&desc, sizeof(desc), &from))) { 2139 vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n", 2140 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); 2141 return -EINVAL; 2142 } 2143 if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) { 2144 vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n", 2145 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); 2146 return -EINVAL; 2147 } 2148 2149 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) 2150 access = VHOST_ACCESS_WO; 2151 else 2152 access = VHOST_ACCESS_RO; 2153 2154 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), 2155 vhost32_to_cpu(vq, desc.len), iov + iov_count, 2156 iov_size - iov_count, access); 2157 if (unlikely(ret < 0)) { 2158 if (ret != -EAGAIN) 2159 vq_err(vq, "Translation failure %d indirect idx %d\n", 2160 ret, i); 2161 return ret; 2162 } 2163 /* If this is an input descriptor, increment that count. */ 2164 if (access == VHOST_ACCESS_WO) { 2165 *in_num += ret; 2166 if (unlikely(log && ret)) { 2167 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); 2168 log[*log_num].len = vhost32_to_cpu(vq, desc.len); 2169 ++*log_num; 2170 } 2171 } else { 2172 /* If it's an output descriptor, they're all supposed 2173 * to come before any input descriptors. */ 2174 if (unlikely(*in_num)) { 2175 vq_err(vq, "Indirect descriptor " 2176 "has out after in: idx %d\n", i); 2177 return -EINVAL; 2178 } 2179 *out_num += ret; 2180 } 2181 } while ((i = next_desc(vq, &desc)) != -1); 2182 return 0; 2183 } 2184 2185 /* This looks in the virtqueue and for the first available buffer, and converts 2186 * it to an iovec for convenient access. Since descriptors consist of some 2187 * number of output then some number of input descriptors, it's actually two 2188 * iovecs, but we pack them into one and note how many of each there were. 2189 * 2190 * This function returns the descriptor number found, or vq->num (which is 2191 * never a valid descriptor number) if none was found. A negative code is 2192 * returned on error. */ 2193 int vhost_get_vq_desc(struct vhost_virtqueue *vq, 2194 struct iovec iov[], unsigned int iov_size, 2195 unsigned int *out_num, unsigned int *in_num, 2196 struct vhost_log *log, unsigned int *log_num) 2197 { 2198 struct vring_desc desc; 2199 unsigned int i, head, found = 0; 2200 u16 last_avail_idx; 2201 __virtio16 avail_idx; 2202 __virtio16 ring_head; 2203 int ret, access; 2204 2205 /* Check it isn't doing very strange things with descriptor numbers. */ 2206 last_avail_idx = vq->last_avail_idx; 2207 2208 if (vq->avail_idx == vq->last_avail_idx) { 2209 if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) { 2210 vq_err(vq, "Failed to access avail idx at %p\n", 2211 &vq->avail->idx); 2212 return -EFAULT; 2213 } 2214 vq->avail_idx = vhost16_to_cpu(vq, avail_idx); 2215 2216 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) { 2217 vq_err(vq, "Guest moved used index from %u to %u", 2218 last_avail_idx, vq->avail_idx); 2219 return -EFAULT; 2220 } 2221 2222 /* If there's nothing new since last we looked, return 2223 * invalid. 2224 */ 2225 if (vq->avail_idx == last_avail_idx) 2226 return vq->num; 2227 2228 /* Only get avail ring entries after they have been 2229 * exposed by guest. 2230 */ 2231 smp_rmb(); 2232 } 2233 2234 /* Grab the next descriptor number they're advertising, and increment 2235 * the index we've seen. */ 2236 if (unlikely(vhost_get_avail_head(vq, &ring_head, last_avail_idx))) { 2237 vq_err(vq, "Failed to read head: idx %d address %p\n", 2238 last_avail_idx, 2239 &vq->avail->ring[last_avail_idx % vq->num]); 2240 return -EFAULT; 2241 } 2242 2243 head = vhost16_to_cpu(vq, ring_head); 2244 2245 /* If their number is silly, that's an error. */ 2246 if (unlikely(head >= vq->num)) { 2247 vq_err(vq, "Guest says index %u > %u is available", 2248 head, vq->num); 2249 return -EINVAL; 2250 } 2251 2252 /* When we start there are none of either input nor output. */ 2253 *out_num = *in_num = 0; 2254 if (unlikely(log)) 2255 *log_num = 0; 2256 2257 i = head; 2258 do { 2259 unsigned iov_count = *in_num + *out_num; 2260 if (unlikely(i >= vq->num)) { 2261 vq_err(vq, "Desc index is %u > %u, head = %u", 2262 i, vq->num, head); 2263 return -EINVAL; 2264 } 2265 if (unlikely(++found > vq->num)) { 2266 vq_err(vq, "Loop detected: last one at %u " 2267 "vq size %u head %u\n", 2268 i, vq->num, head); 2269 return -EINVAL; 2270 } 2271 ret = vhost_get_desc(vq, &desc, i); 2272 if (unlikely(ret)) { 2273 vq_err(vq, "Failed to get descriptor: idx %d addr %p\n", 2274 i, vq->desc + i); 2275 return -EFAULT; 2276 } 2277 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) { 2278 ret = get_indirect(vq, iov, iov_size, 2279 out_num, in_num, 2280 log, log_num, &desc); 2281 if (unlikely(ret < 0)) { 2282 if (ret != -EAGAIN) 2283 vq_err(vq, "Failure detected " 2284 "in indirect descriptor at idx %d\n", i); 2285 return ret; 2286 } 2287 continue; 2288 } 2289 2290 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) 2291 access = VHOST_ACCESS_WO; 2292 else 2293 access = VHOST_ACCESS_RO; 2294 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), 2295 vhost32_to_cpu(vq, desc.len), iov + iov_count, 2296 iov_size - iov_count, access); 2297 if (unlikely(ret < 0)) { 2298 if (ret != -EAGAIN) 2299 vq_err(vq, "Translation failure %d descriptor idx %d\n", 2300 ret, i); 2301 return ret; 2302 } 2303 if (access == VHOST_ACCESS_WO) { 2304 /* If this is an input descriptor, 2305 * increment that count. */ 2306 *in_num += ret; 2307 if (unlikely(log && ret)) { 2308 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); 2309 log[*log_num].len = vhost32_to_cpu(vq, desc.len); 2310 ++*log_num; 2311 } 2312 } else { 2313 /* If it's an output descriptor, they're all supposed 2314 * to come before any input descriptors. */ 2315 if (unlikely(*in_num)) { 2316 vq_err(vq, "Descriptor has out after in: " 2317 "idx %d\n", i); 2318 return -EINVAL; 2319 } 2320 *out_num += ret; 2321 } 2322 } while ((i = next_desc(vq, &desc)) != -1); 2323 2324 /* On success, increment avail index. */ 2325 vq->last_avail_idx++; 2326 2327 /* Assume notifications from guest are disabled at this point, 2328 * if they aren't we would need to update avail_event index. */ 2329 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY)); 2330 return head; 2331 } 2332 EXPORT_SYMBOL_GPL(vhost_get_vq_desc); 2333 2334 /* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */ 2335 void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n) 2336 { 2337 vq->last_avail_idx -= n; 2338 } 2339 EXPORT_SYMBOL_GPL(vhost_discard_vq_desc); 2340 2341 /* After we've used one of their buffers, we tell them about it. We'll then 2342 * want to notify the guest, using eventfd. */ 2343 int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len) 2344 { 2345 struct vring_used_elem heads = { 2346 cpu_to_vhost32(vq, head), 2347 cpu_to_vhost32(vq, len) 2348 }; 2349 2350 return vhost_add_used_n(vq, &heads, 1); 2351 } 2352 EXPORT_SYMBOL_GPL(vhost_add_used); 2353 2354 static int __vhost_add_used_n(struct vhost_virtqueue *vq, 2355 struct vring_used_elem *heads, 2356 unsigned count) 2357 { 2358 vring_used_elem_t __user *used; 2359 u16 old, new; 2360 int start; 2361 2362 start = vq->last_used_idx & (vq->num - 1); 2363 used = vq->used->ring + start; 2364 if (vhost_put_used(vq, heads, start, count)) { 2365 vq_err(vq, "Failed to write used"); 2366 return -EFAULT; 2367 } 2368 if (unlikely(vq->log_used)) { 2369 /* Make sure data is seen before log. */ 2370 smp_wmb(); 2371 /* Log used ring entry write. */ 2372 log_used(vq, ((void __user *)used - (void __user *)vq->used), 2373 count * sizeof *used); 2374 } 2375 old = vq->last_used_idx; 2376 new = (vq->last_used_idx += count); 2377 /* If the driver never bothers to signal in a very long while, 2378 * used index might wrap around. If that happens, invalidate 2379 * signalled_used index we stored. TODO: make sure driver 2380 * signals at least once in 2^16 and remove this. */ 2381 if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old))) 2382 vq->signalled_used_valid = false; 2383 return 0; 2384 } 2385 2386 /* After we've used one of their buffers, we tell them about it. We'll then 2387 * want to notify the guest, using eventfd. */ 2388 int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, 2389 unsigned count) 2390 { 2391 int start, n, r; 2392 2393 start = vq->last_used_idx & (vq->num - 1); 2394 n = vq->num - start; 2395 if (n < count) { 2396 r = __vhost_add_used_n(vq, heads, n); 2397 if (r < 0) 2398 return r; 2399 heads += n; 2400 count -= n; 2401 } 2402 r = __vhost_add_used_n(vq, heads, count); 2403 2404 /* Make sure buffer is written before we update index. */ 2405 smp_wmb(); 2406 if (vhost_put_used_idx(vq)) { 2407 vq_err(vq, "Failed to increment used idx"); 2408 return -EFAULT; 2409 } 2410 if (unlikely(vq->log_used)) { 2411 /* Make sure used idx is seen before log. */ 2412 smp_wmb(); 2413 /* Log used index update. */ 2414 log_used(vq, offsetof(struct vring_used, idx), 2415 sizeof vq->used->idx); 2416 if (vq->log_ctx) 2417 eventfd_signal(vq->log_ctx, 1); 2418 } 2419 return r; 2420 } 2421 EXPORT_SYMBOL_GPL(vhost_add_used_n); 2422 2423 static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2424 { 2425 __u16 old, new; 2426 __virtio16 event; 2427 bool v; 2428 /* Flush out used index updates. This is paired 2429 * with the barrier that the Guest executes when enabling 2430 * interrupts. */ 2431 smp_mb(); 2432 2433 if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) && 2434 unlikely(vq->avail_idx == vq->last_avail_idx)) 2435 return true; 2436 2437 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { 2438 __virtio16 flags; 2439 if (vhost_get_avail_flags(vq, &flags)) { 2440 vq_err(vq, "Failed to get flags"); 2441 return true; 2442 } 2443 return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT)); 2444 } 2445 old = vq->signalled_used; 2446 v = vq->signalled_used_valid; 2447 new = vq->signalled_used = vq->last_used_idx; 2448 vq->signalled_used_valid = true; 2449 2450 if (unlikely(!v)) 2451 return true; 2452 2453 if (vhost_get_used_event(vq, &event)) { 2454 vq_err(vq, "Failed to get used event idx"); 2455 return true; 2456 } 2457 return vring_need_event(vhost16_to_cpu(vq, event), new, old); 2458 } 2459 2460 /* This actually signals the guest, using eventfd. */ 2461 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2462 { 2463 /* Signal the Guest tell them we used something up. */ 2464 if (vq->call_ctx.ctx && vhost_notify(dev, vq)) 2465 eventfd_signal(vq->call_ctx.ctx, 1); 2466 } 2467 EXPORT_SYMBOL_GPL(vhost_signal); 2468 2469 /* And here's the combo meal deal. Supersize me! */ 2470 void vhost_add_used_and_signal(struct vhost_dev *dev, 2471 struct vhost_virtqueue *vq, 2472 unsigned int head, int len) 2473 { 2474 vhost_add_used(vq, head, len); 2475 vhost_signal(dev, vq); 2476 } 2477 EXPORT_SYMBOL_GPL(vhost_add_used_and_signal); 2478 2479 /* multi-buffer version of vhost_add_used_and_signal */ 2480 void vhost_add_used_and_signal_n(struct vhost_dev *dev, 2481 struct vhost_virtqueue *vq, 2482 struct vring_used_elem *heads, unsigned count) 2483 { 2484 vhost_add_used_n(vq, heads, count); 2485 vhost_signal(dev, vq); 2486 } 2487 EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n); 2488 2489 /* return true if we're sure that avaiable ring is empty */ 2490 bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2491 { 2492 __virtio16 avail_idx; 2493 int r; 2494 2495 if (vq->avail_idx != vq->last_avail_idx) 2496 return false; 2497 2498 r = vhost_get_avail_idx(vq, &avail_idx); 2499 if (unlikely(r)) 2500 return false; 2501 vq->avail_idx = vhost16_to_cpu(vq, avail_idx); 2502 2503 return vq->avail_idx == vq->last_avail_idx; 2504 } 2505 EXPORT_SYMBOL_GPL(vhost_vq_avail_empty); 2506 2507 /* OK, now we need to know about added descriptors. */ 2508 bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2509 { 2510 __virtio16 avail_idx; 2511 int r; 2512 2513 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) 2514 return false; 2515 vq->used_flags &= ~VRING_USED_F_NO_NOTIFY; 2516 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { 2517 r = vhost_update_used_flags(vq); 2518 if (r) { 2519 vq_err(vq, "Failed to enable notification at %p: %d\n", 2520 &vq->used->flags, r); 2521 return false; 2522 } 2523 } else { 2524 r = vhost_update_avail_event(vq, vq->avail_idx); 2525 if (r) { 2526 vq_err(vq, "Failed to update avail event index at %p: %d\n", 2527 vhost_avail_event(vq), r); 2528 return false; 2529 } 2530 } 2531 /* They could have slipped one in as we were doing that: make 2532 * sure it's written, then check again. */ 2533 smp_mb(); 2534 r = vhost_get_avail_idx(vq, &avail_idx); 2535 if (r) { 2536 vq_err(vq, "Failed to check avail idx at %p: %d\n", 2537 &vq->avail->idx, r); 2538 return false; 2539 } 2540 2541 return vhost16_to_cpu(vq, avail_idx) != vq->avail_idx; 2542 } 2543 EXPORT_SYMBOL_GPL(vhost_enable_notify); 2544 2545 /* We don't need to be notified again. */ 2546 void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2547 { 2548 int r; 2549 2550 if (vq->used_flags & VRING_USED_F_NO_NOTIFY) 2551 return; 2552 vq->used_flags |= VRING_USED_F_NO_NOTIFY; 2553 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { 2554 r = vhost_update_used_flags(vq); 2555 if (r) 2556 vq_err(vq, "Failed to disable notification at %p: %d\n", 2557 &vq->used->flags, r); 2558 } 2559 } 2560 EXPORT_SYMBOL_GPL(vhost_disable_notify); 2561 2562 /* Create a new message. */ 2563 struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type) 2564 { 2565 struct vhost_msg_node *node = kmalloc(sizeof *node, GFP_KERNEL); 2566 if (!node) 2567 return NULL; 2568 2569 /* Make sure all padding within the structure is initialized. */ 2570 memset(&node->msg, 0, sizeof node->msg); 2571 node->vq = vq; 2572 node->msg.type = type; 2573 return node; 2574 } 2575 EXPORT_SYMBOL_GPL(vhost_new_msg); 2576 2577 void vhost_enqueue_msg(struct vhost_dev *dev, struct list_head *head, 2578 struct vhost_msg_node *node) 2579 { 2580 spin_lock(&dev->iotlb_lock); 2581 list_add_tail(&node->node, head); 2582 spin_unlock(&dev->iotlb_lock); 2583 2584 wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM); 2585 } 2586 EXPORT_SYMBOL_GPL(vhost_enqueue_msg); 2587 2588 struct vhost_msg_node *vhost_dequeue_msg(struct vhost_dev *dev, 2589 struct list_head *head) 2590 { 2591 struct vhost_msg_node *node = NULL; 2592 2593 spin_lock(&dev->iotlb_lock); 2594 if (!list_empty(head)) { 2595 node = list_first_entry(head, struct vhost_msg_node, 2596 node); 2597 list_del(&node->node); 2598 } 2599 spin_unlock(&dev->iotlb_lock); 2600 2601 return node; 2602 } 2603 EXPORT_SYMBOL_GPL(vhost_dequeue_msg); 2604 2605 void vhost_set_backend_features(struct vhost_dev *dev, u64 features) 2606 { 2607 struct vhost_virtqueue *vq; 2608 int i; 2609 2610 mutex_lock(&dev->mutex); 2611 for (i = 0; i < dev->nvqs; ++i) { 2612 vq = dev->vqs[i]; 2613 mutex_lock(&vq->mutex); 2614 vq->acked_backend_features = features; 2615 mutex_unlock(&vq->mutex); 2616 } 2617 mutex_unlock(&dev->mutex); 2618 } 2619 EXPORT_SYMBOL_GPL(vhost_set_backend_features); 2620 2621 static int __init vhost_init(void) 2622 { 2623 return 0; 2624 } 2625 2626 static void __exit vhost_exit(void) 2627 { 2628 } 2629 2630 module_init(vhost_init); 2631 module_exit(vhost_exit); 2632 2633 MODULE_VERSION("0.0.1"); 2634 MODULE_LICENSE("GPL v2"); 2635 MODULE_AUTHOR("Michael S. Tsirkin"); 2636 MODULE_DESCRIPTION("Host kernel accelerator for virtio"); 2637