1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDUSE: vDPA Device in Userspace 4 * 5 * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved. 6 * 7 * Author: Xie Yongji <xieyongji@bytedance.com> 8 * 9 */ 10 11 #include "linux/virtio_net.h" 12 #include <linux/bits.h> 13 #include <linux/cleanup.h> 14 #include <linux/init.h> 15 #include <linux/module.h> 16 #include <linux/cdev.h> 17 #include <linux/device.h> 18 #include <linux/eventfd.h> 19 #include <linux/slab.h> 20 #include <linux/wait.h> 21 #include <linux/dma-map-ops.h> 22 #include <linux/poll.h> 23 #include <linux/file.h> 24 #include <linux/uio.h> 25 #include <linux/vdpa.h> 26 #include <linux/nospec.h> 27 #include <linux/virtio.h> 28 #include <linux/vmalloc.h> 29 #include <linux/sched/mm.h> 30 #include <uapi/linux/vduse.h> 31 #include <uapi/linux/vdpa.h> 32 #include <uapi/linux/virtio_config.h> 33 #include <uapi/linux/virtio_ids.h> 34 #include <uapi/linux/virtio_blk.h> 35 #include <uapi/linux/virtio_ring.h> 36 37 #include "iova_domain.h" 38 39 #define DRV_AUTHOR "Yongji Xie <xieyongji@bytedance.com>" 40 #define DRV_DESC "vDPA Device in Userspace" 41 #define DRV_LICENSE "GPL v2" 42 43 #define VDUSE_DEV_MAX (1U << MINORBITS) 44 #define VDUSE_DEV_MAX_GROUPS 0xffff 45 #define VDUSE_DEV_MAX_AS 0xffff 46 #define VDUSE_MAX_BOUNCE_SIZE (1024 * 1024 * 1024) 47 #define VDUSE_MIN_BOUNCE_SIZE (1024 * 1024) 48 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024) 49 /* 128 MB reserved for virtqueue creation */ 50 #define VDUSE_IOVA_SIZE (VDUSE_MAX_BOUNCE_SIZE + 128 * 1024 * 1024) 51 #define VDUSE_MSG_DEFAULT_TIMEOUT 30 52 53 #define IRQ_UNBOUND -1 54 55 /* Supported VDUSE features */ 56 static const uint64_t vduse_features = BIT_U64(VDUSE_F_QUEUE_READY) | 57 BIT_U64(VDUSE_F_SUSPEND); 58 59 /* 60 * VDUSE instance have not asked the vduse API version, so assume 0. 61 * 62 * Old devices may not ask for the device version and assume it is 0. Keep 63 * this value for these. From the moment the VDUSE instance ask for the 64 * version, convert to the latests supported one and continue regular flow 65 */ 66 #define VDUSE_API_VERSION_NOT_ASKED U64_MAX 67 68 struct vduse_virtqueue { 69 u16 index; 70 u16 num_max; 71 u32 num; 72 u64 desc_addr; 73 u64 driver_addr; 74 u64 device_addr; 75 struct vdpa_vq_state state; 76 bool ready; 77 bool kicked; 78 u32 group; 79 spinlock_t kick_lock; 80 spinlock_t irq_lock; 81 spinlock_t ready_lock; 82 struct eventfd_ctx *kickfd; 83 struct vdpa_callback cb; 84 struct work_struct inject; 85 struct work_struct kick; 86 int irq_effective_cpu; 87 struct cpumask irq_affinity; 88 struct kobject kobj; 89 struct vduse_dev *dev; 90 }; 91 92 struct vduse_dev; 93 94 struct vduse_vdpa { 95 struct vdpa_device vdpa; 96 struct vduse_dev *dev; 97 }; 98 99 struct vduse_umem { 100 unsigned long iova; 101 unsigned long npages; 102 struct page **pages; 103 struct mm_struct *mm; 104 }; 105 106 struct vduse_as { 107 struct vduse_iova_domain *domain; 108 struct vduse_umem *umem; 109 struct mutex mem_lock; 110 }; 111 112 struct vduse_vq_group { 113 rwlock_t as_lock; 114 struct vduse_as *as; /* Protected by as_lock */ 115 struct vduse_dev *dev; 116 }; 117 118 struct vduse_dev { 119 struct vduse_vdpa *vdev; 120 struct device *dev; 121 struct vduse_virtqueue **vqs; 122 struct vduse_as *as; 123 char *name; 124 struct mutex lock; 125 spinlock_t msg_lock; 126 u64 vduse_features; 127 u64 msg_unique; 128 u32 msg_timeout; 129 wait_queue_head_t waitq; 130 struct list_head send_list; 131 struct list_head recv_list; 132 struct vdpa_callback config_cb; 133 struct work_struct inject; 134 spinlock_t irq_lock; 135 struct rw_semaphore rwsem; 136 int minor; 137 bool broken; 138 bool connected; 139 bool suspended; 140 u64 api_version; 141 u64 device_features; 142 u64 driver_features; 143 u32 device_id; 144 u32 vendor_id; 145 u32 generation; 146 u32 config_size; 147 void *config; 148 u8 status; 149 u32 vq_num; 150 u32 vq_align; 151 u32 ngroups; 152 u32 nas; 153 struct vduse_vq_group *groups; 154 unsigned int bounce_size; 155 struct mutex domain_lock; 156 }; 157 158 struct vduse_dev_msg { 159 struct vduse_dev_request req; 160 struct vduse_dev_response resp; 161 struct list_head list; 162 wait_queue_head_t waitq; 163 bool completed; 164 }; 165 166 struct vduse_control { 167 u64 api_version; 168 u64 vduse_features; 169 }; 170 171 static DEFINE_MUTEX(vduse_lock); 172 static DEFINE_IDR(vduse_idr); 173 174 static dev_t vduse_major; 175 static struct cdev vduse_ctrl_cdev; 176 static const struct device *vduse_ctrl_dev; 177 static struct cdev vduse_cdev; 178 static struct workqueue_struct *vduse_irq_wq; 179 static struct workqueue_struct *vduse_irq_bound_wq; 180 181 static u32 allowed_device_id[] = { 182 VIRTIO_ID_BLOCK, 183 VIRTIO_ID_NET, 184 VIRTIO_ID_FS, 185 }; 186 187 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa) 188 { 189 struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa); 190 191 return vdev->dev; 192 } 193 194 static inline struct vduse_dev *dev_to_vduse(struct device *dev) 195 { 196 struct vdpa_device *vdpa = dev_to_vdpa(dev); 197 198 return vdpa_to_vduse(vdpa); 199 } 200 201 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head, 202 uint32_t request_id) 203 { 204 struct vduse_dev_msg *msg; 205 206 list_for_each_entry(msg, head, list) { 207 if (msg->req.request_id == request_id) { 208 list_del(&msg->list); 209 return msg; 210 } 211 } 212 213 return NULL; 214 } 215 216 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head) 217 { 218 struct vduse_dev_msg *msg = NULL; 219 220 if (!list_empty(head)) { 221 msg = list_first_entry(head, struct vduse_dev_msg, list); 222 list_del(&msg->list); 223 } 224 225 return msg; 226 } 227 228 static void vduse_enqueue_msg(struct list_head *head, 229 struct vduse_dev_msg *msg) 230 { 231 list_add_tail(&msg->list, head); 232 } 233 234 static void vduse_enqueue_msg_head(struct list_head *head, 235 struct vduse_dev_msg *msg) 236 { 237 list_add(&msg->list, head); 238 } 239 240 static void vduse_dev_broken(struct vduse_dev *dev) 241 { 242 struct vduse_dev_msg *msg, *tmp; 243 244 if (unlikely(dev->broken)) 245 return; 246 247 list_splice_init(&dev->recv_list, &dev->send_list); 248 list_for_each_entry_safe(msg, tmp, &dev->send_list, list) { 249 list_del(&msg->list); 250 msg->completed = 1; 251 msg->resp.result = VDUSE_REQ_RESULT_FAILED; 252 wake_up(&msg->waitq); 253 } 254 dev->broken = true; 255 wake_up(&dev->waitq); 256 } 257 258 static int vduse_dev_msg_sync(struct vduse_dev *dev, 259 struct vduse_dev_msg *msg) 260 { 261 int ret; 262 263 if (unlikely(dev->broken)) 264 return -EIO; 265 266 init_waitqueue_head(&msg->waitq); 267 spin_lock(&dev->msg_lock); 268 if (unlikely(dev->broken)) { 269 spin_unlock(&dev->msg_lock); 270 return -EIO; 271 } 272 msg->req.request_id = dev->msg_unique++; 273 vduse_enqueue_msg(&dev->send_list, msg); 274 wake_up(&dev->waitq); 275 spin_unlock(&dev->msg_lock); 276 if (dev->msg_timeout) 277 ret = wait_event_killable_timeout(msg->waitq, msg->completed, 278 (long)dev->msg_timeout * HZ); 279 else 280 ret = wait_event_killable(msg->waitq, msg->completed); 281 282 spin_lock(&dev->msg_lock); 283 if (!msg->completed) { 284 list_del(&msg->list); 285 msg->resp.result = VDUSE_REQ_RESULT_FAILED; 286 /* Mark the device as malfunction when there is a timeout */ 287 if (!ret) 288 vduse_dev_broken(dev); 289 } 290 ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO; 291 spin_unlock(&dev->msg_lock); 292 293 return ret; 294 } 295 296 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev, 297 struct vduse_virtqueue *vq, 298 struct vdpa_vq_state_packed *packed) 299 { 300 struct vduse_dev_msg msg = { 0 }; 301 int ret; 302 303 msg.req.type = VDUSE_GET_VQ_STATE; 304 msg.req.vq_state.index = vq->index; 305 306 ret = vduse_dev_msg_sync(dev, &msg); 307 if (ret) 308 return ret; 309 310 packed->last_avail_counter = 311 msg.resp.vq_state.packed.last_avail_counter & 0x0001; 312 packed->last_avail_idx = 313 msg.resp.vq_state.packed.last_avail_idx & 0x7FFF; 314 packed->last_used_counter = 315 msg.resp.vq_state.packed.last_used_counter & 0x0001; 316 packed->last_used_idx = 317 msg.resp.vq_state.packed.last_used_idx & 0x7FFF; 318 319 return 0; 320 } 321 322 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev, 323 struct vduse_virtqueue *vq, 324 struct vdpa_vq_state_split *split) 325 { 326 struct vduse_dev_msg msg = { 0 }; 327 int ret; 328 329 msg.req.type = VDUSE_GET_VQ_STATE; 330 msg.req.vq_state.index = vq->index; 331 332 ret = vduse_dev_msg_sync(dev, &msg); 333 if (ret) 334 return ret; 335 336 split->avail_index = msg.resp.vq_state.split.avail_index; 337 338 return 0; 339 } 340 341 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status) 342 { 343 struct vduse_dev_msg msg = { 0 }; 344 345 msg.req.type = VDUSE_SET_STATUS; 346 msg.req.s.status = status; 347 348 return vduse_dev_msg_sync(dev, &msg); 349 } 350 351 static int vduse_dev_update_iotlb(struct vduse_dev *dev, u32 asid, 352 u64 start, u64 last) 353 { 354 struct vduse_dev_msg msg = { 0 }; 355 356 if (last < start) 357 return -EINVAL; 358 359 msg.req.type = VDUSE_UPDATE_IOTLB; 360 if (dev->api_version < VDUSE_API_VERSION_1) { 361 msg.req.iova.start = start; 362 msg.req.iova.last = last; 363 } else { 364 msg.req.iova_v2.start = start; 365 msg.req.iova_v2.last = last; 366 msg.req.iova_v2.asid = asid; 367 } 368 369 return vduse_dev_msg_sync(dev, &msg); 370 } 371 372 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to) 373 { 374 struct file *file = iocb->ki_filp; 375 struct vduse_dev *dev = file->private_data; 376 struct vduse_dev_msg *msg; 377 struct vduse_dev_request req; 378 int size = sizeof(struct vduse_dev_request); 379 ssize_t ret; 380 381 if (iov_iter_count(to) < size) 382 return -EINVAL; 383 384 spin_lock(&dev->msg_lock); 385 while (1) { 386 msg = vduse_dequeue_msg(&dev->send_list); 387 if (msg) 388 break; 389 spin_unlock(&dev->msg_lock); 390 391 if (file->f_flags & O_NONBLOCK) 392 return -EAGAIN; 393 394 ret = wait_event_interruptible_exclusive(dev->waitq, 395 !list_empty(&dev->send_list)); 396 if (ret) 397 return ret; 398 399 spin_lock(&dev->msg_lock); 400 } 401 402 memcpy(&req, &msg->req, sizeof(req)); 403 /* 404 * We must ensure vduse_msg is on send_list or recv_list before unlock 405 * dev->msg_lock. Because vduse_dev_msg_sync() may be timeout when we 406 * copy data to userspace, and will call list_del() for this msg. 407 */ 408 vduse_enqueue_msg(&dev->recv_list, msg); 409 spin_unlock(&dev->msg_lock); 410 411 ret = copy_to_iter(&req, size, to); 412 if (ret != size) { 413 /* 414 * Roll back: move msg back to send_list if still pending. 415 * 416 * NOTE: 417 * vduse_find_msg() must use req.request_id instead of `msg`. 418 * A malicious userspace may reply to this request, and wake up 419 * the caller, after which `msg` will have already been freed. 420 * And here vduse_find_msg() will return NULL then do nothing. 421 */ 422 spin_lock(&dev->msg_lock); 423 msg = vduse_find_msg(&dev->recv_list, req.request_id); 424 if (msg) 425 vduse_enqueue_msg_head(&dev->send_list, msg); 426 spin_unlock(&dev->msg_lock); 427 ret = -EFAULT; 428 } 429 430 return ret; 431 } 432 433 static bool is_mem_zero(const char *ptr, int size) 434 { 435 int i; 436 437 for (i = 0; i < size; i++) { 438 if (ptr[i]) 439 return false; 440 } 441 return true; 442 } 443 444 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from) 445 { 446 struct file *file = iocb->ki_filp; 447 struct vduse_dev *dev = file->private_data; 448 struct vduse_dev_response resp; 449 struct vduse_dev_msg *msg; 450 size_t ret; 451 452 ret = copy_from_iter(&resp, sizeof(resp), from); 453 if (ret != sizeof(resp)) 454 return -EINVAL; 455 456 if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved))) 457 return -EINVAL; 458 459 spin_lock(&dev->msg_lock); 460 msg = vduse_find_msg(&dev->recv_list, resp.request_id); 461 if (!msg) { 462 ret = -ENOENT; 463 goto unlock; 464 } 465 466 memcpy(&msg->resp, &resp, sizeof(resp)); 467 msg->completed = 1; 468 wake_up(&msg->waitq); 469 unlock: 470 spin_unlock(&dev->msg_lock); 471 472 return ret; 473 } 474 475 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait) 476 { 477 struct vduse_dev *dev = file->private_data; 478 __poll_t mask = 0; 479 480 poll_wait(file, &dev->waitq, wait); 481 482 spin_lock(&dev->msg_lock); 483 484 if (unlikely(dev->broken)) 485 mask |= EPOLLERR; 486 if (!list_empty(&dev->send_list)) 487 mask |= EPOLLIN | EPOLLRDNORM; 488 if (!list_empty(&dev->recv_list)) 489 mask |= EPOLLOUT | EPOLLWRNORM; 490 491 spin_unlock(&dev->msg_lock); 492 493 return mask; 494 } 495 496 static void vduse_dev_reset(struct vduse_dev *dev) 497 { 498 int i; 499 500 /* The coherent mappings are handled in vduse_dev_free_coherent() */ 501 for (i = 0; i < dev->nas; i++) { 502 struct vduse_iova_domain *domain = dev->as[i].domain; 503 504 if (domain && domain->bounce_map) 505 vduse_domain_reset_bounce_map(domain); 506 } 507 508 scoped_guard(rwsem_write, &dev->rwsem) { 509 dev->suspended = false; 510 dev->status = 0; 511 dev->driver_features = 0; 512 dev->generation++; 513 spin_lock(&dev->irq_lock); 514 dev->config_cb.callback = NULL; 515 dev->config_cb.private = NULL; 516 spin_unlock(&dev->irq_lock); 517 518 for (i = 0; i < dev->vq_num; i++) { 519 struct vduse_virtqueue *vq = dev->vqs[i]; 520 521 scoped_guard(spinlock_bh, &vq->ready_lock) { 522 vq->ready = false; 523 } 524 vq->desc_addr = 0; 525 vq->driver_addr = 0; 526 vq->device_addr = 0; 527 vq->num = 0; 528 memset(&vq->state, 0, sizeof(vq->state)); 529 530 spin_lock(&vq->kick_lock); 531 vq->kicked = false; 532 if (vq->kickfd) 533 eventfd_ctx_put(vq->kickfd); 534 vq->kickfd = NULL; 535 spin_unlock(&vq->kick_lock); 536 537 spin_lock(&vq->irq_lock); 538 vq->cb.callback = NULL; 539 vq->cb.private = NULL; 540 vq->cb.trigger = NULL; 541 spin_unlock(&vq->irq_lock); 542 } 543 } 544 545 flush_work(&dev->inject); 546 for (i = 0; i < dev->vq_num; i++) { 547 struct vduse_virtqueue *vq = dev->vqs[i]; 548 549 flush_work(&vq->inject); 550 flush_work(&vq->kick); 551 } 552 } 553 554 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx, 555 u64 desc_area, u64 driver_area, 556 u64 device_area) 557 { 558 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 559 struct vduse_virtqueue *vq = dev->vqs[idx]; 560 561 vq->desc_addr = desc_area; 562 vq->driver_addr = driver_area; 563 vq->device_addr = device_area; 564 565 return 0; 566 } 567 568 static void vduse_vq_kick(struct vduse_virtqueue *vq) 569 { 570 guard(rwsem_read)(&vq->dev->rwsem); 571 if (vq->dev->suspended) 572 return; 573 574 guard(spinlock)(&vq->kick_lock); 575 scoped_guard(spinlock_bh, &vq->ready_lock) 576 if (!vq->ready) 577 return; 578 579 if (vq->kickfd) 580 eventfd_signal(vq->kickfd); 581 else 582 vq->kicked = true; 583 } 584 585 static void vduse_vq_kick_work(struct work_struct *work) 586 { 587 struct vduse_virtqueue *vq = container_of(work, 588 struct vduse_virtqueue, kick); 589 590 vduse_vq_kick(vq); 591 } 592 593 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx) 594 { 595 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 596 struct vduse_virtqueue *vq = dev->vqs[idx]; 597 598 if (!eventfd_signal_allowed()) { 599 schedule_work(&vq->kick); 600 return; 601 } 602 vduse_vq_kick(vq); 603 } 604 605 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx, 606 struct vdpa_callback *cb) 607 { 608 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 609 struct vduse_virtqueue *vq = dev->vqs[idx]; 610 611 spin_lock(&vq->irq_lock); 612 vq->cb.callback = cb->callback; 613 vq->cb.private = cb->private; 614 vq->cb.trigger = cb->trigger; 615 spin_unlock(&vq->irq_lock); 616 } 617 618 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) 619 { 620 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 621 struct vduse_virtqueue *vq = dev->vqs[idx]; 622 623 vq->num = num; 624 } 625 626 static u16 vduse_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 idx) 627 { 628 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 629 struct vduse_virtqueue *vq = dev->vqs[idx]; 630 631 if (vq->num) 632 return vq->num; 633 else 634 return vq->num_max; 635 } 636 637 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa, 638 u16 idx, bool ready) 639 { 640 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 641 struct vduse_virtqueue *vq = dev->vqs[idx]; 642 struct vduse_dev_msg msg = { 0 }; 643 int r; 644 645 if (dev->vduse_features & BIT_U64(VDUSE_F_QUEUE_READY)) { 646 msg.req.type = VDUSE_SET_VQ_READY; 647 msg.req.vq_ready.num = idx; 648 msg.req.vq_ready.ready = !!ready; 649 650 r = vduse_dev_msg_sync(dev, &msg); 651 652 if (r < 0) { 653 dev_dbg(&vdpa->dev, "device refuses to set vq %u ready %u", 654 idx, ready); 655 656 /* We can't do better than break the device in this case */ 657 spin_lock(&dev->msg_lock); 658 vduse_dev_broken(dev); 659 spin_unlock(&dev->msg_lock); 660 661 return; 662 } 663 } 664 665 guard(spinlock_bh)(&vq->ready_lock); 666 vq->ready = ready; 667 } 668 669 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx) 670 { 671 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 672 struct vduse_virtqueue *vq = dev->vqs[idx]; 673 674 guard(spinlock_bh)(&vq->ready_lock); 675 return vq->ready; 676 } 677 678 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx, 679 const struct vdpa_vq_state *state) 680 { 681 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 682 struct vduse_virtqueue *vq = dev->vqs[idx]; 683 684 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 685 vq->state.packed.last_avail_counter = 686 state->packed.last_avail_counter; 687 vq->state.packed.last_avail_idx = state->packed.last_avail_idx; 688 vq->state.packed.last_used_counter = 689 state->packed.last_used_counter; 690 vq->state.packed.last_used_idx = state->packed.last_used_idx; 691 } else 692 vq->state.split.avail_index = state->split.avail_index; 693 694 return 0; 695 } 696 697 static u32 vduse_get_vq_group(struct vdpa_device *vdpa, u16 idx) 698 { 699 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 700 701 if (dev->api_version < VDUSE_API_VERSION_1) 702 return 0; 703 704 return dev->vqs[idx]->group; 705 } 706 707 static union virtio_map vduse_get_vq_map(struct vdpa_device *vdpa, u16 idx) 708 { 709 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 710 u32 vq_group = vduse_get_vq_group(vdpa, idx); 711 union virtio_map ret = { 712 .group = &dev->groups[vq_group], 713 }; 714 715 return ret; 716 } 717 718 DEFINE_GUARD(vq_group_as_read_lock, struct vduse_vq_group *, 719 if (_T->dev->nas > 1) 720 read_lock(&_T->as_lock), 721 if (_T->dev->nas > 1) 722 read_unlock(&_T->as_lock)) 723 724 DEFINE_GUARD(vq_group_as_write_lock, struct vduse_vq_group *, 725 if (_T->dev->nas > 1) 726 write_lock(&_T->as_lock), 727 if (_T->dev->nas > 1) 728 write_unlock(&_T->as_lock)) 729 730 static int vduse_set_group_asid(struct vdpa_device *vdpa, unsigned int group, 731 unsigned int asid) 732 { 733 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 734 struct vduse_dev_msg msg = { 0 }; 735 int r; 736 737 if (dev->api_version < VDUSE_API_VERSION_1) 738 return -EINVAL; 739 740 msg.req.type = VDUSE_SET_VQ_GROUP_ASID; 741 msg.req.vq_group_asid.group = group; 742 msg.req.vq_group_asid.asid = asid; 743 744 r = vduse_dev_msg_sync(dev, &msg); 745 if (r < 0) 746 return r; 747 748 guard(vq_group_as_write_lock)(&dev->groups[group]); 749 dev->groups[group].as = &dev->as[asid]; 750 751 return 0; 752 } 753 754 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx, 755 struct vdpa_vq_state *state) 756 { 757 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 758 struct vduse_virtqueue *vq = dev->vqs[idx]; 759 760 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) 761 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed); 762 763 return vduse_dev_get_vq_state_split(dev, vq, &state->split); 764 } 765 766 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa) 767 { 768 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 769 770 return dev->vq_align; 771 } 772 773 static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa) 774 { 775 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 776 777 return dev->device_features; 778 } 779 780 static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features) 781 { 782 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 783 784 dev->driver_features = features; 785 return 0; 786 } 787 788 static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa) 789 { 790 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 791 792 return dev->driver_features; 793 } 794 795 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa, 796 struct vdpa_callback *cb) 797 { 798 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 799 800 spin_lock(&dev->irq_lock); 801 dev->config_cb.callback = cb->callback; 802 dev->config_cb.private = cb->private; 803 spin_unlock(&dev->irq_lock); 804 } 805 806 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa) 807 { 808 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 809 u16 num_max = 0; 810 int i; 811 812 for (i = 0; i < dev->vq_num; i++) 813 if (num_max < dev->vqs[i]->num_max) 814 num_max = dev->vqs[i]->num_max; 815 816 return num_max; 817 } 818 819 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa) 820 { 821 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 822 823 return dev->device_id; 824 } 825 826 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa) 827 { 828 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 829 830 return dev->vendor_id; 831 } 832 833 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa) 834 { 835 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 836 837 return dev->status; 838 } 839 840 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status) 841 { 842 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 843 844 if (vduse_dev_set_status(dev, status)) 845 return; 846 847 dev->status = status; 848 } 849 850 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa) 851 { 852 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 853 854 return dev->config_size; 855 } 856 857 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset, 858 void *buf, unsigned int len) 859 { 860 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 861 862 /* Initialize the buffer in case of partial copy. */ 863 memset(buf, 0, len); 864 865 if (offset > dev->config_size) 866 return; 867 868 if (len > dev->config_size - offset) 869 len = dev->config_size - offset; 870 871 memcpy(buf, dev->config + offset, len); 872 } 873 874 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset, 875 const void *buf, unsigned int len) 876 { 877 /* Now we only support read-only configuration space */ 878 } 879 880 static int vduse_vdpa_reset(struct vdpa_device *vdpa) 881 { 882 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 883 int ret = vduse_dev_set_status(dev, 0); 884 885 vduse_dev_reset(dev); 886 887 return ret; 888 } 889 890 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa) 891 { 892 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 893 894 return dev->generation; 895 } 896 897 static int vduse_vdpa_set_vq_affinity(struct vdpa_device *vdpa, u16 idx, 898 const struct cpumask *cpu_mask) 899 { 900 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 901 902 if (cpu_mask) 903 cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask); 904 else 905 cpumask_setall(&dev->vqs[idx]->irq_affinity); 906 907 return 0; 908 } 909 910 static const struct cpumask * 911 vduse_vdpa_get_vq_affinity(struct vdpa_device *vdpa, u16 idx) 912 { 913 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 914 915 return &dev->vqs[idx]->irq_affinity; 916 } 917 918 static int vduse_vdpa_set_map(struct vdpa_device *vdpa, 919 unsigned int asid, 920 struct vhost_iotlb *iotlb) 921 { 922 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 923 int ret; 924 925 ret = vduse_domain_set_map(dev->as[asid].domain, iotlb); 926 if (ret) 927 return ret; 928 929 ret = vduse_dev_update_iotlb(dev, asid, 0ULL, ULLONG_MAX); 930 if (ret) { 931 vduse_domain_clear_map(dev->as[asid].domain, iotlb); 932 return ret; 933 } 934 935 return 0; 936 } 937 938 static int vduse_vdpa_suspend(struct vdpa_device *vdpa) 939 { 940 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 941 struct vduse_dev_msg msg = { 0 }; 942 int ret; 943 944 msg.req.type = VDUSE_SUSPEND; 945 946 ret = vduse_dev_msg_sync(dev, &msg); 947 if (ret == 0) { 948 scoped_guard(rwsem_write, &dev->rwsem) 949 dev->suspended = true; 950 951 cancel_work_sync(&dev->inject); 952 for (u32 i = 0; i < dev->vq_num; i++) 953 cancel_work_sync(&dev->vqs[i]->inject); 954 } 955 956 return ret; 957 } 958 959 static void vduse_vdpa_free(struct vdpa_device *vdpa) 960 { 961 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 962 963 dev->vdev = NULL; 964 } 965 966 static const struct vdpa_config_ops vduse_vdpa_config_ops = { 967 .set_vq_address = vduse_vdpa_set_vq_address, 968 .kick_vq = vduse_vdpa_kick_vq, 969 .set_vq_cb = vduse_vdpa_set_vq_cb, 970 .set_vq_num = vduse_vdpa_set_vq_num, 971 .get_vq_size = vduse_vdpa_get_vq_size, 972 .get_vq_group = vduse_get_vq_group, 973 .set_vq_ready = vduse_vdpa_set_vq_ready, 974 .get_vq_ready = vduse_vdpa_get_vq_ready, 975 .set_vq_state = vduse_vdpa_set_vq_state, 976 .get_vq_state = vduse_vdpa_get_vq_state, 977 .get_vq_align = vduse_vdpa_get_vq_align, 978 .get_device_features = vduse_vdpa_get_device_features, 979 .set_driver_features = vduse_vdpa_set_driver_features, 980 .get_driver_features = vduse_vdpa_get_driver_features, 981 .set_config_cb = vduse_vdpa_set_config_cb, 982 .get_vq_num_max = vduse_vdpa_get_vq_num_max, 983 .get_device_id = vduse_vdpa_get_device_id, 984 .get_vendor_id = vduse_vdpa_get_vendor_id, 985 .get_status = vduse_vdpa_get_status, 986 .set_status = vduse_vdpa_set_status, 987 .get_config_size = vduse_vdpa_get_config_size, 988 .get_config = vduse_vdpa_get_config, 989 .set_config = vduse_vdpa_set_config, 990 .get_generation = vduse_vdpa_get_generation, 991 .set_vq_affinity = vduse_vdpa_set_vq_affinity, 992 .get_vq_affinity = vduse_vdpa_get_vq_affinity, 993 .reset = vduse_vdpa_reset, 994 .set_map = vduse_vdpa_set_map, 995 .set_group_asid = vduse_set_group_asid, 996 .get_vq_map = vduse_get_vq_map, 997 .free = vduse_vdpa_free, 998 }; 999 1000 static const struct vdpa_config_ops vduse_vdpa_config_ops_with_suspend = { 1001 .set_vq_address = vduse_vdpa_set_vq_address, 1002 .kick_vq = vduse_vdpa_kick_vq, 1003 .set_vq_cb = vduse_vdpa_set_vq_cb, 1004 .set_vq_num = vduse_vdpa_set_vq_num, 1005 .get_vq_size = vduse_vdpa_get_vq_size, 1006 .get_vq_group = vduse_get_vq_group, 1007 .set_vq_ready = vduse_vdpa_set_vq_ready, 1008 .get_vq_ready = vduse_vdpa_get_vq_ready, 1009 .set_vq_state = vduse_vdpa_set_vq_state, 1010 .get_vq_state = vduse_vdpa_get_vq_state, 1011 .get_vq_align = vduse_vdpa_get_vq_align, 1012 .get_device_features = vduse_vdpa_get_device_features, 1013 .set_driver_features = vduse_vdpa_set_driver_features, 1014 .get_driver_features = vduse_vdpa_get_driver_features, 1015 .set_config_cb = vduse_vdpa_set_config_cb, 1016 .get_vq_num_max = vduse_vdpa_get_vq_num_max, 1017 .get_device_id = vduse_vdpa_get_device_id, 1018 .get_vendor_id = vduse_vdpa_get_vendor_id, 1019 .get_status = vduse_vdpa_get_status, 1020 .set_status = vduse_vdpa_set_status, 1021 .get_config_size = vduse_vdpa_get_config_size, 1022 .get_config = vduse_vdpa_get_config, 1023 .set_config = vduse_vdpa_set_config, 1024 .get_generation = vduse_vdpa_get_generation, 1025 .set_vq_affinity = vduse_vdpa_set_vq_affinity, 1026 .get_vq_affinity = vduse_vdpa_get_vq_affinity, 1027 .reset = vduse_vdpa_reset, 1028 .set_map = vduse_vdpa_set_map, 1029 .set_group_asid = vduse_set_group_asid, 1030 .get_vq_map = vduse_get_vq_map, 1031 .suspend = vduse_vdpa_suspend, 1032 .free = vduse_vdpa_free, 1033 }; 1034 1035 static void vduse_dev_sync_single_for_device(union virtio_map token, 1036 dma_addr_t dma_addr, size_t size, 1037 enum dma_data_direction dir) 1038 { 1039 struct vduse_iova_domain *domain; 1040 1041 if (!token.group) 1042 return; 1043 1044 guard(vq_group_as_read_lock)(token.group); 1045 domain = token.group->as->domain; 1046 vduse_domain_sync_single_for_device(domain, dma_addr, size, dir); 1047 } 1048 1049 static void vduse_dev_sync_single_for_cpu(union virtio_map token, 1050 dma_addr_t dma_addr, size_t size, 1051 enum dma_data_direction dir) 1052 { 1053 struct vduse_iova_domain *domain; 1054 1055 if (!token.group) 1056 return; 1057 1058 guard(vq_group_as_read_lock)(token.group); 1059 domain = token.group->as->domain; 1060 vduse_domain_sync_single_for_cpu(domain, dma_addr, size, dir); 1061 } 1062 1063 static dma_addr_t vduse_dev_map_page(union virtio_map token, struct page *page, 1064 unsigned long offset, size_t size, 1065 enum dma_data_direction dir, 1066 unsigned long attrs) 1067 { 1068 struct vduse_iova_domain *domain; 1069 1070 if (!token.group) 1071 return DMA_MAPPING_ERROR; 1072 1073 guard(vq_group_as_read_lock)(token.group); 1074 domain = token.group->as->domain; 1075 return vduse_domain_map_page(domain, page, offset, size, dir, attrs); 1076 } 1077 1078 static void vduse_dev_unmap_page(union virtio_map token, dma_addr_t dma_addr, 1079 size_t size, enum dma_data_direction dir, 1080 unsigned long attrs) 1081 { 1082 struct vduse_iova_domain *domain; 1083 1084 if (!token.group) 1085 return; 1086 1087 guard(vq_group_as_read_lock)(token.group); 1088 domain = token.group->as->domain; 1089 vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs); 1090 } 1091 1092 static void *vduse_dev_alloc_coherent(union virtio_map token, size_t size, 1093 dma_addr_t *dma_addr, gfp_t flag) 1094 { 1095 void *addr; 1096 1097 *dma_addr = DMA_MAPPING_ERROR; 1098 if (!token.group) 1099 return NULL; 1100 1101 addr = alloc_pages_exact(size, flag | __GFP_ZERO); 1102 if (!addr) 1103 return NULL; 1104 1105 { 1106 struct vduse_iova_domain *domain; 1107 1108 guard(vq_group_as_read_lock)(token.group); 1109 domain = token.group->as->domain; 1110 *dma_addr = vduse_domain_alloc_coherent(domain, size, addr); 1111 if (*dma_addr == DMA_MAPPING_ERROR) 1112 goto err; 1113 } 1114 1115 return addr; 1116 1117 err: 1118 free_pages_exact(addr, size); 1119 return NULL; 1120 } 1121 1122 static void vduse_dev_free_coherent(union virtio_map token, size_t size, 1123 void *vaddr, dma_addr_t dma_addr, 1124 unsigned long attrs) 1125 { 1126 if (!token.group) 1127 return; 1128 1129 { 1130 struct vduse_iova_domain *domain; 1131 1132 guard(vq_group_as_read_lock)(token.group); 1133 domain = token.group->as->domain; 1134 vduse_domain_free_coherent(domain, size, dma_addr, attrs); 1135 } 1136 1137 free_pages_exact(vaddr, size); 1138 } 1139 1140 static bool vduse_dev_need_sync(union virtio_map token, dma_addr_t dma_addr) 1141 { 1142 if (!token.group) 1143 return false; 1144 1145 guard(vq_group_as_read_lock)(token.group); 1146 return dma_addr < token.group->as->domain->bounce_size; 1147 } 1148 1149 static int vduse_dev_mapping_error(union virtio_map token, dma_addr_t dma_addr) 1150 { 1151 if (unlikely(dma_addr == DMA_MAPPING_ERROR)) 1152 return -ENOMEM; 1153 return 0; 1154 } 1155 1156 static size_t vduse_dev_max_mapping_size(union virtio_map token) 1157 { 1158 if (!token.group) 1159 return 0; 1160 1161 guard(vq_group_as_read_lock)(token.group); 1162 return token.group->as->domain->bounce_size; 1163 } 1164 1165 static const struct virtio_map_ops vduse_map_ops = { 1166 .sync_single_for_device = vduse_dev_sync_single_for_device, 1167 .sync_single_for_cpu = vduse_dev_sync_single_for_cpu, 1168 .map_page = vduse_dev_map_page, 1169 .unmap_page = vduse_dev_unmap_page, 1170 .alloc = vduse_dev_alloc_coherent, 1171 .free = vduse_dev_free_coherent, 1172 .need_sync = vduse_dev_need_sync, 1173 .mapping_error = vduse_dev_mapping_error, 1174 .max_mapping_size = vduse_dev_max_mapping_size, 1175 }; 1176 1177 static unsigned int perm_to_file_flags(u8 perm) 1178 { 1179 unsigned int flags = 0; 1180 1181 switch (perm) { 1182 case VDUSE_ACCESS_WO: 1183 flags |= O_WRONLY; 1184 break; 1185 case VDUSE_ACCESS_RO: 1186 flags |= O_RDONLY; 1187 break; 1188 case VDUSE_ACCESS_RW: 1189 flags |= O_RDWR; 1190 break; 1191 default: 1192 WARN(1, "invalidate vhost IOTLB permission\n"); 1193 break; 1194 } 1195 1196 return flags; 1197 } 1198 1199 static int vduse_kickfd_setup(struct vduse_dev *dev, 1200 struct vduse_vq_eventfd *eventfd) 1201 { 1202 struct eventfd_ctx *ctx = NULL; 1203 struct vduse_virtqueue *vq; 1204 u32 index; 1205 1206 if (eventfd->index >= dev->vq_num) 1207 return -EINVAL; 1208 1209 index = array_index_nospec(eventfd->index, dev->vq_num); 1210 vq = dev->vqs[index]; 1211 if (eventfd->fd >= 0) { 1212 ctx = eventfd_ctx_fdget(eventfd->fd); 1213 if (IS_ERR(ctx)) 1214 return PTR_ERR(ctx); 1215 } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN) 1216 return 0; 1217 1218 guard(spinlock)(&vq->kick_lock); 1219 if (vq->kickfd) 1220 eventfd_ctx_put(vq->kickfd); 1221 vq->kickfd = ctx; 1222 1223 guard(spinlock_bh)(&vq->ready_lock); 1224 if (vq->ready && vq->kicked && vq->kickfd) { 1225 eventfd_signal(vq->kickfd); 1226 vq->kicked = false; 1227 } 1228 1229 return 0; 1230 } 1231 1232 static bool vduse_dev_is_ready(struct vduse_dev *dev) 1233 { 1234 int i; 1235 1236 for (i = 0; i < dev->vq_num; i++) 1237 if (!dev->vqs[i]->num_max) 1238 return false; 1239 1240 return true; 1241 } 1242 1243 static void vduse_dev_irq_inject(struct work_struct *work) 1244 { 1245 struct vduse_dev *dev = container_of(work, struct vduse_dev, inject); 1246 1247 guard(rwsem_read)(&dev->rwsem); 1248 if (dev->suspended) 1249 return; 1250 1251 spin_lock_bh(&dev->irq_lock); 1252 if (dev->config_cb.callback) 1253 dev->config_cb.callback(dev->config_cb.private); 1254 spin_unlock_bh(&dev->irq_lock); 1255 } 1256 1257 static void vduse_vq_irq_inject(struct work_struct *work) 1258 { 1259 struct vduse_virtqueue *vq = container_of(work, 1260 struct vduse_virtqueue, inject); 1261 1262 guard(rwsem_read)(&vq->dev->rwsem); 1263 if (vq->dev->suspended) 1264 return; 1265 1266 guard(spinlock_bh)(&vq->irq_lock); 1267 guard(spinlock_bh)(&vq->ready_lock); 1268 if (vq->ready && vq->cb.callback) 1269 vq->cb.callback(vq->cb.private); 1270 } 1271 1272 static bool vduse_vq_signal_irqfd(struct vduse_virtqueue *vq) 1273 { 1274 bool signal = false; 1275 1276 guard(rwsem_read)(&vq->dev->rwsem); 1277 if (vq->dev->suspended) 1278 return false; 1279 1280 if (!vq->cb.trigger) 1281 return false; 1282 1283 guard(spinlock_irq)(&vq->irq_lock); 1284 guard(spinlock_irq)(&vq->ready_lock); 1285 if (vq->ready && vq->cb.trigger) { 1286 eventfd_signal(vq->cb.trigger); 1287 signal = true; 1288 } 1289 1290 return signal; 1291 } 1292 1293 static int vduse_dev_queue_irq_work(struct vduse_dev *dev, 1294 struct work_struct *irq_work, 1295 int irq_effective_cpu) 1296 { 1297 int ret = -EINVAL; 1298 1299 guard(rwsem_read)(&dev->rwsem); 1300 if (dev->suspended || !(dev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 1301 return ret; 1302 1303 ret = 0; 1304 if (irq_effective_cpu == IRQ_UNBOUND) 1305 queue_work(vduse_irq_wq, irq_work); 1306 else 1307 queue_work_on(irq_effective_cpu, 1308 vduse_irq_bound_wq, irq_work); 1309 1310 return ret; 1311 } 1312 1313 static int vduse_dev_dereg_umem(struct vduse_dev *dev, u32 asid, 1314 u64 iova, u64 size) 1315 { 1316 int ret; 1317 1318 mutex_lock(&dev->as[asid].mem_lock); 1319 ret = -ENOENT; 1320 if (!dev->as[asid].umem) 1321 goto unlock; 1322 1323 ret = -EINVAL; 1324 if (!dev->as[asid].domain) 1325 goto unlock; 1326 1327 if (dev->as[asid].umem->iova != iova || 1328 size != dev->as[asid].domain->bounce_size) 1329 goto unlock; 1330 1331 vduse_domain_remove_user_bounce_pages(dev->as[asid].domain); 1332 unpin_user_pages_dirty_lock(dev->as[asid].umem->pages, 1333 dev->as[asid].umem->npages, true); 1334 atomic64_sub(dev->as[asid].umem->npages, &dev->as[asid].umem->mm->pinned_vm); 1335 mmdrop(dev->as[asid].umem->mm); 1336 vfree(dev->as[asid].umem->pages); 1337 kfree(dev->as[asid].umem); 1338 dev->as[asid].umem = NULL; 1339 ret = 0; 1340 unlock: 1341 mutex_unlock(&dev->as[asid].mem_lock); 1342 return ret; 1343 } 1344 1345 static int vduse_dev_reg_umem(struct vduse_dev *dev, 1346 u32 asid, u64 iova, u64 uaddr, u64 size) 1347 { 1348 struct page **page_list = NULL; 1349 struct vduse_umem *umem = NULL; 1350 long pinned = 0; 1351 unsigned long npages, lock_limit; 1352 int ret; 1353 1354 if (!dev->as[asid].domain || !dev->as[asid].domain->bounce_map || 1355 size != dev->as[asid].domain->bounce_size || 1356 iova != 0 || uaddr & ~PAGE_MASK) 1357 return -EINVAL; 1358 1359 mutex_lock(&dev->as[asid].mem_lock); 1360 ret = -EEXIST; 1361 if (dev->as[asid].umem) 1362 goto unlock; 1363 1364 ret = -ENOMEM; 1365 npages = size >> PAGE_SHIFT; 1366 page_list = __vmalloc(array_size(npages, sizeof(struct page *)), 1367 GFP_KERNEL_ACCOUNT); 1368 umem = kzalloc_obj(*umem); 1369 if (!page_list || !umem) 1370 goto unlock; 1371 1372 mmap_read_lock(current->mm); 1373 1374 lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK)); 1375 if (npages + atomic64_read(¤t->mm->pinned_vm) > lock_limit) 1376 goto out; 1377 1378 pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM | FOLL_WRITE, 1379 page_list); 1380 if (pinned != npages) { 1381 ret = pinned < 0 ? pinned : -ENOMEM; 1382 goto out; 1383 } 1384 1385 ret = vduse_domain_add_user_bounce_pages(dev->as[asid].domain, 1386 page_list, pinned); 1387 if (ret) 1388 goto out; 1389 1390 atomic64_add(npages, ¤t->mm->pinned_vm); 1391 1392 umem->pages = page_list; 1393 umem->npages = pinned; 1394 umem->iova = iova; 1395 umem->mm = current->mm; 1396 mmgrab(current->mm); 1397 1398 dev->as[asid].umem = umem; 1399 out: 1400 if (ret && pinned > 0) 1401 unpin_user_pages(page_list, pinned); 1402 1403 mmap_read_unlock(current->mm); 1404 unlock: 1405 if (ret) { 1406 vfree(page_list); 1407 kfree(umem); 1408 } 1409 mutex_unlock(&dev->as[asid].mem_lock); 1410 return ret; 1411 } 1412 1413 static void vduse_vq_update_effective_cpu(struct vduse_virtqueue *vq) 1414 { 1415 int curr_cpu = vq->irq_effective_cpu; 1416 1417 while (true) { 1418 curr_cpu = cpumask_next(curr_cpu, &vq->irq_affinity); 1419 if (cpu_online(curr_cpu)) 1420 break; 1421 1422 if (curr_cpu >= nr_cpu_ids) 1423 curr_cpu = IRQ_UNBOUND; 1424 } 1425 1426 vq->irq_effective_cpu = curr_cpu; 1427 } 1428 1429 static int vduse_dev_iotlb_entry(struct vduse_dev *dev, 1430 struct vduse_iotlb_entry_v2 *entry, 1431 struct file **f, uint64_t *capability) 1432 { 1433 u32 asid; 1434 int r = -EINVAL; 1435 struct vhost_iotlb_map *map; 1436 1437 if (entry->start > entry->last || entry->asid >= dev->nas) 1438 return -EINVAL; 1439 1440 asid = array_index_nospec(entry->asid, dev->nas); 1441 mutex_lock(&dev->domain_lock); 1442 1443 if (!dev->as[asid].domain) 1444 goto out; 1445 1446 spin_lock(&dev->as[asid].domain->iotlb_lock); 1447 map = vhost_iotlb_itree_first(dev->as[asid].domain->iotlb, 1448 entry->start, entry->last); 1449 if (map) { 1450 if (f) { 1451 const struct vdpa_map_file *map_file; 1452 1453 map_file = (struct vdpa_map_file *)map->opaque; 1454 entry->offset = map_file->offset; 1455 *f = get_file(map_file->file); 1456 } 1457 entry->start = map->start; 1458 entry->last = map->last; 1459 entry->perm = map->perm; 1460 if (capability) { 1461 *capability = 0; 1462 1463 if (dev->as[asid].domain->bounce_map && map->start == 0 && 1464 map->last == dev->as[asid].domain->bounce_size - 1) 1465 *capability |= VDUSE_IOVA_CAP_UMEM; 1466 } 1467 1468 r = 0; 1469 } 1470 spin_unlock(&dev->as[asid].domain->iotlb_lock); 1471 1472 out: 1473 mutex_unlock(&dev->domain_lock); 1474 return r; 1475 } 1476 1477 static long vduse_dev_ioctl(struct file *file, unsigned int cmd, 1478 unsigned long arg) 1479 { 1480 struct vduse_dev *dev = file->private_data; 1481 void __user *argp = (void __user *)arg; 1482 int ret; 1483 1484 if (unlikely(dev->broken)) 1485 return -EPERM; 1486 1487 switch (cmd) { 1488 case VDUSE_IOTLB_GET_FD: 1489 case VDUSE_IOTLB_GET_FD2: { 1490 struct vduse_iotlb_entry_v2 entry = {0}; 1491 struct file *f = NULL; 1492 1493 ret = -ENOIOCTLCMD; 1494 if (dev->api_version < VDUSE_API_VERSION_1 && 1495 cmd == VDUSE_IOTLB_GET_FD2) 1496 break; 1497 1498 ret = -EFAULT; 1499 if (copy_from_user(&entry, argp, _IOC_SIZE(cmd))) 1500 break; 1501 1502 ret = -EINVAL; 1503 if (!is_mem_zero((const char *)entry.reserved, 1504 sizeof(entry.reserved))) 1505 break; 1506 1507 ret = vduse_dev_iotlb_entry(dev, &entry, &f, NULL); 1508 if (ret) 1509 break; 1510 1511 ret = -EINVAL; 1512 if (!f) 1513 break; 1514 1515 ret = copy_to_user(argp, &entry, _IOC_SIZE(cmd)); 1516 if (ret) { 1517 ret = -EFAULT; 1518 fput(f); 1519 break; 1520 } 1521 ret = receive_fd(f, NULL, perm_to_file_flags(entry.perm)); 1522 fput(f); 1523 break; 1524 } 1525 case VDUSE_DEV_GET_FEATURES: 1526 /* 1527 * Just mirror what driver wrote here. 1528 * The driver is expected to check FEATURE_OK later. 1529 */ 1530 ret = put_user(dev->driver_features, (u64 __user *)argp); 1531 break; 1532 case VDUSE_DEV_SET_CONFIG: { 1533 struct vduse_config_data config; 1534 unsigned long size = offsetof(struct vduse_config_data, 1535 buffer); 1536 1537 ret = -EFAULT; 1538 if (copy_from_user(&config, argp, size)) 1539 break; 1540 1541 ret = -EINVAL; 1542 if (config.offset > dev->config_size || 1543 config.length == 0 || 1544 config.length > dev->config_size - config.offset) 1545 break; 1546 1547 ret = -EFAULT; 1548 if (copy_from_user(dev->config + config.offset, argp + size, 1549 config.length)) 1550 break; 1551 1552 ret = 0; 1553 break; 1554 } 1555 case VDUSE_DEV_INJECT_CONFIG_IRQ: 1556 ret = vduse_dev_queue_irq_work(dev, &dev->inject, IRQ_UNBOUND); 1557 break; 1558 case VDUSE_VQ_SETUP: { 1559 struct vduse_vq_config config; 1560 u32 index; 1561 1562 ret = -EFAULT; 1563 if (copy_from_user(&config, argp, sizeof(config))) 1564 break; 1565 1566 ret = -EINVAL; 1567 if (config.index >= dev->vq_num) 1568 break; 1569 1570 if (dev->api_version < VDUSE_API_VERSION_1) { 1571 if (config.group) 1572 break; 1573 } else { 1574 if (config.group >= dev->ngroups) 1575 break; 1576 if (dev->status & VIRTIO_CONFIG_S_DRIVER_OK) 1577 break; 1578 } 1579 1580 if (config.reserved1 || 1581 !is_mem_zero((const char *)config.reserved2, 1582 sizeof(config.reserved2))) 1583 break; 1584 1585 index = array_index_nospec(config.index, dev->vq_num); 1586 dev->vqs[index]->num_max = config.max_size; 1587 dev->vqs[index]->group = config.group; 1588 ret = 0; 1589 break; 1590 } 1591 case VDUSE_VQ_GET_INFO: { 1592 struct vduse_vq_info vq_info; 1593 struct vduse_virtqueue *vq; 1594 u32 index; 1595 1596 ret = -EFAULT; 1597 if (copy_from_user(&vq_info, argp, sizeof(vq_info))) 1598 break; 1599 1600 ret = -EINVAL; 1601 if (vq_info.index >= dev->vq_num) 1602 break; 1603 1604 index = array_index_nospec(vq_info.index, dev->vq_num); 1605 vq = dev->vqs[index]; 1606 vq_info.desc_addr = vq->desc_addr; 1607 vq_info.driver_addr = vq->driver_addr; 1608 vq_info.device_addr = vq->device_addr; 1609 vq_info.num = vq->num; 1610 1611 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 1612 vq_info.packed.last_avail_counter = 1613 vq->state.packed.last_avail_counter; 1614 vq_info.packed.last_avail_idx = 1615 vq->state.packed.last_avail_idx; 1616 vq_info.packed.last_used_counter = 1617 vq->state.packed.last_used_counter; 1618 vq_info.packed.last_used_idx = 1619 vq->state.packed.last_used_idx; 1620 } else 1621 vq_info.split.avail_index = 1622 vq->state.split.avail_index; 1623 1624 scoped_guard(spinlock_bh, &vq->ready_lock) { 1625 vq_info.ready = vq->ready; 1626 } 1627 1628 ret = -EFAULT; 1629 if (copy_to_user(argp, &vq_info, sizeof(vq_info))) 1630 break; 1631 1632 ret = 0; 1633 break; 1634 } 1635 case VDUSE_VQ_SETUP_KICKFD: { 1636 struct vduse_vq_eventfd eventfd; 1637 1638 ret = -EFAULT; 1639 if (copy_from_user(&eventfd, argp, sizeof(eventfd))) 1640 break; 1641 1642 ret = vduse_kickfd_setup(dev, &eventfd); 1643 break; 1644 } 1645 case VDUSE_VQ_INJECT_IRQ: { 1646 u32 index; 1647 1648 ret = -EFAULT; 1649 if (get_user(index, (u32 __user *)argp)) 1650 break; 1651 1652 ret = -EINVAL; 1653 if (index >= dev->vq_num) 1654 break; 1655 1656 ret = 0; 1657 index = array_index_nospec(index, dev->vq_num); 1658 if (!vduse_vq_signal_irqfd(dev->vqs[index])) { 1659 vduse_vq_update_effective_cpu(dev->vqs[index]); 1660 ret = vduse_dev_queue_irq_work(dev, 1661 &dev->vqs[index]->inject, 1662 dev->vqs[index]->irq_effective_cpu); 1663 } 1664 break; 1665 } 1666 case VDUSE_IOTLB_REG_UMEM: { 1667 struct vduse_iova_umem umem; 1668 u32 asid; 1669 1670 ret = -EFAULT; 1671 if (copy_from_user(&umem, argp, sizeof(umem))) 1672 break; 1673 1674 ret = -EINVAL; 1675 if (!is_mem_zero((const char *)umem.reserved, 1676 sizeof(umem.reserved)) || 1677 (dev->api_version < VDUSE_API_VERSION_1 && 1678 umem.asid != 0) || umem.asid >= dev->nas) 1679 break; 1680 1681 mutex_lock(&dev->domain_lock); 1682 asid = array_index_nospec(umem.asid, dev->nas); 1683 ret = vduse_dev_reg_umem(dev, asid, umem.iova, 1684 umem.uaddr, umem.size); 1685 mutex_unlock(&dev->domain_lock); 1686 break; 1687 } 1688 case VDUSE_IOTLB_DEREG_UMEM: { 1689 struct vduse_iova_umem umem; 1690 u32 asid; 1691 1692 ret = -EFAULT; 1693 if (copy_from_user(&umem, argp, sizeof(umem))) 1694 break; 1695 1696 ret = -EINVAL; 1697 if (!is_mem_zero((const char *)umem.reserved, 1698 sizeof(umem.reserved)) || 1699 (dev->api_version < VDUSE_API_VERSION_1 && 1700 umem.asid != 0) || 1701 umem.asid >= dev->nas) 1702 break; 1703 1704 mutex_lock(&dev->domain_lock); 1705 asid = array_index_nospec(umem.asid, dev->nas); 1706 ret = vduse_dev_dereg_umem(dev, asid, umem.iova, 1707 umem.size); 1708 mutex_unlock(&dev->domain_lock); 1709 break; 1710 } 1711 case VDUSE_IOTLB_GET_INFO: { 1712 struct vduse_iova_info info; 1713 struct vduse_iotlb_entry_v2 entry; 1714 1715 ret = -EFAULT; 1716 if (copy_from_user(&info, argp, sizeof(info))) 1717 break; 1718 1719 if (!is_mem_zero((const char *)info.reserved, 1720 sizeof(info.reserved))) 1721 break; 1722 1723 if (dev->api_version < VDUSE_API_VERSION_1) { 1724 if (info.asid) 1725 break; 1726 } else if (info.asid >= dev->nas) 1727 break; 1728 1729 entry.start = info.start; 1730 entry.last = info.last; 1731 entry.asid = info.asid; 1732 ret = vduse_dev_iotlb_entry(dev, &entry, NULL, 1733 &info.capability); 1734 if (ret < 0) 1735 break; 1736 1737 info.start = entry.start; 1738 info.last = entry.last; 1739 info.asid = entry.asid; 1740 1741 ret = -EFAULT; 1742 if (copy_to_user(argp, &info, sizeof(info))) 1743 break; 1744 1745 ret = 0; 1746 break; 1747 } 1748 default: 1749 ret = -ENOIOCTLCMD; 1750 break; 1751 } 1752 1753 return ret; 1754 } 1755 1756 #ifdef CONFIG_COMPAT_FOR_U64_ALIGNMENT 1757 /* 1758 * i386 has different alignment constraints than x86_64, 1759 * so there are only 3 bytes of padding instead of 7. 1760 */ 1761 struct compat_vduse_iotlb_entry { 1762 compat_u64 offset; 1763 compat_u64 start; 1764 compat_u64 last; 1765 __u8 perm; 1766 __u8 padding[3]; 1767 }; 1768 #define COMPAT_VDUSE_IOTLB_GET_FD _IOWR(VDUSE_BASE, 0x10, struct compat_vduse_iotlb_entry) 1769 1770 struct compat_vduse_vq_info { 1771 __u32 index; 1772 __u32 num; 1773 compat_u64 desc_addr; 1774 compat_u64 driver_addr; 1775 compat_u64 device_addr; 1776 union { 1777 struct vduse_vq_state_split split; 1778 struct vduse_vq_state_packed packed; 1779 }; 1780 __u8 ready; 1781 __u8 padding[3]; 1782 }; 1783 #define COMPAT_VDUSE_VQ_GET_INFO _IOWR(VDUSE_BASE, 0x15, struct compat_vduse_vq_info) 1784 1785 static long vduse_dev_compat_ioctl(struct file *file, unsigned int cmd, 1786 unsigned long arg) 1787 { 1788 struct vduse_dev *dev = file->private_data; 1789 void __user *argp = (void __user *)arg; 1790 int ret; 1791 1792 if (unlikely(dev->broken)) 1793 return -EPERM; 1794 1795 switch (cmd) { 1796 case COMPAT_VDUSE_IOTLB_GET_FD: { 1797 struct vduse_iotlb_entry_v2 entry = {0}; 1798 struct file *f = NULL; 1799 1800 ret = -EFAULT; 1801 if (copy_from_user(&entry, argp, _IOC_SIZE(cmd))) 1802 break; 1803 1804 ret = vduse_dev_iotlb_entry(dev, &entry, &f, NULL); 1805 if (ret) 1806 break; 1807 1808 ret = -EINVAL; 1809 if (!f) 1810 break; 1811 1812 ret = copy_to_user(argp, &entry, _IOC_SIZE(cmd)); 1813 if (ret) { 1814 ret = -EFAULT; 1815 fput(f); 1816 break; 1817 } 1818 ret = receive_fd(f, NULL, perm_to_file_flags(entry.perm)); 1819 fput(f); 1820 break; 1821 } 1822 case COMPAT_VDUSE_VQ_GET_INFO: { 1823 struct vduse_vq_info vq_info = {}; 1824 struct vduse_virtqueue *vq; 1825 u32 index; 1826 1827 ret = -EFAULT; 1828 if (copy_from_user(&vq_info, argp, 1829 sizeof(struct compat_vduse_vq_info))) 1830 break; 1831 1832 ret = -EINVAL; 1833 if (vq_info.index >= dev->vq_num) 1834 break; 1835 1836 index = array_index_nospec(vq_info.index, dev->vq_num); 1837 vq = dev->vqs[index]; 1838 vq_info.desc_addr = vq->desc_addr; 1839 vq_info.driver_addr = vq->driver_addr; 1840 vq_info.device_addr = vq->device_addr; 1841 vq_info.num = vq->num; 1842 1843 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 1844 vq_info.packed.last_avail_counter = 1845 vq->state.packed.last_avail_counter; 1846 vq_info.packed.last_avail_idx = 1847 vq->state.packed.last_avail_idx; 1848 vq_info.packed.last_used_counter = 1849 vq->state.packed.last_used_counter; 1850 vq_info.packed.last_used_idx = 1851 vq->state.packed.last_used_idx; 1852 } else 1853 vq_info.split.avail_index = 1854 vq->state.split.avail_index; 1855 1856 scoped_guard(spinlock_bh, &vq->ready_lock) { 1857 vq_info.ready = vq->ready; 1858 } 1859 1860 ret = -EFAULT; 1861 if (copy_to_user(argp, &vq_info, 1862 sizeof(struct compat_vduse_vq_info))) 1863 break; 1864 1865 ret = 0; 1866 break; 1867 } 1868 default: 1869 ret = -ENOIOCTLCMD; 1870 break; 1871 } 1872 1873 return vduse_dev_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); 1874 } 1875 #else 1876 #define vduse_dev_compat_ioctl compat_ptr_ioctl 1877 #endif 1878 1879 static int vduse_dev_release(struct inode *inode, struct file *file) 1880 { 1881 struct vduse_dev *dev = file->private_data; 1882 1883 mutex_lock(&dev->domain_lock); 1884 for (int i = 0; i < dev->nas; i++) 1885 if (dev->as[i].domain) 1886 vduse_dev_dereg_umem(dev, i, 0, 1887 dev->as[i].domain->bounce_size); 1888 mutex_unlock(&dev->domain_lock); 1889 spin_lock(&dev->msg_lock); 1890 /* Make sure the inflight messages can processed after reconncection */ 1891 list_splice_init(&dev->recv_list, &dev->send_list); 1892 spin_unlock(&dev->msg_lock); 1893 dev->connected = false; 1894 1895 return 0; 1896 } 1897 1898 static int vduse_dev_open(struct inode *inode, struct file *file) 1899 { 1900 int ret = -EBUSY; 1901 struct vduse_dev *dev; 1902 1903 mutex_lock(&vduse_lock); 1904 dev = idr_find(&vduse_idr, iminor(inode)); 1905 if (!dev) { 1906 mutex_unlock(&vduse_lock); 1907 return -ENODEV; 1908 } 1909 1910 mutex_lock(&dev->lock); 1911 if (dev->connected) 1912 goto unlock; 1913 1914 ret = 0; 1915 dev->connected = true; 1916 file->private_data = dev; 1917 unlock: 1918 mutex_unlock(&dev->lock); 1919 mutex_unlock(&vduse_lock); 1920 1921 return ret; 1922 } 1923 1924 static const struct file_operations vduse_dev_fops = { 1925 .owner = THIS_MODULE, 1926 .open = vduse_dev_open, 1927 .release = vduse_dev_release, 1928 .read_iter = vduse_dev_read_iter, 1929 .write_iter = vduse_dev_write_iter, 1930 .poll = vduse_dev_poll, 1931 .unlocked_ioctl = vduse_dev_ioctl, 1932 .compat_ioctl = vduse_dev_compat_ioctl, 1933 .llseek = noop_llseek, 1934 }; 1935 1936 static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf) 1937 { 1938 return sprintf(buf, "%*pb\n", cpumask_pr_args(&vq->irq_affinity)); 1939 } 1940 1941 static ssize_t irq_cb_affinity_store(struct vduse_virtqueue *vq, 1942 const char *buf, size_t count) 1943 { 1944 cpumask_var_t new_value; 1945 int ret; 1946 1947 if (!zalloc_cpumask_var(&new_value, GFP_KERNEL)) 1948 return -ENOMEM; 1949 1950 ret = cpumask_parse(buf, new_value); 1951 if (ret) 1952 goto free_mask; 1953 1954 ret = -EINVAL; 1955 if (!cpumask_intersects(new_value, cpu_online_mask)) 1956 goto free_mask; 1957 1958 cpumask_copy(&vq->irq_affinity, new_value); 1959 ret = count; 1960 free_mask: 1961 free_cpumask_var(new_value); 1962 return ret; 1963 } 1964 1965 struct vq_sysfs_entry { 1966 struct attribute attr; 1967 ssize_t (*show)(struct vduse_virtqueue *vq, char *buf); 1968 ssize_t (*store)(struct vduse_virtqueue *vq, const char *buf, 1969 size_t count); 1970 }; 1971 1972 static struct vq_sysfs_entry irq_cb_affinity_attr = __ATTR_RW(irq_cb_affinity); 1973 1974 static struct attribute *vq_attrs[] = { 1975 &irq_cb_affinity_attr.attr, 1976 NULL, 1977 }; 1978 ATTRIBUTE_GROUPS(vq); 1979 1980 static ssize_t vq_attr_show(struct kobject *kobj, struct attribute *attr, 1981 char *buf) 1982 { 1983 struct vduse_virtqueue *vq = container_of(kobj, 1984 struct vduse_virtqueue, kobj); 1985 struct vq_sysfs_entry *entry = container_of(attr, 1986 struct vq_sysfs_entry, attr); 1987 1988 if (!entry->show) 1989 return -EIO; 1990 1991 return entry->show(vq, buf); 1992 } 1993 1994 static ssize_t vq_attr_store(struct kobject *kobj, struct attribute *attr, 1995 const char *buf, size_t count) 1996 { 1997 struct vduse_virtqueue *vq = container_of(kobj, 1998 struct vduse_virtqueue, kobj); 1999 struct vq_sysfs_entry *entry = container_of(attr, 2000 struct vq_sysfs_entry, attr); 2001 2002 if (!entry->store) 2003 return -EIO; 2004 2005 return entry->store(vq, buf, count); 2006 } 2007 2008 static const struct sysfs_ops vq_sysfs_ops = { 2009 .show = vq_attr_show, 2010 .store = vq_attr_store, 2011 }; 2012 2013 static void vq_release(struct kobject *kobj) 2014 { 2015 struct vduse_virtqueue *vq = container_of(kobj, 2016 struct vduse_virtqueue, kobj); 2017 kfree(vq); 2018 } 2019 2020 static const struct kobj_type vq_type = { 2021 .release = vq_release, 2022 .sysfs_ops = &vq_sysfs_ops, 2023 .default_groups = vq_groups, 2024 }; 2025 2026 static char *vduse_devnode(const struct device *dev, umode_t *mode) 2027 { 2028 return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev)); 2029 } 2030 2031 static const struct class vduse_class = { 2032 .name = "vduse", 2033 .devnode = vduse_devnode, 2034 }; 2035 2036 static void vduse_dev_deinit_vqs(struct vduse_dev *dev) 2037 { 2038 int i; 2039 2040 if (!dev->vqs) 2041 return; 2042 2043 for (i = 0; i < dev->vq_num; i++) 2044 kobject_put(&dev->vqs[i]->kobj); 2045 kfree(dev->vqs); 2046 } 2047 2048 static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num) 2049 { 2050 int ret, i; 2051 2052 dev->vq_align = vq_align; 2053 dev->vq_num = vq_num; 2054 dev->vqs = kzalloc_objs(*dev->vqs, dev->vq_num); 2055 if (!dev->vqs) 2056 return -ENOMEM; 2057 2058 for (i = 0; i < vq_num; i++) { 2059 dev->vqs[i] = kzalloc_obj(*dev->vqs[i]); 2060 if (!dev->vqs[i]) { 2061 ret = -ENOMEM; 2062 goto err; 2063 } 2064 2065 dev->vqs[i]->index = i; 2066 dev->vqs[i]->dev = dev; 2067 dev->vqs[i]->irq_effective_cpu = IRQ_UNBOUND; 2068 INIT_WORK(&dev->vqs[i]->inject, vduse_vq_irq_inject); 2069 INIT_WORK(&dev->vqs[i]->kick, vduse_vq_kick_work); 2070 spin_lock_init(&dev->vqs[i]->kick_lock); 2071 spin_lock_init(&dev->vqs[i]->irq_lock); 2072 spin_lock_init(&dev->vqs[i]->ready_lock); 2073 cpumask_setall(&dev->vqs[i]->irq_affinity); 2074 2075 kobject_init(&dev->vqs[i]->kobj, &vq_type); 2076 ret = kobject_add(&dev->vqs[i]->kobj, 2077 &dev->dev->kobj, "vq%d", i); 2078 if (ret) { 2079 kfree(dev->vqs[i]); 2080 goto err; 2081 } 2082 } 2083 2084 return 0; 2085 err: 2086 while (i--) 2087 kobject_put(&dev->vqs[i]->kobj); 2088 kfree(dev->vqs); 2089 dev->vqs = NULL; 2090 return ret; 2091 } 2092 2093 static struct vduse_dev *vduse_dev_create(void) 2094 { 2095 struct vduse_dev *dev = kzalloc_obj(*dev); 2096 2097 if (!dev) 2098 return NULL; 2099 2100 mutex_init(&dev->lock); 2101 mutex_init(&dev->domain_lock); 2102 spin_lock_init(&dev->msg_lock); 2103 INIT_LIST_HEAD(&dev->send_list); 2104 INIT_LIST_HEAD(&dev->recv_list); 2105 spin_lock_init(&dev->irq_lock); 2106 init_rwsem(&dev->rwsem); 2107 2108 INIT_WORK(&dev->inject, vduse_dev_irq_inject); 2109 init_waitqueue_head(&dev->waitq); 2110 2111 return dev; 2112 } 2113 2114 static void vduse_dev_destroy(struct vduse_dev *dev) 2115 { 2116 kfree(dev); 2117 } 2118 2119 static struct vduse_dev *vduse_find_dev(const char *name) 2120 { 2121 struct vduse_dev *dev; 2122 int id; 2123 2124 idr_for_each_entry(&vduse_idr, dev, id) 2125 if (!strcmp(dev->name, name)) 2126 return dev; 2127 2128 return NULL; 2129 } 2130 2131 static int vduse_destroy_dev(char *name) 2132 { 2133 struct vduse_dev *dev = vduse_find_dev(name); 2134 2135 if (!dev) 2136 return -EINVAL; 2137 2138 mutex_lock(&dev->lock); 2139 if (dev->vdev || dev->connected) { 2140 mutex_unlock(&dev->lock); 2141 return -EBUSY; 2142 } 2143 dev->connected = true; 2144 mutex_unlock(&dev->lock); 2145 2146 vduse_dev_reset(dev); 2147 device_destroy(&vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); 2148 idr_remove(&vduse_idr, dev->minor); 2149 kvfree(dev->config); 2150 vduse_dev_deinit_vqs(dev); 2151 for (int i = 0; i < dev->nas; i++) { 2152 if (dev->as[i].domain) 2153 vduse_domain_destroy(dev->as[i].domain); 2154 } 2155 kfree(dev->as); 2156 kfree(dev->name); 2157 kfree(dev->groups); 2158 vduse_dev_destroy(dev); 2159 module_put(THIS_MODULE); 2160 2161 return 0; 2162 } 2163 2164 static bool device_is_allowed(u32 device_id) 2165 { 2166 int i; 2167 2168 for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++) 2169 if (allowed_device_id[i] == device_id) 2170 return true; 2171 2172 return false; 2173 } 2174 2175 static bool features_is_valid(struct vduse_dev_config *config) 2176 { 2177 if (!(config->features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) 2178 return false; 2179 2180 /* Now we only support read-only configuration space */ 2181 if ((config->device_id == VIRTIO_ID_BLOCK) && 2182 (config->features & BIT_ULL(VIRTIO_BLK_F_CONFIG_WCE))) 2183 return false; 2184 else if ((config->device_id == VIRTIO_ID_NET) && 2185 (config->features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 2186 return false; 2187 2188 if ((config->device_id == VIRTIO_ID_NET) && 2189 !(config->features & BIT_ULL(VIRTIO_F_VERSION_1))) 2190 return false; 2191 2192 return true; 2193 } 2194 2195 static bool vduse_validate_config(struct vduse_dev_config *config, 2196 u64 api_version) 2197 { 2198 if (!is_mem_zero((const char *)config->reserved, 2199 sizeof(config->reserved))) 2200 return false; 2201 2202 if (api_version < VDUSE_API_VERSION_1 && 2203 (config->ngroups || config->nas)) 2204 return false; 2205 2206 if (api_version >= VDUSE_API_VERSION_1) { 2207 if (!config->ngroups || config->ngroups > VDUSE_DEV_MAX_GROUPS) 2208 return false; 2209 2210 if (!config->nas || config->nas > VDUSE_DEV_MAX_AS) 2211 return false; 2212 } 2213 2214 if (config->vq_align > PAGE_SIZE) 2215 return false; 2216 2217 if (config->config_size > PAGE_SIZE) 2218 return false; 2219 2220 if (config->vq_num > 0xffff) 2221 return false; 2222 2223 if (!config->name[0]) 2224 return false; 2225 2226 if (!device_is_allowed(config->device_id)) 2227 return false; 2228 2229 if (!features_is_valid(config)) 2230 return false; 2231 2232 return true; 2233 } 2234 2235 static ssize_t msg_timeout_show(struct device *device, 2236 struct device_attribute *attr, char *buf) 2237 { 2238 struct vduse_dev *dev = dev_get_drvdata(device); 2239 2240 return sysfs_emit(buf, "%u\n", dev->msg_timeout); 2241 } 2242 2243 static ssize_t msg_timeout_store(struct device *device, 2244 struct device_attribute *attr, 2245 const char *buf, size_t count) 2246 { 2247 struct vduse_dev *dev = dev_get_drvdata(device); 2248 int ret; 2249 2250 ret = kstrtouint(buf, 10, &dev->msg_timeout); 2251 if (ret < 0) 2252 return ret; 2253 2254 return count; 2255 } 2256 2257 static DEVICE_ATTR_RW(msg_timeout); 2258 2259 static ssize_t bounce_size_show(struct device *device, 2260 struct device_attribute *attr, char *buf) 2261 { 2262 struct vduse_dev *dev = dev_get_drvdata(device); 2263 2264 return sysfs_emit(buf, "%u\n", dev->bounce_size); 2265 } 2266 2267 static ssize_t bounce_size_store(struct device *device, 2268 struct device_attribute *attr, 2269 const char *buf, size_t count) 2270 { 2271 struct vduse_dev *dev = dev_get_drvdata(device); 2272 unsigned int bounce_size; 2273 int ret; 2274 2275 ret = -EPERM; 2276 mutex_lock(&dev->domain_lock); 2277 /* Assuming that if the first domain is allocated, all are allocated */ 2278 if (dev->as[0].domain) 2279 goto unlock; 2280 2281 ret = kstrtouint(buf, 10, &bounce_size); 2282 if (ret < 0) 2283 goto unlock; 2284 2285 ret = -EINVAL; 2286 if (bounce_size > VDUSE_MAX_BOUNCE_SIZE || 2287 bounce_size < VDUSE_MIN_BOUNCE_SIZE) 2288 goto unlock; 2289 2290 dev->bounce_size = bounce_size & PAGE_MASK; 2291 ret = count; 2292 unlock: 2293 mutex_unlock(&dev->domain_lock); 2294 return ret; 2295 } 2296 2297 static DEVICE_ATTR_RW(bounce_size); 2298 2299 static struct attribute *vduse_dev_attrs[] = { 2300 &dev_attr_msg_timeout.attr, 2301 &dev_attr_bounce_size.attr, 2302 NULL 2303 }; 2304 2305 ATTRIBUTE_GROUPS(vduse_dev); 2306 2307 static int vduse_create_dev(struct vduse_dev_config *config, 2308 void *config_buf, u64 api_version, 2309 uint64_t vduse_features) 2310 { 2311 int ret; 2312 struct vduse_dev *dev; 2313 2314 ret = -EPERM; 2315 if ((config->device_id == VIRTIO_ID_NET) && !capable(CAP_NET_ADMIN)) 2316 goto err; 2317 2318 ret = -EEXIST; 2319 if (vduse_find_dev(config->name)) 2320 goto err; 2321 2322 ret = -ENOMEM; 2323 dev = vduse_dev_create(); 2324 if (!dev) 2325 goto err; 2326 2327 dev->api_version = api_version; 2328 dev->device_features = config->features; 2329 dev->device_id = config->device_id; 2330 dev->vendor_id = config->vendor_id; 2331 dev->vduse_features = vduse_features; 2332 dev_dbg(vduse_ctrl_dev, "Creating device %s with features 0x%llx", 2333 config->name, vduse_features); 2334 2335 dev->nas = (dev->api_version < VDUSE_API_VERSION_1) ? 1 : config->nas; 2336 dev->as = kzalloc_objs(dev->as[0], dev->nas); 2337 if (!dev->as) 2338 goto err_as; 2339 for (int i = 0; i < dev->nas; i++) 2340 mutex_init(&dev->as[i].mem_lock); 2341 2342 dev->ngroups = (dev->api_version < VDUSE_API_VERSION_1) 2343 ? 1 2344 : config->ngroups; 2345 dev->groups = kzalloc_objs(dev->groups[0], dev->ngroups); 2346 if (!dev->groups) 2347 goto err_vq_groups; 2348 for (u32 i = 0; i < dev->ngroups; ++i) { 2349 dev->groups[i].dev = dev; 2350 rwlock_init(&dev->groups[i].as_lock); 2351 dev->groups[i].as = &dev->as[0]; 2352 } 2353 2354 dev->name = kstrdup(config->name, GFP_KERNEL); 2355 if (!dev->name) 2356 goto err_str; 2357 2358 dev->bounce_size = VDUSE_BOUNCE_SIZE; 2359 dev->config = config_buf; 2360 dev->config_size = config->config_size; 2361 2362 ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL); 2363 if (ret < 0) 2364 goto err_idr; 2365 2366 dev->minor = ret; 2367 dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT; 2368 dev->dev = device_create_with_groups(&vduse_class, NULL, 2369 MKDEV(MAJOR(vduse_major), dev->minor), 2370 dev, vduse_dev_groups, "%s", config->name); 2371 if (IS_ERR(dev->dev)) { 2372 ret = PTR_ERR(dev->dev); 2373 goto err_dev; 2374 } 2375 2376 ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num); 2377 if (ret) 2378 goto err_vqs; 2379 2380 __module_get(THIS_MODULE); 2381 2382 return 0; 2383 err_vqs: 2384 device_destroy(&vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); 2385 err_dev: 2386 idr_remove(&vduse_idr, dev->minor); 2387 err_idr: 2388 kfree(dev->name); 2389 err_str: 2390 kfree(dev->groups); 2391 err_vq_groups: 2392 kfree(dev->as); 2393 err_as: 2394 vduse_dev_destroy(dev); 2395 err: 2396 return ret; 2397 } 2398 2399 static long vduse_ioctl(struct file *file, unsigned int cmd, 2400 unsigned long arg) 2401 { 2402 int ret; 2403 void __user *argp = (void __user *)arg; 2404 struct vduse_control *control = file->private_data; 2405 2406 mutex_lock(&vduse_lock); 2407 switch (cmd) { 2408 case VDUSE_GET_API_VERSION: 2409 if (control->api_version == VDUSE_API_VERSION_NOT_ASKED) 2410 control->api_version = VDUSE_API_VERSION_1; 2411 ret = put_user(control->api_version, (u64 __user *)argp); 2412 break; 2413 case VDUSE_SET_API_VERSION: { 2414 u64 api_version; 2415 2416 ret = -EFAULT; 2417 if (get_user(api_version, (u64 __user *)argp)) 2418 break; 2419 2420 ret = -EINVAL; 2421 if (api_version > VDUSE_API_VERSION_1) 2422 break; 2423 2424 ret = 0; 2425 control->api_version = api_version; 2426 break; 2427 } 2428 case VDUSE_CREATE_DEV: { 2429 struct vduse_dev_config config; 2430 unsigned long size = offsetof(struct vduse_dev_config, config); 2431 void *buf; 2432 2433 ret = -EFAULT; 2434 if (copy_from_user(&config, argp, size)) 2435 break; 2436 2437 ret = -EINVAL; 2438 if (control->api_version == VDUSE_API_VERSION_NOT_ASKED) 2439 control->api_version = VDUSE_API_VERSION; 2440 if (!vduse_validate_config(&config, control->api_version)) 2441 break; 2442 2443 buf = vmemdup_user(argp + size, config.config_size); 2444 if (IS_ERR(buf)) { 2445 ret = PTR_ERR(buf); 2446 break; 2447 } 2448 config.name[VDUSE_NAME_MAX - 1] = '\0'; 2449 ret = vduse_create_dev(&config, buf, control->api_version, 2450 control->vduse_features); 2451 if (ret) 2452 kvfree(buf); 2453 break; 2454 } 2455 case VDUSE_DESTROY_DEV: { 2456 char name[VDUSE_NAME_MAX]; 2457 2458 ret = -EFAULT; 2459 if (copy_from_user(name, argp, VDUSE_NAME_MAX)) 2460 break; 2461 2462 name[VDUSE_NAME_MAX - 1] = '\0'; 2463 ret = vduse_destroy_dev(name); 2464 break; 2465 } 2466 case VDUSE_GET_FEATURES: 2467 ret = put_user(vduse_features, (u64 __user *)argp); 2468 break; 2469 case VDUSE_SET_FEATURES: { 2470 u64 features; 2471 2472 ret = -EFAULT; 2473 if (get_user(features, (u64 __user *)argp)) { 2474 dev_dbg(vduse_ctrl_dev, "Could not get vduse features"); 2475 break; 2476 } 2477 2478 ret = -EINVAL; 2479 if (features & ~vduse_features) { 2480 dev_dbg(vduse_ctrl_dev, 2481 "Invalid features in %llx, expected %llx", 2482 features, vduse_features); 2483 break; 2484 } 2485 2486 ret = 0; 2487 control->vduse_features = features; 2488 dev_dbg(vduse_ctrl_dev, "Set features %llx", features); 2489 2490 break; 2491 } 2492 default: 2493 ret = -EINVAL; 2494 break; 2495 } 2496 mutex_unlock(&vduse_lock); 2497 2498 return ret; 2499 } 2500 2501 static int vduse_release(struct inode *inode, struct file *file) 2502 { 2503 struct vduse_control *control = file->private_data; 2504 2505 kfree(control); 2506 return 0; 2507 } 2508 2509 static int vduse_open(struct inode *inode, struct file *file) 2510 { 2511 struct vduse_control *control; 2512 2513 control = kmalloc_obj(struct vduse_control); 2514 if (!control) 2515 return -ENOMEM; 2516 2517 control->api_version = VDUSE_API_VERSION_NOT_ASKED; 2518 control->vduse_features = 0; 2519 file->private_data = control; 2520 2521 return 0; 2522 } 2523 2524 static const struct file_operations vduse_ctrl_fops = { 2525 .owner = THIS_MODULE, 2526 .open = vduse_open, 2527 .release = vduse_release, 2528 .unlocked_ioctl = vduse_ioctl, 2529 .compat_ioctl = compat_ptr_ioctl, 2530 .llseek = noop_llseek, 2531 }; 2532 2533 struct vduse_mgmt_dev { 2534 struct vdpa_mgmt_dev mgmt_dev; 2535 struct device dev; 2536 }; 2537 2538 static struct vduse_mgmt_dev *vduse_mgmt; 2539 2540 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name) 2541 { 2542 struct vduse_vdpa *vdev; 2543 const struct vdpa_config_ops *ops; 2544 2545 if (dev->vdev) 2546 return -EEXIST; 2547 2548 if (dev->vduse_features & BIT_U64(VDUSE_F_SUSPEND)) 2549 ops = &vduse_vdpa_config_ops_with_suspend; 2550 else 2551 ops = &vduse_vdpa_config_ops; 2552 2553 vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev, 2554 ops, &vduse_map_ops, 2555 dev->ngroups, dev->nas, name, true); 2556 if (IS_ERR(vdev)) 2557 return PTR_ERR(vdev); 2558 2559 dev->vdev = vdev; 2560 vdev->dev = dev; 2561 vdev->vdpa.mdev = &vduse_mgmt->mgmt_dev; 2562 2563 return 0; 2564 } 2565 2566 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, 2567 const struct vdpa_dev_set_config *config) 2568 { 2569 struct vduse_dev *dev; 2570 size_t domain_bounce_size; 2571 int ret, i; 2572 2573 mutex_lock(&vduse_lock); 2574 dev = vduse_find_dev(name); 2575 if (!dev || !vduse_dev_is_ready(dev)) { 2576 mutex_unlock(&vduse_lock); 2577 return -EINVAL; 2578 } 2579 ret = vduse_dev_init_vdpa(dev, name); 2580 mutex_unlock(&vduse_lock); 2581 if (ret) 2582 return ret; 2583 2584 mutex_lock(&dev->domain_lock); 2585 ret = 0; 2586 2587 domain_bounce_size = dev->bounce_size / dev->nas; 2588 for (i = 0; i < dev->nas; ++i) { 2589 dev->as[i].domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1, 2590 domain_bounce_size); 2591 if (!dev->as[i].domain) { 2592 ret = -ENOMEM; 2593 goto err; 2594 } 2595 } 2596 2597 mutex_unlock(&dev->domain_lock); 2598 2599 ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num); 2600 if (ret) 2601 goto err_register; 2602 2603 return 0; 2604 2605 err_register: 2606 mutex_lock(&dev->domain_lock); 2607 2608 err: 2609 for (int j = 0; j < i; j++) { 2610 if (dev->as[j].domain) { 2611 vduse_domain_destroy(dev->as[j].domain); 2612 dev->as[j].domain = NULL; 2613 } 2614 } 2615 mutex_unlock(&dev->domain_lock); 2616 2617 put_device(&dev->vdev->vdpa.dev); 2618 2619 return ret; 2620 } 2621 2622 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev) 2623 { 2624 _vdpa_unregister_device(dev); 2625 } 2626 2627 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = { 2628 .dev_add = vdpa_dev_add, 2629 .dev_del = vdpa_dev_del, 2630 }; 2631 2632 static struct virtio_device_id id_table[] = { 2633 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, 2634 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 2635 { 0 }, 2636 }; 2637 2638 static void vduse_mgmtdev_release(struct device *dev) 2639 { 2640 struct vduse_mgmt_dev *mgmt_dev; 2641 2642 mgmt_dev = container_of(dev, struct vduse_mgmt_dev, dev); 2643 kfree(mgmt_dev); 2644 } 2645 2646 static int vduse_mgmtdev_init(void) 2647 { 2648 int ret; 2649 2650 vduse_mgmt = kzalloc_obj(*vduse_mgmt); 2651 if (!vduse_mgmt) 2652 return -ENOMEM; 2653 2654 ret = dev_set_name(&vduse_mgmt->dev, "vduse"); 2655 if (ret) { 2656 kfree(vduse_mgmt); 2657 return ret; 2658 } 2659 2660 vduse_mgmt->dev.release = vduse_mgmtdev_release; 2661 2662 ret = device_register(&vduse_mgmt->dev); 2663 if (ret) 2664 goto dev_reg_err; 2665 2666 vduse_mgmt->mgmt_dev.id_table = id_table; 2667 vduse_mgmt->mgmt_dev.ops = &vdpa_dev_mgmtdev_ops; 2668 vduse_mgmt->mgmt_dev.device = &vduse_mgmt->dev; 2669 ret = vdpa_mgmtdev_register(&vduse_mgmt->mgmt_dev); 2670 if (ret) 2671 device_unregister(&vduse_mgmt->dev); 2672 2673 return ret; 2674 2675 dev_reg_err: 2676 put_device(&vduse_mgmt->dev); 2677 return ret; 2678 } 2679 2680 static void vduse_mgmtdev_exit(void) 2681 { 2682 vdpa_mgmtdev_unregister(&vduse_mgmt->mgmt_dev); 2683 device_unregister(&vduse_mgmt->dev); 2684 } 2685 2686 static int vduse_init(void) 2687 { 2688 int ret; 2689 2690 ret = class_register(&vduse_class); 2691 if (ret) 2692 return ret; 2693 2694 ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse"); 2695 if (ret) 2696 goto err_chardev_region; 2697 2698 /* /dev/vduse/control */ 2699 cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops); 2700 vduse_ctrl_cdev.owner = THIS_MODULE; 2701 ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1); 2702 if (ret) 2703 goto err_ctrl_cdev; 2704 2705 vduse_ctrl_dev = device_create(&vduse_class, NULL, vduse_major, NULL, "control"); 2706 if (IS_ERR(vduse_ctrl_dev)) { 2707 ret = PTR_ERR(vduse_ctrl_dev); 2708 vduse_ctrl_dev = NULL; 2709 goto err_device; 2710 } 2711 2712 /* /dev/vduse/$DEVICE */ 2713 cdev_init(&vduse_cdev, &vduse_dev_fops); 2714 vduse_cdev.owner = THIS_MODULE; 2715 ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1), 2716 VDUSE_DEV_MAX - 1); 2717 if (ret) 2718 goto err_cdev; 2719 2720 ret = -ENOMEM; 2721 vduse_irq_wq = alloc_workqueue("vduse-irq", 2722 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0); 2723 if (!vduse_irq_wq) 2724 goto err_wq; 2725 2726 vduse_irq_bound_wq = alloc_workqueue("vduse-irq-bound", 2727 WQ_HIGHPRI | WQ_PERCPU, 0); 2728 if (!vduse_irq_bound_wq) 2729 goto err_bound_wq; 2730 2731 ret = vduse_domain_init(); 2732 if (ret) 2733 goto err_domain; 2734 2735 ret = vduse_mgmtdev_init(); 2736 if (ret) 2737 goto err_mgmtdev; 2738 2739 return 0; 2740 err_mgmtdev: 2741 vduse_domain_exit(); 2742 err_domain: 2743 destroy_workqueue(vduse_irq_bound_wq); 2744 err_bound_wq: 2745 destroy_workqueue(vduse_irq_wq); 2746 err_wq: 2747 cdev_del(&vduse_cdev); 2748 err_cdev: 2749 device_destroy(&vduse_class, vduse_major); 2750 err_device: 2751 cdev_del(&vduse_ctrl_cdev); 2752 err_ctrl_cdev: 2753 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX); 2754 err_chardev_region: 2755 class_unregister(&vduse_class); 2756 return ret; 2757 } 2758 module_init(vduse_init); 2759 2760 static void vduse_exit(void) 2761 { 2762 vduse_mgmtdev_exit(); 2763 vduse_domain_exit(); 2764 destroy_workqueue(vduse_irq_bound_wq); 2765 destroy_workqueue(vduse_irq_wq); 2766 cdev_del(&vduse_cdev); 2767 device_destroy(&vduse_class, vduse_major); 2768 cdev_del(&vduse_ctrl_cdev); 2769 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX); 2770 class_unregister(&vduse_class); 2771 idr_destroy(&vduse_idr); 2772 } 2773 module_exit(vduse_exit); 2774 2775 MODULE_LICENSE(DRV_LICENSE); 2776 MODULE_AUTHOR(DRV_AUTHOR); 2777 MODULE_DESCRIPTION(DRV_DESC); 2778