1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * vhost transport for vsock 4 * 5 * Copyright (C) 2013-2015 Red Hat, Inc. 6 * Author: Asias He <asias@redhat.com> 7 * Stefan Hajnoczi <stefanha@redhat.com> 8 */ 9 #include <linux/miscdevice.h> 10 #include <linux/atomic.h> 11 #include <linux/module.h> 12 #include <linux/mutex.h> 13 #include <linux/vmalloc.h> 14 #include <net/sock.h> 15 #include <linux/virtio_vsock.h> 16 #include <linux/vhost.h> 17 #include <linux/hashtable.h> 18 19 #include <net/af_vsock.h> 20 #include "vhost.h" 21 22 #define VHOST_VSOCK_DEFAULT_HOST_CID 2 23 /* Max number of bytes transferred before requeueing the job. 24 * Using this limit prevents one virtqueue from starving others. */ 25 #define VHOST_VSOCK_WEIGHT 0x80000 26 /* Max number of packets transferred before requeueing the job. 27 * Using this limit prevents one virtqueue from starving others with 28 * small pkts. 29 */ 30 #define VHOST_VSOCK_PKT_WEIGHT 256 31 32 static const int vhost_vsock_bits[] = { 33 VHOST_FEATURES, 34 VIRTIO_F_ACCESS_PLATFORM, 35 VIRTIO_VSOCK_F_SEQPACKET 36 }; 37 38 #define VHOST_VSOCK_FEATURES VHOST_FEATURES_U64(vhost_vsock_bits, 0) 39 40 enum { 41 VHOST_VSOCK_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) 42 }; 43 44 /* Used to track all the vhost_vsock instances on the system. */ 45 static DEFINE_MUTEX(vhost_vsock_mutex); 46 static DEFINE_READ_MOSTLY_HASHTABLE(vhost_vsock_hash, 8); 47 48 struct vhost_vsock { 49 struct vhost_dev dev; 50 struct vhost_virtqueue vqs[2]; 51 52 /* Link to global vhost_vsock_hash, writes use vhost_vsock_mutex */ 53 struct hlist_node hash; 54 55 struct vhost_work send_pkt_work; 56 struct sk_buff_head send_pkt_queue; /* host->guest pending packets */ 57 58 atomic_t queued_replies; 59 60 u32 guest_cid; 61 bool seqpacket_allow; 62 }; 63 64 static u32 vhost_transport_get_local_cid(void) 65 { 66 return VHOST_VSOCK_DEFAULT_HOST_CID; 67 } 68 69 /* Callers that dereference the return value must hold vhost_vsock_mutex or the 70 * RCU read lock. 71 */ 72 static struct vhost_vsock *vhost_vsock_get(u32 guest_cid) 73 { 74 struct vhost_vsock *vsock; 75 76 hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid) { 77 u32 other_cid = vsock->guest_cid; 78 79 /* Skip instances that have no CID yet */ 80 if (other_cid == 0) 81 continue; 82 83 if (other_cid == guest_cid) 84 return vsock; 85 86 } 87 88 return NULL; 89 } 90 91 static void 92 vhost_transport_do_send_pkt(struct vhost_vsock *vsock, 93 struct vhost_virtqueue *vq) 94 { 95 struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX]; 96 int pkts = 0, total_len = 0; 97 bool added = false; 98 bool restart_tx = false; 99 100 mutex_lock(&vq->mutex); 101 102 if (!vhost_vq_get_backend(vq)) 103 goto out; 104 105 if (!vq_meta_prefetch(vq)) 106 goto out; 107 108 /* Avoid further vmexits, we're already processing the virtqueue */ 109 vhost_disable_notify(&vsock->dev, vq); 110 111 do { 112 struct virtio_vsock_hdr *hdr; 113 size_t iov_len, payload_len; 114 struct iov_iter iov_iter; 115 u32 flags_to_restore = 0; 116 struct sk_buff *skb; 117 unsigned out, in; 118 size_t nbytes; 119 u32 offset; 120 int head; 121 122 skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue); 123 124 if (!skb) { 125 vhost_enable_notify(&vsock->dev, vq); 126 break; 127 } 128 129 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), 130 &out, &in, NULL, NULL); 131 if (head < 0) { 132 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 133 break; 134 } 135 136 if (head == vq->num) { 137 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 138 /* We cannot finish yet if more buffers snuck in while 139 * re-enabling notify. 140 */ 141 if (unlikely(vhost_enable_notify(&vsock->dev, vq))) { 142 vhost_disable_notify(&vsock->dev, vq); 143 continue; 144 } 145 break; 146 } 147 148 if (out) { 149 kfree_skb(skb); 150 vq_err(vq, "Expected 0 output buffers, got %u\n", out); 151 break; 152 } 153 154 iov_len = iov_length(&vq->iov[out], in); 155 if (iov_len < sizeof(*hdr)) { 156 kfree_skb(skb); 157 vq_err(vq, "Buffer len [%zu] too small\n", iov_len); 158 break; 159 } 160 161 iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[out], in, iov_len); 162 offset = VIRTIO_VSOCK_SKB_CB(skb)->offset; 163 payload_len = skb->len - offset; 164 hdr = virtio_vsock_hdr(skb); 165 166 /* If the packet is greater than the space available in the 167 * buffer, we split it using multiple buffers. 168 */ 169 if (payload_len > iov_len - sizeof(*hdr)) { 170 payload_len = iov_len - sizeof(*hdr); 171 172 /* As we are copying pieces of large packet's buffer to 173 * small rx buffers, headers of packets in rx queue are 174 * created dynamically and are initialized with header 175 * of current packet(except length). But in case of 176 * SOCK_SEQPACKET, we also must clear message delimeter 177 * bit (VIRTIO_VSOCK_SEQ_EOM) and MSG_EOR bit 178 * (VIRTIO_VSOCK_SEQ_EOR) if set. Otherwise, 179 * there will be sequence of packets with these 180 * bits set. After initialized header will be copied to 181 * rx buffer, these required bits will be restored. 182 */ 183 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) { 184 hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM); 185 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOM; 186 187 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) { 188 hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR); 189 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOR; 190 } 191 } 192 } 193 194 /* Set the correct length in the header */ 195 hdr->len = cpu_to_le32(payload_len); 196 197 nbytes = copy_to_iter(hdr, sizeof(*hdr), &iov_iter); 198 if (nbytes != sizeof(*hdr)) { 199 kfree_skb(skb); 200 vq_err(vq, "Faulted on copying pkt hdr\n"); 201 break; 202 } 203 204 if (skb_copy_datagram_iter(skb, 205 offset, 206 &iov_iter, 207 payload_len)) { 208 kfree_skb(skb); 209 vq_err(vq, "Faulted on copying pkt buf\n"); 210 break; 211 } 212 213 /* Deliver to monitoring devices all packets that we 214 * will transmit. 215 */ 216 virtio_transport_deliver_tap_pkt(skb); 217 218 vhost_add_used(vq, head, sizeof(*hdr) + payload_len); 219 added = true; 220 221 VIRTIO_VSOCK_SKB_CB(skb)->offset += payload_len; 222 total_len += payload_len; 223 224 /* If we didn't send all the payload we can requeue the packet 225 * to send it with the next available buffer. 226 */ 227 if (VIRTIO_VSOCK_SKB_CB(skb)->offset < skb->len) { 228 hdr->flags |= cpu_to_le32(flags_to_restore); 229 230 /* We are queueing the same skb to handle 231 * the remaining bytes, and we want to deliver it 232 * to monitoring devices in the next iteration. 233 */ 234 virtio_vsock_skb_clear_tap_delivered(skb); 235 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 236 } else { 237 if (virtio_vsock_skb_reply(skb)) { 238 int val; 239 240 val = atomic_dec_return(&vsock->queued_replies); 241 242 /* Do we have resources to resume tx 243 * processing? 244 */ 245 if (val + 1 == tx_vq->num) 246 restart_tx = true; 247 } 248 249 virtio_transport_consume_skb_sent(skb, true); 250 } 251 } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len))); 252 if (added) 253 vhost_signal(&vsock->dev, vq); 254 255 out: 256 mutex_unlock(&vq->mutex); 257 258 if (restart_tx) 259 vhost_poll_queue(&tx_vq->poll); 260 } 261 262 static void vhost_transport_send_pkt_work(struct vhost_work *work) 263 { 264 struct vhost_virtqueue *vq; 265 struct vhost_vsock *vsock; 266 267 vsock = container_of(work, struct vhost_vsock, send_pkt_work); 268 vq = &vsock->vqs[VSOCK_VQ_RX]; 269 270 vhost_transport_do_send_pkt(vsock, vq); 271 } 272 273 static int 274 vhost_transport_send_pkt(struct sk_buff *skb) 275 { 276 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 277 struct vhost_vsock *vsock; 278 int len = skb->len; 279 280 rcu_read_lock(); 281 282 /* Find the vhost_vsock according to guest context id */ 283 vsock = vhost_vsock_get(le64_to_cpu(hdr->dst_cid)); 284 if (!vsock) { 285 rcu_read_unlock(); 286 kfree_skb(skb); 287 return -ENODEV; 288 } 289 290 if (virtio_vsock_skb_reply(skb)) 291 atomic_inc(&vsock->queued_replies); 292 293 virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb); 294 vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work); 295 296 rcu_read_unlock(); 297 return len; 298 } 299 300 static int 301 vhost_transport_cancel_pkt(struct vsock_sock *vsk) 302 { 303 struct vhost_vsock *vsock; 304 int cnt = 0; 305 int ret = -ENODEV; 306 307 rcu_read_lock(); 308 309 /* Find the vhost_vsock according to guest context id */ 310 vsock = vhost_vsock_get(vsk->remote_addr.svm_cid); 311 if (!vsock) 312 goto out; 313 314 cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue); 315 316 if (cnt) { 317 struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX]; 318 int new_cnt; 319 320 new_cnt = atomic_sub_return(cnt, &vsock->queued_replies); 321 if (new_cnt + cnt >= tx_vq->num && new_cnt < tx_vq->num) 322 vhost_poll_queue(&tx_vq->poll); 323 } 324 325 ret = 0; 326 out: 327 rcu_read_unlock(); 328 return ret; 329 } 330 331 static struct sk_buff * 332 vhost_vsock_alloc_skb(struct vhost_virtqueue *vq, 333 unsigned int out, unsigned int in) 334 { 335 struct virtio_vsock_hdr *hdr; 336 struct iov_iter iov_iter; 337 struct sk_buff *skb; 338 size_t payload_len; 339 size_t nbytes; 340 size_t len; 341 342 if (in != 0) { 343 vq_err(vq, "Expected 0 input buffers, got %u\n", in); 344 return NULL; 345 } 346 347 len = iov_length(vq->iov, out); 348 349 if (len < VIRTIO_VSOCK_SKB_HEADROOM || 350 len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE + VIRTIO_VSOCK_SKB_HEADROOM) 351 return NULL; 352 353 /* len contains both payload and hdr */ 354 skb = virtio_vsock_alloc_skb(len, GFP_KERNEL); 355 if (!skb) 356 return NULL; 357 358 iov_iter_init(&iov_iter, ITER_SOURCE, vq->iov, out, len); 359 360 hdr = virtio_vsock_hdr(skb); 361 nbytes = copy_from_iter(hdr, sizeof(*hdr), &iov_iter); 362 if (nbytes != sizeof(*hdr)) { 363 vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n", 364 sizeof(*hdr), nbytes); 365 kfree_skb(skb); 366 return NULL; 367 } 368 369 payload_len = le32_to_cpu(hdr->len); 370 371 /* No payload */ 372 if (!payload_len) 373 return skb; 374 375 /* The pkt is too big or the length in the header is invalid */ 376 if (payload_len + sizeof(*hdr) > len) { 377 kfree_skb(skb); 378 return NULL; 379 } 380 381 virtio_vsock_skb_put(skb, payload_len); 382 383 if (skb_copy_datagram_from_iter(skb, 0, &iov_iter, payload_len)) { 384 vq_err(vq, "Failed to copy %zu byte payload\n", payload_len); 385 kfree_skb(skb); 386 return NULL; 387 } 388 389 return skb; 390 } 391 392 /* Is there space left for replies to rx packets? */ 393 static bool vhost_vsock_more_replies(struct vhost_vsock *vsock) 394 { 395 struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX]; 396 int val; 397 398 smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */ 399 val = atomic_read(&vsock->queued_replies); 400 401 return val < vq->num; 402 } 403 404 static bool vhost_transport_msgzerocopy_allow(void) 405 { 406 return true; 407 } 408 409 static bool vhost_transport_seqpacket_allow(u32 remote_cid); 410 411 static struct virtio_transport vhost_transport = { 412 .transport = { 413 .module = THIS_MODULE, 414 415 .get_local_cid = vhost_transport_get_local_cid, 416 417 .init = virtio_transport_do_socket_init, 418 .destruct = virtio_transport_destruct, 419 .release = virtio_transport_release, 420 .connect = virtio_transport_connect, 421 .shutdown = virtio_transport_shutdown, 422 .cancel_pkt = vhost_transport_cancel_pkt, 423 424 .dgram_enqueue = virtio_transport_dgram_enqueue, 425 .dgram_dequeue = virtio_transport_dgram_dequeue, 426 .dgram_bind = virtio_transport_dgram_bind, 427 .dgram_allow = virtio_transport_dgram_allow, 428 429 .stream_enqueue = virtio_transport_stream_enqueue, 430 .stream_dequeue = virtio_transport_stream_dequeue, 431 .stream_has_data = virtio_transport_stream_has_data, 432 .stream_has_space = virtio_transport_stream_has_space, 433 .stream_rcvhiwat = virtio_transport_stream_rcvhiwat, 434 .stream_is_active = virtio_transport_stream_is_active, 435 .stream_allow = virtio_transport_stream_allow, 436 437 .seqpacket_dequeue = virtio_transport_seqpacket_dequeue, 438 .seqpacket_enqueue = virtio_transport_seqpacket_enqueue, 439 .seqpacket_allow = vhost_transport_seqpacket_allow, 440 .seqpacket_has_data = virtio_transport_seqpacket_has_data, 441 442 .msgzerocopy_allow = vhost_transport_msgzerocopy_allow, 443 444 .notify_poll_in = virtio_transport_notify_poll_in, 445 .notify_poll_out = virtio_transport_notify_poll_out, 446 .notify_recv_init = virtio_transport_notify_recv_init, 447 .notify_recv_pre_block = virtio_transport_notify_recv_pre_block, 448 .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue, 449 .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue, 450 .notify_send_init = virtio_transport_notify_send_init, 451 .notify_send_pre_block = virtio_transport_notify_send_pre_block, 452 .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, 453 .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, 454 .notify_buffer_size = virtio_transport_notify_buffer_size, 455 .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat, 456 457 .unsent_bytes = virtio_transport_unsent_bytes, 458 459 .read_skb = virtio_transport_read_skb, 460 }, 461 462 .send_pkt = vhost_transport_send_pkt, 463 }; 464 465 static bool vhost_transport_seqpacket_allow(u32 remote_cid) 466 { 467 struct vhost_vsock *vsock; 468 bool seqpacket_allow = false; 469 470 rcu_read_lock(); 471 vsock = vhost_vsock_get(remote_cid); 472 473 if (vsock) 474 seqpacket_allow = vsock->seqpacket_allow; 475 476 rcu_read_unlock(); 477 478 return seqpacket_allow; 479 } 480 481 static void vhost_vsock_handle_tx_kick(struct vhost_work *work) 482 { 483 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 484 poll.work); 485 struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock, 486 dev); 487 int head, pkts = 0, total_len = 0; 488 unsigned int out, in; 489 struct sk_buff *skb; 490 bool added = false; 491 492 mutex_lock(&vq->mutex); 493 494 if (!vhost_vq_get_backend(vq)) 495 goto out; 496 497 if (!vq_meta_prefetch(vq)) 498 goto out; 499 500 vhost_disable_notify(&vsock->dev, vq); 501 do { 502 struct virtio_vsock_hdr *hdr; 503 504 if (!vhost_vsock_more_replies(vsock)) { 505 /* Stop tx until the device processes already 506 * pending replies. Leave tx virtqueue 507 * callbacks disabled. 508 */ 509 goto no_more_replies; 510 } 511 512 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), 513 &out, &in, NULL, NULL); 514 if (head < 0) 515 break; 516 517 if (head == vq->num) { 518 if (unlikely(vhost_enable_notify(&vsock->dev, vq))) { 519 vhost_disable_notify(&vsock->dev, vq); 520 continue; 521 } 522 break; 523 } 524 525 skb = vhost_vsock_alloc_skb(vq, out, in); 526 if (!skb) { 527 vq_err(vq, "Faulted on pkt\n"); 528 continue; 529 } 530 531 total_len += sizeof(*hdr) + skb->len; 532 533 /* Deliver to monitoring devices all received packets */ 534 virtio_transport_deliver_tap_pkt(skb); 535 536 hdr = virtio_vsock_hdr(skb); 537 538 /* Only accept correctly addressed packets */ 539 if (le64_to_cpu(hdr->src_cid) == vsock->guest_cid && 540 le64_to_cpu(hdr->dst_cid) == 541 vhost_transport_get_local_cid()) 542 virtio_transport_recv_pkt(&vhost_transport, skb); 543 else 544 kfree_skb(skb); 545 546 vhost_add_used(vq, head, 0); 547 added = true; 548 } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len))); 549 550 no_more_replies: 551 if (added) 552 vhost_signal(&vsock->dev, vq); 553 554 out: 555 mutex_unlock(&vq->mutex); 556 } 557 558 static void vhost_vsock_handle_rx_kick(struct vhost_work *work) 559 { 560 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 561 poll.work); 562 struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock, 563 dev); 564 565 vhost_transport_do_send_pkt(vsock, vq); 566 } 567 568 static int vhost_vsock_start(struct vhost_vsock *vsock) 569 { 570 struct vhost_virtqueue *vq; 571 size_t i; 572 int ret; 573 574 mutex_lock(&vsock->dev.mutex); 575 576 ret = vhost_dev_check_owner(&vsock->dev); 577 if (ret) 578 goto err; 579 580 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 581 vq = &vsock->vqs[i]; 582 583 mutex_lock(&vq->mutex); 584 585 if (!vhost_vq_access_ok(vq)) { 586 ret = -EFAULT; 587 goto err_vq; 588 } 589 590 if (!vhost_vq_get_backend(vq)) { 591 vhost_vq_set_backend(vq, vsock); 592 ret = vhost_vq_init_access(vq); 593 if (ret) 594 goto err_vq; 595 } 596 597 mutex_unlock(&vq->mutex); 598 } 599 600 /* Some packets may have been queued before the device was started, 601 * let's kick the send worker to send them. 602 */ 603 vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work); 604 605 mutex_unlock(&vsock->dev.mutex); 606 return 0; 607 608 err_vq: 609 vhost_vq_set_backend(vq, NULL); 610 mutex_unlock(&vq->mutex); 611 612 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 613 vq = &vsock->vqs[i]; 614 615 mutex_lock(&vq->mutex); 616 vhost_vq_set_backend(vq, NULL); 617 mutex_unlock(&vq->mutex); 618 } 619 err: 620 mutex_unlock(&vsock->dev.mutex); 621 return ret; 622 } 623 624 static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner) 625 { 626 size_t i; 627 int ret = 0; 628 629 mutex_lock(&vsock->dev.mutex); 630 631 if (check_owner) { 632 ret = vhost_dev_check_owner(&vsock->dev); 633 if (ret) 634 goto err; 635 } 636 637 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 638 struct vhost_virtqueue *vq = &vsock->vqs[i]; 639 640 mutex_lock(&vq->mutex); 641 vhost_vq_set_backend(vq, NULL); 642 mutex_unlock(&vq->mutex); 643 } 644 645 err: 646 mutex_unlock(&vsock->dev.mutex); 647 return ret; 648 } 649 650 static void vhost_vsock_free(struct vhost_vsock *vsock) 651 { 652 kvfree(vsock); 653 } 654 655 static int vhost_vsock_dev_open(struct inode *inode, struct file *file) 656 { 657 struct vhost_virtqueue **vqs; 658 struct vhost_vsock *vsock; 659 int ret; 660 661 /* This struct is large and allocation could fail, fall back to vmalloc 662 * if there is no other way. 663 */ 664 vsock = kvmalloc(sizeof(*vsock), GFP_KERNEL | __GFP_RETRY_MAYFAIL); 665 if (!vsock) 666 return -ENOMEM; 667 668 vqs = kmalloc_array(ARRAY_SIZE(vsock->vqs), sizeof(*vqs), GFP_KERNEL); 669 if (!vqs) { 670 ret = -ENOMEM; 671 goto out; 672 } 673 674 vsock->guest_cid = 0; /* no CID assigned yet */ 675 vsock->seqpacket_allow = false; 676 677 atomic_set(&vsock->queued_replies, 0); 678 679 vqs[VSOCK_VQ_TX] = &vsock->vqs[VSOCK_VQ_TX]; 680 vqs[VSOCK_VQ_RX] = &vsock->vqs[VSOCK_VQ_RX]; 681 vsock->vqs[VSOCK_VQ_TX].handle_kick = vhost_vsock_handle_tx_kick; 682 vsock->vqs[VSOCK_VQ_RX].handle_kick = vhost_vsock_handle_rx_kick; 683 684 vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs), 685 UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT, 686 VHOST_VSOCK_WEIGHT, true, NULL); 687 688 file->private_data = vsock; 689 skb_queue_head_init(&vsock->send_pkt_queue); 690 vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work); 691 return 0; 692 693 out: 694 vhost_vsock_free(vsock); 695 return ret; 696 } 697 698 static void vhost_vsock_flush(struct vhost_vsock *vsock) 699 { 700 vhost_dev_flush(&vsock->dev); 701 } 702 703 static void vhost_vsock_reset_orphans(struct sock *sk) 704 { 705 struct vsock_sock *vsk = vsock_sk(sk); 706 707 /* vmci_transport.c doesn't take sk_lock here either. At least we're 708 * under vsock_table_lock so the sock cannot disappear while we're 709 * executing. 710 */ 711 712 /* If the peer is still valid, no need to reset connection */ 713 if (vhost_vsock_get(vsk->remote_addr.svm_cid)) 714 return; 715 716 /* If the close timeout is pending, let it expire. This avoids races 717 * with the timeout callback. 718 */ 719 if (vsk->close_work_scheduled) 720 return; 721 722 sock_set_flag(sk, SOCK_DONE); 723 vsk->peer_shutdown = SHUTDOWN_MASK; 724 sk->sk_state = SS_UNCONNECTED; 725 sk->sk_err = ECONNRESET; 726 sk_error_report(sk); 727 } 728 729 static int vhost_vsock_dev_release(struct inode *inode, struct file *file) 730 { 731 struct vhost_vsock *vsock = file->private_data; 732 733 mutex_lock(&vhost_vsock_mutex); 734 if (vsock->guest_cid) 735 hash_del_rcu(&vsock->hash); 736 mutex_unlock(&vhost_vsock_mutex); 737 738 /* Wait for other CPUs to finish using vsock */ 739 synchronize_rcu(); 740 741 /* Iterating over all connections for all CIDs to find orphans is 742 * inefficient. Room for improvement here. */ 743 vsock_for_each_connected_socket(&vhost_transport.transport, 744 vhost_vsock_reset_orphans); 745 746 /* Don't check the owner, because we are in the release path, so we 747 * need to stop the vsock device in any case. 748 * vhost_vsock_stop() can not fail in this case, so we don't need to 749 * check the return code. 750 */ 751 vhost_vsock_stop(vsock, false); 752 vhost_vsock_flush(vsock); 753 vhost_dev_stop(&vsock->dev); 754 755 virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue); 756 757 vhost_dev_cleanup(&vsock->dev); 758 kfree(vsock->dev.vqs); 759 vhost_vsock_free(vsock); 760 return 0; 761 } 762 763 static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid) 764 { 765 struct vhost_vsock *other; 766 767 /* Refuse reserved CIDs */ 768 if (guest_cid <= VMADDR_CID_HOST || 769 guest_cid == U32_MAX) 770 return -EINVAL; 771 772 /* 64-bit CIDs are not yet supported */ 773 if (guest_cid > U32_MAX) 774 return -EINVAL; 775 776 /* Refuse if CID is assigned to the guest->host transport (i.e. nested 777 * VM), to make the loopback work. 778 */ 779 if (vsock_find_cid(guest_cid)) 780 return -EADDRINUSE; 781 782 /* Refuse if CID is already in use */ 783 mutex_lock(&vhost_vsock_mutex); 784 other = vhost_vsock_get(guest_cid); 785 if (other && other != vsock) { 786 mutex_unlock(&vhost_vsock_mutex); 787 return -EADDRINUSE; 788 } 789 790 if (vsock->guest_cid) 791 hash_del_rcu(&vsock->hash); 792 793 vsock->guest_cid = guest_cid; 794 hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid); 795 mutex_unlock(&vhost_vsock_mutex); 796 797 return 0; 798 } 799 800 static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features) 801 { 802 struct vhost_virtqueue *vq; 803 int i; 804 805 if (features & ~VHOST_VSOCK_FEATURES) 806 return -EOPNOTSUPP; 807 808 mutex_lock(&vsock->dev.mutex); 809 if ((features & (1 << VHOST_F_LOG_ALL)) && 810 !vhost_log_access_ok(&vsock->dev)) { 811 goto err; 812 } 813 814 if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) { 815 if (vhost_init_device_iotlb(&vsock->dev)) 816 goto err; 817 } 818 819 vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET); 820 821 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 822 vq = &vsock->vqs[i]; 823 mutex_lock(&vq->mutex); 824 vq->acked_features = features; 825 mutex_unlock(&vq->mutex); 826 } 827 mutex_unlock(&vsock->dev.mutex); 828 return 0; 829 830 err: 831 mutex_unlock(&vsock->dev.mutex); 832 return -EFAULT; 833 } 834 835 static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl, 836 unsigned long arg) 837 { 838 struct vhost_vsock *vsock = f->private_data; 839 void __user *argp = (void __user *)arg; 840 u64 guest_cid; 841 u64 features; 842 int start; 843 int r; 844 845 switch (ioctl) { 846 case VHOST_VSOCK_SET_GUEST_CID: 847 if (copy_from_user(&guest_cid, argp, sizeof(guest_cid))) 848 return -EFAULT; 849 return vhost_vsock_set_cid(vsock, guest_cid); 850 case VHOST_VSOCK_SET_RUNNING: 851 if (copy_from_user(&start, argp, sizeof(start))) 852 return -EFAULT; 853 if (start) 854 return vhost_vsock_start(vsock); 855 else 856 return vhost_vsock_stop(vsock, true); 857 case VHOST_GET_FEATURES: 858 features = VHOST_VSOCK_FEATURES; 859 if (copy_to_user(argp, &features, sizeof(features))) 860 return -EFAULT; 861 return 0; 862 case VHOST_SET_FEATURES: 863 if (copy_from_user(&features, argp, sizeof(features))) 864 return -EFAULT; 865 return vhost_vsock_set_features(vsock, features); 866 case VHOST_GET_BACKEND_FEATURES: 867 features = VHOST_VSOCK_BACKEND_FEATURES; 868 if (copy_to_user(argp, &features, sizeof(features))) 869 return -EFAULT; 870 return 0; 871 case VHOST_SET_BACKEND_FEATURES: 872 if (copy_from_user(&features, argp, sizeof(features))) 873 return -EFAULT; 874 if (features & ~VHOST_VSOCK_BACKEND_FEATURES) 875 return -EOPNOTSUPP; 876 vhost_set_backend_features(&vsock->dev, features); 877 return 0; 878 default: 879 mutex_lock(&vsock->dev.mutex); 880 r = vhost_dev_ioctl(&vsock->dev, ioctl, argp); 881 if (r == -ENOIOCTLCMD) 882 r = vhost_vring_ioctl(&vsock->dev, ioctl, argp); 883 else 884 vhost_vsock_flush(vsock); 885 mutex_unlock(&vsock->dev.mutex); 886 return r; 887 } 888 } 889 890 static ssize_t vhost_vsock_chr_read_iter(struct kiocb *iocb, struct iov_iter *to) 891 { 892 struct file *file = iocb->ki_filp; 893 struct vhost_vsock *vsock = file->private_data; 894 struct vhost_dev *dev = &vsock->dev; 895 int noblock = file->f_flags & O_NONBLOCK; 896 897 return vhost_chr_read_iter(dev, to, noblock); 898 } 899 900 static ssize_t vhost_vsock_chr_write_iter(struct kiocb *iocb, 901 struct iov_iter *from) 902 { 903 struct file *file = iocb->ki_filp; 904 struct vhost_vsock *vsock = file->private_data; 905 struct vhost_dev *dev = &vsock->dev; 906 907 return vhost_chr_write_iter(dev, from); 908 } 909 910 static __poll_t vhost_vsock_chr_poll(struct file *file, poll_table *wait) 911 { 912 struct vhost_vsock *vsock = file->private_data; 913 struct vhost_dev *dev = &vsock->dev; 914 915 return vhost_chr_poll(file, dev, wait); 916 } 917 918 static const struct file_operations vhost_vsock_fops = { 919 .owner = THIS_MODULE, 920 .open = vhost_vsock_dev_open, 921 .release = vhost_vsock_dev_release, 922 .llseek = noop_llseek, 923 .unlocked_ioctl = vhost_vsock_dev_ioctl, 924 .compat_ioctl = compat_ptr_ioctl, 925 .read_iter = vhost_vsock_chr_read_iter, 926 .write_iter = vhost_vsock_chr_write_iter, 927 .poll = vhost_vsock_chr_poll, 928 }; 929 930 static struct miscdevice vhost_vsock_misc = { 931 .minor = VHOST_VSOCK_MINOR, 932 .name = "vhost-vsock", 933 .fops = &vhost_vsock_fops, 934 }; 935 936 static int __init vhost_vsock_init(void) 937 { 938 int ret; 939 940 ret = vsock_core_register(&vhost_transport.transport, 941 VSOCK_TRANSPORT_F_H2G); 942 if (ret < 0) 943 return ret; 944 945 ret = misc_register(&vhost_vsock_misc); 946 if (ret) { 947 vsock_core_unregister(&vhost_transport.transport); 948 return ret; 949 } 950 951 return 0; 952 }; 953 954 static void __exit vhost_vsock_exit(void) 955 { 956 misc_deregister(&vhost_vsock_misc); 957 vsock_core_unregister(&vhost_transport.transport); 958 }; 959 960 module_init(vhost_vsock_init); 961 module_exit(vhost_vsock_exit); 962 MODULE_LICENSE("GPL v2"); 963 MODULE_AUTHOR("Asias He"); 964 MODULE_DESCRIPTION("vhost transport for vsock "); 965 MODULE_ALIAS_MISCDEV(VHOST_VSOCK_MINOR); 966 MODULE_ALIAS("devname:vhost-vsock"); 967