1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * vhost transport for vsock 4 * 5 * Copyright (C) 2013-2015 Red Hat, Inc. 6 * Author: Asias He <asias@redhat.com> 7 * Stefan Hajnoczi <stefanha@redhat.com> 8 */ 9 #include <linux/miscdevice.h> 10 #include <linux/atomic.h> 11 #include <linux/module.h> 12 #include <linux/mutex.h> 13 #include <linux/vmalloc.h> 14 #include <net/sock.h> 15 #include <linux/virtio_vsock.h> 16 #include <linux/vhost.h> 17 #include <linux/hashtable.h> 18 19 #include <net/af_vsock.h> 20 #include "vhost.h" 21 22 #define VHOST_VSOCK_DEFAULT_HOST_CID 2 23 /* Max number of bytes transferred before requeueing the job. 24 * Using this limit prevents one virtqueue from starving others. */ 25 #define VHOST_VSOCK_WEIGHT 0x80000 26 /* Max number of packets transferred before requeueing the job. 27 * Using this limit prevents one virtqueue from starving others with 28 * small pkts. 29 */ 30 #define VHOST_VSOCK_PKT_WEIGHT 256 31 32 enum { 33 VHOST_VSOCK_FEATURES = VHOST_FEATURES | 34 (1ULL << VIRTIO_F_ACCESS_PLATFORM) | 35 (1ULL << VIRTIO_VSOCK_F_SEQPACKET) 36 }; 37 38 enum { 39 VHOST_VSOCK_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) 40 }; 41 42 /* Used to track all the vhost_vsock instances on the system. */ 43 static DEFINE_MUTEX(vhost_vsock_mutex); 44 static DEFINE_READ_MOSTLY_HASHTABLE(vhost_vsock_hash, 8); 45 46 struct vhost_vsock { 47 struct vhost_dev dev; 48 struct vhost_virtqueue vqs[2]; 49 50 /* Link to global vhost_vsock_hash, writes use vhost_vsock_mutex */ 51 struct hlist_node hash; 52 53 struct vhost_work send_pkt_work; 54 struct sk_buff_head send_pkt_queue; /* host->guest pending packets */ 55 56 atomic_t queued_replies; 57 58 u32 guest_cid; 59 bool seqpacket_allow; 60 }; 61 62 static u32 vhost_transport_get_local_cid(void) 63 { 64 return VHOST_VSOCK_DEFAULT_HOST_CID; 65 } 66 67 /* Callers that dereference the return value must hold vhost_vsock_mutex or the 68 * RCU read lock. 69 */ 70 static struct vhost_vsock *vhost_vsock_get(u32 guest_cid) 71 { 72 struct vhost_vsock *vsock; 73 74 hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid) { 75 u32 other_cid = vsock->guest_cid; 76 77 /* Skip instances that have no CID yet */ 78 if (other_cid == 0) 79 continue; 80 81 if (other_cid == guest_cid) 82 return vsock; 83 84 } 85 86 return NULL; 87 } 88 89 static void 90 vhost_transport_do_send_pkt(struct vhost_vsock *vsock, 91 struct vhost_virtqueue *vq) 92 { 93 struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX]; 94 int pkts = 0, total_len = 0; 95 bool added = false; 96 bool restart_tx = false; 97 98 mutex_lock(&vq->mutex); 99 100 if (!vhost_vq_get_backend(vq)) 101 goto out; 102 103 if (!vq_meta_prefetch(vq)) 104 goto out; 105 106 /* Avoid further vmexits, we're already processing the virtqueue */ 107 vhost_disable_notify(&vsock->dev, vq); 108 109 do { 110 struct virtio_vsock_hdr *hdr; 111 size_t iov_len, payload_len; 112 struct iov_iter iov_iter; 113 u32 flags_to_restore = 0; 114 struct sk_buff *skb; 115 unsigned out, in; 116 size_t nbytes; 117 u32 offset; 118 int head; 119 120 skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue); 121 122 if (!skb) { 123 vhost_enable_notify(&vsock->dev, vq); 124 break; 125 } 126 127 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), 128 &out, &in, NULL, NULL); 129 if (head < 0) { 130 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 131 break; 132 } 133 134 if (head == vq->num) { 135 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 136 /* We cannot finish yet if more buffers snuck in while 137 * re-enabling notify. 138 */ 139 if (unlikely(vhost_enable_notify(&vsock->dev, vq))) { 140 vhost_disable_notify(&vsock->dev, vq); 141 continue; 142 } 143 break; 144 } 145 146 if (out) { 147 kfree_skb(skb); 148 vq_err(vq, "Expected 0 output buffers, got %u\n", out); 149 break; 150 } 151 152 iov_len = iov_length(&vq->iov[out], in); 153 if (iov_len < sizeof(*hdr)) { 154 kfree_skb(skb); 155 vq_err(vq, "Buffer len [%zu] too small\n", iov_len); 156 break; 157 } 158 159 iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[out], in, iov_len); 160 offset = VIRTIO_VSOCK_SKB_CB(skb)->offset; 161 payload_len = skb->len - offset; 162 hdr = virtio_vsock_hdr(skb); 163 164 /* If the packet is greater than the space available in the 165 * buffer, we split it using multiple buffers. 166 */ 167 if (payload_len > iov_len - sizeof(*hdr)) { 168 payload_len = iov_len - sizeof(*hdr); 169 170 /* As we are copying pieces of large packet's buffer to 171 * small rx buffers, headers of packets in rx queue are 172 * created dynamically and are initialized with header 173 * of current packet(except length). But in case of 174 * SOCK_SEQPACKET, we also must clear message delimeter 175 * bit (VIRTIO_VSOCK_SEQ_EOM) and MSG_EOR bit 176 * (VIRTIO_VSOCK_SEQ_EOR) if set. Otherwise, 177 * there will be sequence of packets with these 178 * bits set. After initialized header will be copied to 179 * rx buffer, these required bits will be restored. 180 */ 181 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) { 182 hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM); 183 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOM; 184 185 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) { 186 hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR); 187 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOR; 188 } 189 } 190 } 191 192 /* Set the correct length in the header */ 193 hdr->len = cpu_to_le32(payload_len); 194 195 nbytes = copy_to_iter(hdr, sizeof(*hdr), &iov_iter); 196 if (nbytes != sizeof(*hdr)) { 197 kfree_skb(skb); 198 vq_err(vq, "Faulted on copying pkt hdr\n"); 199 break; 200 } 201 202 if (skb_copy_datagram_iter(skb, 203 offset, 204 &iov_iter, 205 payload_len)) { 206 kfree_skb(skb); 207 vq_err(vq, "Faulted on copying pkt buf\n"); 208 break; 209 } 210 211 /* Deliver to monitoring devices all packets that we 212 * will transmit. 213 */ 214 virtio_transport_deliver_tap_pkt(skb); 215 216 vhost_add_used(vq, head, sizeof(*hdr) + payload_len); 217 added = true; 218 219 VIRTIO_VSOCK_SKB_CB(skb)->offset += payload_len; 220 total_len += payload_len; 221 222 /* If we didn't send all the payload we can requeue the packet 223 * to send it with the next available buffer. 224 */ 225 if (VIRTIO_VSOCK_SKB_CB(skb)->offset < skb->len) { 226 hdr->flags |= cpu_to_le32(flags_to_restore); 227 228 /* We are queueing the same skb to handle 229 * the remaining bytes, and we want to deliver it 230 * to monitoring devices in the next iteration. 231 */ 232 virtio_vsock_skb_clear_tap_delivered(skb); 233 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 234 } else { 235 if (virtio_vsock_skb_reply(skb)) { 236 int val; 237 238 val = atomic_dec_return(&vsock->queued_replies); 239 240 /* Do we have resources to resume tx 241 * processing? 242 */ 243 if (val + 1 == tx_vq->num) 244 restart_tx = true; 245 } 246 247 virtio_transport_consume_skb_sent(skb, true); 248 } 249 } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len))); 250 if (added) 251 vhost_signal(&vsock->dev, vq); 252 253 out: 254 mutex_unlock(&vq->mutex); 255 256 if (restart_tx) 257 vhost_poll_queue(&tx_vq->poll); 258 } 259 260 static void vhost_transport_send_pkt_work(struct vhost_work *work) 261 { 262 struct vhost_virtqueue *vq; 263 struct vhost_vsock *vsock; 264 265 vsock = container_of(work, struct vhost_vsock, send_pkt_work); 266 vq = &vsock->vqs[VSOCK_VQ_RX]; 267 268 vhost_transport_do_send_pkt(vsock, vq); 269 } 270 271 static int 272 vhost_transport_send_pkt(struct sk_buff *skb) 273 { 274 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 275 struct vhost_vsock *vsock; 276 int len = skb->len; 277 278 rcu_read_lock(); 279 280 /* Find the vhost_vsock according to guest context id */ 281 vsock = vhost_vsock_get(le64_to_cpu(hdr->dst_cid)); 282 if (!vsock) { 283 rcu_read_unlock(); 284 kfree_skb(skb); 285 return -ENODEV; 286 } 287 288 if (virtio_vsock_skb_reply(skb)) 289 atomic_inc(&vsock->queued_replies); 290 291 virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb); 292 vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work); 293 294 rcu_read_unlock(); 295 return len; 296 } 297 298 static int 299 vhost_transport_cancel_pkt(struct vsock_sock *vsk) 300 { 301 struct vhost_vsock *vsock; 302 int cnt = 0; 303 int ret = -ENODEV; 304 305 rcu_read_lock(); 306 307 /* Find the vhost_vsock according to guest context id */ 308 vsock = vhost_vsock_get(vsk->remote_addr.svm_cid); 309 if (!vsock) 310 goto out; 311 312 cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue); 313 314 if (cnt) { 315 struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX]; 316 int new_cnt; 317 318 new_cnt = atomic_sub_return(cnt, &vsock->queued_replies); 319 if (new_cnt + cnt >= tx_vq->num && new_cnt < tx_vq->num) 320 vhost_poll_queue(&tx_vq->poll); 321 } 322 323 ret = 0; 324 out: 325 rcu_read_unlock(); 326 return ret; 327 } 328 329 static struct sk_buff * 330 vhost_vsock_alloc_skb(struct vhost_virtqueue *vq, 331 unsigned int out, unsigned int in) 332 { 333 struct virtio_vsock_hdr *hdr; 334 struct iov_iter iov_iter; 335 struct sk_buff *skb; 336 size_t payload_len; 337 size_t nbytes; 338 size_t len; 339 340 if (in != 0) { 341 vq_err(vq, "Expected 0 input buffers, got %u\n", in); 342 return NULL; 343 } 344 345 len = iov_length(vq->iov, out); 346 347 if (len < VIRTIO_VSOCK_SKB_HEADROOM || 348 len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE + VIRTIO_VSOCK_SKB_HEADROOM) 349 return NULL; 350 351 /* len contains both payload and hdr */ 352 skb = virtio_vsock_alloc_skb(len, GFP_KERNEL); 353 if (!skb) 354 return NULL; 355 356 iov_iter_init(&iov_iter, ITER_SOURCE, vq->iov, out, len); 357 358 hdr = virtio_vsock_hdr(skb); 359 nbytes = copy_from_iter(hdr, sizeof(*hdr), &iov_iter); 360 if (nbytes != sizeof(*hdr)) { 361 vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n", 362 sizeof(*hdr), nbytes); 363 kfree_skb(skb); 364 return NULL; 365 } 366 367 payload_len = le32_to_cpu(hdr->len); 368 369 /* No payload */ 370 if (!payload_len) 371 return skb; 372 373 /* The pkt is too big or the length in the header is invalid */ 374 if (payload_len + sizeof(*hdr) > len) { 375 kfree_skb(skb); 376 return NULL; 377 } 378 379 virtio_vsock_skb_put(skb, payload_len); 380 381 if (skb_copy_datagram_from_iter(skb, 0, &iov_iter, payload_len)) { 382 vq_err(vq, "Failed to copy %zu byte payload\n", payload_len); 383 kfree_skb(skb); 384 return NULL; 385 } 386 387 return skb; 388 } 389 390 /* Is there space left for replies to rx packets? */ 391 static bool vhost_vsock_more_replies(struct vhost_vsock *vsock) 392 { 393 struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX]; 394 int val; 395 396 smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */ 397 val = atomic_read(&vsock->queued_replies); 398 399 return val < vq->num; 400 } 401 402 static bool vhost_transport_msgzerocopy_allow(void) 403 { 404 return true; 405 } 406 407 static bool vhost_transport_seqpacket_allow(u32 remote_cid); 408 409 static struct virtio_transport vhost_transport = { 410 .transport = { 411 .module = THIS_MODULE, 412 413 .get_local_cid = vhost_transport_get_local_cid, 414 415 .init = virtio_transport_do_socket_init, 416 .destruct = virtio_transport_destruct, 417 .release = virtio_transport_release, 418 .connect = virtio_transport_connect, 419 .shutdown = virtio_transport_shutdown, 420 .cancel_pkt = vhost_transport_cancel_pkt, 421 422 .dgram_enqueue = virtio_transport_dgram_enqueue, 423 .dgram_dequeue = virtio_transport_dgram_dequeue, 424 .dgram_bind = virtio_transport_dgram_bind, 425 .dgram_allow = virtio_transport_dgram_allow, 426 427 .stream_enqueue = virtio_transport_stream_enqueue, 428 .stream_dequeue = virtio_transport_stream_dequeue, 429 .stream_has_data = virtio_transport_stream_has_data, 430 .stream_has_space = virtio_transport_stream_has_space, 431 .stream_rcvhiwat = virtio_transport_stream_rcvhiwat, 432 .stream_is_active = virtio_transport_stream_is_active, 433 .stream_allow = virtio_transport_stream_allow, 434 435 .seqpacket_dequeue = virtio_transport_seqpacket_dequeue, 436 .seqpacket_enqueue = virtio_transport_seqpacket_enqueue, 437 .seqpacket_allow = vhost_transport_seqpacket_allow, 438 .seqpacket_has_data = virtio_transport_seqpacket_has_data, 439 440 .msgzerocopy_allow = vhost_transport_msgzerocopy_allow, 441 442 .notify_poll_in = virtio_transport_notify_poll_in, 443 .notify_poll_out = virtio_transport_notify_poll_out, 444 .notify_recv_init = virtio_transport_notify_recv_init, 445 .notify_recv_pre_block = virtio_transport_notify_recv_pre_block, 446 .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue, 447 .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue, 448 .notify_send_init = virtio_transport_notify_send_init, 449 .notify_send_pre_block = virtio_transport_notify_send_pre_block, 450 .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, 451 .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, 452 .notify_buffer_size = virtio_transport_notify_buffer_size, 453 .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat, 454 455 .unsent_bytes = virtio_transport_unsent_bytes, 456 457 .read_skb = virtio_transport_read_skb, 458 }, 459 460 .send_pkt = vhost_transport_send_pkt, 461 }; 462 463 static bool vhost_transport_seqpacket_allow(u32 remote_cid) 464 { 465 struct vhost_vsock *vsock; 466 bool seqpacket_allow = false; 467 468 rcu_read_lock(); 469 vsock = vhost_vsock_get(remote_cid); 470 471 if (vsock) 472 seqpacket_allow = vsock->seqpacket_allow; 473 474 rcu_read_unlock(); 475 476 return seqpacket_allow; 477 } 478 479 static void vhost_vsock_handle_tx_kick(struct vhost_work *work) 480 { 481 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 482 poll.work); 483 struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock, 484 dev); 485 int head, pkts = 0, total_len = 0; 486 unsigned int out, in; 487 struct sk_buff *skb; 488 bool added = false; 489 490 mutex_lock(&vq->mutex); 491 492 if (!vhost_vq_get_backend(vq)) 493 goto out; 494 495 if (!vq_meta_prefetch(vq)) 496 goto out; 497 498 vhost_disable_notify(&vsock->dev, vq); 499 do { 500 struct virtio_vsock_hdr *hdr; 501 502 if (!vhost_vsock_more_replies(vsock)) { 503 /* Stop tx until the device processes already 504 * pending replies. Leave tx virtqueue 505 * callbacks disabled. 506 */ 507 goto no_more_replies; 508 } 509 510 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), 511 &out, &in, NULL, NULL); 512 if (head < 0) 513 break; 514 515 if (head == vq->num) { 516 if (unlikely(vhost_enable_notify(&vsock->dev, vq))) { 517 vhost_disable_notify(&vsock->dev, vq); 518 continue; 519 } 520 break; 521 } 522 523 skb = vhost_vsock_alloc_skb(vq, out, in); 524 if (!skb) { 525 vq_err(vq, "Faulted on pkt\n"); 526 continue; 527 } 528 529 total_len += sizeof(*hdr) + skb->len; 530 531 /* Deliver to monitoring devices all received packets */ 532 virtio_transport_deliver_tap_pkt(skb); 533 534 hdr = virtio_vsock_hdr(skb); 535 536 /* Only accept correctly addressed packets */ 537 if (le64_to_cpu(hdr->src_cid) == vsock->guest_cid && 538 le64_to_cpu(hdr->dst_cid) == 539 vhost_transport_get_local_cid()) 540 virtio_transport_recv_pkt(&vhost_transport, skb); 541 else 542 kfree_skb(skb); 543 544 vhost_add_used(vq, head, 0); 545 added = true; 546 } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len))); 547 548 no_more_replies: 549 if (added) 550 vhost_signal(&vsock->dev, vq); 551 552 out: 553 mutex_unlock(&vq->mutex); 554 } 555 556 static void vhost_vsock_handle_rx_kick(struct vhost_work *work) 557 { 558 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 559 poll.work); 560 struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock, 561 dev); 562 563 vhost_transport_do_send_pkt(vsock, vq); 564 } 565 566 static int vhost_vsock_start(struct vhost_vsock *vsock) 567 { 568 struct vhost_virtqueue *vq; 569 size_t i; 570 int ret; 571 572 mutex_lock(&vsock->dev.mutex); 573 574 ret = vhost_dev_check_owner(&vsock->dev); 575 if (ret) 576 goto err; 577 578 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 579 vq = &vsock->vqs[i]; 580 581 mutex_lock(&vq->mutex); 582 583 if (!vhost_vq_access_ok(vq)) { 584 ret = -EFAULT; 585 goto err_vq; 586 } 587 588 if (!vhost_vq_get_backend(vq)) { 589 vhost_vq_set_backend(vq, vsock); 590 ret = vhost_vq_init_access(vq); 591 if (ret) 592 goto err_vq; 593 } 594 595 mutex_unlock(&vq->mutex); 596 } 597 598 /* Some packets may have been queued before the device was started, 599 * let's kick the send worker to send them. 600 */ 601 vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work); 602 603 mutex_unlock(&vsock->dev.mutex); 604 return 0; 605 606 err_vq: 607 vhost_vq_set_backend(vq, NULL); 608 mutex_unlock(&vq->mutex); 609 610 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 611 vq = &vsock->vqs[i]; 612 613 mutex_lock(&vq->mutex); 614 vhost_vq_set_backend(vq, NULL); 615 mutex_unlock(&vq->mutex); 616 } 617 err: 618 mutex_unlock(&vsock->dev.mutex); 619 return ret; 620 } 621 622 static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner) 623 { 624 size_t i; 625 int ret = 0; 626 627 mutex_lock(&vsock->dev.mutex); 628 629 if (check_owner) { 630 ret = vhost_dev_check_owner(&vsock->dev); 631 if (ret) 632 goto err; 633 } 634 635 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 636 struct vhost_virtqueue *vq = &vsock->vqs[i]; 637 638 mutex_lock(&vq->mutex); 639 vhost_vq_set_backend(vq, NULL); 640 mutex_unlock(&vq->mutex); 641 } 642 643 err: 644 mutex_unlock(&vsock->dev.mutex); 645 return ret; 646 } 647 648 static void vhost_vsock_free(struct vhost_vsock *vsock) 649 { 650 kvfree(vsock); 651 } 652 653 static int vhost_vsock_dev_open(struct inode *inode, struct file *file) 654 { 655 struct vhost_virtqueue **vqs; 656 struct vhost_vsock *vsock; 657 int ret; 658 659 /* This struct is large and allocation could fail, fall back to vmalloc 660 * if there is no other way. 661 */ 662 vsock = kvmalloc(sizeof(*vsock), GFP_KERNEL | __GFP_RETRY_MAYFAIL); 663 if (!vsock) 664 return -ENOMEM; 665 666 vqs = kmalloc_array(ARRAY_SIZE(vsock->vqs), sizeof(*vqs), GFP_KERNEL); 667 if (!vqs) { 668 ret = -ENOMEM; 669 goto out; 670 } 671 672 vsock->guest_cid = 0; /* no CID assigned yet */ 673 vsock->seqpacket_allow = false; 674 675 atomic_set(&vsock->queued_replies, 0); 676 677 vqs[VSOCK_VQ_TX] = &vsock->vqs[VSOCK_VQ_TX]; 678 vqs[VSOCK_VQ_RX] = &vsock->vqs[VSOCK_VQ_RX]; 679 vsock->vqs[VSOCK_VQ_TX].handle_kick = vhost_vsock_handle_tx_kick; 680 vsock->vqs[VSOCK_VQ_RX].handle_kick = vhost_vsock_handle_rx_kick; 681 682 vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs), 683 UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT, 684 VHOST_VSOCK_WEIGHT, true, NULL); 685 686 file->private_data = vsock; 687 skb_queue_head_init(&vsock->send_pkt_queue); 688 vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work); 689 return 0; 690 691 out: 692 vhost_vsock_free(vsock); 693 return ret; 694 } 695 696 static void vhost_vsock_flush(struct vhost_vsock *vsock) 697 { 698 vhost_dev_flush(&vsock->dev); 699 } 700 701 static void vhost_vsock_reset_orphans(struct sock *sk) 702 { 703 struct vsock_sock *vsk = vsock_sk(sk); 704 705 /* vmci_transport.c doesn't take sk_lock here either. At least we're 706 * under vsock_table_lock so the sock cannot disappear while we're 707 * executing. 708 */ 709 710 /* If the peer is still valid, no need to reset connection */ 711 if (vhost_vsock_get(vsk->remote_addr.svm_cid)) 712 return; 713 714 /* If the close timeout is pending, let it expire. This avoids races 715 * with the timeout callback. 716 */ 717 if (vsk->close_work_scheduled) 718 return; 719 720 sock_set_flag(sk, SOCK_DONE); 721 vsk->peer_shutdown = SHUTDOWN_MASK; 722 sk->sk_state = SS_UNCONNECTED; 723 sk->sk_err = ECONNRESET; 724 sk_error_report(sk); 725 } 726 727 static int vhost_vsock_dev_release(struct inode *inode, struct file *file) 728 { 729 struct vhost_vsock *vsock = file->private_data; 730 731 mutex_lock(&vhost_vsock_mutex); 732 if (vsock->guest_cid) 733 hash_del_rcu(&vsock->hash); 734 mutex_unlock(&vhost_vsock_mutex); 735 736 /* Wait for other CPUs to finish using vsock */ 737 synchronize_rcu(); 738 739 /* Iterating over all connections for all CIDs to find orphans is 740 * inefficient. Room for improvement here. */ 741 vsock_for_each_connected_socket(&vhost_transport.transport, 742 vhost_vsock_reset_orphans); 743 744 /* Don't check the owner, because we are in the release path, so we 745 * need to stop the vsock device in any case. 746 * vhost_vsock_stop() can not fail in this case, so we don't need to 747 * check the return code. 748 */ 749 vhost_vsock_stop(vsock, false); 750 vhost_vsock_flush(vsock); 751 vhost_dev_stop(&vsock->dev); 752 753 virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue); 754 755 vhost_dev_cleanup(&vsock->dev); 756 kfree(vsock->dev.vqs); 757 vhost_vsock_free(vsock); 758 return 0; 759 } 760 761 static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid) 762 { 763 struct vhost_vsock *other; 764 765 /* Refuse reserved CIDs */ 766 if (guest_cid <= VMADDR_CID_HOST || 767 guest_cid == U32_MAX) 768 return -EINVAL; 769 770 /* 64-bit CIDs are not yet supported */ 771 if (guest_cid > U32_MAX) 772 return -EINVAL; 773 774 /* Refuse if CID is assigned to the guest->host transport (i.e. nested 775 * VM), to make the loopback work. 776 */ 777 if (vsock_find_cid(guest_cid)) 778 return -EADDRINUSE; 779 780 /* Refuse if CID is already in use */ 781 mutex_lock(&vhost_vsock_mutex); 782 other = vhost_vsock_get(guest_cid); 783 if (other && other != vsock) { 784 mutex_unlock(&vhost_vsock_mutex); 785 return -EADDRINUSE; 786 } 787 788 if (vsock->guest_cid) 789 hash_del_rcu(&vsock->hash); 790 791 vsock->guest_cid = guest_cid; 792 hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid); 793 mutex_unlock(&vhost_vsock_mutex); 794 795 return 0; 796 } 797 798 static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features) 799 { 800 struct vhost_virtqueue *vq; 801 int i; 802 803 if (features & ~VHOST_VSOCK_FEATURES) 804 return -EOPNOTSUPP; 805 806 mutex_lock(&vsock->dev.mutex); 807 if ((features & (1 << VHOST_F_LOG_ALL)) && 808 !vhost_log_access_ok(&vsock->dev)) { 809 goto err; 810 } 811 812 if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) { 813 if (vhost_init_device_iotlb(&vsock->dev)) 814 goto err; 815 } 816 817 vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET); 818 819 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 820 vq = &vsock->vqs[i]; 821 mutex_lock(&vq->mutex); 822 vq->acked_features = features; 823 mutex_unlock(&vq->mutex); 824 } 825 mutex_unlock(&vsock->dev.mutex); 826 return 0; 827 828 err: 829 mutex_unlock(&vsock->dev.mutex); 830 return -EFAULT; 831 } 832 833 static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl, 834 unsigned long arg) 835 { 836 struct vhost_vsock *vsock = f->private_data; 837 void __user *argp = (void __user *)arg; 838 u64 guest_cid; 839 u64 features; 840 int start; 841 int r; 842 843 switch (ioctl) { 844 case VHOST_VSOCK_SET_GUEST_CID: 845 if (copy_from_user(&guest_cid, argp, sizeof(guest_cid))) 846 return -EFAULT; 847 return vhost_vsock_set_cid(vsock, guest_cid); 848 case VHOST_VSOCK_SET_RUNNING: 849 if (copy_from_user(&start, argp, sizeof(start))) 850 return -EFAULT; 851 if (start) 852 return vhost_vsock_start(vsock); 853 else 854 return vhost_vsock_stop(vsock, true); 855 case VHOST_GET_FEATURES: 856 features = VHOST_VSOCK_FEATURES; 857 if (copy_to_user(argp, &features, sizeof(features))) 858 return -EFAULT; 859 return 0; 860 case VHOST_SET_FEATURES: 861 if (copy_from_user(&features, argp, sizeof(features))) 862 return -EFAULT; 863 return vhost_vsock_set_features(vsock, features); 864 case VHOST_GET_BACKEND_FEATURES: 865 features = VHOST_VSOCK_BACKEND_FEATURES; 866 if (copy_to_user(argp, &features, sizeof(features))) 867 return -EFAULT; 868 return 0; 869 case VHOST_SET_BACKEND_FEATURES: 870 if (copy_from_user(&features, argp, sizeof(features))) 871 return -EFAULT; 872 if (features & ~VHOST_VSOCK_BACKEND_FEATURES) 873 return -EOPNOTSUPP; 874 vhost_set_backend_features(&vsock->dev, features); 875 return 0; 876 default: 877 mutex_lock(&vsock->dev.mutex); 878 r = vhost_dev_ioctl(&vsock->dev, ioctl, argp); 879 if (r == -ENOIOCTLCMD) 880 r = vhost_vring_ioctl(&vsock->dev, ioctl, argp); 881 else 882 vhost_vsock_flush(vsock); 883 mutex_unlock(&vsock->dev.mutex); 884 return r; 885 } 886 } 887 888 static ssize_t vhost_vsock_chr_read_iter(struct kiocb *iocb, struct iov_iter *to) 889 { 890 struct file *file = iocb->ki_filp; 891 struct vhost_vsock *vsock = file->private_data; 892 struct vhost_dev *dev = &vsock->dev; 893 int noblock = file->f_flags & O_NONBLOCK; 894 895 return vhost_chr_read_iter(dev, to, noblock); 896 } 897 898 static ssize_t vhost_vsock_chr_write_iter(struct kiocb *iocb, 899 struct iov_iter *from) 900 { 901 struct file *file = iocb->ki_filp; 902 struct vhost_vsock *vsock = file->private_data; 903 struct vhost_dev *dev = &vsock->dev; 904 905 return vhost_chr_write_iter(dev, from); 906 } 907 908 static __poll_t vhost_vsock_chr_poll(struct file *file, poll_table *wait) 909 { 910 struct vhost_vsock *vsock = file->private_data; 911 struct vhost_dev *dev = &vsock->dev; 912 913 return vhost_chr_poll(file, dev, wait); 914 } 915 916 static const struct file_operations vhost_vsock_fops = { 917 .owner = THIS_MODULE, 918 .open = vhost_vsock_dev_open, 919 .release = vhost_vsock_dev_release, 920 .llseek = noop_llseek, 921 .unlocked_ioctl = vhost_vsock_dev_ioctl, 922 .compat_ioctl = compat_ptr_ioctl, 923 .read_iter = vhost_vsock_chr_read_iter, 924 .write_iter = vhost_vsock_chr_write_iter, 925 .poll = vhost_vsock_chr_poll, 926 }; 927 928 static struct miscdevice vhost_vsock_misc = { 929 .minor = VHOST_VSOCK_MINOR, 930 .name = "vhost-vsock", 931 .fops = &vhost_vsock_fops, 932 }; 933 934 static int __init vhost_vsock_init(void) 935 { 936 int ret; 937 938 ret = vsock_core_register(&vhost_transport.transport, 939 VSOCK_TRANSPORT_F_H2G); 940 if (ret < 0) 941 return ret; 942 943 ret = misc_register(&vhost_vsock_misc); 944 if (ret) { 945 vsock_core_unregister(&vhost_transport.transport); 946 return ret; 947 } 948 949 return 0; 950 }; 951 952 static void __exit vhost_vsock_exit(void) 953 { 954 misc_deregister(&vhost_vsock_misc); 955 vsock_core_unregister(&vhost_transport.transport); 956 }; 957 958 module_init(vhost_vsock_init); 959 module_exit(vhost_vsock_exit); 960 MODULE_LICENSE("GPL v2"); 961 MODULE_AUTHOR("Asias He"); 962 MODULE_DESCRIPTION("vhost transport for vsock "); 963 MODULE_ALIAS_MISCDEV(VHOST_VSOCK_MINOR); 964 MODULE_ALIAS("devname:vhost-vsock"); 965