1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * vhost transport for vsock 4 * 5 * Copyright (C) 2013-2015 Red Hat, Inc. 6 * Author: Asias He <asias@redhat.com> 7 * Stefan Hajnoczi <stefanha@redhat.com> 8 */ 9 #include <linux/miscdevice.h> 10 #include <linux/atomic.h> 11 #include <linux/module.h> 12 #include <linux/mutex.h> 13 #include <linux/vmalloc.h> 14 #include <net/sock.h> 15 #include <linux/virtio_vsock.h> 16 #include <linux/vhost.h> 17 #include <linux/hashtable.h> 18 19 #include <net/af_vsock.h> 20 #include "vhost.h" 21 22 #define VHOST_VSOCK_DEFAULT_HOST_CID 2 23 /* Max number of bytes transferred before requeueing the job. 24 * Using this limit prevents one virtqueue from starving others. */ 25 #define VHOST_VSOCK_WEIGHT 0x80000 26 /* Max number of packets transferred before requeueing the job. 27 * Using this limit prevents one virtqueue from starving others with 28 * small pkts. 29 */ 30 #define VHOST_VSOCK_PKT_WEIGHT 256 31 32 static const int vhost_vsock_bits[] = { 33 VHOST_FEATURES, 34 VIRTIO_F_ACCESS_PLATFORM, 35 VIRTIO_VSOCK_F_SEQPACKET 36 }; 37 38 #define VHOST_VSOCK_FEATURES VHOST_FEATURES_U64(vhost_vsock_bits, 0) 39 40 enum { 41 VHOST_VSOCK_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) 42 }; 43 44 /* Used to track all the vhost_vsock instances on the system. */ 45 static DEFINE_MUTEX(vhost_vsock_mutex); 46 static DEFINE_READ_MOSTLY_HASHTABLE(vhost_vsock_hash, 8); 47 48 struct vhost_vsock { 49 struct vhost_dev dev; 50 struct vhost_virtqueue vqs[2]; 51 struct net *net; 52 netns_tracker ns_tracker; 53 54 /* Link to global vhost_vsock_hash, writes use vhost_vsock_mutex */ 55 struct hlist_node hash; 56 57 struct vhost_work send_pkt_work; 58 struct sk_buff_head send_pkt_queue; /* host->guest pending packets */ 59 60 atomic_t queued_replies; 61 62 u32 guest_cid; 63 bool seqpacket_allow; 64 bool ever_started; /* set on first SET_RUNNING(1); never cleared */ 65 }; 66 67 static u32 vhost_transport_get_local_cid(void) 68 { 69 return VHOST_VSOCK_DEFAULT_HOST_CID; 70 } 71 72 /* Callers must be in an RCU read section or hold the vhost_vsock_mutex. 73 * The return value can only be dereferenced while within the section. 74 */ 75 static struct vhost_vsock *vhost_vsock_get(u32 guest_cid, struct net *net) 76 { 77 struct vhost_vsock *vsock; 78 79 hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid, 80 lockdep_is_held(&vhost_vsock_mutex)) { 81 u32 other_cid = vsock->guest_cid; 82 83 /* Skip instances that have no CID yet */ 84 if (other_cid == 0) 85 continue; 86 87 if (other_cid == guest_cid && 88 vsock_net_check_mode(net, vsock->net)) 89 return vsock; 90 } 91 92 return NULL; 93 } 94 95 static bool vhost_transport_has_remote_cid(struct vsock_sock *vsk, u32 cid) 96 { 97 struct sock *sk = sk_vsock(vsk); 98 struct net *net = sock_net(sk); 99 bool found; 100 101 rcu_read_lock(); 102 found = !!vhost_vsock_get(cid, net); 103 rcu_read_unlock(); 104 return found; 105 } 106 107 static void 108 vhost_transport_do_send_pkt(struct vhost_vsock *vsock, 109 struct vhost_virtqueue *vq) 110 { 111 struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX]; 112 int pkts = 0, total_len = 0; 113 bool added = false; 114 bool restart_tx = false; 115 116 mutex_lock(&vq->mutex); 117 118 if (!vhost_vq_get_backend(vq)) 119 goto out; 120 121 if (!vq_meta_prefetch(vq)) 122 goto out; 123 124 /* Avoid further vmexits, we're already processing the virtqueue */ 125 vhost_disable_notify(&vsock->dev, vq); 126 127 do { 128 struct virtio_vsock_hdr *hdr; 129 size_t iov_len, payload_len; 130 struct iov_iter iov_iter; 131 u32 flags_to_restore = 0; 132 struct sk_buff *skb; 133 unsigned out, in; 134 size_t nbytes; 135 u32 offset; 136 int head; 137 138 skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue); 139 140 if (!skb) { 141 vhost_enable_notify(&vsock->dev, vq); 142 break; 143 } 144 145 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), 146 &out, &in, NULL, NULL); 147 if (head < 0) { 148 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 149 break; 150 } 151 152 if (head == vq->num) { 153 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 154 /* We cannot finish yet if more buffers snuck in while 155 * re-enabling notify. 156 */ 157 if (unlikely(vhost_enable_notify(&vsock->dev, vq))) { 158 vhost_disable_notify(&vsock->dev, vq); 159 continue; 160 } 161 break; 162 } 163 164 if (out) { 165 kfree_skb(skb); 166 vq_err(vq, "Expected 0 output buffers, got %u\n", out); 167 break; 168 } 169 170 iov_len = iov_length(&vq->iov[out], in); 171 if (iov_len < sizeof(*hdr)) { 172 kfree_skb(skb); 173 vq_err(vq, "Buffer len [%zu] too small\n", iov_len); 174 break; 175 } 176 177 iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[out], in, iov_len); 178 offset = VIRTIO_VSOCK_SKB_CB(skb)->offset; 179 payload_len = skb->len - offset; 180 hdr = virtio_vsock_hdr(skb); 181 182 /* If the packet is greater than the space available in the 183 * buffer, we split it using multiple buffers. 184 */ 185 if (payload_len > iov_len - sizeof(*hdr)) { 186 payload_len = iov_len - sizeof(*hdr); 187 188 /* As we are copying pieces of large packet's buffer to 189 * small rx buffers, headers of packets in rx queue are 190 * created dynamically and are initialized with header 191 * of current packet(except length). But in case of 192 * SOCK_SEQPACKET, we also must clear message delimeter 193 * bit (VIRTIO_VSOCK_SEQ_EOM) and MSG_EOR bit 194 * (VIRTIO_VSOCK_SEQ_EOR) if set. Otherwise, 195 * there will be sequence of packets with these 196 * bits set. After initialized header will be copied to 197 * rx buffer, these required bits will be restored. 198 */ 199 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) { 200 hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM); 201 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOM; 202 203 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) { 204 hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR); 205 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOR; 206 } 207 } 208 } 209 210 /* Set the correct length in the header */ 211 hdr->len = cpu_to_le32(payload_len); 212 213 nbytes = copy_to_iter(hdr, sizeof(*hdr), &iov_iter); 214 if (nbytes != sizeof(*hdr)) { 215 kfree_skb(skb); 216 vq_err(vq, "Faulted on copying pkt hdr\n"); 217 break; 218 } 219 220 if (skb_copy_datagram_iter(skb, 221 offset, 222 &iov_iter, 223 payload_len)) { 224 kfree_skb(skb); 225 vq_err(vq, "Faulted on copying pkt buf\n"); 226 break; 227 } 228 229 /* Deliver to monitoring devices all packets that we 230 * will transmit. 231 */ 232 virtio_transport_deliver_tap_pkt(skb); 233 234 vhost_add_used(vq, head, sizeof(*hdr) + payload_len); 235 added = true; 236 237 VIRTIO_VSOCK_SKB_CB(skb)->offset += payload_len; 238 total_len += payload_len; 239 240 /* If we didn't send all the payload we can requeue the packet 241 * to send it with the next available buffer. 242 */ 243 if (VIRTIO_VSOCK_SKB_CB(skb)->offset < skb->len) { 244 hdr->flags |= cpu_to_le32(flags_to_restore); 245 246 /* We are queueing the same skb to handle 247 * the remaining bytes, and we want to deliver it 248 * to monitoring devices in the next iteration. 249 */ 250 virtio_vsock_skb_clear_tap_delivered(skb); 251 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 252 } else { 253 if (virtio_vsock_skb_reply(skb)) { 254 int val; 255 256 val = atomic_dec_return(&vsock->queued_replies); 257 258 /* Do we have resources to resume tx 259 * processing? 260 */ 261 if (val + 1 == tx_vq->num) 262 restart_tx = true; 263 } 264 265 virtio_transport_consume_skb_sent(skb, true); 266 } 267 } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len))); 268 if (added) 269 vhost_signal(&vsock->dev, vq); 270 271 out: 272 mutex_unlock(&vq->mutex); 273 274 if (restart_tx) 275 vhost_poll_queue(&tx_vq->poll); 276 } 277 278 static void vhost_transport_send_pkt_work(struct vhost_work *work) 279 { 280 struct vhost_virtqueue *vq; 281 struct vhost_vsock *vsock; 282 283 vsock = container_of(work, struct vhost_vsock, send_pkt_work); 284 vq = &vsock->vqs[VSOCK_VQ_RX]; 285 286 vhost_transport_do_send_pkt(vsock, vq); 287 } 288 289 static int 290 vhost_transport_send_pkt(struct sk_buff *skb, struct net *net) 291 { 292 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 293 struct vhost_vsock *vsock; 294 int len = skb->len; 295 296 rcu_read_lock(); 297 298 /* Find the vhost_vsock according to guest context id */ 299 vsock = vhost_vsock_get(le64_to_cpu(hdr->dst_cid), net); 300 if (!vsock) { 301 rcu_read_unlock(); 302 kfree_skb(skb); 303 return -ENODEV; 304 } 305 306 /* Fast-fail until the guest first enables the device (SET_RUNNING(1)). 307 * Before that there is no listener, so queuing is pointless. 308 * 'ever_started' is never cleared, so once we're up we keep queuing 309 * across later stop / CPR-pause windows. 310 */ 311 if (unlikely(!READ_ONCE(vsock->ever_started))) { 312 rcu_read_unlock(); 313 kfree_skb(skb); 314 return -EHOSTUNREACH; 315 } 316 317 if (virtio_vsock_skb_reply(skb)) 318 atomic_inc(&vsock->queued_replies); 319 320 virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb); 321 vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work); 322 323 rcu_read_unlock(); 324 return len; 325 } 326 327 static int 328 vhost_transport_cancel_pkt(struct vsock_sock *vsk) 329 { 330 struct vhost_vsock *vsock; 331 int cnt = 0; 332 int ret = -ENODEV; 333 334 rcu_read_lock(); 335 336 /* Find the vhost_vsock according to guest context id */ 337 vsock = vhost_vsock_get(vsk->remote_addr.svm_cid, 338 sock_net(sk_vsock(vsk))); 339 if (!vsock) 340 goto out; 341 342 cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue); 343 344 if (cnt) { 345 struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX]; 346 int new_cnt; 347 348 new_cnt = atomic_sub_return(cnt, &vsock->queued_replies); 349 if (new_cnt + cnt >= tx_vq->num && new_cnt < tx_vq->num) 350 vhost_poll_queue(&tx_vq->poll); 351 } 352 353 ret = 0; 354 out: 355 rcu_read_unlock(); 356 return ret; 357 } 358 359 static struct sk_buff * 360 vhost_vsock_alloc_skb(struct vhost_virtqueue *vq, 361 unsigned int out, unsigned int in) 362 { 363 struct virtio_vsock_hdr hdr; 364 struct iov_iter iov_iter; 365 struct sk_buff *skb; 366 size_t payload_len; 367 size_t nbytes; 368 size_t len; 369 370 if (in != 0) { 371 vq_err(vq, "Expected 0 input buffers, got %u\n", in); 372 return NULL; 373 } 374 375 len = iov_length(vq->iov, out); 376 377 if (len < VIRTIO_VSOCK_SKB_HEADROOM || 378 len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE + VIRTIO_VSOCK_SKB_HEADROOM) 379 return NULL; 380 381 iov_iter_init(&iov_iter, ITER_SOURCE, vq->iov, out, len); 382 383 nbytes = copy_from_iter(&hdr, sizeof(hdr), &iov_iter); 384 if (nbytes != sizeof(hdr)) { 385 vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n", 386 sizeof(hdr), nbytes); 387 return NULL; 388 } 389 390 payload_len = le32_to_cpu(hdr.len); 391 392 /* The pkt is too big or the length in the header is invalid */ 393 if (payload_len > len - sizeof(hdr)) 394 return NULL; 395 396 /* Allocate only for the payload declared in the header. */ 397 skb = virtio_vsock_alloc_skb(payload_len + sizeof(hdr), GFP_KERNEL); 398 if (!skb) 399 return NULL; 400 401 memcpy(virtio_vsock_hdr(skb), &hdr, sizeof(hdr)); 402 403 /* No payload */ 404 if (!payload_len) 405 return skb; 406 407 virtio_vsock_skb_put(skb, payload_len); 408 409 if (skb_copy_datagram_from_iter(skb, 0, &iov_iter, payload_len)) { 410 vq_err(vq, "Failed to copy %zu byte payload\n", payload_len); 411 kfree_skb(skb); 412 return NULL; 413 } 414 415 return skb; 416 } 417 418 /* Is there space left for replies to rx packets? */ 419 static bool vhost_vsock_more_replies(struct vhost_vsock *vsock) 420 { 421 struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX]; 422 int val; 423 424 smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */ 425 val = atomic_read(&vsock->queued_replies); 426 427 return val < vq->num; 428 } 429 430 static bool vhost_transport_msgzerocopy_allow(void) 431 { 432 return true; 433 } 434 435 static bool vhost_transport_seqpacket_allow(struct vsock_sock *vsk, 436 u32 remote_cid); 437 438 static bool 439 vhost_transport_stream_allow(struct vsock_sock *vsk, u32 cid, u32 port) 440 { 441 return true; 442 } 443 444 static struct virtio_transport vhost_transport = { 445 .transport = { 446 .module = THIS_MODULE, 447 448 .get_local_cid = vhost_transport_get_local_cid, 449 .has_remote_cid = vhost_transport_has_remote_cid, 450 451 .init = virtio_transport_do_socket_init, 452 .destruct = virtio_transport_destruct, 453 .release = virtio_transport_release, 454 .connect = virtio_transport_connect, 455 .shutdown = virtio_transport_shutdown, 456 .cancel_pkt = vhost_transport_cancel_pkt, 457 458 .dgram_enqueue = virtio_transport_dgram_enqueue, 459 .dgram_dequeue = virtio_transport_dgram_dequeue, 460 .dgram_bind = virtio_transport_dgram_bind, 461 .dgram_allow = virtio_transport_dgram_allow, 462 463 .stream_enqueue = virtio_transport_stream_enqueue, 464 .stream_dequeue = virtio_transport_stream_dequeue, 465 .stream_has_data = virtio_transport_stream_has_data, 466 .stream_has_space = virtio_transport_stream_has_space, 467 .stream_rcvhiwat = virtio_transport_stream_rcvhiwat, 468 .stream_is_active = virtio_transport_stream_is_active, 469 .stream_allow = vhost_transport_stream_allow, 470 471 .seqpacket_dequeue = virtio_transport_seqpacket_dequeue, 472 .seqpacket_enqueue = virtio_transport_seqpacket_enqueue, 473 .seqpacket_allow = vhost_transport_seqpacket_allow, 474 .seqpacket_has_data = virtio_transport_seqpacket_has_data, 475 476 .msgzerocopy_allow = vhost_transport_msgzerocopy_allow, 477 478 .notify_poll_in = virtio_transport_notify_poll_in, 479 .notify_poll_out = virtio_transport_notify_poll_out, 480 .notify_recv_init = virtio_transport_notify_recv_init, 481 .notify_recv_pre_block = virtio_transport_notify_recv_pre_block, 482 .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue, 483 .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue, 484 .notify_send_init = virtio_transport_notify_send_init, 485 .notify_send_pre_block = virtio_transport_notify_send_pre_block, 486 .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, 487 .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, 488 .notify_buffer_size = virtio_transport_notify_buffer_size, 489 .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat, 490 491 .unsent_bytes = virtio_transport_unsent_bytes, 492 493 .read_skb = virtio_transport_read_skb, 494 }, 495 496 .send_pkt = vhost_transport_send_pkt, 497 }; 498 499 static bool vhost_transport_seqpacket_allow(struct vsock_sock *vsk, 500 u32 remote_cid) 501 { 502 struct net *net = sock_net(sk_vsock(vsk)); 503 struct vhost_vsock *vsock; 504 bool seqpacket_allow = false; 505 506 rcu_read_lock(); 507 vsock = vhost_vsock_get(remote_cid, net); 508 509 if (vsock) 510 seqpacket_allow = vsock->seqpacket_allow; 511 512 rcu_read_unlock(); 513 514 return seqpacket_allow; 515 } 516 517 static void vhost_vsock_handle_tx_kick(struct vhost_work *work) 518 { 519 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 520 poll.work); 521 struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock, 522 dev); 523 int head, pkts = 0, total_len = 0; 524 unsigned int out, in; 525 struct sk_buff *skb; 526 bool added = false; 527 528 mutex_lock(&vq->mutex); 529 530 if (!vhost_vq_get_backend(vq)) 531 goto out; 532 533 if (!vq_meta_prefetch(vq)) 534 goto out; 535 536 vhost_disable_notify(&vsock->dev, vq); 537 do { 538 struct virtio_vsock_hdr *hdr; 539 540 if (!vhost_vsock_more_replies(vsock)) { 541 /* Stop tx until the device processes already 542 * pending replies. Leave tx virtqueue 543 * callbacks disabled. 544 */ 545 goto no_more_replies; 546 } 547 548 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), 549 &out, &in, NULL, NULL); 550 if (head < 0) 551 break; 552 553 if (head == vq->num) { 554 if (unlikely(vhost_enable_notify(&vsock->dev, vq))) { 555 vhost_disable_notify(&vsock->dev, vq); 556 continue; 557 } 558 break; 559 } 560 561 skb = vhost_vsock_alloc_skb(vq, out, in); 562 if (!skb) { 563 vq_err(vq, "Faulted on pkt\n"); 564 continue; 565 } 566 567 total_len += sizeof(*hdr) + skb->len; 568 569 /* Deliver to monitoring devices all received packets */ 570 virtio_transport_deliver_tap_pkt(skb); 571 572 hdr = virtio_vsock_hdr(skb); 573 574 /* Only accept correctly addressed packets */ 575 if (le64_to_cpu(hdr->src_cid) == vsock->guest_cid && 576 le64_to_cpu(hdr->dst_cid) == 577 vhost_transport_get_local_cid()) 578 virtio_transport_recv_pkt(&vhost_transport, skb, 579 vsock->net); 580 else 581 kfree_skb(skb); 582 583 vhost_add_used(vq, head, 0); 584 added = true; 585 } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len))); 586 587 no_more_replies: 588 if (added) 589 vhost_signal(&vsock->dev, vq); 590 591 out: 592 mutex_unlock(&vq->mutex); 593 } 594 595 static void vhost_vsock_handle_rx_kick(struct vhost_work *work) 596 { 597 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 598 poll.work); 599 struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock, 600 dev); 601 602 vhost_transport_do_send_pkt(vsock, vq); 603 } 604 605 static int vhost_vsock_start(struct vhost_vsock *vsock) 606 { 607 struct vhost_virtqueue *vq; 608 size_t i; 609 int ret; 610 611 mutex_lock(&vsock->dev.mutex); 612 613 ret = vhost_dev_check_owner(&vsock->dev); 614 if (ret) 615 goto err; 616 617 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 618 vq = &vsock->vqs[i]; 619 620 mutex_lock(&vq->mutex); 621 622 if (!vhost_vq_access_ok(vq)) { 623 ret = -EFAULT; 624 goto err_vq; 625 } 626 627 if (!vhost_vq_get_backend(vq)) { 628 vhost_vq_set_backend(vq, vsock); 629 ret = vhost_vq_init_access(vq); 630 if (ret) 631 goto err_vq; 632 } 633 634 mutex_unlock(&vq->mutex); 635 } 636 637 /* Set 'ever_started' flag on the first start; never cleared, so send_pkt 638 * keeps queuing (instead of fast-failing) on later stop / CPR pauses. 639 */ 640 WRITE_ONCE(vsock->ever_started, true); 641 642 /* Some packets may have been queued before the device was started, 643 * let's kick the send worker to send them. 644 */ 645 vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work); 646 647 /* The guest may have added TX buffers while the device was stopped 648 * (e.g. across VHOST_RESET_OWNER) and their kicks got consumed by 649 * the NULL-backend window. Re-scan the TX VQ, mirroring the RX 650 * send-worker kick above. 651 */ 652 vhost_poll_queue(&vsock->vqs[VSOCK_VQ_TX].poll); 653 654 mutex_unlock(&vsock->dev.mutex); 655 return 0; 656 657 err_vq: 658 vhost_vq_set_backend(vq, NULL); 659 mutex_unlock(&vq->mutex); 660 661 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 662 vq = &vsock->vqs[i]; 663 664 mutex_lock(&vq->mutex); 665 vhost_vq_set_backend(vq, NULL); 666 mutex_unlock(&vq->mutex); 667 } 668 err: 669 mutex_unlock(&vsock->dev.mutex); 670 return ret; 671 } 672 673 static void vhost_vsock_drop_backends(struct vhost_vsock *vsock) 674 { 675 struct vhost_virtqueue *vq; 676 size_t i; 677 678 lockdep_assert_held(&vsock->dev.mutex); 679 680 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 681 vq = &vsock->vqs[i]; 682 683 mutex_lock(&vq->mutex); 684 vhost_vq_set_backend(vq, NULL); 685 mutex_unlock(&vq->mutex); 686 } 687 } 688 689 static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner) 690 { 691 int ret = 0; 692 693 mutex_lock(&vsock->dev.mutex); 694 695 if (check_owner) { 696 ret = vhost_dev_check_owner(&vsock->dev); 697 if (ret) 698 goto err; 699 } 700 701 vhost_vsock_drop_backends(vsock); 702 err: 703 mutex_unlock(&vsock->dev.mutex); 704 return ret; 705 } 706 707 static void vhost_vsock_free(struct vhost_vsock *vsock) 708 { 709 kvfree(vsock); 710 } 711 712 static int vhost_vsock_dev_open(struct inode *inode, struct file *file) 713 { 714 struct vhost_virtqueue **vqs; 715 struct vhost_vsock *vsock; 716 struct net *net; 717 int ret; 718 719 /* This struct is large and allocation could fail, fall back to vmalloc 720 * if there is no other way. 721 */ 722 vsock = kvmalloc_obj(*vsock, GFP_KERNEL | __GFP_RETRY_MAYFAIL); 723 if (!vsock) 724 return -ENOMEM; 725 726 vqs = kmalloc_objs(*vqs, ARRAY_SIZE(vsock->vqs)); 727 if (!vqs) { 728 ret = -ENOMEM; 729 goto out; 730 } 731 732 net = current->nsproxy->net_ns; 733 vsock->net = get_net_track(net, &vsock->ns_tracker, GFP_KERNEL); 734 735 vsock->guest_cid = 0; /* no CID assigned yet */ 736 vsock->seqpacket_allow = false; 737 vsock->ever_started = false; 738 739 atomic_set(&vsock->queued_replies, 0); 740 741 vqs[VSOCK_VQ_TX] = &vsock->vqs[VSOCK_VQ_TX]; 742 vqs[VSOCK_VQ_RX] = &vsock->vqs[VSOCK_VQ_RX]; 743 vsock->vqs[VSOCK_VQ_TX].handle_kick = vhost_vsock_handle_tx_kick; 744 vsock->vqs[VSOCK_VQ_RX].handle_kick = vhost_vsock_handle_rx_kick; 745 746 vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs), 747 UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT, 748 VHOST_VSOCK_WEIGHT, true, NULL); 749 750 file->private_data = vsock; 751 skb_queue_head_init(&vsock->send_pkt_queue); 752 vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work); 753 return 0; 754 755 out: 756 vhost_vsock_free(vsock); 757 return ret; 758 } 759 760 static void vhost_vsock_flush(struct vhost_vsock *vsock) 761 { 762 vhost_dev_flush(&vsock->dev); 763 } 764 765 static void vhost_vsock_reset_orphans(struct sock *sk) 766 { 767 struct vsock_sock *vsk = vsock_sk(sk); 768 769 /* vmci_transport.c doesn't take sk_lock here either. At least we're 770 * under vsock_table_lock so the sock cannot disappear while we're 771 * executing. 772 */ 773 774 rcu_read_lock(); 775 776 /* If the peer is still valid, no need to reset connection */ 777 if (vhost_vsock_get(vsk->remote_addr.svm_cid, sock_net(sk))) { 778 rcu_read_unlock(); 779 return; 780 } 781 782 rcu_read_unlock(); 783 784 /* If the close timeout is pending, let it expire. This avoids races 785 * with the timeout callback. 786 */ 787 if (vsk->close_work_scheduled) 788 return; 789 790 sock_set_flag(sk, SOCK_DONE); 791 vsk->peer_shutdown = SHUTDOWN_MASK; 792 sk->sk_state = SS_UNCONNECTED; 793 sk->sk_err = ECONNRESET; 794 sk_error_report(sk); 795 } 796 797 static int vhost_vsock_dev_release(struct inode *inode, struct file *file) 798 { 799 struct vhost_vsock *vsock = file->private_data; 800 801 mutex_lock(&vhost_vsock_mutex); 802 if (vsock->guest_cid) 803 hash_del_rcu(&vsock->hash); 804 mutex_unlock(&vhost_vsock_mutex); 805 806 /* Wait for other CPUs to finish using vsock */ 807 synchronize_rcu(); 808 809 /* Iterating over all connections for all CIDs to find orphans is 810 * inefficient. Room for improvement here. */ 811 vsock_for_each_connected_socket(&vhost_transport.transport, 812 vhost_vsock_reset_orphans); 813 814 /* Don't check the owner, because we are in the release path, so we 815 * need to stop the vsock device in any case. 816 * vhost_vsock_stop() can not fail in this case, so we don't need to 817 * check the return code. 818 */ 819 vhost_vsock_stop(vsock, false); 820 vhost_vsock_flush(vsock); 821 vhost_dev_stop(&vsock->dev); 822 823 virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue); 824 825 vhost_dev_cleanup(&vsock->dev, false); 826 put_net_track(vsock->net, &vsock->ns_tracker); 827 kfree(vsock->dev.vqs); 828 vhost_vsock_free(vsock); 829 return 0; 830 } 831 832 static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid) 833 { 834 struct vhost_vsock *other; 835 836 /* Refuse reserved CIDs */ 837 if (guest_cid <= VMADDR_CID_HOST || 838 guest_cid == U32_MAX) 839 return -EINVAL; 840 841 /* 64-bit CIDs are not yet supported */ 842 if (guest_cid > U32_MAX) 843 return -EINVAL; 844 845 /* Refuse if CID is assigned to the guest->host transport (i.e. nested 846 * VM), to make the loopback work. 847 */ 848 if (vsock_find_cid(guest_cid)) 849 return -EADDRINUSE; 850 851 /* Refuse if CID is already in use */ 852 mutex_lock(&vhost_vsock_mutex); 853 other = vhost_vsock_get(guest_cid, vsock->net); 854 if (other && other != vsock) { 855 mutex_unlock(&vhost_vsock_mutex); 856 return -EADDRINUSE; 857 } 858 859 if (vsock->guest_cid) 860 hash_del_rcu(&vsock->hash); 861 862 vsock->guest_cid = guest_cid; 863 hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid); 864 mutex_unlock(&vhost_vsock_mutex); 865 866 return 0; 867 } 868 869 static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features) 870 { 871 struct vhost_virtqueue *vq; 872 int i; 873 874 if (features & ~VHOST_VSOCK_FEATURES) 875 return -EOPNOTSUPP; 876 877 mutex_lock(&vsock->dev.mutex); 878 if ((features & (1 << VHOST_F_LOG_ALL)) && 879 !vhost_log_access_ok(&vsock->dev)) { 880 goto err; 881 } 882 883 if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) { 884 if (vhost_init_device_iotlb(&vsock->dev)) 885 goto err; 886 } else { 887 vhost_clear_device_iotlb(&vsock->dev); 888 } 889 890 vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET); 891 892 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 893 vq = &vsock->vqs[i]; 894 mutex_lock(&vq->mutex); 895 vq->acked_features = features; 896 mutex_unlock(&vq->mutex); 897 } 898 mutex_unlock(&vsock->dev.mutex); 899 return 0; 900 901 err: 902 mutex_unlock(&vsock->dev.mutex); 903 return -EFAULT; 904 } 905 906 static long vhost_vsock_reset_owner(struct vhost_vsock *vsock) 907 { 908 struct vhost_iotlb *umem; 909 long err; 910 911 mutex_lock(&vsock->dev.mutex); 912 err = vhost_dev_check_owner(&vsock->dev); 913 if (err) 914 goto done; 915 umem = vhost_dev_reset_owner_prepare(); 916 if (!umem) { 917 err = -ENOMEM; 918 goto done; 919 } 920 vhost_vsock_drop_backends(vsock); 921 vhost_vsock_flush(vsock); 922 vhost_dev_stop(&vsock->dev); 923 924 /* 925 * vsock keeps the CID hashed across RESET_OWNER, so a lockless sender 926 * can still reach the device while its workers are freed. Pass 927 * sync=true so vhost_workers_free() fences RCU readers and flushes 928 * before freeing them. 929 */ 930 vhost_dev_reset_owner(&vsock->dev, umem, true); 931 done: 932 mutex_unlock(&vsock->dev.mutex); 933 return err; 934 } 935 936 static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl, 937 unsigned long arg) 938 { 939 struct vhost_vsock *vsock = f->private_data; 940 void __user *argp = (void __user *)arg; 941 u64 guest_cid; 942 u64 features; 943 int start; 944 int r; 945 946 switch (ioctl) { 947 case VHOST_VSOCK_SET_GUEST_CID: 948 if (copy_from_user(&guest_cid, argp, sizeof(guest_cid))) 949 return -EFAULT; 950 return vhost_vsock_set_cid(vsock, guest_cid); 951 case VHOST_VSOCK_SET_RUNNING: 952 if (copy_from_user(&start, argp, sizeof(start))) 953 return -EFAULT; 954 if (start) 955 return vhost_vsock_start(vsock); 956 else 957 return vhost_vsock_stop(vsock, true); 958 case VHOST_GET_FEATURES: 959 features = VHOST_VSOCK_FEATURES; 960 if (copy_to_user(argp, &features, sizeof(features))) 961 return -EFAULT; 962 return 0; 963 case VHOST_SET_FEATURES: 964 if (copy_from_user(&features, argp, sizeof(features))) 965 return -EFAULT; 966 return vhost_vsock_set_features(vsock, features); 967 case VHOST_GET_BACKEND_FEATURES: 968 features = VHOST_VSOCK_BACKEND_FEATURES; 969 if (copy_to_user(argp, &features, sizeof(features))) 970 return -EFAULT; 971 return 0; 972 case VHOST_SET_BACKEND_FEATURES: 973 if (copy_from_user(&features, argp, sizeof(features))) 974 return -EFAULT; 975 if (features & ~VHOST_VSOCK_BACKEND_FEATURES) 976 return -EOPNOTSUPP; 977 vhost_set_backend_features(&vsock->dev, features); 978 return 0; 979 case VHOST_RESET_OWNER: 980 return vhost_vsock_reset_owner(vsock); 981 default: 982 mutex_lock(&vsock->dev.mutex); 983 r = vhost_dev_ioctl(&vsock->dev, ioctl, argp); 984 if (r == -ENOIOCTLCMD) 985 r = vhost_vring_ioctl(&vsock->dev, ioctl, argp); 986 else 987 vhost_vsock_flush(vsock); 988 mutex_unlock(&vsock->dev.mutex); 989 return r; 990 } 991 } 992 993 static ssize_t vhost_vsock_chr_read_iter(struct kiocb *iocb, struct iov_iter *to) 994 { 995 struct file *file = iocb->ki_filp; 996 struct vhost_vsock *vsock = file->private_data; 997 struct vhost_dev *dev = &vsock->dev; 998 int noblock = file->f_flags & O_NONBLOCK; 999 1000 return vhost_chr_read_iter(dev, to, noblock); 1001 } 1002 1003 static ssize_t vhost_vsock_chr_write_iter(struct kiocb *iocb, 1004 struct iov_iter *from) 1005 { 1006 struct file *file = iocb->ki_filp; 1007 struct vhost_vsock *vsock = file->private_data; 1008 struct vhost_dev *dev = &vsock->dev; 1009 1010 return vhost_chr_write_iter(dev, from); 1011 } 1012 1013 static __poll_t vhost_vsock_chr_poll(struct file *file, poll_table *wait) 1014 { 1015 struct vhost_vsock *vsock = file->private_data; 1016 struct vhost_dev *dev = &vsock->dev; 1017 1018 return vhost_chr_poll(file, dev, wait); 1019 } 1020 1021 static const struct file_operations vhost_vsock_fops = { 1022 .owner = THIS_MODULE, 1023 .open = vhost_vsock_dev_open, 1024 .release = vhost_vsock_dev_release, 1025 .llseek = noop_llseek, 1026 .unlocked_ioctl = vhost_vsock_dev_ioctl, 1027 .compat_ioctl = compat_ptr_ioctl, 1028 .read_iter = vhost_vsock_chr_read_iter, 1029 .write_iter = vhost_vsock_chr_write_iter, 1030 .poll = vhost_vsock_chr_poll, 1031 }; 1032 1033 static struct miscdevice vhost_vsock_misc = { 1034 .minor = VHOST_VSOCK_MINOR, 1035 .name = "vhost-vsock", 1036 .fops = &vhost_vsock_fops, 1037 }; 1038 1039 static int __init vhost_vsock_init(void) 1040 { 1041 int ret; 1042 1043 ret = vsock_core_register(&vhost_transport.transport, 1044 VSOCK_TRANSPORT_F_H2G); 1045 if (ret < 0) 1046 return ret; 1047 1048 ret = misc_register(&vhost_vsock_misc); 1049 if (ret) { 1050 vsock_core_unregister(&vhost_transport.transport); 1051 return ret; 1052 } 1053 1054 return 0; 1055 }; 1056 1057 static void __exit vhost_vsock_exit(void) 1058 { 1059 misc_deregister(&vhost_vsock_misc); 1060 vsock_core_unregister(&vhost_transport.transport); 1061 }; 1062 1063 module_init(vhost_vsock_init); 1064 module_exit(vhost_vsock_exit); 1065 MODULE_LICENSE("GPL v2"); 1066 MODULE_AUTHOR("Asias He"); 1067 MODULE_DESCRIPTION("vhost transport for vsock "); 1068 MODULE_ALIAS_MISCDEV(VHOST_VSOCK_MINOR); 1069 MODULE_ALIAS("devname:vhost-vsock"); 1070