1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * virtio transport for vsock 4 * 5 * Copyright (C) 2013-2015 Red Hat, Inc. 6 * Author: Asias He <asias@redhat.com> 7 * Stefan Hajnoczi <stefanha@redhat.com> 8 * 9 * Some of the code is take from Gerd Hoffmann <kraxel@redhat.com>'s 10 * early virtio-vsock proof-of-concept bits. 11 */ 12 #include <linux/spinlock.h> 13 #include <linux/module.h> 14 #include <linux/list.h> 15 #include <linux/atomic.h> 16 #include <linux/virtio.h> 17 #include <linux/virtio_ids.h> 18 #include <linux/virtio_config.h> 19 #include <linux/virtio_vsock.h> 20 #include <linux/dma-mapping.h> 21 #include <net/sock.h> 22 #include <linux/mutex.h> 23 #include <net/af_vsock.h> 24 25 static struct workqueue_struct *virtio_vsock_workqueue; 26 static struct virtio_vsock __rcu *the_virtio_vsock; 27 static DEFINE_MUTEX(the_virtio_vsock_mutex); /* protects the_virtio_vsock */ 28 static struct virtio_transport virtio_transport; /* forward declaration */ 29 30 struct virtio_vsock { 31 struct virtio_device *vdev; 32 struct virtqueue *vqs[VSOCK_VQ_MAX]; 33 34 /* Virtqueue processing is deferred to a workqueue */ 35 struct work_struct tx_work; 36 struct work_struct rx_work; 37 struct work_struct event_work; 38 39 /* The following fields are protected by tx_lock. vqs[VSOCK_VQ_TX] 40 * must be accessed with tx_lock held. 41 */ 42 struct mutex tx_lock; 43 bool tx_run; 44 45 struct work_struct send_pkt_work; 46 struct sk_buff_head send_pkt_queue; 47 48 atomic_t queued_replies; 49 50 /* The following fields are protected by rx_lock. vqs[VSOCK_VQ_RX] 51 * must be accessed with rx_lock held. 52 */ 53 struct mutex rx_lock; 54 bool rx_run; 55 int rx_buf_nr; 56 int rx_buf_max_nr; 57 58 u32 guest_cid; 59 bool seqpacket_allow; 60 61 /* These fields are used only in tx path in function 62 * 'virtio_transport_send_pkt_work()', so to save 63 * stack space in it, place both of them here. Each 64 * pointer from 'out_sgs' points to the corresponding 65 * element in 'out_bufs' - this is initialized in 66 * 'virtio_vsock_probe()'. Both fields are protected 67 * by 'tx_lock'. +1 is needed for packet header. 68 */ 69 struct scatterlist *out_sgs[MAX_SKB_FRAGS + 1]; 70 struct scatterlist out_bufs[MAX_SKB_FRAGS + 1]; 71 72 /* The following fields are protected by event_lock. 73 * vqs[VSOCK_VQ_EVENT] must be accessed with event_lock held. 74 */ 75 struct mutex event_lock; 76 bool event_run; 77 __dma_from_device_group_begin(); 78 struct virtio_vsock_event event_list[8]; 79 __dma_from_device_group_end(); 80 }; 81 82 static u32 virtio_transport_get_local_cid(void) 83 { 84 struct virtio_vsock *vsock; 85 u32 ret; 86 87 rcu_read_lock(); 88 vsock = rcu_dereference(the_virtio_vsock); 89 if (!vsock) { 90 ret = VMADDR_CID_ANY; 91 goto out_rcu; 92 } 93 94 ret = vsock->guest_cid; 95 out_rcu: 96 rcu_read_unlock(); 97 return ret; 98 } 99 100 /* Caller need to hold vsock->tx_lock on vq */ 101 static int virtio_transport_send_skb(struct sk_buff *skb, struct virtqueue *vq, 102 struct virtio_vsock *vsock, gfp_t gfp) 103 { 104 int ret, in_sg = 0, out_sg = 0; 105 struct scatterlist **sgs; 106 107 sgs = vsock->out_sgs; 108 sg_init_one(sgs[out_sg], virtio_vsock_hdr(skb), 109 sizeof(*virtio_vsock_hdr(skb))); 110 out_sg++; 111 112 if (!skb_is_nonlinear(skb)) { 113 if (skb->len > 0) { 114 sg_init_one(sgs[out_sg], skb->data, skb->len); 115 out_sg++; 116 } 117 } else { 118 struct skb_shared_info *si; 119 int i; 120 121 /* If skb is nonlinear, then its buffer must contain 122 * only header and nothing more. Data is stored in 123 * the fragged part. 124 */ 125 WARN_ON_ONCE(skb_headroom(skb) != sizeof(*virtio_vsock_hdr(skb))); 126 127 si = skb_shinfo(skb); 128 129 for (i = 0; i < si->nr_frags; i++) { 130 skb_frag_t *skb_frag = &si->frags[i]; 131 void *va; 132 133 /* We will use 'page_to_virt()' for the userspace page 134 * here, because virtio or dma-mapping layers will call 135 * 'virt_to_phys()' later to fill the buffer descriptor. 136 * We don't touch memory at "virtual" address of this page. 137 */ 138 va = page_to_virt(skb_frag_page(skb_frag)); 139 sg_init_one(sgs[out_sg], 140 va + skb_frag_off(skb_frag), 141 skb_frag_size(skb_frag)); 142 out_sg++; 143 } 144 } 145 146 ret = virtqueue_add_sgs(vq, sgs, out_sg, in_sg, skb, gfp); 147 /* Usually this means that there is no more space available in 148 * the vq 149 */ 150 if (ret < 0) 151 return ret; 152 153 virtio_transport_deliver_tap_pkt(skb); 154 return 0; 155 } 156 157 static void 158 virtio_transport_send_pkt_work(struct work_struct *work) 159 { 160 struct virtio_vsock *vsock = 161 container_of(work, struct virtio_vsock, send_pkt_work); 162 struct virtqueue *vq; 163 bool added = false; 164 bool restart_rx = false; 165 166 mutex_lock(&vsock->tx_lock); 167 168 if (!vsock->tx_run) 169 goto out; 170 171 vq = vsock->vqs[VSOCK_VQ_TX]; 172 173 for (;;) { 174 struct sk_buff *skb; 175 bool reply; 176 int ret; 177 178 skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue); 179 if (!skb) 180 break; 181 182 reply = virtio_vsock_skb_reply(skb); 183 184 ret = virtio_transport_send_skb(skb, vq, vsock, GFP_KERNEL); 185 if (ret < 0) { 186 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); 187 break; 188 } 189 190 if (reply) { 191 struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX]; 192 int val; 193 194 val = atomic_dec_return(&vsock->queued_replies); 195 196 /* Do we now have resources to resume rx processing? */ 197 if (val + 1 == virtqueue_get_vring_size(rx_vq)) 198 restart_rx = true; 199 } 200 201 added = true; 202 } 203 204 if (added) 205 virtqueue_kick(vq); 206 207 out: 208 mutex_unlock(&vsock->tx_lock); 209 210 if (restart_rx) 211 queue_work(virtio_vsock_workqueue, &vsock->rx_work); 212 } 213 214 /* Caller need to hold RCU for vsock. 215 * Returns 0 if the packet is successfully put on the vq. 216 */ 217 static int virtio_transport_send_skb_fast_path(struct virtio_vsock *vsock, struct sk_buff *skb) 218 { 219 struct virtqueue *vq = vsock->vqs[VSOCK_VQ_TX]; 220 int ret; 221 222 /* Inside RCU, can't sleep! */ 223 ret = mutex_trylock(&vsock->tx_lock); 224 if (unlikely(ret == 0)) 225 return -EBUSY; 226 227 ret = virtio_transport_send_skb(skb, vq, vsock, GFP_ATOMIC); 228 if (ret == 0) 229 virtqueue_kick(vq); 230 231 mutex_unlock(&vsock->tx_lock); 232 233 return ret; 234 } 235 236 static int 237 virtio_transport_send_pkt(struct sk_buff *skb, struct net *net) 238 { 239 struct virtio_vsock_hdr *hdr; 240 struct virtio_vsock *vsock; 241 int len = skb->len; 242 243 hdr = virtio_vsock_hdr(skb); 244 245 rcu_read_lock(); 246 vsock = rcu_dereference(the_virtio_vsock); 247 if (!vsock) { 248 kfree_skb(skb); 249 len = -ENODEV; 250 goto out_rcu; 251 } 252 253 if (le64_to_cpu(hdr->dst_cid) == vsock->guest_cid) { 254 kfree_skb(skb); 255 len = -ENODEV; 256 goto out_rcu; 257 } 258 259 /* If send_pkt_queue is empty, we can safely bypass this queue 260 * because packet order is maintained and (try) to put the packet 261 * on the virtqueue using virtio_transport_send_skb_fast_path. 262 * If this fails we simply put the packet on the intermediate 263 * queue and schedule the worker. 264 */ 265 if (!skb_queue_empty_lockless(&vsock->send_pkt_queue) || 266 virtio_transport_send_skb_fast_path(vsock, skb)) { 267 if (virtio_vsock_skb_reply(skb)) 268 atomic_inc(&vsock->queued_replies); 269 270 virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb); 271 queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work); 272 } 273 274 out_rcu: 275 rcu_read_unlock(); 276 return len; 277 } 278 279 static int 280 virtio_transport_cancel_pkt(struct vsock_sock *vsk) 281 { 282 struct virtio_vsock *vsock; 283 int cnt = 0, ret; 284 285 rcu_read_lock(); 286 vsock = rcu_dereference(the_virtio_vsock); 287 if (!vsock) { 288 ret = -ENODEV; 289 goto out_rcu; 290 } 291 292 cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue); 293 294 if (cnt) { 295 struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX]; 296 int new_cnt; 297 298 new_cnt = atomic_sub_return(cnt, &vsock->queued_replies); 299 if (new_cnt + cnt >= virtqueue_get_vring_size(rx_vq) && 300 new_cnt < virtqueue_get_vring_size(rx_vq)) 301 queue_work(virtio_vsock_workqueue, &vsock->rx_work); 302 } 303 304 ret = 0; 305 306 out_rcu: 307 rcu_read_unlock(); 308 return ret; 309 } 310 311 static void virtio_vsock_rx_fill(struct virtio_vsock *vsock) 312 { 313 int total_len = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE; 314 struct scatterlist pkt, *p; 315 struct virtqueue *vq; 316 struct sk_buff *skb; 317 int ret; 318 319 vq = vsock->vqs[VSOCK_VQ_RX]; 320 321 do { 322 skb = virtio_vsock_alloc_linear_skb(total_len, GFP_KERNEL); 323 if (!skb) 324 break; 325 326 memset(skb->head, 0, VIRTIO_VSOCK_SKB_HEADROOM); 327 sg_init_one(&pkt, virtio_vsock_hdr(skb), total_len); 328 p = &pkt; 329 ret = virtqueue_add_sgs(vq, &p, 0, 1, skb, GFP_KERNEL); 330 if (ret < 0) { 331 kfree_skb(skb); 332 break; 333 } 334 335 vsock->rx_buf_nr++; 336 } while (vq->num_free); 337 if (vsock->rx_buf_nr > vsock->rx_buf_max_nr) 338 vsock->rx_buf_max_nr = vsock->rx_buf_nr; 339 virtqueue_kick(vq); 340 } 341 342 static void virtio_transport_tx_work(struct work_struct *work) 343 { 344 struct virtio_vsock *vsock = 345 container_of(work, struct virtio_vsock, tx_work); 346 struct virtqueue *vq; 347 bool added = false; 348 349 mutex_lock(&vsock->tx_lock); 350 351 if (!vsock->tx_run) 352 goto out; 353 354 vq = vsock->vqs[VSOCK_VQ_TX]; 355 356 do { 357 struct sk_buff *skb; 358 unsigned int len; 359 360 virtqueue_disable_cb(vq); 361 while ((skb = virtqueue_get_buf(vq, &len)) != NULL) { 362 virtio_transport_consume_skb_sent(skb, true); 363 added = true; 364 } 365 } while (!virtqueue_enable_cb(vq)); 366 367 out: 368 mutex_unlock(&vsock->tx_lock); 369 370 if (added) 371 queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work); 372 } 373 374 /* Is there space left for replies to rx packets? */ 375 static bool virtio_transport_more_replies(struct virtio_vsock *vsock) 376 { 377 struct virtqueue *vq = vsock->vqs[VSOCK_VQ_RX]; 378 int val; 379 380 smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */ 381 val = atomic_read(&vsock->queued_replies); 382 383 return val < virtqueue_get_vring_size(vq); 384 } 385 386 /* event_lock must be held */ 387 static int virtio_vsock_event_fill_one(struct virtio_vsock *vsock, 388 struct virtio_vsock_event *event) 389 { 390 struct scatterlist sg; 391 struct virtqueue *vq; 392 393 vq = vsock->vqs[VSOCK_VQ_EVENT]; 394 395 sg_init_one(&sg, event, sizeof(*event)); 396 397 return virtqueue_add_inbuf_cache_clean(vq, &sg, 1, event, GFP_KERNEL); 398 } 399 400 /* event_lock must be held */ 401 static void virtio_vsock_event_fill(struct virtio_vsock *vsock) 402 { 403 size_t i; 404 405 for (i = 0; i < ARRAY_SIZE(vsock->event_list); i++) { 406 struct virtio_vsock_event *event = &vsock->event_list[i]; 407 408 virtio_vsock_event_fill_one(vsock, event); 409 } 410 411 virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]); 412 } 413 414 static void virtio_vsock_reset_sock(struct sock *sk) 415 { 416 /* vmci_transport.c doesn't take sk_lock here either. At least we're 417 * under vsock_table_lock so the sock cannot disappear while we're 418 * executing. 419 */ 420 421 sk->sk_state = TCP_CLOSE; 422 sk->sk_err = ECONNRESET; 423 sk_error_report(sk); 424 } 425 426 static void virtio_vsock_update_guest_cid(struct virtio_vsock *vsock) 427 { 428 struct virtio_device *vdev = vsock->vdev; 429 __le64 guest_cid; 430 431 vdev->config->get(vdev, offsetof(struct virtio_vsock_config, guest_cid), 432 &guest_cid, sizeof(guest_cid)); 433 vsock->guest_cid = le64_to_cpu(guest_cid); 434 } 435 436 /* event_lock must be held */ 437 static void virtio_vsock_event_handle(struct virtio_vsock *vsock, 438 struct virtio_vsock_event *event) 439 { 440 switch (le32_to_cpu(event->id)) { 441 case VIRTIO_VSOCK_EVENT_TRANSPORT_RESET: 442 virtio_vsock_update_guest_cid(vsock); 443 vsock_for_each_connected_socket(&virtio_transport.transport, 444 virtio_vsock_reset_sock); 445 break; 446 } 447 } 448 449 static void virtio_transport_event_work(struct work_struct *work) 450 { 451 struct virtio_vsock *vsock = 452 container_of(work, struct virtio_vsock, event_work); 453 struct virtqueue *vq; 454 455 mutex_lock(&vsock->event_lock); 456 457 if (!vsock->event_run) 458 goto out; 459 460 vq = vsock->vqs[VSOCK_VQ_EVENT]; 461 462 do { 463 struct virtio_vsock_event *event; 464 unsigned int len; 465 466 virtqueue_disable_cb(vq); 467 while ((event = virtqueue_get_buf(vq, &len)) != NULL) { 468 if (len == sizeof(*event)) 469 virtio_vsock_event_handle(vsock, event); 470 471 virtio_vsock_event_fill_one(vsock, event); 472 } 473 } while (!virtqueue_enable_cb(vq)); 474 475 virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]); 476 out: 477 mutex_unlock(&vsock->event_lock); 478 } 479 480 static void virtio_vsock_event_done(struct virtqueue *vq) 481 { 482 struct virtio_vsock *vsock = vq->vdev->priv; 483 484 if (!vsock) 485 return; 486 queue_work(virtio_vsock_workqueue, &vsock->event_work); 487 } 488 489 static void virtio_vsock_tx_done(struct virtqueue *vq) 490 { 491 struct virtio_vsock *vsock = vq->vdev->priv; 492 493 if (!vsock) 494 return; 495 queue_work(virtio_vsock_workqueue, &vsock->tx_work); 496 } 497 498 static void virtio_vsock_rx_done(struct virtqueue *vq) 499 { 500 struct virtio_vsock *vsock = vq->vdev->priv; 501 502 if (!vsock) 503 return; 504 queue_work(virtio_vsock_workqueue, &vsock->rx_work); 505 } 506 507 static bool virtio_transport_can_msgzerocopy(int bufs_num) 508 { 509 struct virtio_vsock *vsock; 510 bool res = false; 511 512 rcu_read_lock(); 513 514 vsock = rcu_dereference(the_virtio_vsock); 515 if (vsock) { 516 struct virtqueue *vq = vsock->vqs[VSOCK_VQ_TX]; 517 518 /* Check that tx queue is large enough to keep whole 519 * data to send. This is needed, because when there is 520 * not enough free space in the queue, current skb to 521 * send will be reinserted to the head of tx list of 522 * the socket to retry transmission later, so if skb 523 * is bigger than whole queue, it will be reinserted 524 * again and again, thus blocking other skbs to be sent. 525 * Each page of the user provided buffer will be added 526 * as a single buffer to the tx virtqueue, so compare 527 * number of pages against maximum capacity of the queue. 528 */ 529 if (bufs_num <= vq->num_max) 530 res = true; 531 } 532 533 rcu_read_unlock(); 534 535 return res; 536 } 537 538 static bool virtio_transport_msgzerocopy_allow(void) 539 { 540 return true; 541 } 542 543 bool virtio_transport_stream_allow(struct vsock_sock *vsk, u32 cid, u32 port) 544 { 545 return vsock_net_mode_global(vsk); 546 } 547 548 static bool virtio_transport_seqpacket_allow(struct vsock_sock *vsk, 549 u32 remote_cid); 550 551 static bool virtio_transport_has_remote_cid(struct vsock_sock *vsk, u32 cid) 552 { 553 /* The CID could be implemented by the host. Always assume it is. */ 554 return true; 555 } 556 557 static struct virtio_transport virtio_transport = { 558 .transport = { 559 .module = THIS_MODULE, 560 561 .get_local_cid = virtio_transport_get_local_cid, 562 .has_remote_cid = virtio_transport_has_remote_cid, 563 564 .init = virtio_transport_do_socket_init, 565 .destruct = virtio_transport_destruct, 566 .release = virtio_transport_release, 567 .connect = virtio_transport_connect, 568 .shutdown = virtio_transport_shutdown, 569 .cancel_pkt = virtio_transport_cancel_pkt, 570 571 .dgram_bind = virtio_transport_dgram_bind, 572 .dgram_dequeue = virtio_transport_dgram_dequeue, 573 .dgram_enqueue = virtio_transport_dgram_enqueue, 574 .dgram_allow = virtio_transport_dgram_allow, 575 576 .stream_dequeue = virtio_transport_stream_dequeue, 577 .stream_enqueue = virtio_transport_stream_enqueue, 578 .stream_has_data = virtio_transport_stream_has_data, 579 .stream_has_space = virtio_transport_stream_has_space, 580 .stream_rcvhiwat = virtio_transport_stream_rcvhiwat, 581 .stream_is_active = virtio_transport_stream_is_active, 582 .stream_allow = virtio_transport_stream_allow, 583 584 .seqpacket_dequeue = virtio_transport_seqpacket_dequeue, 585 .seqpacket_enqueue = virtio_transport_seqpacket_enqueue, 586 .seqpacket_allow = virtio_transport_seqpacket_allow, 587 .seqpacket_has_data = virtio_transport_seqpacket_has_data, 588 589 .msgzerocopy_allow = virtio_transport_msgzerocopy_allow, 590 591 .notify_poll_in = virtio_transport_notify_poll_in, 592 .notify_poll_out = virtio_transport_notify_poll_out, 593 .notify_recv_init = virtio_transport_notify_recv_init, 594 .notify_recv_pre_block = virtio_transport_notify_recv_pre_block, 595 .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue, 596 .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue, 597 .notify_send_init = virtio_transport_notify_send_init, 598 .notify_send_pre_block = virtio_transport_notify_send_pre_block, 599 .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, 600 .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, 601 .notify_buffer_size = virtio_transport_notify_buffer_size, 602 .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat, 603 604 .unsent_bytes = virtio_transport_unsent_bytes, 605 606 .read_skb = virtio_transport_read_skb, 607 }, 608 609 .send_pkt = virtio_transport_send_pkt, 610 .can_msgzerocopy = virtio_transport_can_msgzerocopy, 611 }; 612 613 static bool 614 virtio_transport_seqpacket_allow(struct vsock_sock *vsk, u32 remote_cid) 615 { 616 struct virtio_vsock *vsock; 617 bool seqpacket_allow; 618 619 if (!vsock_net_mode_global(vsk)) 620 return false; 621 622 seqpacket_allow = false; 623 rcu_read_lock(); 624 vsock = rcu_dereference(the_virtio_vsock); 625 if (vsock) 626 seqpacket_allow = vsock->seqpacket_allow; 627 rcu_read_unlock(); 628 629 return seqpacket_allow; 630 } 631 632 static void virtio_transport_rx_work(struct work_struct *work) 633 { 634 struct virtio_vsock *vsock = 635 container_of(work, struct virtio_vsock, rx_work); 636 struct virtqueue *vq; 637 638 mutex_lock(&vsock->rx_lock); 639 640 if (!vsock->rx_run) 641 goto out_nofill; 642 643 vq = vsock->vqs[VSOCK_VQ_RX]; 644 645 do { 646 virtqueue_disable_cb(vq); 647 for (;;) { 648 unsigned int len, payload_len; 649 struct virtio_vsock_hdr *hdr; 650 struct sk_buff *skb; 651 652 if (!virtio_transport_more_replies(vsock)) { 653 /* Stop rx until the device processes already 654 * pending replies. Leave rx virtqueue 655 * callbacks disabled. 656 */ 657 goto out; 658 } 659 660 skb = virtqueue_get_buf(vq, &len); 661 if (!skb) 662 break; 663 664 vsock->rx_buf_nr--; 665 666 /* Drop short/long packets */ 667 if (unlikely(len < sizeof(*hdr) || 668 len > virtio_vsock_skb_len(skb))) { 669 kfree_skb(skb); 670 continue; 671 } 672 673 hdr = virtio_vsock_hdr(skb); 674 payload_len = le32_to_cpu(hdr->len); 675 if (unlikely(payload_len > len - sizeof(*hdr))) { 676 kfree_skb(skb); 677 continue; 678 } 679 680 if (payload_len) 681 virtio_vsock_skb_put(skb, payload_len); 682 683 virtio_transport_deliver_tap_pkt(skb); 684 685 /* Force virtio-transport into global mode since it 686 * does not yet support local-mode namespacing. 687 */ 688 virtio_transport_recv_pkt(&virtio_transport, skb, NULL); 689 } 690 } while (!virtqueue_enable_cb(vq)); 691 692 out: 693 if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2) 694 virtio_vsock_rx_fill(vsock); 695 out_nofill: 696 mutex_unlock(&vsock->rx_lock); 697 } 698 699 static int virtio_vsock_vqs_init(struct virtio_vsock *vsock) 700 { 701 struct virtio_device *vdev = vsock->vdev; 702 struct virtqueue_info vqs_info[] = { 703 { "rx", virtio_vsock_rx_done }, 704 { "tx", virtio_vsock_tx_done }, 705 { "event", virtio_vsock_event_done }, 706 }; 707 int ret; 708 709 mutex_lock(&vsock->rx_lock); 710 vsock->rx_buf_nr = 0; 711 vsock->rx_buf_max_nr = 0; 712 mutex_unlock(&vsock->rx_lock); 713 714 atomic_set(&vsock->queued_replies, 0); 715 716 ret = virtio_find_vqs(vdev, VSOCK_VQ_MAX, vsock->vqs, vqs_info, NULL); 717 if (ret < 0) 718 return ret; 719 720 virtio_vsock_update_guest_cid(vsock); 721 722 virtio_device_ready(vdev); 723 724 return 0; 725 } 726 727 static void virtio_vsock_vqs_start(struct virtio_vsock *vsock) 728 { 729 mutex_lock(&vsock->tx_lock); 730 vsock->tx_run = true; 731 mutex_unlock(&vsock->tx_lock); 732 733 mutex_lock(&vsock->rx_lock); 734 virtio_vsock_rx_fill(vsock); 735 vsock->rx_run = true; 736 mutex_unlock(&vsock->rx_lock); 737 738 mutex_lock(&vsock->event_lock); 739 virtio_vsock_event_fill(vsock); 740 vsock->event_run = true; 741 mutex_unlock(&vsock->event_lock); 742 743 /* virtio_transport_send_pkt() can queue packets once 744 * the_virtio_vsock is set, but they won't be processed until 745 * vsock->tx_run is set to true. We queue vsock->send_pkt_work 746 * when initialization finishes to send those packets queued 747 * earlier. 748 * We don't need to queue the other workers (rx, event) because 749 * as long as we don't fill the queues with empty buffers, the 750 * host can't send us any notification. 751 */ 752 queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work); 753 } 754 755 static void virtio_vsock_vqs_del(struct virtio_vsock *vsock) 756 { 757 struct virtio_device *vdev = vsock->vdev; 758 struct sk_buff *skb; 759 760 /* Reset all connected sockets when the VQs disappear */ 761 vsock_for_each_connected_socket(&virtio_transport.transport, 762 virtio_vsock_reset_sock); 763 764 /* Stop all work handlers to make sure no one is accessing the device, 765 * so we can safely call virtio_reset_device(). 766 */ 767 mutex_lock(&vsock->rx_lock); 768 vsock->rx_run = false; 769 mutex_unlock(&vsock->rx_lock); 770 771 mutex_lock(&vsock->tx_lock); 772 vsock->tx_run = false; 773 mutex_unlock(&vsock->tx_lock); 774 775 mutex_lock(&vsock->event_lock); 776 vsock->event_run = false; 777 mutex_unlock(&vsock->event_lock); 778 779 /* Flush all device writes and interrupts, device will not use any 780 * more buffers. 781 */ 782 virtio_reset_device(vdev); 783 784 mutex_lock(&vsock->rx_lock); 785 while ((skb = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX]))) 786 kfree_skb(skb); 787 mutex_unlock(&vsock->rx_lock); 788 789 mutex_lock(&vsock->tx_lock); 790 while ((skb = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX]))) 791 kfree_skb(skb); 792 mutex_unlock(&vsock->tx_lock); 793 794 virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue); 795 796 /* Delete virtqueues and flush outstanding callbacks if any */ 797 vdev->config->del_vqs(vdev); 798 } 799 800 static int virtio_vsock_probe(struct virtio_device *vdev) 801 { 802 struct virtio_vsock *vsock = NULL; 803 int ret; 804 int i; 805 806 ret = mutex_lock_interruptible(&the_virtio_vsock_mutex); 807 if (ret) 808 return ret; 809 810 /* Only one virtio-vsock device per guest is supported */ 811 if (rcu_dereference_protected(the_virtio_vsock, 812 lockdep_is_held(&the_virtio_vsock_mutex))) { 813 ret = -EBUSY; 814 goto out; 815 } 816 817 vsock = kzalloc_obj(*vsock); 818 if (!vsock) { 819 ret = -ENOMEM; 820 goto out; 821 } 822 823 vsock->vdev = vdev; 824 825 826 mutex_init(&vsock->tx_lock); 827 mutex_init(&vsock->rx_lock); 828 mutex_init(&vsock->event_lock); 829 skb_queue_head_init(&vsock->send_pkt_queue); 830 INIT_WORK(&vsock->rx_work, virtio_transport_rx_work); 831 INIT_WORK(&vsock->tx_work, virtio_transport_tx_work); 832 INIT_WORK(&vsock->event_work, virtio_transport_event_work); 833 INIT_WORK(&vsock->send_pkt_work, virtio_transport_send_pkt_work); 834 835 if (virtio_has_feature(vdev, VIRTIO_VSOCK_F_SEQPACKET)) 836 vsock->seqpacket_allow = true; 837 838 vdev->priv = vsock; 839 840 ret = virtio_vsock_vqs_init(vsock); 841 if (ret < 0) 842 goto out; 843 844 for (i = 0; i < ARRAY_SIZE(vsock->out_sgs); i++) 845 vsock->out_sgs[i] = &vsock->out_bufs[i]; 846 847 rcu_assign_pointer(the_virtio_vsock, vsock); 848 virtio_vsock_vqs_start(vsock); 849 850 mutex_unlock(&the_virtio_vsock_mutex); 851 852 return 0; 853 854 out: 855 kfree(vsock); 856 mutex_unlock(&the_virtio_vsock_mutex); 857 return ret; 858 } 859 860 static void virtio_vsock_remove(struct virtio_device *vdev) 861 { 862 struct virtio_vsock *vsock = vdev->priv; 863 864 mutex_lock(&the_virtio_vsock_mutex); 865 866 vdev->priv = NULL; 867 rcu_assign_pointer(the_virtio_vsock, NULL); 868 synchronize_rcu(); 869 870 virtio_vsock_vqs_del(vsock); 871 872 /* Other works can be queued before 'config->del_vqs()', so we flush 873 * all works before to free the vsock object to avoid use after free. 874 */ 875 flush_work(&vsock->tx_work); 876 flush_work(&vsock->event_work); 877 flush_work(&vsock->send_pkt_work); 878 flush_work(&vsock->rx_work); 879 880 mutex_unlock(&the_virtio_vsock_mutex); 881 882 kfree(vsock); 883 } 884 885 #ifdef CONFIG_PM_SLEEP 886 static int virtio_vsock_freeze(struct virtio_device *vdev) 887 { 888 struct virtio_vsock *vsock = vdev->priv; 889 890 mutex_lock(&the_virtio_vsock_mutex); 891 892 rcu_assign_pointer(the_virtio_vsock, NULL); 893 synchronize_rcu(); 894 895 virtio_vsock_vqs_del(vsock); 896 897 mutex_unlock(&the_virtio_vsock_mutex); 898 899 return 0; 900 } 901 902 static int virtio_vsock_restore(struct virtio_device *vdev) 903 { 904 struct virtio_vsock *vsock = vdev->priv; 905 int ret; 906 907 mutex_lock(&the_virtio_vsock_mutex); 908 909 /* Only one virtio-vsock device per guest is supported */ 910 if (rcu_dereference_protected(the_virtio_vsock, 911 lockdep_is_held(&the_virtio_vsock_mutex))) { 912 ret = -EBUSY; 913 goto out; 914 } 915 916 ret = virtio_vsock_vqs_init(vsock); 917 if (ret < 0) 918 goto out; 919 920 rcu_assign_pointer(the_virtio_vsock, vsock); 921 virtio_vsock_vqs_start(vsock); 922 923 out: 924 mutex_unlock(&the_virtio_vsock_mutex); 925 return ret; 926 } 927 #endif /* CONFIG_PM_SLEEP */ 928 929 static struct virtio_device_id id_table[] = { 930 { VIRTIO_ID_VSOCK, VIRTIO_DEV_ANY_ID }, 931 { 0 }, 932 }; 933 934 static unsigned int features[] = { 935 VIRTIO_VSOCK_F_SEQPACKET 936 }; 937 938 static struct virtio_driver virtio_vsock_driver = { 939 .feature_table = features, 940 .feature_table_size = ARRAY_SIZE(features), 941 .driver.name = KBUILD_MODNAME, 942 .id_table = id_table, 943 .probe = virtio_vsock_probe, 944 .remove = virtio_vsock_remove, 945 #ifdef CONFIG_PM_SLEEP 946 .freeze = virtio_vsock_freeze, 947 .restore = virtio_vsock_restore, 948 #endif 949 }; 950 951 static int __init virtio_vsock_init(void) 952 { 953 int ret; 954 955 virtio_vsock_workqueue = alloc_workqueue("virtio_vsock", WQ_PERCPU, 0); 956 if (!virtio_vsock_workqueue) 957 return -ENOMEM; 958 959 ret = vsock_core_register(&virtio_transport.transport, 960 VSOCK_TRANSPORT_F_G2H); 961 if (ret) 962 goto out_wq; 963 964 ret = register_virtio_driver(&virtio_vsock_driver); 965 if (ret) 966 goto out_vci; 967 968 return 0; 969 970 out_vci: 971 vsock_core_unregister(&virtio_transport.transport); 972 out_wq: 973 destroy_workqueue(virtio_vsock_workqueue); 974 return ret; 975 } 976 977 static void __exit virtio_vsock_exit(void) 978 { 979 unregister_virtio_driver(&virtio_vsock_driver); 980 vsock_core_unregister(&virtio_transport.transport); 981 destroy_workqueue(virtio_vsock_workqueue); 982 } 983 984 module_init(virtio_vsock_init); 985 module_exit(virtio_vsock_exit); 986 MODULE_LICENSE("GPL v2"); 987 MODULE_AUTHOR("Asias He"); 988 MODULE_DESCRIPTION("virtio transport for vsock"); 989 MODULE_DEVICE_TABLE(virtio, id_table); 990