1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2009 Red Hat, Inc. 3 * Author: Michael S. Tsirkin <mst@redhat.com> 4 * 5 * virtio-net server in host kernel. 6 */ 7 8 #include <linux/compat.h> 9 #include <linux/eventfd.h> 10 #include <linux/vhost.h> 11 #include <linux/virtio_net.h> 12 #include <linux/miscdevice.h> 13 #include <linux/module.h> 14 #include <linux/moduleparam.h> 15 #include <linux/mutex.h> 16 #include <linux/workqueue.h> 17 #include <linux/file.h> 18 #include <linux/slab.h> 19 #include <linux/sched/clock.h> 20 #include <linux/sched/signal.h> 21 #include <linux/vmalloc.h> 22 23 #include <linux/net.h> 24 #include <linux/if_packet.h> 25 #include <linux/if_arp.h> 26 #include <linux/if_tun.h> 27 #include <linux/if_macvlan.h> 28 #include <linux/if_tap.h> 29 #include <linux/if_vlan.h> 30 #include <linux/skb_array.h> 31 #include <linux/skbuff.h> 32 33 #include <net/sock.h> 34 #include <net/xdp.h> 35 36 #include "vhost.h" 37 38 static int experimental_zcopytx = 0; 39 module_param(experimental_zcopytx, int, 0444); 40 MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;" 41 " 1 -Enable; 0 - Disable"); 42 43 /* Max number of bytes transferred before requeueing the job. 44 * Using this limit prevents one virtqueue from starving others. */ 45 #define VHOST_NET_WEIGHT 0x80000 46 47 /* Max number of packets transferred before requeueing the job. 48 * Using this limit prevents one virtqueue from starving others with small 49 * pkts. 50 */ 51 #define VHOST_NET_PKT_WEIGHT 256 52 53 /* MAX number of TX used buffers for outstanding zerocopy */ 54 #define VHOST_MAX_PEND 128 55 #define VHOST_GOODCOPY_LEN 256 56 57 /* 58 * For transmit, used buffer len is unused; we override it to track buffer 59 * status internally; used for zerocopy tx only. 60 */ 61 /* Lower device DMA failed */ 62 #define VHOST_DMA_FAILED_LEN ((__force __virtio32)3) 63 /* Lower device DMA done */ 64 #define VHOST_DMA_DONE_LEN ((__force __virtio32)2) 65 /* Lower device DMA in progress */ 66 #define VHOST_DMA_IN_PROGRESS ((__force __virtio32)1) 67 /* Buffer unused */ 68 #define VHOST_DMA_CLEAR_LEN ((__force __virtio32)0) 69 70 #define VHOST_DMA_IS_DONE(len) ((__force u32)(len) >= (__force u32)VHOST_DMA_DONE_LEN) 71 72 static const u64 vhost_net_features[VIRTIO_FEATURES_DWORDS] = { 73 VHOST_FEATURES | 74 (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) | 75 (1ULL << VIRTIO_NET_F_MRG_RXBUF) | 76 (1ULL << VIRTIO_F_ACCESS_PLATFORM) | 77 (1ULL << VIRTIO_F_RING_RESET) | 78 (1ULL << VIRTIO_F_IN_ORDER), 79 VIRTIO_BIT(VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO) | 80 VIRTIO_BIT(VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO), 81 }; 82 83 enum { 84 VHOST_NET_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) 85 }; 86 87 enum { 88 VHOST_NET_VQ_RX = 0, 89 VHOST_NET_VQ_TX = 1, 90 VHOST_NET_VQ_MAX = 2, 91 }; 92 93 struct vhost_net_ubuf_ref { 94 /* refcount follows semantics similar to kref: 95 * 0: object is released 96 * 1: no outstanding ubufs 97 * >1: outstanding ubufs 98 */ 99 atomic_t refcount; 100 wait_queue_head_t wait; 101 struct vhost_virtqueue *vq; 102 struct rcu_head rcu; 103 }; 104 105 #define VHOST_NET_BATCH 64 106 struct vhost_net_buf { 107 void **queue; 108 int tail; 109 int head; 110 }; 111 112 struct vhost_net_virtqueue { 113 struct vhost_virtqueue vq; 114 size_t vhost_hlen; 115 size_t sock_hlen; 116 /* vhost zerocopy support fields below: */ 117 /* last used idx for outstanding DMA zerocopy buffers */ 118 int upend_idx; 119 /* For TX, first used idx for DMA done zerocopy buffers 120 * For RX, number of batched heads 121 */ 122 int done_idx; 123 /* Number of XDP frames batched */ 124 int batched_xdp; 125 /* an array of userspace buffers info */ 126 struct ubuf_info_msgzc *ubuf_info; 127 /* Reference counting for outstanding ubufs. 128 * Protected by vq mutex. Writers must also take device mutex. */ 129 struct vhost_net_ubuf_ref *ubufs; 130 struct ptr_ring *rx_ring; 131 struct vhost_net_buf rxq; 132 /* Batched XDP buffs */ 133 struct xdp_buff *xdp; 134 }; 135 136 struct vhost_net { 137 struct vhost_dev dev; 138 struct vhost_net_virtqueue vqs[VHOST_NET_VQ_MAX]; 139 struct vhost_poll poll[VHOST_NET_VQ_MAX]; 140 /* Number of TX recently submitted. 141 * Protected by tx vq lock. */ 142 unsigned tx_packets; 143 /* Number of times zerocopy TX recently failed. 144 * Protected by tx vq lock. */ 145 unsigned tx_zcopy_err; 146 /* Flush in progress. Protected by tx vq lock. */ 147 bool tx_flush; 148 /* Private page frag cache */ 149 struct page_frag_cache pf_cache; 150 }; 151 152 static unsigned vhost_net_zcopy_mask __read_mostly; 153 154 static void *vhost_net_buf_get_ptr(struct vhost_net_buf *rxq) 155 { 156 if (rxq->tail != rxq->head) 157 return rxq->queue[rxq->head]; 158 else 159 return NULL; 160 } 161 162 static int vhost_net_buf_get_size(struct vhost_net_buf *rxq) 163 { 164 return rxq->tail - rxq->head; 165 } 166 167 static int vhost_net_buf_is_empty(struct vhost_net_buf *rxq) 168 { 169 return rxq->tail == rxq->head; 170 } 171 172 static void *vhost_net_buf_consume(struct vhost_net_buf *rxq) 173 { 174 void *ret = vhost_net_buf_get_ptr(rxq); 175 ++rxq->head; 176 return ret; 177 } 178 179 static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq) 180 { 181 struct vhost_net_buf *rxq = &nvq->rxq; 182 183 rxq->head = 0; 184 rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue, 185 VHOST_NET_BATCH); 186 return rxq->tail; 187 } 188 189 static void vhost_net_buf_unproduce(struct vhost_net_virtqueue *nvq) 190 { 191 struct vhost_net_buf *rxq = &nvq->rxq; 192 193 if (nvq->rx_ring && !vhost_net_buf_is_empty(rxq)) { 194 ptr_ring_unconsume(nvq->rx_ring, rxq->queue + rxq->head, 195 vhost_net_buf_get_size(rxq), 196 tun_ptr_free); 197 rxq->head = rxq->tail = 0; 198 } 199 } 200 201 static int vhost_net_buf_peek_len(void *ptr) 202 { 203 if (tun_is_xdp_frame(ptr)) { 204 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 205 206 return xdpf->len; 207 } 208 209 return __skb_array_len_with_tag(ptr); 210 } 211 212 static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq) 213 { 214 struct vhost_net_buf *rxq = &nvq->rxq; 215 216 if (!vhost_net_buf_is_empty(rxq)) 217 goto out; 218 219 if (!vhost_net_buf_produce(nvq)) 220 return 0; 221 222 out: 223 return vhost_net_buf_peek_len(vhost_net_buf_get_ptr(rxq)); 224 } 225 226 static void vhost_net_buf_init(struct vhost_net_buf *rxq) 227 { 228 rxq->head = rxq->tail = 0; 229 } 230 231 static void vhost_net_enable_zcopy(int vq) 232 { 233 vhost_net_zcopy_mask |= 0x1 << vq; 234 } 235 236 static struct vhost_net_ubuf_ref * 237 vhost_net_ubuf_alloc(struct vhost_virtqueue *vq, bool zcopy) 238 { 239 struct vhost_net_ubuf_ref *ubufs; 240 /* No zero copy backend? Nothing to count. */ 241 if (!zcopy) 242 return NULL; 243 ubufs = kmalloc(sizeof(*ubufs), GFP_KERNEL); 244 if (!ubufs) 245 return ERR_PTR(-ENOMEM); 246 atomic_set(&ubufs->refcount, 1); 247 init_waitqueue_head(&ubufs->wait); 248 ubufs->vq = vq; 249 return ubufs; 250 } 251 252 static int vhost_net_ubuf_put(struct vhost_net_ubuf_ref *ubufs) 253 { 254 int r; 255 256 rcu_read_lock(); 257 r = atomic_sub_return(1, &ubufs->refcount); 258 if (unlikely(!r)) 259 wake_up(&ubufs->wait); 260 rcu_read_unlock(); 261 return r; 262 } 263 264 static void vhost_net_ubuf_put_and_wait(struct vhost_net_ubuf_ref *ubufs) 265 { 266 vhost_net_ubuf_put(ubufs); 267 wait_event(ubufs->wait, !atomic_read(&ubufs->refcount)); 268 } 269 270 static void vhost_net_ubuf_put_wait_and_free(struct vhost_net_ubuf_ref *ubufs) 271 { 272 vhost_net_ubuf_put_and_wait(ubufs); 273 kfree_rcu(ubufs, rcu); 274 } 275 276 static void vhost_net_clear_ubuf_info(struct vhost_net *n) 277 { 278 int i; 279 280 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) { 281 kfree(n->vqs[i].ubuf_info); 282 n->vqs[i].ubuf_info = NULL; 283 } 284 } 285 286 static int vhost_net_set_ubuf_info(struct vhost_net *n) 287 { 288 bool zcopy; 289 int i; 290 291 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) { 292 zcopy = vhost_net_zcopy_mask & (0x1 << i); 293 if (!zcopy) 294 continue; 295 n->vqs[i].ubuf_info = 296 kmalloc_array(UIO_MAXIOV, 297 sizeof(*n->vqs[i].ubuf_info), 298 GFP_KERNEL); 299 if (!n->vqs[i].ubuf_info) 300 goto err; 301 } 302 return 0; 303 304 err: 305 vhost_net_clear_ubuf_info(n); 306 return -ENOMEM; 307 } 308 309 static void vhost_net_vq_reset(struct vhost_net *n) 310 { 311 int i; 312 313 vhost_net_clear_ubuf_info(n); 314 315 for (i = 0; i < VHOST_NET_VQ_MAX; i++) { 316 n->vqs[i].done_idx = 0; 317 n->vqs[i].upend_idx = 0; 318 n->vqs[i].ubufs = NULL; 319 n->vqs[i].vhost_hlen = 0; 320 n->vqs[i].sock_hlen = 0; 321 vhost_net_buf_init(&n->vqs[i].rxq); 322 } 323 324 } 325 326 static void vhost_net_tx_packet(struct vhost_net *net) 327 { 328 ++net->tx_packets; 329 if (net->tx_packets < 1024) 330 return; 331 net->tx_packets = 0; 332 net->tx_zcopy_err = 0; 333 } 334 335 static void vhost_net_tx_err(struct vhost_net *net) 336 { 337 ++net->tx_zcopy_err; 338 } 339 340 static bool vhost_net_tx_select_zcopy(struct vhost_net *net) 341 { 342 /* TX flush waits for outstanding DMAs to be done. 343 * Don't start new DMAs. 344 */ 345 return !net->tx_flush && 346 net->tx_packets / 64 >= net->tx_zcopy_err; 347 } 348 349 static bool vhost_sock_zcopy(struct socket *sock) 350 { 351 return unlikely(experimental_zcopytx) && 352 sock_flag(sock->sk, SOCK_ZEROCOPY); 353 } 354 355 static bool vhost_sock_xdp(struct socket *sock) 356 { 357 return sock_flag(sock->sk, SOCK_XDP); 358 } 359 360 /* In case of DMA done not in order in lower device driver for some reason. 361 * upend_idx is used to track end of used idx, done_idx is used to track head 362 * of used idx. Once lower device DMA done contiguously, we will signal KVM 363 * guest used idx. 364 */ 365 static void vhost_zerocopy_signal_used(struct vhost_net *net, 366 struct vhost_virtqueue *vq) 367 { 368 struct vhost_net_virtqueue *nvq = 369 container_of(vq, struct vhost_net_virtqueue, vq); 370 int i, add; 371 int j = 0; 372 373 for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) { 374 if (vq->heads[i].len == VHOST_DMA_FAILED_LEN) 375 vhost_net_tx_err(net); 376 if (VHOST_DMA_IS_DONE(vq->heads[i].len)) { 377 vq->heads[i].len = VHOST_DMA_CLEAR_LEN; 378 ++j; 379 } else 380 break; 381 } 382 while (j) { 383 add = min(UIO_MAXIOV - nvq->done_idx, j); 384 vhost_add_used_and_signal_n(vq->dev, vq, 385 &vq->heads[nvq->done_idx], 386 NULL, add); 387 nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV; 388 j -= add; 389 } 390 } 391 392 static void vhost_zerocopy_complete(struct sk_buff *skb, 393 struct ubuf_info *ubuf_base, bool success) 394 { 395 struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base); 396 struct vhost_net_ubuf_ref *ubufs = ubuf->ctx; 397 struct vhost_virtqueue *vq = ubufs->vq; 398 int cnt; 399 400 rcu_read_lock_bh(); 401 402 /* set len to mark this desc buffers done DMA */ 403 vq->heads[ubuf->desc].len = success ? 404 VHOST_DMA_DONE_LEN : VHOST_DMA_FAILED_LEN; 405 cnt = vhost_net_ubuf_put(ubufs); 406 407 /* 408 * Trigger polling thread if guest stopped submitting new buffers: 409 * in this case, the refcount after decrement will eventually reach 1. 410 * We also trigger polling periodically after each 16 packets 411 * (the value 16 here is more or less arbitrary, it's tuned to trigger 412 * less than 10% of times). 413 */ 414 if (cnt <= 1 || !(cnt % 16)) 415 vhost_poll_queue(&vq->poll); 416 417 rcu_read_unlock_bh(); 418 } 419 420 static const struct ubuf_info_ops vhost_ubuf_ops = { 421 .complete = vhost_zerocopy_complete, 422 }; 423 424 static inline unsigned long busy_clock(void) 425 { 426 return local_clock() >> 10; 427 } 428 429 static bool vhost_can_busy_poll(unsigned long endtime) 430 { 431 return likely(!need_resched() && !time_after(busy_clock(), endtime) && 432 !signal_pending(current)); 433 } 434 435 static void vhost_net_disable_vq(struct vhost_net *n, 436 struct vhost_virtqueue *vq) 437 { 438 struct vhost_net_virtqueue *nvq = 439 container_of(vq, struct vhost_net_virtqueue, vq); 440 struct vhost_poll *poll = n->poll + (nvq - n->vqs); 441 if (!vhost_vq_get_backend(vq)) 442 return; 443 vhost_poll_stop(poll); 444 } 445 446 static int vhost_net_enable_vq(struct vhost_net *n, 447 struct vhost_virtqueue *vq) 448 { 449 struct vhost_net_virtqueue *nvq = 450 container_of(vq, struct vhost_net_virtqueue, vq); 451 struct vhost_poll *poll = n->poll + (nvq - n->vqs); 452 struct socket *sock; 453 454 sock = vhost_vq_get_backend(vq); 455 if (!sock) 456 return 0; 457 458 return vhost_poll_start(poll, sock->file); 459 } 460 461 static void vhost_net_signal_used(struct vhost_net_virtqueue *nvq, 462 unsigned int count) 463 { 464 struct vhost_virtqueue *vq = &nvq->vq; 465 struct vhost_dev *dev = vq->dev; 466 467 if (!nvq->done_idx) 468 return; 469 470 vhost_add_used_and_signal_n(dev, vq, vq->heads, 471 vq->nheads, count); 472 nvq->done_idx = 0; 473 } 474 475 static void vhost_tx_batch(struct vhost_net *net, 476 struct vhost_net_virtqueue *nvq, 477 struct socket *sock, 478 struct msghdr *msghdr) 479 { 480 struct vhost_virtqueue *vq = &nvq->vq; 481 bool in_order = vhost_has_feature(vq, VIRTIO_F_IN_ORDER); 482 struct tun_msg_ctl ctl = { 483 .type = TUN_MSG_PTR, 484 .num = nvq->batched_xdp, 485 .ptr = nvq->xdp, 486 }; 487 int i, err; 488 489 if (in_order) { 490 vq->heads[0].len = 0; 491 vq->nheads[0] = nvq->done_idx; 492 } 493 494 if (nvq->batched_xdp == 0) 495 goto signal_used; 496 497 msghdr->msg_control = &ctl; 498 msghdr->msg_controllen = sizeof(ctl); 499 err = sock->ops->sendmsg(sock, msghdr, 0); 500 if (unlikely(err < 0)) { 501 vq_err(&nvq->vq, "Fail to batch sending packets\n"); 502 503 /* free pages owned by XDP; since this is an unlikely error path, 504 * keep it simple and avoid more complex bulk update for the 505 * used pages 506 */ 507 for (i = 0; i < nvq->batched_xdp; ++i) 508 put_page(virt_to_head_page(nvq->xdp[i].data)); 509 nvq->batched_xdp = 0; 510 nvq->done_idx = 0; 511 return; 512 } 513 514 signal_used: 515 vhost_net_signal_used(nvq, in_order ? 1 : nvq->done_idx); 516 nvq->batched_xdp = 0; 517 } 518 519 static int sock_has_rx_data(struct socket *sock) 520 { 521 if (unlikely(!sock)) 522 return 0; 523 524 if (sock->ops->peek_len) 525 return sock->ops->peek_len(sock); 526 527 return skb_queue_empty(&sock->sk->sk_receive_queue); 528 } 529 530 static void vhost_net_busy_poll_try_queue(struct vhost_net *net, 531 struct vhost_virtqueue *vq) 532 { 533 if (!vhost_vq_avail_empty(&net->dev, vq)) { 534 vhost_poll_queue(&vq->poll); 535 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { 536 vhost_disable_notify(&net->dev, vq); 537 vhost_poll_queue(&vq->poll); 538 } 539 } 540 541 static void vhost_net_busy_poll(struct vhost_net *net, 542 struct vhost_virtqueue *rvq, 543 struct vhost_virtqueue *tvq, 544 bool *busyloop_intr, 545 bool poll_rx) 546 { 547 unsigned long busyloop_timeout; 548 unsigned long endtime; 549 struct socket *sock; 550 struct vhost_virtqueue *vq = poll_rx ? tvq : rvq; 551 552 /* Try to hold the vq mutex of the paired virtqueue. We can't 553 * use mutex_lock() here since we could not guarantee a 554 * consistenet lock ordering. 555 */ 556 if (!mutex_trylock(&vq->mutex)) 557 return; 558 559 vhost_disable_notify(&net->dev, vq); 560 sock = vhost_vq_get_backend(rvq); 561 562 busyloop_timeout = poll_rx ? rvq->busyloop_timeout: 563 tvq->busyloop_timeout; 564 565 preempt_disable(); 566 endtime = busy_clock() + busyloop_timeout; 567 568 while (vhost_can_busy_poll(endtime)) { 569 if (vhost_vq_has_work(vq)) { 570 *busyloop_intr = true; 571 break; 572 } 573 574 if ((sock_has_rx_data(sock) && 575 !vhost_vq_avail_empty(&net->dev, rvq)) || 576 !vhost_vq_avail_empty(&net->dev, tvq)) 577 break; 578 579 cpu_relax(); 580 } 581 582 preempt_enable(); 583 584 if (poll_rx || sock_has_rx_data(sock)) 585 vhost_net_busy_poll_try_queue(net, vq); 586 else if (!poll_rx) /* On tx here, sock has no rx data. */ 587 vhost_enable_notify(&net->dev, rvq); 588 589 mutex_unlock(&vq->mutex); 590 } 591 592 static int vhost_net_tx_get_vq_desc(struct vhost_net *net, 593 struct vhost_net_virtqueue *tnvq, 594 unsigned int *out_num, unsigned int *in_num, 595 struct msghdr *msghdr, bool *busyloop_intr) 596 { 597 struct vhost_net_virtqueue *rnvq = &net->vqs[VHOST_NET_VQ_RX]; 598 struct vhost_virtqueue *rvq = &rnvq->vq; 599 struct vhost_virtqueue *tvq = &tnvq->vq; 600 601 int r = vhost_get_vq_desc(tvq, tvq->iov, ARRAY_SIZE(tvq->iov), 602 out_num, in_num, NULL, NULL); 603 604 if (r == tvq->num && tvq->busyloop_timeout) { 605 /* Flush batched packets first */ 606 if (!vhost_sock_zcopy(vhost_vq_get_backend(tvq))) 607 vhost_tx_batch(net, tnvq, 608 vhost_vq_get_backend(tvq), 609 msghdr); 610 611 vhost_net_busy_poll(net, rvq, tvq, busyloop_intr, false); 612 613 r = vhost_get_vq_desc(tvq, tvq->iov, ARRAY_SIZE(tvq->iov), 614 out_num, in_num, NULL, NULL); 615 } 616 617 return r; 618 } 619 620 static bool vhost_exceeds_maxpend(struct vhost_net *net) 621 { 622 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; 623 struct vhost_virtqueue *vq = &nvq->vq; 624 625 return (nvq->upend_idx + UIO_MAXIOV - nvq->done_idx) % UIO_MAXIOV > 626 min_t(unsigned int, VHOST_MAX_PEND, vq->num >> 2); 627 } 628 629 static size_t init_iov_iter(struct vhost_virtqueue *vq, struct iov_iter *iter, 630 size_t hdr_size, int out) 631 { 632 /* Skip header. TODO: support TSO. */ 633 size_t len = iov_length(vq->iov, out); 634 635 iov_iter_init(iter, ITER_SOURCE, vq->iov, out, len); 636 iov_iter_advance(iter, hdr_size); 637 638 return iov_iter_count(iter); 639 } 640 641 static int get_tx_bufs(struct vhost_net *net, 642 struct vhost_net_virtqueue *nvq, 643 struct msghdr *msg, 644 unsigned int *out, unsigned int *in, 645 size_t *len, bool *busyloop_intr) 646 { 647 struct vhost_virtqueue *vq = &nvq->vq; 648 int ret; 649 650 ret = vhost_net_tx_get_vq_desc(net, nvq, out, in, msg, busyloop_intr); 651 652 if (ret < 0 || ret == vq->num) 653 return ret; 654 655 if (*in) { 656 vq_err(vq, "Unexpected descriptor format for TX: out %d, int %d\n", 657 *out, *in); 658 return -EFAULT; 659 } 660 661 /* Sanity check */ 662 *len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out); 663 if (*len == 0) { 664 vq_err(vq, "Unexpected header len for TX: %zd expected %zd\n", 665 *len, nvq->vhost_hlen); 666 return -EFAULT; 667 } 668 669 return ret; 670 } 671 672 static bool tx_can_batch(struct vhost_virtqueue *vq, size_t total_len) 673 { 674 return total_len < VHOST_NET_WEIGHT && 675 !vhost_vq_avail_empty(vq->dev, vq); 676 } 677 678 #define VHOST_NET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) 679 680 static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq, 681 struct iov_iter *from) 682 { 683 struct vhost_virtqueue *vq = &nvq->vq; 684 struct vhost_net *net = container_of(vq->dev, struct vhost_net, 685 dev); 686 struct socket *sock = vhost_vq_get_backend(vq); 687 struct virtio_net_hdr *gso; 688 struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp]; 689 size_t len = iov_iter_count(from); 690 int headroom = vhost_sock_xdp(sock) ? XDP_PACKET_HEADROOM : 0; 691 int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 692 int pad = SKB_DATA_ALIGN(VHOST_NET_RX_PAD + headroom + nvq->sock_hlen); 693 int sock_hlen = nvq->sock_hlen; 694 void *buf; 695 int copied; 696 int ret; 697 698 if (unlikely(len < nvq->sock_hlen)) 699 return -EFAULT; 700 701 if (SKB_DATA_ALIGN(len + pad) + 702 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE) 703 return -ENOSPC; 704 705 buflen += SKB_DATA_ALIGN(len + pad); 706 buf = page_frag_alloc_align(&net->pf_cache, buflen, GFP_KERNEL, 707 SMP_CACHE_BYTES); 708 if (unlikely(!buf)) 709 return -ENOMEM; 710 711 copied = copy_from_iter(buf + pad - sock_hlen, len, from); 712 if (copied != len) { 713 ret = -EFAULT; 714 goto err; 715 } 716 717 gso = buf + pad - sock_hlen; 718 719 if (!sock_hlen) 720 memset(buf, 0, pad); 721 722 if ((gso->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && 723 vhost16_to_cpu(vq, gso->csum_start) + 724 vhost16_to_cpu(vq, gso->csum_offset) + 2 > 725 vhost16_to_cpu(vq, gso->hdr_len)) { 726 gso->hdr_len = cpu_to_vhost16(vq, 727 vhost16_to_cpu(vq, gso->csum_start) + 728 vhost16_to_cpu(vq, gso->csum_offset) + 2); 729 730 if (vhost16_to_cpu(vq, gso->hdr_len) > len) { 731 ret = -EINVAL; 732 goto err; 733 } 734 } 735 736 /* pad contains sock_hlen */ 737 memcpy(buf, buf + pad - sock_hlen, sock_hlen); 738 739 xdp_init_buff(xdp, buflen, NULL); 740 xdp_prepare_buff(xdp, buf, pad, len - sock_hlen, true); 741 742 ++nvq->batched_xdp; 743 744 return 0; 745 746 err: 747 page_frag_free(buf); 748 return ret; 749 } 750 751 static void handle_tx_copy(struct vhost_net *net, struct socket *sock) 752 { 753 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; 754 struct vhost_virtqueue *vq = &nvq->vq; 755 unsigned out, in; 756 int head; 757 struct msghdr msg = { 758 .msg_name = NULL, 759 .msg_namelen = 0, 760 .msg_control = NULL, 761 .msg_controllen = 0, 762 .msg_flags = MSG_DONTWAIT, 763 }; 764 size_t len, total_len = 0; 765 int err; 766 int sent_pkts = 0; 767 bool sock_can_batch = (sock->sk->sk_sndbuf == INT_MAX); 768 bool busyloop_intr; 769 bool in_order = vhost_has_feature(vq, VIRTIO_F_IN_ORDER); 770 771 do { 772 busyloop_intr = false; 773 if (nvq->done_idx == VHOST_NET_BATCH) 774 vhost_tx_batch(net, nvq, sock, &msg); 775 776 head = get_tx_bufs(net, nvq, &msg, &out, &in, &len, 777 &busyloop_intr); 778 /* On error, stop handling until the next kick. */ 779 if (unlikely(head < 0)) 780 break; 781 /* Nothing new? Wait for eventfd to tell us they refilled. */ 782 if (head == vq->num) { 783 /* Kicks are disabled at this point, break loop and 784 * process any remaining batched packets. Queue will 785 * be re-enabled afterwards. 786 */ 787 break; 788 } 789 790 total_len += len; 791 792 /* For simplicity, TX batching is only enabled if 793 * sndbuf is unlimited. 794 */ 795 if (sock_can_batch) { 796 err = vhost_net_build_xdp(nvq, &msg.msg_iter); 797 if (!err) { 798 goto done; 799 } else if (unlikely(err != -ENOSPC)) { 800 vhost_tx_batch(net, nvq, sock, &msg); 801 vhost_discard_vq_desc(vq, 1); 802 vhost_net_enable_vq(net, vq); 803 break; 804 } 805 806 if (nvq->batched_xdp) { 807 /* We can't build XDP buff, go for single 808 * packet path but let's flush batched 809 * packets. 810 */ 811 vhost_tx_batch(net, nvq, sock, &msg); 812 } 813 msg.msg_control = NULL; 814 } else { 815 if (tx_can_batch(vq, total_len)) 816 msg.msg_flags |= MSG_MORE; 817 else 818 msg.msg_flags &= ~MSG_MORE; 819 } 820 821 err = sock->ops->sendmsg(sock, &msg, len); 822 if (unlikely(err < 0)) { 823 if (err == -EAGAIN || err == -ENOMEM || err == -ENOBUFS) { 824 vhost_discard_vq_desc(vq, 1); 825 vhost_net_enable_vq(net, vq); 826 break; 827 } 828 pr_debug("Fail to send packet: err %d", err); 829 } else if (unlikely(err != len)) 830 pr_debug("Truncated TX packet: len %d != %zd\n", 831 err, len); 832 done: 833 if (in_order) { 834 vq->heads[0].id = cpu_to_vhost32(vq, head); 835 } else { 836 vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head); 837 vq->heads[nvq->done_idx].len = 0; 838 } 839 ++nvq->done_idx; 840 } while (likely(!vhost_exceeds_weight(vq, ++sent_pkts, total_len))); 841 842 /* Kicks are still disabled, dispatch any remaining batched msgs. */ 843 vhost_tx_batch(net, nvq, sock, &msg); 844 845 if (unlikely(busyloop_intr)) 846 /* If interrupted while doing busy polling, requeue the 847 * handler to be fair handle_rx as well as other tasks 848 * waiting on cpu. 849 */ 850 vhost_poll_queue(&vq->poll); 851 else 852 /* All of our work has been completed; however, before 853 * leaving the TX handler, do one last check for work, 854 * and requeue handler if necessary. If there is no work, 855 * queue will be reenabled. 856 */ 857 vhost_net_busy_poll_try_queue(net, vq); 858 } 859 860 static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock) 861 { 862 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; 863 struct vhost_virtqueue *vq = &nvq->vq; 864 unsigned out, in; 865 int head; 866 struct msghdr msg = { 867 .msg_name = NULL, 868 .msg_namelen = 0, 869 .msg_control = NULL, 870 .msg_controllen = 0, 871 .msg_flags = MSG_DONTWAIT, 872 }; 873 struct tun_msg_ctl ctl; 874 size_t len, total_len = 0; 875 int err; 876 struct vhost_net_ubuf_ref *ubufs; 877 struct ubuf_info_msgzc *ubuf; 878 bool zcopy_used; 879 int sent_pkts = 0; 880 881 do { 882 bool busyloop_intr; 883 884 /* Release DMAs done buffers first */ 885 vhost_zerocopy_signal_used(net, vq); 886 887 busyloop_intr = false; 888 head = get_tx_bufs(net, nvq, &msg, &out, &in, &len, 889 &busyloop_intr); 890 /* On error, stop handling until the next kick. */ 891 if (unlikely(head < 0)) 892 break; 893 /* Nothing new? Wait for eventfd to tell us they refilled. */ 894 if (head == vq->num) { 895 if (unlikely(busyloop_intr)) { 896 vhost_poll_queue(&vq->poll); 897 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { 898 vhost_disable_notify(&net->dev, vq); 899 continue; 900 } 901 break; 902 } 903 904 zcopy_used = len >= VHOST_GOODCOPY_LEN 905 && !vhost_exceeds_maxpend(net) 906 && vhost_net_tx_select_zcopy(net); 907 908 /* use msg_control to pass vhost zerocopy ubuf info to skb */ 909 if (zcopy_used) { 910 ubuf = nvq->ubuf_info + nvq->upend_idx; 911 vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head); 912 vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS; 913 ubuf->ctx = nvq->ubufs; 914 ubuf->desc = nvq->upend_idx; 915 ubuf->ubuf.ops = &vhost_ubuf_ops; 916 ubuf->ubuf.flags = SKBFL_ZEROCOPY_FRAG; 917 refcount_set(&ubuf->ubuf.refcnt, 1); 918 msg.msg_control = &ctl; 919 ctl.type = TUN_MSG_UBUF; 920 ctl.ptr = &ubuf->ubuf; 921 msg.msg_controllen = sizeof(ctl); 922 ubufs = nvq->ubufs; 923 atomic_inc(&ubufs->refcount); 924 nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV; 925 } else { 926 msg.msg_control = NULL; 927 ubufs = NULL; 928 } 929 total_len += len; 930 if (tx_can_batch(vq, total_len) && 931 likely(!vhost_exceeds_maxpend(net))) { 932 msg.msg_flags |= MSG_MORE; 933 } else { 934 msg.msg_flags &= ~MSG_MORE; 935 } 936 937 err = sock->ops->sendmsg(sock, &msg, len); 938 if (unlikely(err < 0)) { 939 bool retry = err == -EAGAIN || err == -ENOMEM || err == -ENOBUFS; 940 941 if (zcopy_used) { 942 if (vq->heads[ubuf->desc].len == VHOST_DMA_IN_PROGRESS) 943 vhost_net_ubuf_put(ubufs); 944 if (retry) 945 nvq->upend_idx = ((unsigned)nvq->upend_idx - 1) 946 % UIO_MAXIOV; 947 else 948 vq->heads[ubuf->desc].len = VHOST_DMA_DONE_LEN; 949 } 950 if (retry) { 951 vhost_discard_vq_desc(vq, 1); 952 vhost_net_enable_vq(net, vq); 953 break; 954 } 955 pr_debug("Fail to send packet: err %d", err); 956 } else if (unlikely(err != len)) 957 pr_debug("Truncated TX packet: " 958 " len %d != %zd\n", err, len); 959 if (!zcopy_used) 960 vhost_add_used_and_signal(&net->dev, vq, head, 0); 961 else 962 vhost_zerocopy_signal_used(net, vq); 963 vhost_net_tx_packet(net); 964 } while (likely(!vhost_exceeds_weight(vq, ++sent_pkts, total_len))); 965 } 966 967 /* Expects to be always run from workqueue - which acts as 968 * read-size critical section for our kind of RCU. */ 969 static void handle_tx(struct vhost_net *net) 970 { 971 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; 972 struct vhost_virtqueue *vq = &nvq->vq; 973 struct socket *sock; 974 975 mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_TX); 976 sock = vhost_vq_get_backend(vq); 977 if (!sock) 978 goto out; 979 980 if (!vq_meta_prefetch(vq)) 981 goto out; 982 983 vhost_disable_notify(&net->dev, vq); 984 vhost_net_disable_vq(net, vq); 985 986 if (vhost_sock_zcopy(sock)) 987 handle_tx_zerocopy(net, sock); 988 else 989 handle_tx_copy(net, sock); 990 991 out: 992 mutex_unlock(&vq->mutex); 993 } 994 995 static int peek_head_len(struct vhost_net_virtqueue *rvq, struct sock *sk) 996 { 997 struct sk_buff *head; 998 int len = 0; 999 unsigned long flags; 1000 1001 if (rvq->rx_ring) 1002 return vhost_net_buf_peek(rvq); 1003 1004 spin_lock_irqsave(&sk->sk_receive_queue.lock, flags); 1005 head = skb_peek(&sk->sk_receive_queue); 1006 if (likely(head)) { 1007 len = head->len; 1008 if (skb_vlan_tag_present(head)) 1009 len += VLAN_HLEN; 1010 } 1011 1012 spin_unlock_irqrestore(&sk->sk_receive_queue.lock, flags); 1013 return len; 1014 } 1015 1016 static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk, 1017 bool *busyloop_intr, unsigned int count) 1018 { 1019 struct vhost_net_virtqueue *rnvq = &net->vqs[VHOST_NET_VQ_RX]; 1020 struct vhost_net_virtqueue *tnvq = &net->vqs[VHOST_NET_VQ_TX]; 1021 struct vhost_virtqueue *rvq = &rnvq->vq; 1022 struct vhost_virtqueue *tvq = &tnvq->vq; 1023 int len = peek_head_len(rnvq, sk); 1024 1025 if (!len && rvq->busyloop_timeout) { 1026 /* Flush batched heads first */ 1027 vhost_net_signal_used(rnvq, count); 1028 /* Both tx vq and rx socket were polled here */ 1029 vhost_net_busy_poll(net, rvq, tvq, busyloop_intr, true); 1030 1031 len = peek_head_len(rnvq, sk); 1032 } 1033 1034 return len; 1035 } 1036 1037 /* This is a multi-buffer version of vhost_get_desc, that works if 1038 * vq has read descriptors only. 1039 * @nvq - the relevant vhost_net virtqueue 1040 * @datalen - data length we'll be reading 1041 * @iovcount - returned count of io vectors we fill 1042 * @log - vhost log 1043 * @log_num - log offset 1044 * @quota - headcount quota, 1 for big buffer 1045 * returns number of buffer heads allocated, negative on error 1046 */ 1047 static int get_rx_bufs(struct vhost_net_virtqueue *nvq, 1048 struct vring_used_elem *heads, 1049 u16 *nheads, 1050 int datalen, 1051 unsigned *iovcount, 1052 struct vhost_log *log, 1053 unsigned *log_num, 1054 unsigned int quota) 1055 { 1056 struct vhost_virtqueue *vq = &nvq->vq; 1057 bool in_order = vhost_has_feature(vq, VIRTIO_F_IN_ORDER); 1058 unsigned int out, in; 1059 int seg = 0; 1060 int headcount = 0; 1061 unsigned d; 1062 int r, nlogs = 0; 1063 /* len is always initialized before use since we are always called with 1064 * datalen > 0. 1065 */ 1066 u32 len; 1067 1068 while (datalen > 0 && headcount < quota) { 1069 if (unlikely(seg >= UIO_MAXIOV)) { 1070 r = -ENOBUFS; 1071 goto err; 1072 } 1073 r = vhost_get_vq_desc(vq, vq->iov + seg, 1074 ARRAY_SIZE(vq->iov) - seg, &out, 1075 &in, log, log_num); 1076 if (unlikely(r < 0)) 1077 goto err; 1078 1079 d = r; 1080 if (d == vq->num) { 1081 r = 0; 1082 goto err; 1083 } 1084 if (unlikely(out || in <= 0)) { 1085 vq_err(vq, "unexpected descriptor format for RX: " 1086 "out %d, in %d\n", out, in); 1087 r = -EINVAL; 1088 goto err; 1089 } 1090 if (unlikely(log)) { 1091 nlogs += *log_num; 1092 log += *log_num; 1093 } 1094 len = iov_length(vq->iov + seg, in); 1095 if (!in_order) { 1096 heads[headcount].id = cpu_to_vhost32(vq, d); 1097 heads[headcount].len = cpu_to_vhost32(vq, len); 1098 } 1099 ++headcount; 1100 datalen -= len; 1101 seg += in; 1102 } 1103 1104 *iovcount = seg; 1105 if (unlikely(log)) 1106 *log_num = nlogs; 1107 1108 /* Detect overrun */ 1109 if (unlikely(datalen > 0)) { 1110 r = UIO_MAXIOV + 1; 1111 goto err; 1112 } 1113 1114 if (!in_order) 1115 heads[headcount - 1].len = cpu_to_vhost32(vq, len + datalen); 1116 else { 1117 heads[0].len = cpu_to_vhost32(vq, len + datalen); 1118 heads[0].id = cpu_to_vhost32(vq, d); 1119 nheads[0] = headcount; 1120 } 1121 1122 return headcount; 1123 err: 1124 vhost_discard_vq_desc(vq, headcount); 1125 return r; 1126 } 1127 1128 /* Expects to be always run from workqueue - which acts as 1129 * read-size critical section for our kind of RCU. */ 1130 static void handle_rx(struct vhost_net *net) 1131 { 1132 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX]; 1133 struct vhost_virtqueue *vq = &nvq->vq; 1134 bool in_order = vhost_has_feature(vq, VIRTIO_F_IN_ORDER); 1135 unsigned int count = 0; 1136 unsigned in, log; 1137 struct vhost_log *vq_log; 1138 struct msghdr msg = { 1139 .msg_name = NULL, 1140 .msg_namelen = 0, 1141 .msg_control = NULL, /* FIXME: get and handle RX aux data. */ 1142 .msg_controllen = 0, 1143 .msg_flags = MSG_DONTWAIT, 1144 }; 1145 struct virtio_net_hdr hdr = { 1146 .flags = 0, 1147 .gso_type = VIRTIO_NET_HDR_GSO_NONE 1148 }; 1149 size_t total_len = 0; 1150 int err, mergeable; 1151 s16 headcount; 1152 size_t vhost_hlen, sock_hlen; 1153 size_t vhost_len, sock_len; 1154 bool busyloop_intr = false; 1155 bool set_num_buffers; 1156 struct socket *sock; 1157 struct iov_iter fixup; 1158 __virtio16 num_buffers; 1159 int recv_pkts = 0; 1160 1161 mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_RX); 1162 sock = vhost_vq_get_backend(vq); 1163 if (!sock) 1164 goto out; 1165 1166 if (!vq_meta_prefetch(vq)) 1167 goto out; 1168 1169 vhost_disable_notify(&net->dev, vq); 1170 vhost_net_disable_vq(net, vq); 1171 1172 vhost_hlen = nvq->vhost_hlen; 1173 sock_hlen = nvq->sock_hlen; 1174 1175 vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ? 1176 vq->log : NULL; 1177 mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF); 1178 set_num_buffers = mergeable || 1179 vhost_has_feature(vq, VIRTIO_F_VERSION_1); 1180 1181 do { 1182 sock_len = vhost_net_rx_peek_head_len(net, sock->sk, 1183 &busyloop_intr, count); 1184 if (!sock_len) 1185 break; 1186 sock_len += sock_hlen; 1187 vhost_len = sock_len + vhost_hlen; 1188 headcount = get_rx_bufs(nvq, vq->heads + count, 1189 vq->nheads + count, 1190 vhost_len, &in, vq_log, &log, 1191 likely(mergeable) ? UIO_MAXIOV : 1); 1192 /* On error, stop handling until the next kick. */ 1193 if (unlikely(headcount < 0)) 1194 goto out; 1195 /* OK, now we need to know about added descriptors. */ 1196 if (!headcount) { 1197 if (unlikely(busyloop_intr)) { 1198 vhost_poll_queue(&vq->poll); 1199 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { 1200 /* They have slipped one in as we were 1201 * doing that: check again. */ 1202 vhost_disable_notify(&net->dev, vq); 1203 continue; 1204 } 1205 /* Nothing new? Wait for eventfd to tell us 1206 * they refilled. */ 1207 goto out; 1208 } 1209 busyloop_intr = false; 1210 if (nvq->rx_ring) 1211 msg.msg_control = vhost_net_buf_consume(&nvq->rxq); 1212 /* On overrun, truncate and discard */ 1213 if (unlikely(headcount > UIO_MAXIOV)) { 1214 iov_iter_init(&msg.msg_iter, ITER_DEST, vq->iov, 1, 1); 1215 err = sock->ops->recvmsg(sock, &msg, 1216 1, MSG_DONTWAIT | MSG_TRUNC); 1217 pr_debug("Discarded rx packet: len %zd\n", sock_len); 1218 continue; 1219 } 1220 /* We don't need to be notified again. */ 1221 iov_iter_init(&msg.msg_iter, ITER_DEST, vq->iov, in, vhost_len); 1222 fixup = msg.msg_iter; 1223 if (unlikely((vhost_hlen))) { 1224 /* We will supply the header ourselves 1225 * TODO: support TSO. 1226 */ 1227 iov_iter_advance(&msg.msg_iter, vhost_hlen); 1228 } 1229 err = sock->ops->recvmsg(sock, &msg, 1230 sock_len, MSG_DONTWAIT | MSG_TRUNC); 1231 /* Userspace might have consumed the packet meanwhile: 1232 * it's not supposed to do this usually, but might be hard 1233 * to prevent. Discard data we got (if any) and keep going. */ 1234 if (unlikely(err != sock_len)) { 1235 pr_debug("Discarded rx packet: " 1236 " len %d, expected %zd\n", err, sock_len); 1237 vhost_discard_vq_desc(vq, headcount); 1238 continue; 1239 } 1240 /* Supply virtio_net_hdr if VHOST_NET_F_VIRTIO_NET_HDR */ 1241 if (unlikely(vhost_hlen)) { 1242 if (copy_to_iter(&hdr, sizeof(hdr), 1243 &fixup) != sizeof(hdr)) { 1244 vq_err(vq, "Unable to write vnet_hdr " 1245 "at addr %p\n", vq->iov->iov_base); 1246 goto out; 1247 } 1248 } else { 1249 /* Header came from socket; we'll need to patch 1250 * ->num_buffers over if VIRTIO_NET_F_MRG_RXBUF 1251 */ 1252 iov_iter_advance(&fixup, sizeof(hdr)); 1253 } 1254 /* TODO: Should check and handle checksum. */ 1255 1256 num_buffers = cpu_to_vhost16(vq, headcount); 1257 if (likely(set_num_buffers) && 1258 copy_to_iter(&num_buffers, sizeof num_buffers, 1259 &fixup) != sizeof num_buffers) { 1260 vq_err(vq, "Failed num_buffers write"); 1261 vhost_discard_vq_desc(vq, headcount); 1262 goto out; 1263 } 1264 nvq->done_idx += headcount; 1265 count += in_order ? 1 : headcount; 1266 if (nvq->done_idx > VHOST_NET_BATCH) { 1267 vhost_net_signal_used(nvq, count); 1268 count = 0; 1269 } 1270 if (unlikely(vq_log)) 1271 vhost_log_write(vq, vq_log, log, vhost_len, 1272 vq->iov, in); 1273 total_len += vhost_len; 1274 } while (likely(!vhost_exceeds_weight(vq, ++recv_pkts, total_len))); 1275 1276 if (unlikely(busyloop_intr)) 1277 vhost_poll_queue(&vq->poll); 1278 else if (!sock_len) 1279 vhost_net_enable_vq(net, vq); 1280 out: 1281 vhost_net_signal_used(nvq, count); 1282 mutex_unlock(&vq->mutex); 1283 } 1284 1285 static void handle_tx_kick(struct vhost_work *work) 1286 { 1287 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 1288 poll.work); 1289 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); 1290 1291 handle_tx(net); 1292 } 1293 1294 static void handle_rx_kick(struct vhost_work *work) 1295 { 1296 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 1297 poll.work); 1298 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); 1299 1300 handle_rx(net); 1301 } 1302 1303 static void handle_tx_net(struct vhost_work *work) 1304 { 1305 struct vhost_net *net = container_of(work, struct vhost_net, 1306 poll[VHOST_NET_VQ_TX].work); 1307 handle_tx(net); 1308 } 1309 1310 static void handle_rx_net(struct vhost_work *work) 1311 { 1312 struct vhost_net *net = container_of(work, struct vhost_net, 1313 poll[VHOST_NET_VQ_RX].work); 1314 handle_rx(net); 1315 } 1316 1317 static int vhost_net_open(struct inode *inode, struct file *f) 1318 { 1319 struct vhost_net *n; 1320 struct vhost_dev *dev; 1321 struct vhost_virtqueue **vqs; 1322 void **queue; 1323 struct xdp_buff *xdp; 1324 int i; 1325 1326 n = kvmalloc(sizeof *n, GFP_KERNEL | __GFP_RETRY_MAYFAIL); 1327 if (!n) 1328 return -ENOMEM; 1329 vqs = kmalloc_array(VHOST_NET_VQ_MAX, sizeof(*vqs), GFP_KERNEL); 1330 if (!vqs) { 1331 kvfree(n); 1332 return -ENOMEM; 1333 } 1334 1335 queue = kmalloc_array(VHOST_NET_BATCH, sizeof(void *), 1336 GFP_KERNEL); 1337 if (!queue) { 1338 kfree(vqs); 1339 kvfree(n); 1340 return -ENOMEM; 1341 } 1342 n->vqs[VHOST_NET_VQ_RX].rxq.queue = queue; 1343 1344 xdp = kmalloc_array(VHOST_NET_BATCH, sizeof(*xdp), GFP_KERNEL); 1345 if (!xdp) { 1346 kfree(vqs); 1347 kvfree(n); 1348 kfree(queue); 1349 return -ENOMEM; 1350 } 1351 n->vqs[VHOST_NET_VQ_TX].xdp = xdp; 1352 1353 dev = &n->dev; 1354 vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq; 1355 vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq; 1356 n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick; 1357 n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick; 1358 for (i = 0; i < VHOST_NET_VQ_MAX; i++) { 1359 n->vqs[i].ubufs = NULL; 1360 n->vqs[i].ubuf_info = NULL; 1361 n->vqs[i].upend_idx = 0; 1362 n->vqs[i].done_idx = 0; 1363 n->vqs[i].batched_xdp = 0; 1364 n->vqs[i].vhost_hlen = 0; 1365 n->vqs[i].sock_hlen = 0; 1366 n->vqs[i].rx_ring = NULL; 1367 vhost_net_buf_init(&n->vqs[i].rxq); 1368 } 1369 vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX, 1370 UIO_MAXIOV + VHOST_NET_BATCH, 1371 VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT, true, 1372 NULL); 1373 1374 vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, EPOLLOUT, dev, 1375 vqs[VHOST_NET_VQ_TX]); 1376 vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, EPOLLIN, dev, 1377 vqs[VHOST_NET_VQ_RX]); 1378 1379 f->private_data = n; 1380 page_frag_cache_init(&n->pf_cache); 1381 1382 return 0; 1383 } 1384 1385 static struct socket *vhost_net_stop_vq(struct vhost_net *n, 1386 struct vhost_virtqueue *vq) 1387 { 1388 struct socket *sock; 1389 struct vhost_net_virtqueue *nvq = 1390 container_of(vq, struct vhost_net_virtqueue, vq); 1391 1392 mutex_lock(&vq->mutex); 1393 sock = vhost_vq_get_backend(vq); 1394 vhost_net_disable_vq(n, vq); 1395 vhost_vq_set_backend(vq, NULL); 1396 vhost_net_buf_unproduce(nvq); 1397 nvq->rx_ring = NULL; 1398 mutex_unlock(&vq->mutex); 1399 return sock; 1400 } 1401 1402 static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock, 1403 struct socket **rx_sock) 1404 { 1405 *tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq); 1406 *rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq); 1407 } 1408 1409 static void vhost_net_flush(struct vhost_net *n) 1410 { 1411 vhost_dev_flush(&n->dev); 1412 if (n->vqs[VHOST_NET_VQ_TX].ubufs) { 1413 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); 1414 n->tx_flush = true; 1415 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); 1416 /* Wait for all lower device DMAs done. */ 1417 vhost_net_ubuf_put_and_wait(n->vqs[VHOST_NET_VQ_TX].ubufs); 1418 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); 1419 n->tx_flush = false; 1420 atomic_set(&n->vqs[VHOST_NET_VQ_TX].ubufs->refcount, 1); 1421 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); 1422 } 1423 } 1424 1425 static int vhost_net_release(struct inode *inode, struct file *f) 1426 { 1427 struct vhost_net *n = f->private_data; 1428 struct socket *tx_sock; 1429 struct socket *rx_sock; 1430 1431 vhost_net_stop(n, &tx_sock, &rx_sock); 1432 vhost_net_flush(n); 1433 vhost_dev_stop(&n->dev); 1434 vhost_dev_cleanup(&n->dev); 1435 vhost_net_vq_reset(n); 1436 if (tx_sock) 1437 sockfd_put(tx_sock); 1438 if (rx_sock) 1439 sockfd_put(rx_sock); 1440 /* Make sure no callbacks are outstanding */ 1441 synchronize_rcu(); 1442 /* We do an extra flush before freeing memory, 1443 * since jobs can re-queue themselves. */ 1444 vhost_net_flush(n); 1445 kfree(n->vqs[VHOST_NET_VQ_RX].rxq.queue); 1446 kfree(n->vqs[VHOST_NET_VQ_TX].xdp); 1447 kfree(n->dev.vqs); 1448 page_frag_cache_drain(&n->pf_cache); 1449 kvfree(n); 1450 return 0; 1451 } 1452 1453 static struct socket *get_raw_socket(int fd) 1454 { 1455 int r; 1456 struct socket *sock = sockfd_lookup(fd, &r); 1457 1458 if (!sock) 1459 return ERR_PTR(-ENOTSOCK); 1460 1461 /* Parameter checking */ 1462 if (sock->sk->sk_type != SOCK_RAW) { 1463 r = -ESOCKTNOSUPPORT; 1464 goto err; 1465 } 1466 1467 if (sock->sk->sk_family != AF_PACKET) { 1468 r = -EPFNOSUPPORT; 1469 goto err; 1470 } 1471 return sock; 1472 err: 1473 sockfd_put(sock); 1474 return ERR_PTR(r); 1475 } 1476 1477 static struct ptr_ring *get_tap_ptr_ring(struct file *file) 1478 { 1479 struct ptr_ring *ring; 1480 ring = tun_get_tx_ring(file); 1481 if (!IS_ERR(ring)) 1482 goto out; 1483 ring = tap_get_ptr_ring(file); 1484 if (!IS_ERR(ring)) 1485 goto out; 1486 ring = NULL; 1487 out: 1488 return ring; 1489 } 1490 1491 static struct socket *get_tap_socket(int fd) 1492 { 1493 struct file *file = fget(fd); 1494 struct socket *sock; 1495 1496 if (!file) 1497 return ERR_PTR(-EBADF); 1498 sock = tun_get_socket(file); 1499 if (!IS_ERR(sock)) 1500 return sock; 1501 sock = tap_get_socket(file); 1502 if (IS_ERR(sock)) 1503 fput(file); 1504 return sock; 1505 } 1506 1507 static struct socket *get_socket(int fd) 1508 { 1509 struct socket *sock; 1510 1511 /* special case to disable backend */ 1512 if (fd == -1) 1513 return NULL; 1514 sock = get_raw_socket(fd); 1515 if (!IS_ERR(sock)) 1516 return sock; 1517 sock = get_tap_socket(fd); 1518 if (!IS_ERR(sock)) 1519 return sock; 1520 return ERR_PTR(-ENOTSOCK); 1521 } 1522 1523 static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd) 1524 { 1525 struct socket *sock, *oldsock; 1526 struct vhost_virtqueue *vq; 1527 struct vhost_net_virtqueue *nvq; 1528 struct vhost_net_ubuf_ref *ubufs, *oldubufs = NULL; 1529 int r; 1530 1531 mutex_lock(&n->dev.mutex); 1532 r = vhost_dev_check_owner(&n->dev); 1533 if (r) 1534 goto err; 1535 1536 if (index >= VHOST_NET_VQ_MAX) { 1537 r = -ENOBUFS; 1538 goto err; 1539 } 1540 vq = &n->vqs[index].vq; 1541 nvq = &n->vqs[index]; 1542 mutex_lock(&vq->mutex); 1543 1544 if (fd == -1) 1545 vhost_clear_msg(&n->dev); 1546 1547 /* Verify that ring has been setup correctly. */ 1548 if (!vhost_vq_access_ok(vq)) { 1549 r = -EFAULT; 1550 goto err_vq; 1551 } 1552 sock = get_socket(fd); 1553 if (IS_ERR(sock)) { 1554 r = PTR_ERR(sock); 1555 goto err_vq; 1556 } 1557 1558 /* start polling new socket */ 1559 oldsock = vhost_vq_get_backend(vq); 1560 if (sock != oldsock) { 1561 ubufs = vhost_net_ubuf_alloc(vq, 1562 sock && vhost_sock_zcopy(sock)); 1563 if (IS_ERR(ubufs)) { 1564 r = PTR_ERR(ubufs); 1565 goto err_ubufs; 1566 } 1567 1568 vhost_net_disable_vq(n, vq); 1569 vhost_vq_set_backend(vq, sock); 1570 vhost_net_buf_unproduce(nvq); 1571 r = vhost_vq_init_access(vq); 1572 if (r) 1573 goto err_used; 1574 r = vhost_net_enable_vq(n, vq); 1575 if (r) 1576 goto err_used; 1577 if (index == VHOST_NET_VQ_RX) { 1578 if (sock) 1579 nvq->rx_ring = get_tap_ptr_ring(sock->file); 1580 else 1581 nvq->rx_ring = NULL; 1582 } 1583 1584 oldubufs = nvq->ubufs; 1585 nvq->ubufs = ubufs; 1586 1587 n->tx_packets = 0; 1588 n->tx_zcopy_err = 0; 1589 n->tx_flush = false; 1590 } 1591 1592 mutex_unlock(&vq->mutex); 1593 1594 if (oldubufs) { 1595 vhost_net_ubuf_put_wait_and_free(oldubufs); 1596 mutex_lock(&vq->mutex); 1597 vhost_zerocopy_signal_used(n, vq); 1598 mutex_unlock(&vq->mutex); 1599 } 1600 1601 if (oldsock) { 1602 vhost_dev_flush(&n->dev); 1603 sockfd_put(oldsock); 1604 } 1605 1606 mutex_unlock(&n->dev.mutex); 1607 return 0; 1608 1609 err_used: 1610 vhost_vq_set_backend(vq, oldsock); 1611 vhost_net_enable_vq(n, vq); 1612 if (ubufs) 1613 vhost_net_ubuf_put_wait_and_free(ubufs); 1614 err_ubufs: 1615 if (sock) 1616 sockfd_put(sock); 1617 err_vq: 1618 mutex_unlock(&vq->mutex); 1619 err: 1620 mutex_unlock(&n->dev.mutex); 1621 return r; 1622 } 1623 1624 static long vhost_net_reset_owner(struct vhost_net *n) 1625 { 1626 struct socket *tx_sock = NULL; 1627 struct socket *rx_sock = NULL; 1628 long err; 1629 struct vhost_iotlb *umem; 1630 1631 mutex_lock(&n->dev.mutex); 1632 err = vhost_dev_check_owner(&n->dev); 1633 if (err) 1634 goto done; 1635 umem = vhost_dev_reset_owner_prepare(); 1636 if (!umem) { 1637 err = -ENOMEM; 1638 goto done; 1639 } 1640 vhost_net_stop(n, &tx_sock, &rx_sock); 1641 vhost_net_flush(n); 1642 vhost_dev_stop(&n->dev); 1643 vhost_dev_reset_owner(&n->dev, umem); 1644 vhost_net_vq_reset(n); 1645 done: 1646 mutex_unlock(&n->dev.mutex); 1647 if (tx_sock) 1648 sockfd_put(tx_sock); 1649 if (rx_sock) 1650 sockfd_put(rx_sock); 1651 return err; 1652 } 1653 1654 static int vhost_net_set_features(struct vhost_net *n, const u64 *features) 1655 { 1656 size_t vhost_hlen, sock_hlen, hdr_len; 1657 int i; 1658 1659 hdr_len = virtio_features_test_bit(features, VIRTIO_NET_F_MRG_RXBUF) || 1660 virtio_features_test_bit(features, VIRTIO_F_VERSION_1) ? 1661 sizeof(struct virtio_net_hdr_mrg_rxbuf) : 1662 sizeof(struct virtio_net_hdr); 1663 1664 if (virtio_features_test_bit(features, 1665 VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO) || 1666 virtio_features_test_bit(features, 1667 VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO)) 1668 hdr_len = sizeof(struct virtio_net_hdr_v1_hash_tunnel); 1669 1670 if (virtio_features_test_bit(features, VHOST_NET_F_VIRTIO_NET_HDR)) { 1671 /* vhost provides vnet_hdr */ 1672 vhost_hlen = hdr_len; 1673 sock_hlen = 0; 1674 } else { 1675 /* socket provides vnet_hdr */ 1676 vhost_hlen = 0; 1677 sock_hlen = hdr_len; 1678 } 1679 mutex_lock(&n->dev.mutex); 1680 if (virtio_features_test_bit(features, VHOST_F_LOG_ALL) && 1681 !vhost_log_access_ok(&n->dev)) 1682 goto out_unlock; 1683 1684 if (virtio_features_test_bit(features, VIRTIO_F_ACCESS_PLATFORM)) { 1685 if (vhost_init_device_iotlb(&n->dev)) 1686 goto out_unlock; 1687 } 1688 1689 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) { 1690 mutex_lock(&n->vqs[i].vq.mutex); 1691 virtio_features_copy(n->vqs[i].vq.acked_features_array, 1692 features); 1693 n->vqs[i].vhost_hlen = vhost_hlen; 1694 n->vqs[i].sock_hlen = sock_hlen; 1695 mutex_unlock(&n->vqs[i].vq.mutex); 1696 } 1697 mutex_unlock(&n->dev.mutex); 1698 return 0; 1699 1700 out_unlock: 1701 mutex_unlock(&n->dev.mutex); 1702 return -EFAULT; 1703 } 1704 1705 static long vhost_net_set_owner(struct vhost_net *n) 1706 { 1707 int r; 1708 1709 mutex_lock(&n->dev.mutex); 1710 if (vhost_dev_has_owner(&n->dev)) { 1711 r = -EBUSY; 1712 goto out; 1713 } 1714 r = vhost_net_set_ubuf_info(n); 1715 if (r) 1716 goto out; 1717 r = vhost_dev_set_owner(&n->dev); 1718 if (r) 1719 vhost_net_clear_ubuf_info(n); 1720 vhost_net_flush(n); 1721 out: 1722 mutex_unlock(&n->dev.mutex); 1723 return r; 1724 } 1725 1726 static long vhost_net_ioctl(struct file *f, unsigned int ioctl, 1727 unsigned long arg) 1728 { 1729 u64 all_features[VIRTIO_FEATURES_DWORDS]; 1730 struct vhost_net *n = f->private_data; 1731 void __user *argp = (void __user *)arg; 1732 u64 __user *featurep = argp; 1733 struct vhost_vring_file backend; 1734 u64 features, count, copied; 1735 int r, i; 1736 1737 switch (ioctl) { 1738 case VHOST_NET_SET_BACKEND: 1739 if (copy_from_user(&backend, argp, sizeof backend)) 1740 return -EFAULT; 1741 return vhost_net_set_backend(n, backend.index, backend.fd); 1742 case VHOST_GET_FEATURES: 1743 features = vhost_net_features[0]; 1744 if (copy_to_user(featurep, &features, sizeof features)) 1745 return -EFAULT; 1746 return 0; 1747 case VHOST_SET_FEATURES: 1748 if (copy_from_user(&features, featurep, sizeof features)) 1749 return -EFAULT; 1750 if (features & ~vhost_net_features[0]) 1751 return -EOPNOTSUPP; 1752 1753 virtio_features_from_u64(all_features, features); 1754 return vhost_net_set_features(n, all_features); 1755 case VHOST_GET_FEATURES_ARRAY: 1756 if (copy_from_user(&count, featurep, sizeof(count))) 1757 return -EFAULT; 1758 1759 /* Copy the net features, up to the user-provided buffer size */ 1760 argp += sizeof(u64); 1761 copied = min(count, VIRTIO_FEATURES_DWORDS); 1762 if (copy_to_user(argp, vhost_net_features, 1763 copied * sizeof(u64))) 1764 return -EFAULT; 1765 1766 /* Zero the trailing space provided by user-space, if any */ 1767 if (clear_user(argp, size_mul(count - copied, sizeof(u64)))) 1768 return -EFAULT; 1769 return 0; 1770 case VHOST_SET_FEATURES_ARRAY: 1771 if (copy_from_user(&count, featurep, sizeof(count))) 1772 return -EFAULT; 1773 1774 virtio_features_zero(all_features); 1775 argp += sizeof(u64); 1776 copied = min(count, VIRTIO_FEATURES_DWORDS); 1777 if (copy_from_user(all_features, argp, copied * sizeof(u64))) 1778 return -EFAULT; 1779 1780 /* 1781 * Any feature specified by user-space above 1782 * VIRTIO_FEATURES_MAX is not supported by definition. 1783 */ 1784 for (i = copied; i < count; ++i) { 1785 if (copy_from_user(&features, featurep + 1 + i, 1786 sizeof(features))) 1787 return -EFAULT; 1788 if (features) 1789 return -EOPNOTSUPP; 1790 } 1791 1792 for (i = 0; i < VIRTIO_FEATURES_DWORDS; i++) 1793 if (all_features[i] & ~vhost_net_features[i]) 1794 return -EOPNOTSUPP; 1795 1796 return vhost_net_set_features(n, all_features); 1797 case VHOST_GET_BACKEND_FEATURES: 1798 features = VHOST_NET_BACKEND_FEATURES; 1799 if (copy_to_user(featurep, &features, sizeof(features))) 1800 return -EFAULT; 1801 return 0; 1802 case VHOST_SET_BACKEND_FEATURES: 1803 if (copy_from_user(&features, featurep, sizeof(features))) 1804 return -EFAULT; 1805 if (features & ~VHOST_NET_BACKEND_FEATURES) 1806 return -EOPNOTSUPP; 1807 vhost_set_backend_features(&n->dev, features); 1808 return 0; 1809 case VHOST_RESET_OWNER: 1810 return vhost_net_reset_owner(n); 1811 case VHOST_SET_OWNER: 1812 return vhost_net_set_owner(n); 1813 default: 1814 mutex_lock(&n->dev.mutex); 1815 r = vhost_dev_ioctl(&n->dev, ioctl, argp); 1816 if (r == -ENOIOCTLCMD) 1817 r = vhost_vring_ioctl(&n->dev, ioctl, argp); 1818 else 1819 vhost_net_flush(n); 1820 mutex_unlock(&n->dev.mutex); 1821 return r; 1822 } 1823 } 1824 1825 static ssize_t vhost_net_chr_read_iter(struct kiocb *iocb, struct iov_iter *to) 1826 { 1827 struct file *file = iocb->ki_filp; 1828 struct vhost_net *n = file->private_data; 1829 struct vhost_dev *dev = &n->dev; 1830 int noblock = file->f_flags & O_NONBLOCK; 1831 1832 return vhost_chr_read_iter(dev, to, noblock); 1833 } 1834 1835 static ssize_t vhost_net_chr_write_iter(struct kiocb *iocb, 1836 struct iov_iter *from) 1837 { 1838 struct file *file = iocb->ki_filp; 1839 struct vhost_net *n = file->private_data; 1840 struct vhost_dev *dev = &n->dev; 1841 1842 return vhost_chr_write_iter(dev, from); 1843 } 1844 1845 static __poll_t vhost_net_chr_poll(struct file *file, poll_table *wait) 1846 { 1847 struct vhost_net *n = file->private_data; 1848 struct vhost_dev *dev = &n->dev; 1849 1850 return vhost_chr_poll(file, dev, wait); 1851 } 1852 1853 static const struct file_operations vhost_net_fops = { 1854 .owner = THIS_MODULE, 1855 .release = vhost_net_release, 1856 .read_iter = vhost_net_chr_read_iter, 1857 .write_iter = vhost_net_chr_write_iter, 1858 .poll = vhost_net_chr_poll, 1859 .unlocked_ioctl = vhost_net_ioctl, 1860 .compat_ioctl = compat_ptr_ioctl, 1861 .open = vhost_net_open, 1862 .llseek = noop_llseek, 1863 }; 1864 1865 static struct miscdevice vhost_net_misc = { 1866 .minor = VHOST_NET_MINOR, 1867 .name = "vhost-net", 1868 .fops = &vhost_net_fops, 1869 }; 1870 1871 static int __init vhost_net_init(void) 1872 { 1873 if (experimental_zcopytx) 1874 vhost_net_enable_zcopy(VHOST_NET_VQ_TX); 1875 return misc_register(&vhost_net_misc); 1876 } 1877 module_init(vhost_net_init); 1878 1879 static void __exit vhost_net_exit(void) 1880 { 1881 misc_deregister(&vhost_net_misc); 1882 } 1883 module_exit(vhost_net_exit); 1884 1885 MODULE_VERSION("0.0.1"); 1886 MODULE_LICENSE("GPL v2"); 1887 MODULE_AUTHOR("Michael S. Tsirkin"); 1888 MODULE_DESCRIPTION("Host kernel accelerator for virtio net"); 1889 MODULE_ALIAS_MISCDEV(VHOST_NET_MINOR); 1890 MODULE_ALIAS("devname:vhost-net"); 1891