Lines Matching +full:dma +full:- +full:poll +full:- +full:cnt
1 // SPDX-License-Identifier: GPL-2.0-only
5 * virtio-net server in host kernel.
41 " 1 -Enable; 0 - Disable");
61 /* Lower device DMA failed */
63 /* Lower device DMA done */
65 /* Lower device DMA in progress */
117 /* last used idx for outstanding DMA zerocopy buffers */
119 /* For TX, first used idx for DMA done zerocopy buffers
139 struct vhost_poll poll[VHOST_NET_VQ_MAX]; member
156 if (rxq->tail != rxq->head) in vhost_net_buf_get_ptr()
157 return rxq->queue[rxq->head]; in vhost_net_buf_get_ptr()
164 return rxq->tail - rxq->head; in vhost_net_buf_get_size()
169 return rxq->tail == rxq->head; in vhost_net_buf_is_empty()
175 ++rxq->head; in vhost_net_buf_consume()
181 struct vhost_net_buf *rxq = &nvq->rxq; in vhost_net_buf_produce()
183 rxq->head = 0; in vhost_net_buf_produce()
184 rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue, in vhost_net_buf_produce()
186 return rxq->tail; in vhost_net_buf_produce()
191 struct vhost_net_buf *rxq = &nvq->rxq; in vhost_net_buf_unproduce()
193 if (nvq->rx_ring && !vhost_net_buf_is_empty(rxq)) { in vhost_net_buf_unproduce()
194 ptr_ring_unconsume(nvq->rx_ring, rxq->queue + rxq->head, in vhost_net_buf_unproduce()
197 rxq->head = rxq->tail = 0; in vhost_net_buf_unproduce()
206 return xdpf->len; in vhost_net_buf_peek_len()
214 struct vhost_net_buf *rxq = &nvq->rxq; in vhost_net_buf_peek()
228 rxq->head = rxq->tail = 0; in vhost_net_buf_init()
245 return ERR_PTR(-ENOMEM); in vhost_net_ubuf_alloc()
246 atomic_set(&ubufs->refcount, 1); in vhost_net_ubuf_alloc()
247 init_waitqueue_head(&ubufs->wait); in vhost_net_ubuf_alloc()
248 ubufs->vq = vq; in vhost_net_ubuf_alloc()
257 r = atomic_sub_return(1, &ubufs->refcount); in vhost_net_ubuf_put()
259 wake_up(&ubufs->wait); in vhost_net_ubuf_put()
267 wait_event(ubufs->wait, !atomic_read(&ubufs->refcount)); in vhost_net_ubuf_put_and_wait()
281 kfree(n->vqs[i].ubuf_info); in vhost_net_clear_ubuf_info()
282 n->vqs[i].ubuf_info = NULL; in vhost_net_clear_ubuf_info()
295 n->vqs[i].ubuf_info = in vhost_net_set_ubuf_info()
297 sizeof(*n->vqs[i].ubuf_info), in vhost_net_set_ubuf_info()
299 if (!n->vqs[i].ubuf_info) in vhost_net_set_ubuf_info()
306 return -ENOMEM; in vhost_net_set_ubuf_info()
316 n->vqs[i].done_idx = 0; in vhost_net_vq_reset()
317 n->vqs[i].upend_idx = 0; in vhost_net_vq_reset()
318 n->vqs[i].ubufs = NULL; in vhost_net_vq_reset()
319 n->vqs[i].vhost_hlen = 0; in vhost_net_vq_reset()
320 n->vqs[i].sock_hlen = 0; in vhost_net_vq_reset()
321 vhost_net_buf_init(&n->vqs[i].rxq); in vhost_net_vq_reset()
328 ++net->tx_packets; in vhost_net_tx_packet()
329 if (net->tx_packets < 1024) in vhost_net_tx_packet()
331 net->tx_packets = 0; in vhost_net_tx_packet()
332 net->tx_zcopy_err = 0; in vhost_net_tx_packet()
337 ++net->tx_zcopy_err; in vhost_net_tx_err()
345 return !net->tx_flush && in vhost_net_tx_select_zcopy()
346 net->tx_packets / 64 >= net->tx_zcopy_err; in vhost_net_tx_select_zcopy()
352 sock_flag(sock->sk, SOCK_ZEROCOPY); in vhost_sock_zcopy()
357 return sock_flag(sock->sk, SOCK_XDP); in vhost_sock_xdp()
360 /* In case of DMA done not in order in lower device driver for some reason.
362 * of used idx. Once lower device DMA done contiguously, we will signal KVM
373 for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) { in vhost_zerocopy_signal_used()
374 if (vq->heads[i].len == VHOST_DMA_FAILED_LEN) in vhost_zerocopy_signal_used()
376 if (VHOST_DMA_IS_DONE(vq->heads[i].len)) { in vhost_zerocopy_signal_used()
377 vq->heads[i].len = VHOST_DMA_CLEAR_LEN; in vhost_zerocopy_signal_used()
383 add = min(UIO_MAXIOV - nvq->done_idx, j); in vhost_zerocopy_signal_used()
384 vhost_add_used_and_signal_n(vq->dev, vq, in vhost_zerocopy_signal_used()
385 &vq->heads[nvq->done_idx], in vhost_zerocopy_signal_used()
387 nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV; in vhost_zerocopy_signal_used()
388 j -= add; in vhost_zerocopy_signal_used()
396 struct vhost_net_ubuf_ref *ubufs = ubuf->ctx; in vhost_zerocopy_complete()
397 struct vhost_virtqueue *vq = ubufs->vq; in vhost_zerocopy_complete()
398 int cnt; in vhost_zerocopy_complete() local
402 /* set len to mark this desc buffers done DMA */ in vhost_zerocopy_complete()
403 vq->heads[ubuf->desc].len = success ? in vhost_zerocopy_complete()
405 cnt = vhost_net_ubuf_put(ubufs); in vhost_zerocopy_complete()
414 if (cnt <= 1 || !(cnt % 16)) in vhost_zerocopy_complete()
415 vhost_poll_queue(&vq->poll); in vhost_zerocopy_complete()
440 struct vhost_poll *poll = n->poll + (nvq - n->vqs); in vhost_net_disable_vq() local
443 vhost_poll_stop(poll); in vhost_net_disable_vq()
451 struct vhost_poll *poll = n->poll + (nvq - n->vqs); in vhost_net_enable_vq() local
458 return vhost_poll_start(poll, sock->file); in vhost_net_enable_vq()
464 struct vhost_virtqueue *vq = &nvq->vq; in vhost_net_signal_used()
465 struct vhost_dev *dev = vq->dev; in vhost_net_signal_used()
467 if (!nvq->done_idx) in vhost_net_signal_used()
470 vhost_add_used_and_signal_n(dev, vq, vq->heads, in vhost_net_signal_used()
471 vq->nheads, count); in vhost_net_signal_used()
472 nvq->done_idx = 0; in vhost_net_signal_used()
480 struct vhost_virtqueue *vq = &nvq->vq; in vhost_tx_batch()
484 .num = nvq->batched_xdp, in vhost_tx_batch()
485 .ptr = nvq->xdp, in vhost_tx_batch()
490 vq->heads[0].len = 0; in vhost_tx_batch()
491 vq->nheads[0] = nvq->done_idx; in vhost_tx_batch()
494 if (nvq->batched_xdp == 0) in vhost_tx_batch()
497 msghdr->msg_control = &ctl; in vhost_tx_batch()
498 msghdr->msg_controllen = sizeof(ctl); in vhost_tx_batch()
499 err = sock->ops->sendmsg(sock, msghdr, 0); in vhost_tx_batch()
501 vq_err(&nvq->vq, "Fail to batch sending packets\n"); in vhost_tx_batch()
507 for (i = 0; i < nvq->batched_xdp; ++i) in vhost_tx_batch()
508 put_page(virt_to_head_page(nvq->xdp[i].data)); in vhost_tx_batch()
509 nvq->batched_xdp = 0; in vhost_tx_batch()
510 nvq->done_idx = 0; in vhost_tx_batch()
515 vhost_net_signal_used(nvq, in_order ? 1 : nvq->done_idx); in vhost_tx_batch()
516 nvq->batched_xdp = 0; in vhost_tx_batch()
524 if (sock->ops->peek_len) in sock_has_rx_data()
525 return sock->ops->peek_len(sock); in sock_has_rx_data()
527 return skb_queue_empty(&sock->sk->sk_receive_queue); in sock_has_rx_data()
533 if (!vhost_vq_avail_empty(&net->dev, vq)) { in vhost_net_busy_poll_try_queue()
534 vhost_poll_queue(&vq->poll); in vhost_net_busy_poll_try_queue()
535 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { in vhost_net_busy_poll_try_queue()
536 vhost_disable_notify(&net->dev, vq); in vhost_net_busy_poll_try_queue()
537 vhost_poll_queue(&vq->poll); in vhost_net_busy_poll_try_queue()
556 if (!mutex_trylock(&vq->mutex)) in vhost_net_busy_poll()
559 vhost_disable_notify(&net->dev, vq); in vhost_net_busy_poll()
562 busyloop_timeout = poll_rx ? rvq->busyloop_timeout: in vhost_net_busy_poll()
563 tvq->busyloop_timeout; in vhost_net_busy_poll()
575 !vhost_vq_avail_empty(&net->dev, rvq)) || in vhost_net_busy_poll()
576 !vhost_vq_avail_empty(&net->dev, tvq)) in vhost_net_busy_poll()
587 vhost_enable_notify(&net->dev, rvq); in vhost_net_busy_poll()
589 mutex_unlock(&vq->mutex); in vhost_net_busy_poll()
597 struct vhost_net_virtqueue *rnvq = &net->vqs[VHOST_NET_VQ_RX]; in vhost_net_tx_get_vq_desc()
598 struct vhost_virtqueue *rvq = &rnvq->vq; in vhost_net_tx_get_vq_desc()
599 struct vhost_virtqueue *tvq = &tnvq->vq; in vhost_net_tx_get_vq_desc()
601 int r = vhost_get_vq_desc(tvq, tvq->iov, ARRAY_SIZE(tvq->iov), in vhost_net_tx_get_vq_desc()
604 if (r == tvq->num && tvq->busyloop_timeout) { in vhost_net_tx_get_vq_desc()
613 r = vhost_get_vq_desc(tvq, tvq->iov, ARRAY_SIZE(tvq->iov), in vhost_net_tx_get_vq_desc()
622 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; in vhost_exceeds_maxpend()
623 struct vhost_virtqueue *vq = &nvq->vq; in vhost_exceeds_maxpend()
625 return (nvq->upend_idx + UIO_MAXIOV - nvq->done_idx) % UIO_MAXIOV > in vhost_exceeds_maxpend()
626 min_t(unsigned int, VHOST_MAX_PEND, vq->num >> 2); in vhost_exceeds_maxpend()
633 size_t len = iov_length(vq->iov, out); in init_iov_iter()
635 iov_iter_init(iter, ITER_SOURCE, vq->iov, out, len); in init_iov_iter()
647 struct vhost_virtqueue *vq = &nvq->vq; in get_tx_bufs()
652 if (ret < 0 || ret == vq->num) in get_tx_bufs()
658 return -EFAULT; in get_tx_bufs()
662 *len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out); in get_tx_bufs()
665 *len, nvq->vhost_hlen); in get_tx_bufs()
666 return -EFAULT; in get_tx_bufs()
675 !vhost_vq_avail_empty(vq->dev, vq); in tx_can_batch()
683 struct vhost_virtqueue *vq = &nvq->vq; in vhost_net_build_xdp()
684 struct vhost_net *net = container_of(vq->dev, struct vhost_net, in vhost_net_build_xdp()
688 struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp]; in vhost_net_build_xdp()
692 int pad = SKB_DATA_ALIGN(VHOST_NET_RX_PAD + headroom + nvq->sock_hlen); in vhost_net_build_xdp()
693 int sock_hlen = nvq->sock_hlen; in vhost_net_build_xdp()
698 if (unlikely(len < nvq->sock_hlen)) in vhost_net_build_xdp()
699 return -EFAULT; in vhost_net_build_xdp()
703 return -ENOSPC; in vhost_net_build_xdp()
706 buf = page_frag_alloc_align(&net->pf_cache, buflen, GFP_KERNEL, in vhost_net_build_xdp()
709 return -ENOMEM; in vhost_net_build_xdp()
711 copied = copy_from_iter(buf + pad - sock_hlen, len, from); in vhost_net_build_xdp()
713 ret = -EFAULT; in vhost_net_build_xdp()
717 gso = buf + pad - sock_hlen; in vhost_net_build_xdp()
722 if ((gso->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && in vhost_net_build_xdp()
723 vhost16_to_cpu(vq, gso->csum_start) + in vhost_net_build_xdp()
724 vhost16_to_cpu(vq, gso->csum_offset) + 2 > in vhost_net_build_xdp()
725 vhost16_to_cpu(vq, gso->hdr_len)) { in vhost_net_build_xdp()
726 gso->hdr_len = cpu_to_vhost16(vq, in vhost_net_build_xdp()
727 vhost16_to_cpu(vq, gso->csum_start) + in vhost_net_build_xdp()
728 vhost16_to_cpu(vq, gso->csum_offset) + 2); in vhost_net_build_xdp()
730 if (vhost16_to_cpu(vq, gso->hdr_len) > len) { in vhost_net_build_xdp()
731 ret = -EINVAL; in vhost_net_build_xdp()
737 memcpy(buf, buf + pad - sock_hlen, sock_hlen); in vhost_net_build_xdp()
740 xdp_prepare_buff(xdp, buf, pad, len - sock_hlen, true); in vhost_net_build_xdp()
742 ++nvq->batched_xdp; in vhost_net_build_xdp()
753 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; in handle_tx_copy()
754 struct vhost_virtqueue *vq = &nvq->vq; in handle_tx_copy()
767 bool sock_can_batch = (sock->sk->sk_sndbuf == INT_MAX); in handle_tx_copy()
773 if (nvq->done_idx == VHOST_NET_BATCH) in handle_tx_copy()
782 if (head == vq->num) { in handle_tx_copy()
789 vhost_poll_queue(&vq->poll); in handle_tx_copy()
790 } else if (unlikely(vhost_enable_notify(&net->dev, in handle_tx_copy()
792 vhost_disable_notify(&net->dev, vq); in handle_tx_copy()
807 } else if (unlikely(err != -ENOSPC)) { in handle_tx_copy()
814 if (nvq->batched_xdp) { in handle_tx_copy()
829 err = sock->ops->sendmsg(sock, &msg, len); in handle_tx_copy()
831 if (err == -EAGAIN || err == -ENOMEM || err == -ENOBUFS) { in handle_tx_copy()
842 vq->heads[0].id = cpu_to_vhost32(vq, head); in handle_tx_copy()
844 vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head); in handle_tx_copy()
845 vq->heads[nvq->done_idx].len = 0; in handle_tx_copy()
847 ++nvq->done_idx; in handle_tx_copy()
855 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; in handle_tx_zerocopy()
856 struct vhost_virtqueue *vq = &nvq->vq; in handle_tx_zerocopy()
887 if (head == vq->num) { in handle_tx_zerocopy()
889 vhost_poll_queue(&vq->poll); in handle_tx_zerocopy()
890 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { in handle_tx_zerocopy()
891 vhost_disable_notify(&net->dev, vq); in handle_tx_zerocopy()
903 ubuf = nvq->ubuf_info + nvq->upend_idx; in handle_tx_zerocopy()
904 vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head); in handle_tx_zerocopy()
905 vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS; in handle_tx_zerocopy()
906 ubuf->ctx = nvq->ubufs; in handle_tx_zerocopy()
907 ubuf->desc = nvq->upend_idx; in handle_tx_zerocopy()
908 ubuf->ubuf.ops = &vhost_ubuf_ops; in handle_tx_zerocopy()
909 ubuf->ubuf.flags = SKBFL_ZEROCOPY_FRAG; in handle_tx_zerocopy()
910 refcount_set(&ubuf->ubuf.refcnt, 1); in handle_tx_zerocopy()
913 ctl.ptr = &ubuf->ubuf; in handle_tx_zerocopy()
915 ubufs = nvq->ubufs; in handle_tx_zerocopy()
916 atomic_inc(&ubufs->refcount); in handle_tx_zerocopy()
917 nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV; in handle_tx_zerocopy()
930 err = sock->ops->sendmsg(sock, &msg, len); in handle_tx_zerocopy()
932 bool retry = err == -EAGAIN || err == -ENOMEM || err == -ENOBUFS; in handle_tx_zerocopy()
935 if (vq->heads[ubuf->desc].len == VHOST_DMA_IN_PROGRESS) in handle_tx_zerocopy()
938 nvq->upend_idx = ((unsigned)nvq->upend_idx - 1) in handle_tx_zerocopy()
941 vq->heads[ubuf->desc].len = VHOST_DMA_DONE_LEN; in handle_tx_zerocopy()
953 vhost_add_used_and_signal(&net->dev, vq, head, 0); in handle_tx_zerocopy()
960 /* Expects to be always run from workqueue - which acts as
961 * read-size critical section for our kind of RCU. */
964 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; in handle_tx()
965 struct vhost_virtqueue *vq = &nvq->vq; in handle_tx()
968 mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_TX); in handle_tx()
976 vhost_disable_notify(&net->dev, vq); in handle_tx()
985 mutex_unlock(&vq->mutex); in handle_tx()
994 if (rvq->rx_ring) in peek_head_len()
997 spin_lock_irqsave(&sk->sk_receive_queue.lock, flags); in peek_head_len()
998 head = skb_peek(&sk->sk_receive_queue); in peek_head_len()
1000 len = head->len; in peek_head_len()
1005 spin_unlock_irqrestore(&sk->sk_receive_queue.lock, flags); in peek_head_len()
1012 struct vhost_net_virtqueue *rnvq = &net->vqs[VHOST_NET_VQ_RX]; in vhost_net_rx_peek_head_len()
1013 struct vhost_net_virtqueue *tnvq = &net->vqs[VHOST_NET_VQ_TX]; in vhost_net_rx_peek_head_len()
1014 struct vhost_virtqueue *rvq = &rnvq->vq; in vhost_net_rx_peek_head_len()
1015 struct vhost_virtqueue *tvq = &tnvq->vq; in vhost_net_rx_peek_head_len()
1018 if (!len && rvq->busyloop_timeout) { in vhost_net_rx_peek_head_len()
1031 /* This is a multi-buffer version of vhost_get_desc, that works if
1033 * @nvq - the relevant vhost_net virtqueue
1034 * @datalen - data length we'll be reading
1035 * @iovcount - returned count of io vectors we fill
1036 * @log - vhost log
1037 * @log_num - log offset
1038 * @quota - headcount quota, 1 for big buffer
1050 struct vhost_virtqueue *vq = &nvq->vq; in get_rx_bufs()
1064 r = -ENOBUFS; in get_rx_bufs()
1067 r = vhost_get_vq_desc(vq, vq->iov + seg, in get_rx_bufs()
1068 ARRAY_SIZE(vq->iov) - seg, &out, in get_rx_bufs()
1074 if (d == vq->num) { in get_rx_bufs()
1081 r = -EINVAL; in get_rx_bufs()
1088 len = iov_length(vq->iov + seg, in); in get_rx_bufs()
1094 datalen -= len; in get_rx_bufs()
1109 heads[headcount - 1].len = cpu_to_vhost32(vq, len + datalen); in get_rx_bufs()
1122 /* Expects to be always run from workqueue - which acts as
1123 * read-size critical section for our kind of RCU. */
1126 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX]; in handle_rx()
1127 struct vhost_virtqueue *vq = &nvq->vq; in handle_rx()
1155 mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_RX); in handle_rx()
1163 vhost_disable_notify(&net->dev, vq); in handle_rx()
1166 vhost_hlen = nvq->vhost_hlen; in handle_rx()
1167 sock_hlen = nvq->sock_hlen; in handle_rx()
1170 vq->log : NULL; in handle_rx()
1176 sock_len = vhost_net_rx_peek_head_len(net, sock->sk, in handle_rx()
1182 headcount = get_rx_bufs(nvq, vq->heads + count, in handle_rx()
1183 vq->nheads + count, in handle_rx()
1192 vhost_poll_queue(&vq->poll); in handle_rx()
1193 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { in handle_rx()
1196 vhost_disable_notify(&net->dev, vq); in handle_rx()
1204 if (nvq->rx_ring) in handle_rx()
1205 msg.msg_control = vhost_net_buf_consume(&nvq->rxq); in handle_rx()
1208 iov_iter_init(&msg.msg_iter, ITER_DEST, vq->iov, 1, 1); in handle_rx()
1209 err = sock->ops->recvmsg(sock, &msg, in handle_rx()
1215 iov_iter_init(&msg.msg_iter, ITER_DEST, vq->iov, in, vhost_len); in handle_rx()
1223 err = sock->ops->recvmsg(sock, &msg, in handle_rx()
1239 "at addr %p\n", vq->iov->iov_base); in handle_rx()
1244 * ->num_buffers over if VIRTIO_NET_F_MRG_RXBUF in handle_rx()
1258 nvq->done_idx += headcount; in handle_rx()
1260 if (nvq->done_idx > VHOST_NET_BATCH) { in handle_rx()
1266 vq->iov, in); in handle_rx()
1271 vhost_poll_queue(&vq->poll); in handle_rx()
1276 mutex_unlock(&vq->mutex); in handle_rx()
1282 poll.work); in handle_tx_kick()
1283 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); in handle_tx_kick()
1291 poll.work); in handle_rx_kick()
1292 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); in handle_rx_kick()
1300 poll[VHOST_NET_VQ_TX].work); in handle_tx_net()
1307 poll[VHOST_NET_VQ_RX].work); in handle_rx_net()
1322 return -ENOMEM; in vhost_net_open()
1326 return -ENOMEM; in vhost_net_open()
1334 return -ENOMEM; in vhost_net_open()
1336 n->vqs[VHOST_NET_VQ_RX].rxq.queue = queue; in vhost_net_open()
1343 return -ENOMEM; in vhost_net_open()
1345 n->vqs[VHOST_NET_VQ_TX].xdp = xdp; in vhost_net_open()
1347 dev = &n->dev; in vhost_net_open()
1348 vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq; in vhost_net_open()
1349 vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq; in vhost_net_open()
1350 n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick; in vhost_net_open()
1351 n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick; in vhost_net_open()
1353 n->vqs[i].ubufs = NULL; in vhost_net_open()
1354 n->vqs[i].ubuf_info = NULL; in vhost_net_open()
1355 n->vqs[i].upend_idx = 0; in vhost_net_open()
1356 n->vqs[i].done_idx = 0; in vhost_net_open()
1357 n->vqs[i].batched_xdp = 0; in vhost_net_open()
1358 n->vqs[i].vhost_hlen = 0; in vhost_net_open()
1359 n->vqs[i].sock_hlen = 0; in vhost_net_open()
1360 n->vqs[i].rx_ring = NULL; in vhost_net_open()
1361 vhost_net_buf_init(&n->vqs[i].rxq); in vhost_net_open()
1368 vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, EPOLLOUT, dev, in vhost_net_open()
1370 vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, EPOLLIN, dev, in vhost_net_open()
1373 f->private_data = n; in vhost_net_open()
1374 page_frag_cache_init(&n->pf_cache); in vhost_net_open()
1386 mutex_lock(&vq->mutex); in vhost_net_stop_vq()
1391 nvq->rx_ring = NULL; in vhost_net_stop_vq()
1392 mutex_unlock(&vq->mutex); in vhost_net_stop_vq()
1399 *tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq); in vhost_net_stop()
1400 *rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq); in vhost_net_stop()
1405 vhost_dev_flush(&n->dev); in vhost_net_flush()
1406 if (n->vqs[VHOST_NET_VQ_TX].ubufs) { in vhost_net_flush()
1407 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1408 n->tx_flush = true; in vhost_net_flush()
1409 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1411 vhost_net_ubuf_put_and_wait(n->vqs[VHOST_NET_VQ_TX].ubufs); in vhost_net_flush()
1412 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1413 n->tx_flush = false; in vhost_net_flush()
1414 atomic_set(&n->vqs[VHOST_NET_VQ_TX].ubufs->refcount, 1); in vhost_net_flush()
1415 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1421 struct vhost_net *n = f->private_data; in vhost_net_release()
1427 vhost_dev_stop(&n->dev); in vhost_net_release()
1428 vhost_dev_cleanup(&n->dev); in vhost_net_release()
1437 * since jobs can re-queue themselves. */ in vhost_net_release()
1439 kfree(n->vqs[VHOST_NET_VQ_RX].rxq.queue); in vhost_net_release()
1440 kfree(n->vqs[VHOST_NET_VQ_TX].xdp); in vhost_net_release()
1441 kfree(n->dev.vqs); in vhost_net_release()
1442 page_frag_cache_drain(&n->pf_cache); in vhost_net_release()
1453 return ERR_PTR(-ENOTSOCK); in get_raw_socket()
1456 if (sock->sk->sk_type != SOCK_RAW) { in get_raw_socket()
1457 r = -ESOCKTNOSUPPORT; in get_raw_socket()
1461 if (sock->sk->sk_family != AF_PACKET) { in get_raw_socket()
1462 r = -EPFNOSUPPORT; in get_raw_socket()
1491 return ERR_PTR(-EBADF); in get_tap_socket()
1506 if (fd == -1) in get_socket()
1514 return ERR_PTR(-ENOTSOCK); in get_socket()
1525 mutex_lock(&n->dev.mutex); in vhost_net_set_backend()
1526 r = vhost_dev_check_owner(&n->dev); in vhost_net_set_backend()
1531 r = -ENOBUFS; in vhost_net_set_backend()
1534 vq = &n->vqs[index].vq; in vhost_net_set_backend()
1535 nvq = &n->vqs[index]; in vhost_net_set_backend()
1536 mutex_lock(&vq->mutex); in vhost_net_set_backend()
1538 if (fd == -1) in vhost_net_set_backend()
1539 vhost_clear_msg(&n->dev); in vhost_net_set_backend()
1543 r = -EFAULT; in vhost_net_set_backend()
1573 nvq->rx_ring = get_tap_ptr_ring(sock->file); in vhost_net_set_backend()
1575 nvq->rx_ring = NULL; in vhost_net_set_backend()
1578 oldubufs = nvq->ubufs; in vhost_net_set_backend()
1579 nvq->ubufs = ubufs; in vhost_net_set_backend()
1581 n->tx_packets = 0; in vhost_net_set_backend()
1582 n->tx_zcopy_err = 0; in vhost_net_set_backend()
1583 n->tx_flush = false; in vhost_net_set_backend()
1586 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
1590 mutex_lock(&vq->mutex); in vhost_net_set_backend()
1592 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
1596 vhost_dev_flush(&n->dev); in vhost_net_set_backend()
1600 mutex_unlock(&n->dev.mutex); in vhost_net_set_backend()
1612 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
1614 mutex_unlock(&n->dev.mutex); in vhost_net_set_backend()
1625 mutex_lock(&n->dev.mutex); in vhost_net_reset_owner()
1626 err = vhost_dev_check_owner(&n->dev); in vhost_net_reset_owner()
1631 err = -ENOMEM; in vhost_net_reset_owner()
1636 vhost_dev_stop(&n->dev); in vhost_net_reset_owner()
1637 vhost_dev_reset_owner(&n->dev, umem); in vhost_net_reset_owner()
1640 mutex_unlock(&n->dev.mutex); in vhost_net_reset_owner()
1673 mutex_lock(&n->dev.mutex); in vhost_net_set_features()
1675 !vhost_log_access_ok(&n->dev)) in vhost_net_set_features()
1679 if (vhost_init_device_iotlb(&n->dev)) in vhost_net_set_features()
1684 mutex_lock(&n->vqs[i].vq.mutex); in vhost_net_set_features()
1685 virtio_features_copy(n->vqs[i].vq.acked_features_array, in vhost_net_set_features()
1687 n->vqs[i].vhost_hlen = vhost_hlen; in vhost_net_set_features()
1688 n->vqs[i].sock_hlen = sock_hlen; in vhost_net_set_features()
1689 mutex_unlock(&n->vqs[i].vq.mutex); in vhost_net_set_features()
1691 mutex_unlock(&n->dev.mutex); in vhost_net_set_features()
1695 mutex_unlock(&n->dev.mutex); in vhost_net_set_features()
1696 return -EFAULT; in vhost_net_set_features()
1703 mutex_lock(&n->dev.mutex); in vhost_net_set_owner()
1704 if (vhost_dev_has_owner(&n->dev)) { in vhost_net_set_owner()
1705 r = -EBUSY; in vhost_net_set_owner()
1711 r = vhost_dev_set_owner(&n->dev); in vhost_net_set_owner()
1716 mutex_unlock(&n->dev.mutex); in vhost_net_set_owner()
1724 struct vhost_net *n = f->private_data; in vhost_net_ioctl()
1734 return -EFAULT; in vhost_net_ioctl()
1739 return -EFAULT; in vhost_net_ioctl()
1743 return -EFAULT; in vhost_net_ioctl()
1745 return -EOPNOTSUPP; in vhost_net_ioctl()
1751 return -EFAULT; in vhost_net_ioctl()
1753 /* Copy the net features, up to the user-provided buffer size */ in vhost_net_ioctl()
1758 return -EFAULT; in vhost_net_ioctl()
1760 /* Zero the trailing space provided by user-space, if any */ in vhost_net_ioctl()
1761 if (clear_user(argp, size_mul(count - copied, sizeof(u64)))) in vhost_net_ioctl()
1762 return -EFAULT; in vhost_net_ioctl()
1766 return -EFAULT; in vhost_net_ioctl()
1772 return -EFAULT; in vhost_net_ioctl()
1775 * Any feature specified by user-space above in vhost_net_ioctl()
1781 return -EFAULT; in vhost_net_ioctl()
1783 return -EOPNOTSUPP; in vhost_net_ioctl()
1788 return -EOPNOTSUPP; in vhost_net_ioctl()
1794 return -EFAULT; in vhost_net_ioctl()
1798 return -EFAULT; in vhost_net_ioctl()
1800 return -EOPNOTSUPP; in vhost_net_ioctl()
1801 vhost_set_backend_features(&n->dev, features); in vhost_net_ioctl()
1808 mutex_lock(&n->dev.mutex); in vhost_net_ioctl()
1809 r = vhost_dev_ioctl(&n->dev, ioctl, argp); in vhost_net_ioctl()
1810 if (r == -ENOIOCTLCMD) in vhost_net_ioctl()
1811 r = vhost_vring_ioctl(&n->dev, ioctl, argp); in vhost_net_ioctl()
1814 mutex_unlock(&n->dev.mutex); in vhost_net_ioctl()
1821 struct file *file = iocb->ki_filp; in vhost_net_chr_read_iter()
1822 struct vhost_net *n = file->private_data; in vhost_net_chr_read_iter()
1823 struct vhost_dev *dev = &n->dev; in vhost_net_chr_read_iter()
1824 int noblock = file->f_flags & O_NONBLOCK; in vhost_net_chr_read_iter()
1832 struct file *file = iocb->ki_filp; in vhost_net_chr_write_iter()
1833 struct vhost_net *n = file->private_data; in vhost_net_chr_write_iter()
1834 struct vhost_dev *dev = &n->dev; in vhost_net_chr_write_iter()
1841 struct vhost_net *n = file->private_data; in vhost_net_chr_poll()
1842 struct vhost_dev *dev = &n->dev; in vhost_net_chr_poll()
1852 .poll = vhost_net_chr_poll,
1861 .name = "vhost-net",
1884 MODULE_ALIAS("devname:vhost-net");