180 				 struct vhost_net_virtqueue *nvq)  in vhost_net_buf_produce()  argument
183 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_produce()
186 	spin_lock(&nvq->rx_ring->consumer_lock);  in vhost_net_buf_produce()
187 	rxq->tail = __ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,  in vhost_net_buf_produce()
193 	spin_unlock(&nvq->rx_ring->consumer_lock);  in vhost_net_buf_produce()
197 static void vhost_net_buf_unproduce(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_unproduce()  argument
199 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_unproduce()
201 	if (nvq->rx_ring && !vhost_net_buf_is_empty(rxq)) {  in vhost_net_buf_unproduce()
202 		ptr_ring_unconsume(nvq->rx_ring, rxq->queue + rxq->head,  in vhost_net_buf_unproduce()
221 			      struct vhost_net_virtqueue *nvq)  in vhost_net_buf_peek()  argument
223 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_peek()
228 	if (!vhost_net_buf_produce(sk, nvq))  in vhost_net_buf_peek()
375 	struct vhost_net_virtqueue *nvq =  in vhost_zerocopy_signal_used()  local
380 	for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {  in vhost_zerocopy_signal_used()
390 		add = min(UIO_MAXIOV - nvq->done_idx, j);  in vhost_zerocopy_signal_used()
392 					    &vq->heads[nvq->done_idx],  in vhost_zerocopy_signal_used()
394 		nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;  in vhost_zerocopy_signal_used()
452 	struct vhost_net_virtqueue *nvq =  in vhost_net_disable_vq()  local
454 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);  in vhost_net_disable_vq()
463 	struct vhost_net_virtqueue *nvq =  in vhost_net_enable_vq()  local
465 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);  in vhost_net_enable_vq()
475 static void vhost_net_signal_used(struct vhost_net_virtqueue *nvq,  in vhost_net_signal_used()  argument
478 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_net_signal_used()
481 	if (!nvq->done_idx)  in vhost_net_signal_used()
486 	nvq->done_idx = 0;  in vhost_net_signal_used()
490 			   struct vhost_net_virtqueue *nvq,  in vhost_tx_batch()  argument
494 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_tx_batch()
498 		.num = nvq->batched_xdp,  in vhost_tx_batch()
499 		.ptr = nvq->xdp,  in vhost_tx_batch()
505 		vq->nheads[0] = nvq->done_idx;  in vhost_tx_batch()
508 	if (nvq->batched_xdp == 0)  in vhost_tx_batch()
515 		vq_err(&nvq->vq, "Fail to batch sending packets\n");  in vhost_tx_batch()
521 		for (i = 0; i < nvq->batched_xdp; ++i)  in vhost_tx_batch()
522 			put_page(virt_to_head_page(nvq->xdp[i].data));  in vhost_tx_batch()
523 		nvq->batched_xdp = 0;  in vhost_tx_batch()
524 		nvq->done_idx = 0;  in vhost_tx_batch()
529 	vhost_net_signal_used(nvq, in_order ? 1 : nvq->done_idx);  in vhost_tx_batch()
530 	nvq->batched_xdp = 0;  in vhost_tx_batch()
637 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in vhost_exceeds_maxpend()  local
638 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_exceeds_maxpend()
640 	return (nvq->upend_idx + UIO_MAXIOV - nvq->done_idx) % UIO_MAXIOV >  in vhost_exceeds_maxpend()
657 		       struct vhost_net_virtqueue *nvq,  in get_tx_bufs()  argument
663 	struct vhost_virtqueue *vq = &nvq->vq;  in get_tx_bufs()
666 	ret = vhost_net_tx_get_vq_desc(net, nvq, out, in, msg,  in get_tx_bufs()
679 	*len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out);  in get_tx_bufs()
682 			*len, nvq->vhost_hlen);  in get_tx_bufs()
697 static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,  in vhost_net_build_xdp()  argument
700 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_net_build_xdp()
705 	struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp];  in vhost_net_build_xdp()
709 	int pad = SKB_DATA_ALIGN(VHOST_NET_RX_PAD + headroom + nvq->sock_hlen);  in vhost_net_build_xdp()
710 	int sock_hlen = nvq->sock_hlen;  in vhost_net_build_xdp()
715 	if (unlikely(len < nvq->sock_hlen))  in vhost_net_build_xdp()
761 	++nvq->batched_xdp;  in vhost_net_build_xdp()
772 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx_copy()  local
773 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx_copy()
793 		if (nvq->done_idx == VHOST_NET_BATCH)  in handle_tx_copy()
794 			vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
796 		head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,  in handle_tx_copy()
807 			vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
824 			err = vhost_net_build_xdp(nvq, &msg.msg_iter);  in handle_tx_copy()
828 				vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
834 			if (nvq->batched_xdp) {  in handle_tx_copy()
839 				vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
864 			vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head);  in handle_tx_copy()
865 			vq->heads[nvq->done_idx].len = 0;  in handle_tx_copy()
867 		++nvq->done_idx;  in handle_tx_copy()
870 	vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
875 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx_zerocopy()  local
876 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx_zerocopy()
902 		head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,  in handle_tx_zerocopy()
924 			ubuf = nvq->ubuf_info + nvq->upend_idx;  in handle_tx_zerocopy()
925 			vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);  in handle_tx_zerocopy()
926 			vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;  in handle_tx_zerocopy()
927 			ubuf->ctx = nvq->ubufs;  in handle_tx_zerocopy()
928 			ubuf->desc = nvq->upend_idx;  in handle_tx_zerocopy()
936 			ubufs = nvq->ubufs;  in handle_tx_zerocopy()
938 			nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;  in handle_tx_zerocopy()
959 					nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)  in handle_tx_zerocopy()
985 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx()  local
986 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx()
1062 static int get_rx_bufs(struct vhost_net_virtqueue *nvq,  in get_rx_bufs()  argument
1072 	struct vhost_virtqueue *vq = &nvq->vq;  in get_rx_bufs()
1151 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];  in handle_rx()  local
1152 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_rx()
1192 	vhost_hlen = nvq->vhost_hlen;  in handle_rx()
1193 	sock_hlen = nvq->sock_hlen;  in handle_rx()
1208 		headcount = get_rx_bufs(nvq, vq->heads + count,  in handle_rx()
1231 		if (nvq->rx_ring)  in handle_rx()
1232 			msg.msg_control = vhost_net_buf_consume(&nvq->rxq);  in handle_rx()
1285 		nvq->done_idx += headcount;  in handle_rx()
1287 		if (nvq->done_idx > VHOST_NET_BATCH) {  in handle_rx()
1288 			vhost_net_signal_used(nvq, count);  in handle_rx()
1302 	vhost_net_signal_used(nvq, count);  in handle_rx()
1410 	struct vhost_net_virtqueue *nvq =  in vhost_net_stop_vq()  local
1417 	vhost_net_buf_unproduce(nvq);  in vhost_net_stop_vq()
1418 	nvq->rx_ring = NULL;  in vhost_net_stop_vq()
1548 	struct vhost_net_virtqueue *nvq;  in vhost_net_set_backend()  local
1562 	nvq = &n->vqs[index];  in vhost_net_set_backend()
1591 		vhost_net_buf_unproduce(nvq);  in vhost_net_set_backend()
1600 				nvq->rx_ring = get_tap_ptr_ring(sock->file);  in vhost_net_set_backend()
1602 				nvq->rx_ring = NULL;  in vhost_net_set_backend()
1605 		oldubufs = nvq->ubufs;  in vhost_net_set_backend()
1606 		nvq->ubufs = ubufs;  in vhost_net_set_backend()