81 	struct virtio_vsock *vsock;  in virtio_transport_get_local_cid()  local
85 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_get_local_cid()
86 	if (!vsock) {  in virtio_transport_get_local_cid()
91 	ret = vsock->guest_cid;  in virtio_transport_get_local_cid()
99 				     struct virtio_vsock *vsock, gfp_t gfp)  in virtio_transport_send_skb()  argument
104 	sgs = vsock->out_sgs;  in virtio_transport_send_skb()
157 	struct virtio_vsock *vsock =  in virtio_transport_send_pkt_work()  local
163 	mutex_lock(&vsock->tx_lock);  in virtio_transport_send_pkt_work()
165 	if (!vsock->tx_run)  in virtio_transport_send_pkt_work()
168 	vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_send_pkt_work()
175 		skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue);  in virtio_transport_send_pkt_work()
181 		ret = virtio_transport_send_skb(skb, vq, vsock, GFP_KERNEL);  in virtio_transport_send_pkt_work()
183 			virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);  in virtio_transport_send_pkt_work()
188 			struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_send_pkt_work()
191 			val = atomic_dec_return(&vsock->queued_replies);  in virtio_transport_send_pkt_work()
205 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_send_pkt_work()
208 		queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_transport_send_pkt_work()
214 static int virtio_transport_send_skb_fast_path(struct virtio_vsock *vsock, struct sk_buff *skb)  in virtio_transport_send_skb_fast_path()  argument
216 	struct virtqueue *vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_send_skb_fast_path()
220 	ret = mutex_trylock(&vsock->tx_lock);  in virtio_transport_send_skb_fast_path()
224 	ret = virtio_transport_send_skb(skb, vq, vsock, GFP_ATOMIC);  in virtio_transport_send_skb_fast_path()
228 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_send_skb_fast_path()
237 	struct virtio_vsock *vsock;  in virtio_transport_send_pkt()  local
243 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_send_pkt()
244 	if (!vsock) {  in virtio_transport_send_pkt()
250 	if (le64_to_cpu(hdr->dst_cid) == vsock->guest_cid) {  in virtio_transport_send_pkt()
262 	if (!skb_queue_empty_lockless(&vsock->send_pkt_queue) ||  in virtio_transport_send_pkt()
263 	    virtio_transport_send_skb_fast_path(vsock, skb)) {  in virtio_transport_send_pkt()
265 			atomic_inc(&vsock->queued_replies);  in virtio_transport_send_pkt()
267 		virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb);  in virtio_transport_send_pkt()
268 		queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_transport_send_pkt()
279 	struct virtio_vsock *vsock;  in virtio_transport_cancel_pkt()  local
283 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_cancel_pkt()
284 	if (!vsock) {  in virtio_transport_cancel_pkt()
289 	cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue);  in virtio_transport_cancel_pkt()
292 		struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_cancel_pkt()
295 		new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);  in virtio_transport_cancel_pkt()
298 			queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_transport_cancel_pkt()
308 static void virtio_vsock_rx_fill(struct virtio_vsock *vsock)  in virtio_vsock_rx_fill()  argument
316 	vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_vsock_rx_fill()
332 		vsock->rx_buf_nr++;  in virtio_vsock_rx_fill()
334 	if (vsock->rx_buf_nr > vsock->rx_buf_max_nr)  in virtio_vsock_rx_fill()
335 		vsock->rx_buf_max_nr = vsock->rx_buf_nr;  in virtio_vsock_rx_fill()
341 	struct virtio_vsock *vsock =  in virtio_transport_tx_work()  local
346 	vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_tx_work()
347 	mutex_lock(&vsock->tx_lock);  in virtio_transport_tx_work()
349 	if (!vsock->tx_run)  in virtio_transport_tx_work()
364 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_tx_work()
367 		queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_transport_tx_work()
371 static bool virtio_transport_more_replies(struct virtio_vsock *vsock)  in virtio_transport_more_replies()  argument
373 	struct virtqueue *vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_more_replies()
377 	val = atomic_read(&vsock->queued_replies);  in virtio_transport_more_replies()
383 static int virtio_vsock_event_fill_one(struct virtio_vsock *vsock,  in virtio_vsock_event_fill_one()  argument
389 	vq = vsock->vqs[VSOCK_VQ_EVENT];  in virtio_vsock_event_fill_one()
397 static void virtio_vsock_event_fill(struct virtio_vsock *vsock)  in virtio_vsock_event_fill()  argument
401 	for (i = 0; i < ARRAY_SIZE(vsock->event_list); i++) {  in virtio_vsock_event_fill()
402 		struct virtio_vsock_event *event = &vsock->event_list[i];  in virtio_vsock_event_fill()
404 		virtio_vsock_event_fill_one(vsock, event);  in virtio_vsock_event_fill()
407 	virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);  in virtio_vsock_event_fill()
422 static void virtio_vsock_update_guest_cid(struct virtio_vsock *vsock)  in virtio_vsock_update_guest_cid()  argument
424 	struct virtio_device *vdev = vsock->vdev;  in virtio_vsock_update_guest_cid()
429 	vsock->guest_cid = le64_to_cpu(guest_cid);  in virtio_vsock_update_guest_cid()
433 static void virtio_vsock_event_handle(struct virtio_vsock *vsock,  in virtio_vsock_event_handle()  argument
438 		virtio_vsock_update_guest_cid(vsock);  in virtio_vsock_event_handle()
447 	struct virtio_vsock *vsock =  in virtio_transport_event_work()  local
451 	vq = vsock->vqs[VSOCK_VQ_EVENT];  in virtio_transport_event_work()
453 	mutex_lock(&vsock->event_lock);  in virtio_transport_event_work()
455 	if (!vsock->event_run)  in virtio_transport_event_work()
465 				virtio_vsock_event_handle(vsock, event);  in virtio_transport_event_work()
467 			virtio_vsock_event_fill_one(vsock, event);  in virtio_transport_event_work()
471 	virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);  in virtio_transport_event_work()
473 	mutex_unlock(&vsock->event_lock);  in virtio_transport_event_work()
478 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_event_done()  local
480 	if (!vsock)  in virtio_vsock_event_done()
482 	queue_work(virtio_vsock_workqueue, &vsock->event_work);  in virtio_vsock_event_done()
487 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_tx_done()  local
489 	if (!vsock)  in virtio_vsock_tx_done()
491 	queue_work(virtio_vsock_workqueue, &vsock->tx_work);  in virtio_vsock_tx_done()
496 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_rx_done()  local
498 	if (!vsock)  in virtio_vsock_rx_done()
500 	queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_vsock_rx_done()
505 	struct virtio_vsock *vsock;  in virtio_transport_can_msgzerocopy()  local
510 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_can_msgzerocopy()
511 	if (vsock) {  in virtio_transport_can_msgzerocopy()
512 		struct virtqueue *vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_can_msgzerocopy()
598 	struct virtio_vsock *vsock;  in virtio_transport_seqpacket_allow()  local
603 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_seqpacket_allow()
604 	if (vsock)  in virtio_transport_seqpacket_allow()
605 		seqpacket_allow = vsock->seqpacket_allow;  in virtio_transport_seqpacket_allow()
613 	struct virtio_vsock *vsock =  in virtio_transport_rx_work()  local
617 	vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_rx_work()
619 	mutex_lock(&vsock->rx_lock);  in virtio_transport_rx_work()
621 	if (!vsock->rx_run)  in virtio_transport_rx_work()
631 			if (!virtio_transport_more_replies(vsock)) {  in virtio_transport_rx_work()
643 			vsock->rx_buf_nr--;  in virtio_transport_rx_work()
668 	if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2)  in virtio_transport_rx_work()
669 		virtio_vsock_rx_fill(vsock);  in virtio_transport_rx_work()
670 	mutex_unlock(&vsock->rx_lock);  in virtio_transport_rx_work()
673 static int virtio_vsock_vqs_init(struct virtio_vsock *vsock)  in virtio_vsock_vqs_init()  argument
675 	struct virtio_device *vdev = vsock->vdev;  in virtio_vsock_vqs_init()
683 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_vqs_init()
684 	vsock->rx_buf_nr = 0;  in virtio_vsock_vqs_init()
685 	vsock->rx_buf_max_nr = 0;  in virtio_vsock_vqs_init()
686 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_vqs_init()
688 	atomic_set(&vsock->queued_replies, 0);  in virtio_vsock_vqs_init()
690 	ret = virtio_find_vqs(vdev, VSOCK_VQ_MAX, vsock->vqs, vqs_info, NULL);  in virtio_vsock_vqs_init()
694 	virtio_vsock_update_guest_cid(vsock);  in virtio_vsock_vqs_init()
701 static void virtio_vsock_vqs_start(struct virtio_vsock *vsock)  in virtio_vsock_vqs_start()  argument
703 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_vqs_start()
704 	vsock->tx_run = true;  in virtio_vsock_vqs_start()
705 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_vqs_start()
707 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_vqs_start()
708 	virtio_vsock_rx_fill(vsock);  in virtio_vsock_vqs_start()
709 	vsock->rx_run = true;  in virtio_vsock_vqs_start()
710 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_vqs_start()
712 	mutex_lock(&vsock->event_lock);  in virtio_vsock_vqs_start()
713 	virtio_vsock_event_fill(vsock);  in virtio_vsock_vqs_start()
714 	vsock->event_run = true;  in virtio_vsock_vqs_start()
715 	mutex_unlock(&vsock->event_lock);  in virtio_vsock_vqs_start()
726 	queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_vsock_vqs_start()
729 static void virtio_vsock_vqs_del(struct virtio_vsock *vsock)  in virtio_vsock_vqs_del()  argument
731 	struct virtio_device *vdev = vsock->vdev;  in virtio_vsock_vqs_del()
741 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_vqs_del()
742 	vsock->rx_run = false;  in virtio_vsock_vqs_del()
743 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_vqs_del()
745 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_vqs_del()
746 	vsock->tx_run = false;  in virtio_vsock_vqs_del()
747 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_vqs_del()
749 	mutex_lock(&vsock->event_lock);  in virtio_vsock_vqs_del()
750 	vsock->event_run = false;  in virtio_vsock_vqs_del()
751 	mutex_unlock(&vsock->event_lock);  in virtio_vsock_vqs_del()
758 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_vqs_del()
759 	while ((skb = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX])))  in virtio_vsock_vqs_del()
761 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_vqs_del()
763 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_vqs_del()
764 	while ((skb = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX])))  in virtio_vsock_vqs_del()
766 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_vqs_del()
768 	virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue);  in virtio_vsock_vqs_del()
776 	struct virtio_vsock *vsock = NULL;  in virtio_vsock_probe()  local
791 	vsock = kzalloc(sizeof(*vsock), GFP_KERNEL);  in virtio_vsock_probe()
792 	if (!vsock) {  in virtio_vsock_probe()
797 	vsock->vdev = vdev;  in virtio_vsock_probe()
800 	mutex_init(&vsock->tx_lock);  in virtio_vsock_probe()
801 	mutex_init(&vsock->rx_lock);  in virtio_vsock_probe()
802 	mutex_init(&vsock->event_lock);  in virtio_vsock_probe()
803 	skb_queue_head_init(&vsock->send_pkt_queue);  in virtio_vsock_probe()
804 	INIT_WORK(&vsock->rx_work, virtio_transport_rx_work);  in virtio_vsock_probe()
805 	INIT_WORK(&vsock->tx_work, virtio_transport_tx_work);  in virtio_vsock_probe()
806 	INIT_WORK(&vsock->event_work, virtio_transport_event_work);  in virtio_vsock_probe()
807 	INIT_WORK(&vsock->send_pkt_work, virtio_transport_send_pkt_work);  in virtio_vsock_probe()
810 		vsock->seqpacket_allow = true;  in virtio_vsock_probe()
812 	vdev->priv = vsock;  in virtio_vsock_probe()
814 	ret = virtio_vsock_vqs_init(vsock);  in virtio_vsock_probe()
818 	for (i = 0; i < ARRAY_SIZE(vsock->out_sgs); i++)  in virtio_vsock_probe()
819 		vsock->out_sgs[i] = &vsock->out_bufs[i];  in virtio_vsock_probe()
821 	rcu_assign_pointer(the_virtio_vsock, vsock);  in virtio_vsock_probe()
822 	virtio_vsock_vqs_start(vsock);  in virtio_vsock_probe()
829 	kfree(vsock);  in virtio_vsock_probe()
836 	struct virtio_vsock *vsock = vdev->priv;  in virtio_vsock_remove()  local
844 	virtio_vsock_vqs_del(vsock);  in virtio_vsock_remove()
849 	flush_work(&vsock->rx_work);  in virtio_vsock_remove()
850 	flush_work(&vsock->tx_work);  in virtio_vsock_remove()
851 	flush_work(&vsock->event_work);  in virtio_vsock_remove()
852 	flush_work(&vsock->send_pkt_work);  in virtio_vsock_remove()
856 	kfree(vsock);  in virtio_vsock_remove()
862 	struct virtio_vsock *vsock = vdev->priv;  in virtio_vsock_freeze()  local
869 	virtio_vsock_vqs_del(vsock);  in virtio_vsock_freeze()
878 	struct virtio_vsock *vsock = vdev->priv;  in virtio_vsock_restore()  local
890 	ret = virtio_vsock_vqs_init(vsock);  in virtio_vsock_restore()
894 	rcu_assign_pointer(the_virtio_vsock, vsock);  in virtio_vsock_restore()
895 	virtio_vsock_vqs_start(vsock);  in virtio_vsock_restore()