72 	struct vhost_vsock *vsock;  in vhost_vsock_get()  local
74 	hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid) {  in vhost_vsock_get()
75 		u32 other_cid = vsock->guest_cid;  in vhost_vsock_get()
82 			return vsock;  in vhost_vsock_get()
90 vhost_transport_do_send_pkt(struct vhost_vsock *vsock,  in vhost_transport_do_send_pkt()  argument
93 	struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];  in vhost_transport_do_send_pkt()
107 	vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
120 		skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue);  in vhost_transport_do_send_pkt()
123 			vhost_enable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
130 			virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);  in vhost_transport_do_send_pkt()
135 			virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);  in vhost_transport_do_send_pkt()
139 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_transport_do_send_pkt()
140 				vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
233 			virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);  in vhost_transport_do_send_pkt()
238 				val = atomic_dec_return(&vsock->queued_replies);  in vhost_transport_do_send_pkt()
251 		vhost_signal(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
263 	struct vhost_vsock *vsock;  in vhost_transport_send_pkt_work()  local
265 	vsock = container_of(work, struct vhost_vsock, send_pkt_work);  in vhost_transport_send_pkt_work()
266 	vq = &vsock->vqs[VSOCK_VQ_RX];  in vhost_transport_send_pkt_work()
268 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_transport_send_pkt_work()
275 	struct vhost_vsock *vsock;  in vhost_transport_send_pkt()  local
281 	vsock = vhost_vsock_get(le64_to_cpu(hdr->dst_cid));  in vhost_transport_send_pkt()
282 	if (!vsock) {  in vhost_transport_send_pkt()
289 		atomic_inc(&vsock->queued_replies);  in vhost_transport_send_pkt()
291 	virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb);  in vhost_transport_send_pkt()
292 	vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work);  in vhost_transport_send_pkt()
301 	struct vhost_vsock *vsock;  in vhost_transport_cancel_pkt()  local
308 	vsock = vhost_vsock_get(vsk->remote_addr.svm_cid);  in vhost_transport_cancel_pkt()
309 	if (!vsock)  in vhost_transport_cancel_pkt()
312 	cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue);  in vhost_transport_cancel_pkt()
315 		struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];  in vhost_transport_cancel_pkt()
318 		new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);  in vhost_transport_cancel_pkt()
390 static bool vhost_vsock_more_replies(struct vhost_vsock *vsock)  in vhost_vsock_more_replies()  argument
392 	struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];  in vhost_vsock_more_replies()
396 	val = atomic_read(&vsock->queued_replies);  in vhost_vsock_more_replies()
464 	struct vhost_vsock *vsock;  in vhost_transport_seqpacket_allow()  local
468 	vsock = vhost_vsock_get(remote_cid);  in vhost_transport_seqpacket_allow()
470 	if (vsock)  in vhost_transport_seqpacket_allow()
471 		seqpacket_allow = vsock->seqpacket_allow;  in vhost_transport_seqpacket_allow()
482 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_tx_kick()  local
497 	vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
501 		if (!vhost_vsock_more_replies(vsock)) {  in vhost_vsock_handle_tx_kick()
515 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_vsock_handle_tx_kick()
516 				vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
536 		if (le64_to_cpu(hdr->src_cid) == vsock->guest_cid &&  in vhost_vsock_handle_tx_kick()
549 		vhost_signal(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
559 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_rx_kick()  local
562 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_vsock_handle_rx_kick()
565 static int vhost_vsock_start(struct vhost_vsock *vsock)  in vhost_vsock_start()  argument
571 	mutex_lock(&vsock->dev.mutex);  in vhost_vsock_start()
573 	ret = vhost_dev_check_owner(&vsock->dev);  in vhost_vsock_start()
577 	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {  in vhost_vsock_start()
578 		vq = &vsock->vqs[i];  in vhost_vsock_start()
588 			vhost_vq_set_backend(vq, vsock);  in vhost_vsock_start()
600 	vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work);  in vhost_vsock_start()
602 	mutex_unlock(&vsock->dev.mutex);  in vhost_vsock_start()
609 	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {  in vhost_vsock_start()
610 		vq = &vsock->vqs[i];  in vhost_vsock_start()
617 	mutex_unlock(&vsock->dev.mutex);  in vhost_vsock_start()
621 static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner)  in vhost_vsock_stop()  argument
626 	mutex_lock(&vsock->dev.mutex);  in vhost_vsock_stop()
629 		ret = vhost_dev_check_owner(&vsock->dev);  in vhost_vsock_stop()
634 	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {  in vhost_vsock_stop()
635 		struct vhost_virtqueue *vq = &vsock->vqs[i];  in vhost_vsock_stop()
643 	mutex_unlock(&vsock->dev.mutex);  in vhost_vsock_stop()
647 static void vhost_vsock_free(struct vhost_vsock *vsock)  in vhost_vsock_free()  argument
649 	kvfree(vsock);  in vhost_vsock_free()
655 	struct vhost_vsock *vsock;  in vhost_vsock_dev_open()  local
661 	vsock = kvmalloc(sizeof(*vsock), GFP_KERNEL | __GFP_RETRY_MAYFAIL);  in vhost_vsock_dev_open()
662 	if (!vsock)  in vhost_vsock_dev_open()
665 	vqs = kmalloc_array(ARRAY_SIZE(vsock->vqs), sizeof(*vqs), GFP_KERNEL);  in vhost_vsock_dev_open()
671 	vsock->guest_cid = 0; /* no CID assigned yet */  in vhost_vsock_dev_open()
672 	vsock->seqpacket_allow = false;  in vhost_vsock_dev_open()
674 	atomic_set(&vsock->queued_replies, 0);  in vhost_vsock_dev_open()
676 	vqs[VSOCK_VQ_TX] = &vsock->vqs[VSOCK_VQ_TX];  in vhost_vsock_dev_open()
677 	vqs[VSOCK_VQ_RX] = &vsock->vqs[VSOCK_VQ_RX];  in vhost_vsock_dev_open()
678 	vsock->vqs[VSOCK_VQ_TX].handle_kick = vhost_vsock_handle_tx_kick;  in vhost_vsock_dev_open()
679 	vsock->vqs[VSOCK_VQ_RX].handle_kick = vhost_vsock_handle_rx_kick;  in vhost_vsock_dev_open()
681 	vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs),  in vhost_vsock_dev_open()
685 	file->private_data = vsock;  in vhost_vsock_dev_open()
686 	skb_queue_head_init(&vsock->send_pkt_queue);  in vhost_vsock_dev_open()
687 	vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work);  in vhost_vsock_dev_open()
691 	vhost_vsock_free(vsock);  in vhost_vsock_dev_open()
695 static void vhost_vsock_flush(struct vhost_vsock *vsock)  in vhost_vsock_flush()  argument
697 	vhost_dev_flush(&vsock->dev);  in vhost_vsock_flush()
728 	struct vhost_vsock *vsock = file->private_data;  in vhost_vsock_dev_release()  local
731 	if (vsock->guest_cid)  in vhost_vsock_dev_release()
732 		hash_del_rcu(&vsock->hash);  in vhost_vsock_dev_release()
748 	vhost_vsock_stop(vsock, false);  in vhost_vsock_dev_release()
749 	vhost_vsock_flush(vsock);  in vhost_vsock_dev_release()
750 	vhost_dev_stop(&vsock->dev);  in vhost_vsock_dev_release()
752 	virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue);  in vhost_vsock_dev_release()
754 	vhost_dev_cleanup(&vsock->dev);  in vhost_vsock_dev_release()
755 	kfree(vsock->dev.vqs);  in vhost_vsock_dev_release()
756 	vhost_vsock_free(vsock);  in vhost_vsock_dev_release()
760 static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid)  in vhost_vsock_set_cid()  argument
782 	if (other && other != vsock) {  in vhost_vsock_set_cid()
787 	if (vsock->guest_cid)  in vhost_vsock_set_cid()
788 		hash_del_rcu(&vsock->hash);  in vhost_vsock_set_cid()
790 	vsock->guest_cid = guest_cid;  in vhost_vsock_set_cid()
791 	hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid);  in vhost_vsock_set_cid()
797 static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)  in vhost_vsock_set_features()  argument
805 	mutex_lock(&vsock->dev.mutex);  in vhost_vsock_set_features()
807 	    !vhost_log_access_ok(&vsock->dev)) {  in vhost_vsock_set_features()
812 		if (vhost_init_device_iotlb(&vsock->dev))  in vhost_vsock_set_features()
816 	vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET);  in vhost_vsock_set_features()
818 	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {  in vhost_vsock_set_features()
819 		vq = &vsock->vqs[i];  in vhost_vsock_set_features()
824 	mutex_unlock(&vsock->dev.mutex);  in vhost_vsock_set_features()
828 	mutex_unlock(&vsock->dev.mutex);  in vhost_vsock_set_features()
835 	struct vhost_vsock *vsock = f->private_data;  in vhost_vsock_dev_ioctl()  local
846 		return vhost_vsock_set_cid(vsock, guest_cid);  in vhost_vsock_dev_ioctl()
851 			return vhost_vsock_start(vsock);  in vhost_vsock_dev_ioctl()
853 			return vhost_vsock_stop(vsock, true);  in vhost_vsock_dev_ioctl()
862 		return vhost_vsock_set_features(vsock, features);  in vhost_vsock_dev_ioctl()
873 		vhost_set_backend_features(&vsock->dev, features);  in vhost_vsock_dev_ioctl()
876 		mutex_lock(&vsock->dev.mutex);  in vhost_vsock_dev_ioctl()
877 		r = vhost_dev_ioctl(&vsock->dev, ioctl, argp);  in vhost_vsock_dev_ioctl()
879 			r = vhost_vring_ioctl(&vsock->dev, ioctl, argp);  in vhost_vsock_dev_ioctl()
881 			vhost_vsock_flush(vsock);  in vhost_vsock_dev_ioctl()
882 		mutex_unlock(&vsock->dev.mutex);  in vhost_vsock_dev_ioctl()
890 	struct vhost_vsock *vsock = file->private_data;  in vhost_vsock_chr_read_iter()  local
891 	struct vhost_dev *dev = &vsock->dev;  in vhost_vsock_chr_read_iter()
901 	struct vhost_vsock *vsock = file->private_data;  in vhost_vsock_chr_write_iter()  local
902 	struct vhost_dev *dev = &vsock->dev;  in vhost_vsock_chr_write_iter()
909 	struct vhost_vsock *vsock = file->private_data;  in vhost_vsock_chr_poll()  local
910 	struct vhost_dev *dev = &vsock->dev;  in vhost_vsock_chr_poll()