xref: /linux/drivers/vhost/vsock.c (revision d5e912448cd98b261eba8e35ab838f2dc69a79f5)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * vhost transport for vsock
4  *
5  * Copyright (C) 2013-2015 Red Hat, Inc.
6  * Author: Asias He <asias@redhat.com>
7  *         Stefan Hajnoczi <stefanha@redhat.com>
8  */
9 #include <linux/miscdevice.h>
10 #include <linux/atomic.h>
11 #include <linux/module.h>
12 #include <linux/mutex.h>
13 #include <linux/vmalloc.h>
14 #include <net/sock.h>
15 #include <linux/virtio_vsock.h>
16 #include <linux/vhost.h>
17 #include <linux/hashtable.h>
18 
19 #include <net/af_vsock.h>
20 #include "vhost.h"
21 
22 #define VHOST_VSOCK_DEFAULT_HOST_CID	2
23 /* Max number of bytes transferred before requeueing the job.
24  * Using this limit prevents one virtqueue from starving others. */
25 #define VHOST_VSOCK_WEIGHT 0x80000
26 /* Max number of packets transferred before requeueing the job.
27  * Using this limit prevents one virtqueue from starving others with
28  * small pkts.
29  */
30 #define VHOST_VSOCK_PKT_WEIGHT 256
31 
32 static const int vhost_vsock_bits[] = {
33 	VHOST_FEATURES,
34 	VIRTIO_F_ACCESS_PLATFORM,
35 	VIRTIO_VSOCK_F_SEQPACKET
36 };
37 
38 #define VHOST_VSOCK_FEATURES VHOST_FEATURES_U64(vhost_vsock_bits, 0)
39 
40 enum {
41 	VHOST_VSOCK_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2)
42 };
43 
44 /* Used to track all the vhost_vsock instances on the system. */
45 static DEFINE_MUTEX(vhost_vsock_mutex);
46 static DEFINE_READ_MOSTLY_HASHTABLE(vhost_vsock_hash, 8);
47 
48 struct vhost_vsock {
49 	struct vhost_dev dev;
50 	struct vhost_virtqueue vqs[2];
51 	struct net *net;
52 	netns_tracker ns_tracker;
53 
54 	/* Link to global vhost_vsock_hash, writes use vhost_vsock_mutex */
55 	struct hlist_node hash;
56 
57 	struct vhost_work send_pkt_work;
58 	struct sk_buff_head send_pkt_queue; /* host->guest pending packets */
59 
60 	atomic_t queued_replies;
61 
62 	u32 guest_cid;
63 	bool seqpacket_allow;
64 	bool ever_started; /* set on first SET_RUNNING(1); never cleared */
65 };
66 
67 static u32 vhost_transport_get_local_cid(void)
68 {
69 	return VHOST_VSOCK_DEFAULT_HOST_CID;
70 }
71 
72 /* Callers must be in an RCU read section or hold the vhost_vsock_mutex.
73  * The return value can only be dereferenced while within the section.
74  */
75 static struct vhost_vsock *vhost_vsock_get(u32 guest_cid, struct net *net)
76 {
77 	struct vhost_vsock *vsock;
78 
79 	hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid,
80 				   lockdep_is_held(&vhost_vsock_mutex)) {
81 		u32 other_cid = vsock->guest_cid;
82 
83 		/* Skip instances that have no CID yet */
84 		if (other_cid == 0)
85 			continue;
86 
87 		if (other_cid == guest_cid &&
88 		    vsock_net_check_mode(net, vsock->net))
89 			return vsock;
90 	}
91 
92 	return NULL;
93 }
94 
95 static bool vhost_transport_has_remote_cid(struct vsock_sock *vsk, u32 cid)
96 {
97 	struct sock *sk = sk_vsock(vsk);
98 	struct net *net = sock_net(sk);
99 	bool found;
100 
101 	rcu_read_lock();
102 	found = !!vhost_vsock_get(cid, net);
103 	rcu_read_unlock();
104 	return found;
105 }
106 
107 static void
108 vhost_transport_do_send_pkt(struct vhost_vsock *vsock,
109 			    struct vhost_virtqueue *vq)
110 {
111 	struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];
112 	int pkts = 0, total_len = 0;
113 	bool added = false;
114 	bool restart_tx = false;
115 
116 	mutex_lock(&vq->mutex);
117 
118 	if (!vhost_vq_get_backend(vq))
119 		goto out;
120 
121 	if (!vq_meta_prefetch(vq))
122 		goto out;
123 
124 	/* Avoid further vmexits, we're already processing the virtqueue */
125 	vhost_disable_notify(&vsock->dev, vq);
126 
127 	do {
128 		struct virtio_vsock_hdr *hdr;
129 		size_t iov_len, payload_len;
130 		struct iov_iter iov_iter;
131 		u32 flags_to_restore = 0;
132 		struct sk_buff *skb;
133 		unsigned out, in;
134 		size_t nbytes;
135 		u32 offset;
136 		int head;
137 
138 		skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue);
139 
140 		if (!skb) {
141 			vhost_enable_notify(&vsock->dev, vq);
142 			break;
143 		}
144 
145 		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
146 					 &out, &in, NULL, NULL);
147 		if (head < 0) {
148 			virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
149 			break;
150 		}
151 
152 		if (head == vq->num) {
153 			virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
154 			/* We cannot finish yet if more buffers snuck in while
155 			 * re-enabling notify.
156 			 */
157 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {
158 				vhost_disable_notify(&vsock->dev, vq);
159 				continue;
160 			}
161 			break;
162 		}
163 
164 		if (out) {
165 			kfree_skb(skb);
166 			vq_err(vq, "Expected 0 output buffers, got %u\n", out);
167 			break;
168 		}
169 
170 		iov_len = iov_length(&vq->iov[out], in);
171 		if (iov_len < sizeof(*hdr)) {
172 			kfree_skb(skb);
173 			vq_err(vq, "Buffer len [%zu] too small\n", iov_len);
174 			break;
175 		}
176 
177 		iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[out], in, iov_len);
178 		offset = VIRTIO_VSOCK_SKB_CB(skb)->offset;
179 		payload_len = skb->len - offset;
180 		hdr = virtio_vsock_hdr(skb);
181 
182 		/* If the packet is greater than the space available in the
183 		 * buffer, we split it using multiple buffers.
184 		 */
185 		if (payload_len > iov_len - sizeof(*hdr)) {
186 			payload_len = iov_len - sizeof(*hdr);
187 
188 			/* As we are copying pieces of large packet's buffer to
189 			 * small rx buffers, headers of packets in rx queue are
190 			 * created dynamically and are initialized with header
191 			 * of current packet(except length). But in case of
192 			 * SOCK_SEQPACKET, we also must clear message delimeter
193 			 * bit (VIRTIO_VSOCK_SEQ_EOM) and MSG_EOR bit
194 			 * (VIRTIO_VSOCK_SEQ_EOR) if set. Otherwise,
195 			 * there will be sequence of packets with these
196 			 * bits set. After initialized header will be copied to
197 			 * rx buffer, these required bits will be restored.
198 			 */
199 			if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) {
200 				hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM);
201 				flags_to_restore |= VIRTIO_VSOCK_SEQ_EOM;
202 
203 				if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) {
204 					hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR);
205 					flags_to_restore |= VIRTIO_VSOCK_SEQ_EOR;
206 				}
207 			}
208 		}
209 
210 		/* Set the correct length in the header */
211 		hdr->len = cpu_to_le32(payload_len);
212 
213 		nbytes = copy_to_iter(hdr, sizeof(*hdr), &iov_iter);
214 		if (nbytes != sizeof(*hdr)) {
215 			kfree_skb(skb);
216 			vq_err(vq, "Faulted on copying pkt hdr\n");
217 			break;
218 		}
219 
220 		if (skb_copy_datagram_iter(skb,
221 					   offset,
222 					   &iov_iter,
223 					   payload_len)) {
224 			kfree_skb(skb);
225 			vq_err(vq, "Faulted on copying pkt buf\n");
226 			break;
227 		}
228 
229 		/* Deliver to monitoring devices all packets that we
230 		 * will transmit.
231 		 */
232 		virtio_transport_deliver_tap_pkt(skb);
233 
234 		vhost_add_used(vq, head, sizeof(*hdr) + payload_len);
235 		added = true;
236 
237 		VIRTIO_VSOCK_SKB_CB(skb)->offset += payload_len;
238 		total_len += payload_len;
239 
240 		/* If we didn't send all the payload we can requeue the packet
241 		 * to send it with the next available buffer.
242 		 */
243 		if (VIRTIO_VSOCK_SKB_CB(skb)->offset < skb->len) {
244 			hdr->flags |= cpu_to_le32(flags_to_restore);
245 
246 			/* We are queueing the same skb to handle
247 			 * the remaining bytes, and we want to deliver it
248 			 * to monitoring devices in the next iteration.
249 			 */
250 			virtio_vsock_skb_clear_tap_delivered(skb);
251 			virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
252 		} else {
253 			if (virtio_vsock_skb_reply(skb)) {
254 				int val;
255 
256 				val = atomic_dec_return(&vsock->queued_replies);
257 
258 				/* Do we have resources to resume tx
259 				 * processing?
260 				 */
261 				if (val + 1 == tx_vq->num)
262 					restart_tx = true;
263 			}
264 
265 			virtio_transport_consume_skb_sent(skb, true);
266 		}
267 	} while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
268 	if (added)
269 		vhost_signal(&vsock->dev, vq);
270 
271 out:
272 	mutex_unlock(&vq->mutex);
273 
274 	if (restart_tx)
275 		vhost_poll_queue(&tx_vq->poll);
276 }
277 
278 static void vhost_transport_send_pkt_work(struct vhost_work *work)
279 {
280 	struct vhost_virtqueue *vq;
281 	struct vhost_vsock *vsock;
282 
283 	vsock = container_of(work, struct vhost_vsock, send_pkt_work);
284 	vq = &vsock->vqs[VSOCK_VQ_RX];
285 
286 	vhost_transport_do_send_pkt(vsock, vq);
287 }
288 
289 static int
290 vhost_transport_send_pkt(struct sk_buff *skb, struct net *net)
291 {
292 	struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
293 	struct vhost_vsock *vsock;
294 	int len = skb->len;
295 
296 	rcu_read_lock();
297 
298 	/* Find the vhost_vsock according to guest context id  */
299 	vsock = vhost_vsock_get(le64_to_cpu(hdr->dst_cid), net);
300 	if (!vsock) {
301 		rcu_read_unlock();
302 		kfree_skb(skb);
303 		return -ENODEV;
304 	}
305 
306 	/* Fast-fail until the guest first enables the device (SET_RUNNING(1)).
307 	 * Before that there is no listener, so queuing is pointless.
308 	 * 'ever_started' is never cleared, so once we're up we keep queuing
309 	 * across later stop / CPR-pause windows.
310 	 */
311 	if (unlikely(!READ_ONCE(vsock->ever_started))) {
312 		rcu_read_unlock();
313 		kfree_skb(skb);
314 		return -EHOSTUNREACH;
315 	}
316 
317 	if (virtio_vsock_skb_reply(skb))
318 		atomic_inc(&vsock->queued_replies);
319 
320 	virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb);
321 	vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work);
322 
323 	rcu_read_unlock();
324 	return len;
325 }
326 
327 static int
328 vhost_transport_cancel_pkt(struct vsock_sock *vsk)
329 {
330 	struct vhost_vsock *vsock;
331 	int cnt = 0;
332 	int ret = -ENODEV;
333 
334 	rcu_read_lock();
335 
336 	/* Find the vhost_vsock according to guest context id  */
337 	vsock = vhost_vsock_get(vsk->remote_addr.svm_cid,
338 				sock_net(sk_vsock(vsk)));
339 	if (!vsock)
340 		goto out;
341 
342 	cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue);
343 
344 	if (cnt) {
345 		struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];
346 		int new_cnt;
347 
348 		new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);
349 		if (new_cnt + cnt >= tx_vq->num && new_cnt < tx_vq->num)
350 			vhost_poll_queue(&tx_vq->poll);
351 	}
352 
353 	ret = 0;
354 out:
355 	rcu_read_unlock();
356 	return ret;
357 }
358 
359 static struct sk_buff *
360 vhost_vsock_alloc_skb(struct vhost_virtqueue *vq,
361 		      unsigned int out, unsigned int in)
362 {
363 	struct virtio_vsock_hdr hdr;
364 	struct iov_iter iov_iter;
365 	struct sk_buff *skb;
366 	size_t payload_len;
367 	size_t nbytes;
368 	size_t len;
369 
370 	if (in != 0) {
371 		vq_err(vq, "Expected 0 input buffers, got %u\n", in);
372 		return NULL;
373 	}
374 
375 	len = iov_length(vq->iov, out);
376 
377 	if (len < VIRTIO_VSOCK_SKB_HEADROOM ||
378 	    len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE + VIRTIO_VSOCK_SKB_HEADROOM)
379 		return NULL;
380 
381 	iov_iter_init(&iov_iter, ITER_SOURCE, vq->iov, out, len);
382 
383 	nbytes = copy_from_iter(&hdr, sizeof(hdr), &iov_iter);
384 	if (nbytes != sizeof(hdr)) {
385 		vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",
386 		       sizeof(hdr), nbytes);
387 		return NULL;
388 	}
389 
390 	payload_len = le32_to_cpu(hdr.len);
391 
392 	/* The pkt is too big or the length in the header is invalid */
393 	if (payload_len > len - sizeof(hdr))
394 		return NULL;
395 
396 	/* Allocate only for the payload declared in the header. */
397 	skb = virtio_vsock_alloc_skb(payload_len + sizeof(hdr), GFP_KERNEL);
398 	if (!skb)
399 		return NULL;
400 
401 	memcpy(virtio_vsock_hdr(skb), &hdr, sizeof(hdr));
402 
403 	/* No payload */
404 	if (!payload_len)
405 		return skb;
406 
407 	virtio_vsock_skb_put(skb, payload_len);
408 
409 	if (skb_copy_datagram_from_iter(skb, 0, &iov_iter, payload_len)) {
410 		vq_err(vq, "Failed to copy %zu byte payload\n", payload_len);
411 		kfree_skb(skb);
412 		return NULL;
413 	}
414 
415 	return skb;
416 }
417 
418 /* Is there space left for replies to rx packets? */
419 static bool vhost_vsock_more_replies(struct vhost_vsock *vsock)
420 {
421 	struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];
422 	int val;
423 
424 	smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */
425 	val = atomic_read(&vsock->queued_replies);
426 
427 	return val < vq->num;
428 }
429 
430 static bool vhost_transport_msgzerocopy_allow(void)
431 {
432 	return true;
433 }
434 
435 static bool vhost_transport_seqpacket_allow(struct vsock_sock *vsk,
436 					    u32 remote_cid);
437 
438 static bool
439 vhost_transport_stream_allow(struct vsock_sock *vsk, u32 cid, u32 port)
440 {
441 	return true;
442 }
443 
444 static struct virtio_transport vhost_transport = {
445 	.transport = {
446 		.module                   = THIS_MODULE,
447 
448 		.get_local_cid            = vhost_transport_get_local_cid,
449 		.has_remote_cid           = vhost_transport_has_remote_cid,
450 
451 		.init                     = virtio_transport_do_socket_init,
452 		.destruct                 = virtio_transport_destruct,
453 		.release                  = virtio_transport_release,
454 		.connect                  = virtio_transport_connect,
455 		.shutdown                 = virtio_transport_shutdown,
456 		.cancel_pkt               = vhost_transport_cancel_pkt,
457 
458 		.dgram_enqueue            = virtio_transport_dgram_enqueue,
459 		.dgram_dequeue            = virtio_transport_dgram_dequeue,
460 		.dgram_bind               = virtio_transport_dgram_bind,
461 		.dgram_allow              = virtio_transport_dgram_allow,
462 
463 		.stream_enqueue           = virtio_transport_stream_enqueue,
464 		.stream_dequeue           = virtio_transport_stream_dequeue,
465 		.stream_has_data          = virtio_transport_stream_has_data,
466 		.stream_has_space         = virtio_transport_stream_has_space,
467 		.stream_rcvhiwat          = virtio_transport_stream_rcvhiwat,
468 		.stream_is_active         = virtio_transport_stream_is_active,
469 		.stream_allow             = vhost_transport_stream_allow,
470 
471 		.seqpacket_dequeue        = virtio_transport_seqpacket_dequeue,
472 		.seqpacket_enqueue        = virtio_transport_seqpacket_enqueue,
473 		.seqpacket_allow          = vhost_transport_seqpacket_allow,
474 		.seqpacket_has_data       = virtio_transport_seqpacket_has_data,
475 
476 		.msgzerocopy_allow        = vhost_transport_msgzerocopy_allow,
477 
478 		.notify_poll_in           = virtio_transport_notify_poll_in,
479 		.notify_poll_out          = virtio_transport_notify_poll_out,
480 		.notify_recv_init         = virtio_transport_notify_recv_init,
481 		.notify_recv_pre_block    = virtio_transport_notify_recv_pre_block,
482 		.notify_recv_pre_dequeue  = virtio_transport_notify_recv_pre_dequeue,
483 		.notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue,
484 		.notify_send_init         = virtio_transport_notify_send_init,
485 		.notify_send_pre_block    = virtio_transport_notify_send_pre_block,
486 		.notify_send_pre_enqueue  = virtio_transport_notify_send_pre_enqueue,
487 		.notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue,
488 		.notify_buffer_size       = virtio_transport_notify_buffer_size,
489 		.notify_set_rcvlowat      = virtio_transport_notify_set_rcvlowat,
490 
491 		.unsent_bytes             = virtio_transport_unsent_bytes,
492 
493 		.read_skb = virtio_transport_read_skb,
494 	},
495 
496 	.send_pkt = vhost_transport_send_pkt,
497 };
498 
499 static bool vhost_transport_seqpacket_allow(struct vsock_sock *vsk,
500 					    u32 remote_cid)
501 {
502 	struct net *net = sock_net(sk_vsock(vsk));
503 	struct vhost_vsock *vsock;
504 	bool seqpacket_allow = false;
505 
506 	rcu_read_lock();
507 	vsock = vhost_vsock_get(remote_cid, net);
508 
509 	if (vsock)
510 		seqpacket_allow = vsock->seqpacket_allow;
511 
512 	rcu_read_unlock();
513 
514 	return seqpacket_allow;
515 }
516 
517 static void vhost_vsock_handle_tx_kick(struct vhost_work *work)
518 {
519 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
520 						  poll.work);
521 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,
522 						 dev);
523 	int head, pkts = 0, total_len = 0;
524 	unsigned int out, in;
525 	struct sk_buff *skb;
526 	bool added = false;
527 
528 	mutex_lock(&vq->mutex);
529 
530 	if (!vhost_vq_get_backend(vq))
531 		goto out;
532 
533 	if (!vq_meta_prefetch(vq))
534 		goto out;
535 
536 	vhost_disable_notify(&vsock->dev, vq);
537 	do {
538 		struct virtio_vsock_hdr *hdr;
539 
540 		if (!vhost_vsock_more_replies(vsock)) {
541 			/* Stop tx until the device processes already
542 			 * pending replies.  Leave tx virtqueue
543 			 * callbacks disabled.
544 			 */
545 			goto no_more_replies;
546 		}
547 
548 		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
549 					 &out, &in, NULL, NULL);
550 		if (head < 0)
551 			break;
552 
553 		if (head == vq->num) {
554 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {
555 				vhost_disable_notify(&vsock->dev, vq);
556 				continue;
557 			}
558 			break;
559 		}
560 
561 		skb = vhost_vsock_alloc_skb(vq, out, in);
562 		if (!skb) {
563 			vq_err(vq, "Faulted on pkt\n");
564 			continue;
565 		}
566 
567 		total_len += sizeof(*hdr) + skb->len;
568 
569 		/* Deliver to monitoring devices all received packets */
570 		virtio_transport_deliver_tap_pkt(skb);
571 
572 		hdr = virtio_vsock_hdr(skb);
573 
574 		/* Only accept correctly addressed packets */
575 		if (le64_to_cpu(hdr->src_cid) == vsock->guest_cid &&
576 		    le64_to_cpu(hdr->dst_cid) ==
577 		    vhost_transport_get_local_cid())
578 			virtio_transport_recv_pkt(&vhost_transport, skb,
579 						  vsock->net);
580 		else
581 			kfree_skb(skb);
582 
583 		vhost_add_used(vq, head, 0);
584 		added = true;
585 	} while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
586 
587 no_more_replies:
588 	if (added)
589 		vhost_signal(&vsock->dev, vq);
590 
591 out:
592 	mutex_unlock(&vq->mutex);
593 }
594 
595 static void vhost_vsock_handle_rx_kick(struct vhost_work *work)
596 {
597 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
598 						poll.work);
599 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,
600 						 dev);
601 
602 	vhost_transport_do_send_pkt(vsock, vq);
603 }
604 
605 static int vhost_vsock_start(struct vhost_vsock *vsock)
606 {
607 	struct vhost_virtqueue *vq;
608 	size_t i;
609 	int ret;
610 
611 	mutex_lock(&vsock->dev.mutex);
612 
613 	ret = vhost_dev_check_owner(&vsock->dev);
614 	if (ret)
615 		goto err;
616 
617 	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
618 		vq = &vsock->vqs[i];
619 
620 		mutex_lock(&vq->mutex);
621 
622 		if (!vhost_vq_access_ok(vq)) {
623 			ret = -EFAULT;
624 			goto err_vq;
625 		}
626 
627 		if (!vhost_vq_get_backend(vq)) {
628 			vhost_vq_set_backend(vq, vsock);
629 			ret = vhost_vq_init_access(vq);
630 			if (ret)
631 				goto err_vq;
632 		}
633 
634 		mutex_unlock(&vq->mutex);
635 	}
636 
637 	/* Set 'ever_started' flag on the first start; never cleared, so send_pkt
638 	 * keeps queuing (instead of fast-failing) on later stop / CPR pauses.
639 	 */
640 	WRITE_ONCE(vsock->ever_started, true);
641 
642 	/* Some packets may have been queued before the device was started,
643 	 * let's kick the send worker to send them.
644 	 */
645 	vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work);
646 
647 	/* The guest may have added TX buffers while the device was stopped
648 	 * (e.g. across VHOST_RESET_OWNER) and their kicks got consumed by
649 	 * the NULL-backend window.  Re-scan the TX VQ, mirroring the RX
650 	 * send-worker kick above.
651 	 */
652 	vhost_poll_queue(&vsock->vqs[VSOCK_VQ_TX].poll);
653 
654 	mutex_unlock(&vsock->dev.mutex);
655 	return 0;
656 
657 err_vq:
658 	vhost_vq_set_backend(vq, NULL);
659 	mutex_unlock(&vq->mutex);
660 
661 	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
662 		vq = &vsock->vqs[i];
663 
664 		mutex_lock(&vq->mutex);
665 		vhost_vq_set_backend(vq, NULL);
666 		mutex_unlock(&vq->mutex);
667 	}
668 err:
669 	mutex_unlock(&vsock->dev.mutex);
670 	return ret;
671 }
672 
673 static void vhost_vsock_drop_backends(struct vhost_vsock *vsock)
674 {
675 	struct vhost_virtqueue *vq;
676 	size_t i;
677 
678 	lockdep_assert_held(&vsock->dev.mutex);
679 
680 	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
681 		vq = &vsock->vqs[i];
682 
683 		mutex_lock(&vq->mutex);
684 		vhost_vq_set_backend(vq, NULL);
685 		mutex_unlock(&vq->mutex);
686 	}
687 }
688 
689 static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner)
690 {
691 	int ret = 0;
692 
693 	mutex_lock(&vsock->dev.mutex);
694 
695 	if (check_owner) {
696 		ret = vhost_dev_check_owner(&vsock->dev);
697 		if (ret)
698 			goto err;
699 	}
700 
701 	vhost_vsock_drop_backends(vsock);
702 err:
703 	mutex_unlock(&vsock->dev.mutex);
704 	return ret;
705 }
706 
707 static void vhost_vsock_free(struct vhost_vsock *vsock)
708 {
709 	kvfree(vsock);
710 }
711 
712 static int vhost_vsock_dev_open(struct inode *inode, struct file *file)
713 {
714 	struct vhost_virtqueue **vqs;
715 	struct vhost_vsock *vsock;
716 	struct net *net;
717 	int ret;
718 
719 	/* This struct is large and allocation could fail, fall back to vmalloc
720 	 * if there is no other way.
721 	 */
722 	vsock = kvmalloc_obj(*vsock, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
723 	if (!vsock)
724 		return -ENOMEM;
725 
726 	vqs = kmalloc_objs(*vqs, ARRAY_SIZE(vsock->vqs));
727 	if (!vqs) {
728 		ret = -ENOMEM;
729 		goto out;
730 	}
731 
732 	net = current->nsproxy->net_ns;
733 	vsock->net = get_net_track(net, &vsock->ns_tracker, GFP_KERNEL);
734 
735 	vsock->guest_cid = 0; /* no CID assigned yet */
736 	vsock->seqpacket_allow = false;
737 	vsock->ever_started = false;
738 
739 	atomic_set(&vsock->queued_replies, 0);
740 
741 	vqs[VSOCK_VQ_TX] = &vsock->vqs[VSOCK_VQ_TX];
742 	vqs[VSOCK_VQ_RX] = &vsock->vqs[VSOCK_VQ_RX];
743 	vsock->vqs[VSOCK_VQ_TX].handle_kick = vhost_vsock_handle_tx_kick;
744 	vsock->vqs[VSOCK_VQ_RX].handle_kick = vhost_vsock_handle_rx_kick;
745 
746 	vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs),
747 		       UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT,
748 		       VHOST_VSOCK_WEIGHT, true, NULL);
749 
750 	file->private_data = vsock;
751 	skb_queue_head_init(&vsock->send_pkt_queue);
752 	vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work);
753 	return 0;
754 
755 out:
756 	vhost_vsock_free(vsock);
757 	return ret;
758 }
759 
760 static void vhost_vsock_flush(struct vhost_vsock *vsock)
761 {
762 	vhost_dev_flush(&vsock->dev);
763 }
764 
765 static void vhost_vsock_reset_orphans(struct sock *sk)
766 {
767 	struct vsock_sock *vsk = vsock_sk(sk);
768 
769 	/* vmci_transport.c doesn't take sk_lock here either.  At least we're
770 	 * under vsock_table_lock so the sock cannot disappear while we're
771 	 * executing.
772 	 */
773 
774 	rcu_read_lock();
775 
776 	/* If the peer is still valid, no need to reset connection */
777 	if (vhost_vsock_get(vsk->remote_addr.svm_cid, sock_net(sk))) {
778 		rcu_read_unlock();
779 		return;
780 	}
781 
782 	rcu_read_unlock();
783 
784 	/* If the close timeout is pending, let it expire.  This avoids races
785 	 * with the timeout callback.
786 	 */
787 	if (vsk->close_work_scheduled)
788 		return;
789 
790 	sock_set_flag(sk, SOCK_DONE);
791 	vsk->peer_shutdown = SHUTDOWN_MASK;
792 	sk->sk_state = SS_UNCONNECTED;
793 	sk->sk_err = ECONNRESET;
794 	sk_error_report(sk);
795 }
796 
797 static int vhost_vsock_dev_release(struct inode *inode, struct file *file)
798 {
799 	struct vhost_vsock *vsock = file->private_data;
800 
801 	mutex_lock(&vhost_vsock_mutex);
802 	if (vsock->guest_cid)
803 		hash_del_rcu(&vsock->hash);
804 	mutex_unlock(&vhost_vsock_mutex);
805 
806 	/* Wait for other CPUs to finish using vsock */
807 	synchronize_rcu();
808 
809 	/* Iterating over all connections for all CIDs to find orphans is
810 	 * inefficient.  Room for improvement here. */
811 	vsock_for_each_connected_socket(&vhost_transport.transport,
812 					vhost_vsock_reset_orphans);
813 
814 	/* Don't check the owner, because we are in the release path, so we
815 	 * need to stop the vsock device in any case.
816 	 * vhost_vsock_stop() can not fail in this case, so we don't need to
817 	 * check the return code.
818 	 */
819 	vhost_vsock_stop(vsock, false);
820 	vhost_vsock_flush(vsock);
821 	vhost_dev_stop(&vsock->dev);
822 
823 	virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue);
824 
825 	vhost_dev_cleanup(&vsock->dev, false);
826 	put_net_track(vsock->net, &vsock->ns_tracker);
827 	kfree(vsock->dev.vqs);
828 	vhost_vsock_free(vsock);
829 	return 0;
830 }
831 
832 static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid)
833 {
834 	struct vhost_vsock *other;
835 
836 	/* Refuse reserved CIDs */
837 	if (guest_cid <= VMADDR_CID_HOST ||
838 	    guest_cid == U32_MAX)
839 		return -EINVAL;
840 
841 	/* 64-bit CIDs are not yet supported */
842 	if (guest_cid > U32_MAX)
843 		return -EINVAL;
844 
845 	/* Refuse if CID is assigned to the guest->host transport (i.e. nested
846 	 * VM), to make the loopback work.
847 	 */
848 	if (vsock_find_cid(guest_cid))
849 		return -EADDRINUSE;
850 
851 	/* Refuse if CID is already in use */
852 	mutex_lock(&vhost_vsock_mutex);
853 	other = vhost_vsock_get(guest_cid, vsock->net);
854 	if (other && other != vsock) {
855 		mutex_unlock(&vhost_vsock_mutex);
856 		return -EADDRINUSE;
857 	}
858 
859 	if (vsock->guest_cid)
860 		hash_del_rcu(&vsock->hash);
861 
862 	vsock->guest_cid = guest_cid;
863 	hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid);
864 	mutex_unlock(&vhost_vsock_mutex);
865 
866 	return 0;
867 }
868 
869 static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
870 {
871 	struct vhost_virtqueue *vq;
872 	int i;
873 
874 	if (features & ~VHOST_VSOCK_FEATURES)
875 		return -EOPNOTSUPP;
876 
877 	mutex_lock(&vsock->dev.mutex);
878 	if ((features & (1 << VHOST_F_LOG_ALL)) &&
879 	    !vhost_log_access_ok(&vsock->dev)) {
880 		goto err;
881 	}
882 
883 	if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) {
884 		if (vhost_init_device_iotlb(&vsock->dev))
885 			goto err;
886 	} else {
887 		vhost_clear_device_iotlb(&vsock->dev);
888 	}
889 
890 	vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET);
891 
892 	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
893 		vq = &vsock->vqs[i];
894 		mutex_lock(&vq->mutex);
895 		vq->acked_features = features;
896 		mutex_unlock(&vq->mutex);
897 	}
898 	mutex_unlock(&vsock->dev.mutex);
899 	return 0;
900 
901 err:
902 	mutex_unlock(&vsock->dev.mutex);
903 	return -EFAULT;
904 }
905 
906 static long vhost_vsock_reset_owner(struct vhost_vsock *vsock)
907 {
908 	struct vhost_iotlb *umem;
909 	long err;
910 
911 	mutex_lock(&vsock->dev.mutex);
912 	err = vhost_dev_check_owner(&vsock->dev);
913 	if (err)
914 		goto done;
915 	umem = vhost_dev_reset_owner_prepare();
916 	if (!umem) {
917 		err = -ENOMEM;
918 		goto done;
919 	}
920 	vhost_vsock_drop_backends(vsock);
921 	vhost_vsock_flush(vsock);
922 	vhost_dev_stop(&vsock->dev);
923 
924 	/*
925 	 * vsock keeps the CID hashed across RESET_OWNER, so a lockless sender
926 	 * can still reach the device while its workers are freed.  Pass
927 	 * sync=true so vhost_workers_free() fences RCU readers and flushes
928 	 * before freeing them.
929 	 */
930 	vhost_dev_reset_owner(&vsock->dev, umem, true);
931 done:
932 	mutex_unlock(&vsock->dev.mutex);
933 	return err;
934 }
935 
936 static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl,
937 				  unsigned long arg)
938 {
939 	struct vhost_vsock *vsock = f->private_data;
940 	void __user *argp = (void __user *)arg;
941 	u64 guest_cid;
942 	u64 features;
943 	int start;
944 	int r;
945 
946 	switch (ioctl) {
947 	case VHOST_VSOCK_SET_GUEST_CID:
948 		if (copy_from_user(&guest_cid, argp, sizeof(guest_cid)))
949 			return -EFAULT;
950 		return vhost_vsock_set_cid(vsock, guest_cid);
951 	case VHOST_VSOCK_SET_RUNNING:
952 		if (copy_from_user(&start, argp, sizeof(start)))
953 			return -EFAULT;
954 		if (start)
955 			return vhost_vsock_start(vsock);
956 		else
957 			return vhost_vsock_stop(vsock, true);
958 	case VHOST_GET_FEATURES:
959 		features = VHOST_VSOCK_FEATURES;
960 		if (copy_to_user(argp, &features, sizeof(features)))
961 			return -EFAULT;
962 		return 0;
963 	case VHOST_SET_FEATURES:
964 		if (copy_from_user(&features, argp, sizeof(features)))
965 			return -EFAULT;
966 		return vhost_vsock_set_features(vsock, features);
967 	case VHOST_GET_BACKEND_FEATURES:
968 		features = VHOST_VSOCK_BACKEND_FEATURES;
969 		if (copy_to_user(argp, &features, sizeof(features)))
970 			return -EFAULT;
971 		return 0;
972 	case VHOST_SET_BACKEND_FEATURES:
973 		if (copy_from_user(&features, argp, sizeof(features)))
974 			return -EFAULT;
975 		if (features & ~VHOST_VSOCK_BACKEND_FEATURES)
976 			return -EOPNOTSUPP;
977 		vhost_set_backend_features(&vsock->dev, features);
978 		return 0;
979 	case VHOST_RESET_OWNER:
980 		return vhost_vsock_reset_owner(vsock);
981 	default:
982 		mutex_lock(&vsock->dev.mutex);
983 		r = vhost_dev_ioctl(&vsock->dev, ioctl, argp);
984 		if (r == -ENOIOCTLCMD)
985 			r = vhost_vring_ioctl(&vsock->dev, ioctl, argp);
986 		else
987 			vhost_vsock_flush(vsock);
988 		mutex_unlock(&vsock->dev.mutex);
989 		return r;
990 	}
991 }
992 
993 static ssize_t vhost_vsock_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
994 {
995 	struct file *file = iocb->ki_filp;
996 	struct vhost_vsock *vsock = file->private_data;
997 	struct vhost_dev *dev = &vsock->dev;
998 	int noblock = file->f_flags & O_NONBLOCK;
999 
1000 	return vhost_chr_read_iter(dev, to, noblock);
1001 }
1002 
1003 static ssize_t vhost_vsock_chr_write_iter(struct kiocb *iocb,
1004 					struct iov_iter *from)
1005 {
1006 	struct file *file = iocb->ki_filp;
1007 	struct vhost_vsock *vsock = file->private_data;
1008 	struct vhost_dev *dev = &vsock->dev;
1009 
1010 	return vhost_chr_write_iter(dev, from);
1011 }
1012 
1013 static __poll_t vhost_vsock_chr_poll(struct file *file, poll_table *wait)
1014 {
1015 	struct vhost_vsock *vsock = file->private_data;
1016 	struct vhost_dev *dev = &vsock->dev;
1017 
1018 	return vhost_chr_poll(file, dev, wait);
1019 }
1020 
1021 static const struct file_operations vhost_vsock_fops = {
1022 	.owner          = THIS_MODULE,
1023 	.open           = vhost_vsock_dev_open,
1024 	.release        = vhost_vsock_dev_release,
1025 	.llseek		= noop_llseek,
1026 	.unlocked_ioctl = vhost_vsock_dev_ioctl,
1027 	.compat_ioctl   = compat_ptr_ioctl,
1028 	.read_iter      = vhost_vsock_chr_read_iter,
1029 	.write_iter     = vhost_vsock_chr_write_iter,
1030 	.poll           = vhost_vsock_chr_poll,
1031 };
1032 
1033 static struct miscdevice vhost_vsock_misc = {
1034 	.minor = VHOST_VSOCK_MINOR,
1035 	.name = "vhost-vsock",
1036 	.fops = &vhost_vsock_fops,
1037 };
1038 
1039 static int __init vhost_vsock_init(void)
1040 {
1041 	int ret;
1042 
1043 	ret = vsock_core_register(&vhost_transport.transport,
1044 				  VSOCK_TRANSPORT_F_H2G);
1045 	if (ret < 0)
1046 		return ret;
1047 
1048 	ret = misc_register(&vhost_vsock_misc);
1049 	if (ret) {
1050 		vsock_core_unregister(&vhost_transport.transport);
1051 		return ret;
1052 	}
1053 
1054 	return 0;
1055 };
1056 
1057 static void __exit vhost_vsock_exit(void)
1058 {
1059 	misc_deregister(&vhost_vsock_misc);
1060 	vsock_core_unregister(&vhost_transport.transport);
1061 };
1062 
1063 module_init(vhost_vsock_init);
1064 module_exit(vhost_vsock_exit);
1065 MODULE_LICENSE("GPL v2");
1066 MODULE_AUTHOR("Asias He");
1067 MODULE_DESCRIPTION("vhost transport for vsock ");
1068 MODULE_ALIAS_MISCDEV(VHOST_VSOCK_MINOR);
1069 MODULE_ALIAS("devname:vhost-vsock");
1070