1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * vhost transport for vsock
4 *
5 * Copyright (C) 2013-2015 Red Hat, Inc.
6 * Author: Asias He <asias@redhat.com>
7 * Stefan Hajnoczi <stefanha@redhat.com>
8 */
9 #include <linux/miscdevice.h>
10 #include <linux/atomic.h>
11 #include <linux/module.h>
12 #include <linux/mutex.h>
13 #include <linux/vmalloc.h>
14 #include <net/sock.h>
15 #include <linux/virtio_vsock.h>
16 #include <linux/vhost.h>
17 #include <linux/hashtable.h>
18
19 #include <net/af_vsock.h>
20 #include "vhost.h"
21
22 #define VHOST_VSOCK_DEFAULT_HOST_CID 2
23 /* Max number of bytes transferred before requeueing the job.
24 * Using this limit prevents one virtqueue from starving others. */
25 #define VHOST_VSOCK_WEIGHT 0x80000
26 /* Max number of packets transferred before requeueing the job.
27 * Using this limit prevents one virtqueue from starving others with
28 * small pkts.
29 */
30 #define VHOST_VSOCK_PKT_WEIGHT 256
31
32 static const int vhost_vsock_bits[] = {
33 VHOST_FEATURES,
34 VIRTIO_F_ACCESS_PLATFORM,
35 VIRTIO_VSOCK_F_SEQPACKET
36 };
37
38 #define VHOST_VSOCK_FEATURES VHOST_FEATURES_U64(vhost_vsock_bits, 0)
39
40 enum {
41 VHOST_VSOCK_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2)
42 };
43
44 /* Used to track all the vhost_vsock instances on the system. */
45 static DEFINE_MUTEX(vhost_vsock_mutex);
46 static DEFINE_READ_MOSTLY_HASHTABLE(vhost_vsock_hash, 8);
47
48 struct vhost_vsock {
49 struct vhost_dev dev;
50 struct vhost_virtqueue vqs[2];
51
52 /* Link to global vhost_vsock_hash, writes use vhost_vsock_mutex */
53 struct hlist_node hash;
54
55 struct vhost_work send_pkt_work;
56 struct sk_buff_head send_pkt_queue; /* host->guest pending packets */
57
58 atomic_t queued_replies;
59
60 u32 guest_cid;
61 bool seqpacket_allow;
62 };
63
vhost_transport_get_local_cid(void)64 static u32 vhost_transport_get_local_cid(void)
65 {
66 return VHOST_VSOCK_DEFAULT_HOST_CID;
67 }
68
69 /* Callers must be in an RCU read section or hold the vhost_vsock_mutex.
70 * The return value can only be dereferenced while within the section.
71 */
vhost_vsock_get(u32 guest_cid)72 static struct vhost_vsock *vhost_vsock_get(u32 guest_cid)
73 {
74 struct vhost_vsock *vsock;
75
76 hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid,
77 lockdep_is_held(&vhost_vsock_mutex)) {
78 u32 other_cid = vsock->guest_cid;
79
80 /* Skip instances that have no CID yet */
81 if (other_cid == 0)
82 continue;
83
84 if (other_cid == guest_cid)
85 return vsock;
86
87 }
88
89 return NULL;
90 }
91
92 static void
vhost_transport_do_send_pkt(struct vhost_vsock * vsock,struct vhost_virtqueue * vq)93 vhost_transport_do_send_pkt(struct vhost_vsock *vsock,
94 struct vhost_virtqueue *vq)
95 {
96 struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];
97 int pkts = 0, total_len = 0;
98 bool added = false;
99 bool restart_tx = false;
100
101 mutex_lock(&vq->mutex);
102
103 if (!vhost_vq_get_backend(vq))
104 goto out;
105
106 if (!vq_meta_prefetch(vq))
107 goto out;
108
109 /* Avoid further vmexits, we're already processing the virtqueue */
110 vhost_disable_notify(&vsock->dev, vq);
111
112 do {
113 struct virtio_vsock_hdr *hdr;
114 size_t iov_len, payload_len;
115 struct iov_iter iov_iter;
116 u32 flags_to_restore = 0;
117 struct sk_buff *skb;
118 unsigned out, in;
119 size_t nbytes;
120 u32 offset;
121 int head;
122
123 skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue);
124
125 if (!skb) {
126 vhost_enable_notify(&vsock->dev, vq);
127 break;
128 }
129
130 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
131 &out, &in, NULL, NULL);
132 if (head < 0) {
133 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
134 break;
135 }
136
137 if (head == vq->num) {
138 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
139 /* We cannot finish yet if more buffers snuck in while
140 * re-enabling notify.
141 */
142 if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {
143 vhost_disable_notify(&vsock->dev, vq);
144 continue;
145 }
146 break;
147 }
148
149 if (out) {
150 kfree_skb(skb);
151 vq_err(vq, "Expected 0 output buffers, got %u\n", out);
152 break;
153 }
154
155 iov_len = iov_length(&vq->iov[out], in);
156 if (iov_len < sizeof(*hdr)) {
157 kfree_skb(skb);
158 vq_err(vq, "Buffer len [%zu] too small\n", iov_len);
159 break;
160 }
161
162 iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[out], in, iov_len);
163 offset = VIRTIO_VSOCK_SKB_CB(skb)->offset;
164 payload_len = skb->len - offset;
165 hdr = virtio_vsock_hdr(skb);
166
167 /* If the packet is greater than the space available in the
168 * buffer, we split it using multiple buffers.
169 */
170 if (payload_len > iov_len - sizeof(*hdr)) {
171 payload_len = iov_len - sizeof(*hdr);
172
173 /* As we are copying pieces of large packet's buffer to
174 * small rx buffers, headers of packets in rx queue are
175 * created dynamically and are initialized with header
176 * of current packet(except length). But in case of
177 * SOCK_SEQPACKET, we also must clear message delimeter
178 * bit (VIRTIO_VSOCK_SEQ_EOM) and MSG_EOR bit
179 * (VIRTIO_VSOCK_SEQ_EOR) if set. Otherwise,
180 * there will be sequence of packets with these
181 * bits set. After initialized header will be copied to
182 * rx buffer, these required bits will be restored.
183 */
184 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) {
185 hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM);
186 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOM;
187
188 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) {
189 hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR);
190 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOR;
191 }
192 }
193 }
194
195 /* Set the correct length in the header */
196 hdr->len = cpu_to_le32(payload_len);
197
198 nbytes = copy_to_iter(hdr, sizeof(*hdr), &iov_iter);
199 if (nbytes != sizeof(*hdr)) {
200 kfree_skb(skb);
201 vq_err(vq, "Faulted on copying pkt hdr\n");
202 break;
203 }
204
205 if (skb_copy_datagram_iter(skb,
206 offset,
207 &iov_iter,
208 payload_len)) {
209 kfree_skb(skb);
210 vq_err(vq, "Faulted on copying pkt buf\n");
211 break;
212 }
213
214 /* Deliver to monitoring devices all packets that we
215 * will transmit.
216 */
217 virtio_transport_deliver_tap_pkt(skb);
218
219 vhost_add_used(vq, head, sizeof(*hdr) + payload_len);
220 added = true;
221
222 VIRTIO_VSOCK_SKB_CB(skb)->offset += payload_len;
223 total_len += payload_len;
224
225 /* If we didn't send all the payload we can requeue the packet
226 * to send it with the next available buffer.
227 */
228 if (VIRTIO_VSOCK_SKB_CB(skb)->offset < skb->len) {
229 hdr->flags |= cpu_to_le32(flags_to_restore);
230
231 /* We are queueing the same skb to handle
232 * the remaining bytes, and we want to deliver it
233 * to monitoring devices in the next iteration.
234 */
235 virtio_vsock_skb_clear_tap_delivered(skb);
236 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
237 } else {
238 if (virtio_vsock_skb_reply(skb)) {
239 int val;
240
241 val = atomic_dec_return(&vsock->queued_replies);
242
243 /* Do we have resources to resume tx
244 * processing?
245 */
246 if (val + 1 == tx_vq->num)
247 restart_tx = true;
248 }
249
250 virtio_transport_consume_skb_sent(skb, true);
251 }
252 } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
253 if (added)
254 vhost_signal(&vsock->dev, vq);
255
256 out:
257 mutex_unlock(&vq->mutex);
258
259 if (restart_tx)
260 vhost_poll_queue(&tx_vq->poll);
261 }
262
vhost_transport_send_pkt_work(struct vhost_work * work)263 static void vhost_transport_send_pkt_work(struct vhost_work *work)
264 {
265 struct vhost_virtqueue *vq;
266 struct vhost_vsock *vsock;
267
268 vsock = container_of(work, struct vhost_vsock, send_pkt_work);
269 vq = &vsock->vqs[VSOCK_VQ_RX];
270
271 vhost_transport_do_send_pkt(vsock, vq);
272 }
273
274 static int
vhost_transport_send_pkt(struct sk_buff * skb)275 vhost_transport_send_pkt(struct sk_buff *skb)
276 {
277 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
278 struct vhost_vsock *vsock;
279 int len = skb->len;
280
281 rcu_read_lock();
282
283 /* Find the vhost_vsock according to guest context id */
284 vsock = vhost_vsock_get(le64_to_cpu(hdr->dst_cid));
285 if (!vsock) {
286 rcu_read_unlock();
287 kfree_skb(skb);
288 return -ENODEV;
289 }
290
291 if (virtio_vsock_skb_reply(skb))
292 atomic_inc(&vsock->queued_replies);
293
294 virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb);
295 vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work);
296
297 rcu_read_unlock();
298 return len;
299 }
300
301 static int
vhost_transport_cancel_pkt(struct vsock_sock * vsk)302 vhost_transport_cancel_pkt(struct vsock_sock *vsk)
303 {
304 struct vhost_vsock *vsock;
305 int cnt = 0;
306 int ret = -ENODEV;
307
308 rcu_read_lock();
309
310 /* Find the vhost_vsock according to guest context id */
311 vsock = vhost_vsock_get(vsk->remote_addr.svm_cid);
312 if (!vsock)
313 goto out;
314
315 cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue);
316
317 if (cnt) {
318 struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];
319 int new_cnt;
320
321 new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);
322 if (new_cnt + cnt >= tx_vq->num && new_cnt < tx_vq->num)
323 vhost_poll_queue(&tx_vq->poll);
324 }
325
326 ret = 0;
327 out:
328 rcu_read_unlock();
329 return ret;
330 }
331
332 static struct sk_buff *
vhost_vsock_alloc_skb(struct vhost_virtqueue * vq,unsigned int out,unsigned int in)333 vhost_vsock_alloc_skb(struct vhost_virtqueue *vq,
334 unsigned int out, unsigned int in)
335 {
336 struct virtio_vsock_hdr *hdr;
337 struct iov_iter iov_iter;
338 struct sk_buff *skb;
339 size_t payload_len;
340 size_t nbytes;
341 size_t len;
342
343 if (in != 0) {
344 vq_err(vq, "Expected 0 input buffers, got %u\n", in);
345 return NULL;
346 }
347
348 len = iov_length(vq->iov, out);
349
350 if (len < VIRTIO_VSOCK_SKB_HEADROOM ||
351 len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE + VIRTIO_VSOCK_SKB_HEADROOM)
352 return NULL;
353
354 /* len contains both payload and hdr */
355 skb = virtio_vsock_alloc_skb(len, GFP_KERNEL);
356 if (!skb)
357 return NULL;
358
359 iov_iter_init(&iov_iter, ITER_SOURCE, vq->iov, out, len);
360
361 hdr = virtio_vsock_hdr(skb);
362 nbytes = copy_from_iter(hdr, sizeof(*hdr), &iov_iter);
363 if (nbytes != sizeof(*hdr)) {
364 vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",
365 sizeof(*hdr), nbytes);
366 kfree_skb(skb);
367 return NULL;
368 }
369
370 payload_len = le32_to_cpu(hdr->len);
371
372 /* No payload */
373 if (!payload_len)
374 return skb;
375
376 /* The pkt is too big or the length in the header is invalid */
377 if (payload_len + sizeof(*hdr) > len) {
378 kfree_skb(skb);
379 return NULL;
380 }
381
382 virtio_vsock_skb_put(skb, payload_len);
383
384 if (skb_copy_datagram_from_iter(skb, 0, &iov_iter, payload_len)) {
385 vq_err(vq, "Failed to copy %zu byte payload\n", payload_len);
386 kfree_skb(skb);
387 return NULL;
388 }
389
390 return skb;
391 }
392
393 /* Is there space left for replies to rx packets? */
vhost_vsock_more_replies(struct vhost_vsock * vsock)394 static bool vhost_vsock_more_replies(struct vhost_vsock *vsock)
395 {
396 struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];
397 int val;
398
399 smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */
400 val = atomic_read(&vsock->queued_replies);
401
402 return val < vq->num;
403 }
404
vhost_transport_msgzerocopy_allow(void)405 static bool vhost_transport_msgzerocopy_allow(void)
406 {
407 return true;
408 }
409
410 static bool vhost_transport_seqpacket_allow(u32 remote_cid);
411
412 static struct virtio_transport vhost_transport = {
413 .transport = {
414 .module = THIS_MODULE,
415
416 .get_local_cid = vhost_transport_get_local_cid,
417
418 .init = virtio_transport_do_socket_init,
419 .destruct = virtio_transport_destruct,
420 .release = virtio_transport_release,
421 .connect = virtio_transport_connect,
422 .shutdown = virtio_transport_shutdown,
423 .cancel_pkt = vhost_transport_cancel_pkt,
424
425 .dgram_enqueue = virtio_transport_dgram_enqueue,
426 .dgram_dequeue = virtio_transport_dgram_dequeue,
427 .dgram_bind = virtio_transport_dgram_bind,
428 .dgram_allow = virtio_transport_dgram_allow,
429
430 .stream_enqueue = virtio_transport_stream_enqueue,
431 .stream_dequeue = virtio_transport_stream_dequeue,
432 .stream_has_data = virtio_transport_stream_has_data,
433 .stream_has_space = virtio_transport_stream_has_space,
434 .stream_rcvhiwat = virtio_transport_stream_rcvhiwat,
435 .stream_is_active = virtio_transport_stream_is_active,
436 .stream_allow = virtio_transport_stream_allow,
437
438 .seqpacket_dequeue = virtio_transport_seqpacket_dequeue,
439 .seqpacket_enqueue = virtio_transport_seqpacket_enqueue,
440 .seqpacket_allow = vhost_transport_seqpacket_allow,
441 .seqpacket_has_data = virtio_transport_seqpacket_has_data,
442
443 .msgzerocopy_allow = vhost_transport_msgzerocopy_allow,
444
445 .notify_poll_in = virtio_transport_notify_poll_in,
446 .notify_poll_out = virtio_transport_notify_poll_out,
447 .notify_recv_init = virtio_transport_notify_recv_init,
448 .notify_recv_pre_block = virtio_transport_notify_recv_pre_block,
449 .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue,
450 .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue,
451 .notify_send_init = virtio_transport_notify_send_init,
452 .notify_send_pre_block = virtio_transport_notify_send_pre_block,
453 .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue,
454 .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue,
455 .notify_buffer_size = virtio_transport_notify_buffer_size,
456 .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat,
457
458 .unsent_bytes = virtio_transport_unsent_bytes,
459
460 .read_skb = virtio_transport_read_skb,
461 },
462
463 .send_pkt = vhost_transport_send_pkt,
464 };
465
vhost_transport_seqpacket_allow(u32 remote_cid)466 static bool vhost_transport_seqpacket_allow(u32 remote_cid)
467 {
468 struct vhost_vsock *vsock;
469 bool seqpacket_allow = false;
470
471 rcu_read_lock();
472 vsock = vhost_vsock_get(remote_cid);
473
474 if (vsock)
475 seqpacket_allow = vsock->seqpacket_allow;
476
477 rcu_read_unlock();
478
479 return seqpacket_allow;
480 }
481
vhost_vsock_handle_tx_kick(struct vhost_work * work)482 static void vhost_vsock_handle_tx_kick(struct vhost_work *work)
483 {
484 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
485 poll.work);
486 struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,
487 dev);
488 int head, pkts = 0, total_len = 0;
489 unsigned int out, in;
490 struct sk_buff *skb;
491 bool added = false;
492
493 mutex_lock(&vq->mutex);
494
495 if (!vhost_vq_get_backend(vq))
496 goto out;
497
498 if (!vq_meta_prefetch(vq))
499 goto out;
500
501 vhost_disable_notify(&vsock->dev, vq);
502 do {
503 struct virtio_vsock_hdr *hdr;
504
505 if (!vhost_vsock_more_replies(vsock)) {
506 /* Stop tx until the device processes already
507 * pending replies. Leave tx virtqueue
508 * callbacks disabled.
509 */
510 goto no_more_replies;
511 }
512
513 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
514 &out, &in, NULL, NULL);
515 if (head < 0)
516 break;
517
518 if (head == vq->num) {
519 if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {
520 vhost_disable_notify(&vsock->dev, vq);
521 continue;
522 }
523 break;
524 }
525
526 skb = vhost_vsock_alloc_skb(vq, out, in);
527 if (!skb) {
528 vq_err(vq, "Faulted on pkt\n");
529 continue;
530 }
531
532 total_len += sizeof(*hdr) + skb->len;
533
534 /* Deliver to monitoring devices all received packets */
535 virtio_transport_deliver_tap_pkt(skb);
536
537 hdr = virtio_vsock_hdr(skb);
538
539 /* Only accept correctly addressed packets */
540 if (le64_to_cpu(hdr->src_cid) == vsock->guest_cid &&
541 le64_to_cpu(hdr->dst_cid) ==
542 vhost_transport_get_local_cid())
543 virtio_transport_recv_pkt(&vhost_transport, skb);
544 else
545 kfree_skb(skb);
546
547 vhost_add_used(vq, head, 0);
548 added = true;
549 } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
550
551 no_more_replies:
552 if (added)
553 vhost_signal(&vsock->dev, vq);
554
555 out:
556 mutex_unlock(&vq->mutex);
557 }
558
vhost_vsock_handle_rx_kick(struct vhost_work * work)559 static void vhost_vsock_handle_rx_kick(struct vhost_work *work)
560 {
561 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
562 poll.work);
563 struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,
564 dev);
565
566 vhost_transport_do_send_pkt(vsock, vq);
567 }
568
vhost_vsock_start(struct vhost_vsock * vsock)569 static int vhost_vsock_start(struct vhost_vsock *vsock)
570 {
571 struct vhost_virtqueue *vq;
572 size_t i;
573 int ret;
574
575 mutex_lock(&vsock->dev.mutex);
576
577 ret = vhost_dev_check_owner(&vsock->dev);
578 if (ret)
579 goto err;
580
581 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
582 vq = &vsock->vqs[i];
583
584 mutex_lock(&vq->mutex);
585
586 if (!vhost_vq_access_ok(vq)) {
587 ret = -EFAULT;
588 goto err_vq;
589 }
590
591 if (!vhost_vq_get_backend(vq)) {
592 vhost_vq_set_backend(vq, vsock);
593 ret = vhost_vq_init_access(vq);
594 if (ret)
595 goto err_vq;
596 }
597
598 mutex_unlock(&vq->mutex);
599 }
600
601 /* Some packets may have been queued before the device was started,
602 * let's kick the send worker to send them.
603 */
604 vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work);
605
606 mutex_unlock(&vsock->dev.mutex);
607 return 0;
608
609 err_vq:
610 vhost_vq_set_backend(vq, NULL);
611 mutex_unlock(&vq->mutex);
612
613 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
614 vq = &vsock->vqs[i];
615
616 mutex_lock(&vq->mutex);
617 vhost_vq_set_backend(vq, NULL);
618 mutex_unlock(&vq->mutex);
619 }
620 err:
621 mutex_unlock(&vsock->dev.mutex);
622 return ret;
623 }
624
vhost_vsock_stop(struct vhost_vsock * vsock,bool check_owner)625 static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner)
626 {
627 size_t i;
628 int ret = 0;
629
630 mutex_lock(&vsock->dev.mutex);
631
632 if (check_owner) {
633 ret = vhost_dev_check_owner(&vsock->dev);
634 if (ret)
635 goto err;
636 }
637
638 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
639 struct vhost_virtqueue *vq = &vsock->vqs[i];
640
641 mutex_lock(&vq->mutex);
642 vhost_vq_set_backend(vq, NULL);
643 mutex_unlock(&vq->mutex);
644 }
645
646 err:
647 mutex_unlock(&vsock->dev.mutex);
648 return ret;
649 }
650
vhost_vsock_free(struct vhost_vsock * vsock)651 static void vhost_vsock_free(struct vhost_vsock *vsock)
652 {
653 kvfree(vsock);
654 }
655
vhost_vsock_dev_open(struct inode * inode,struct file * file)656 static int vhost_vsock_dev_open(struct inode *inode, struct file *file)
657 {
658 struct vhost_virtqueue **vqs;
659 struct vhost_vsock *vsock;
660 int ret;
661
662 /* This struct is large and allocation could fail, fall back to vmalloc
663 * if there is no other way.
664 */
665 vsock = kvmalloc(sizeof(*vsock), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
666 if (!vsock)
667 return -ENOMEM;
668
669 vqs = kmalloc_array(ARRAY_SIZE(vsock->vqs), sizeof(*vqs), GFP_KERNEL);
670 if (!vqs) {
671 ret = -ENOMEM;
672 goto out;
673 }
674
675 vsock->guest_cid = 0; /* no CID assigned yet */
676 vsock->seqpacket_allow = false;
677
678 atomic_set(&vsock->queued_replies, 0);
679
680 vqs[VSOCK_VQ_TX] = &vsock->vqs[VSOCK_VQ_TX];
681 vqs[VSOCK_VQ_RX] = &vsock->vqs[VSOCK_VQ_RX];
682 vsock->vqs[VSOCK_VQ_TX].handle_kick = vhost_vsock_handle_tx_kick;
683 vsock->vqs[VSOCK_VQ_RX].handle_kick = vhost_vsock_handle_rx_kick;
684
685 vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs),
686 UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT,
687 VHOST_VSOCK_WEIGHT, true, NULL);
688
689 file->private_data = vsock;
690 skb_queue_head_init(&vsock->send_pkt_queue);
691 vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work);
692 return 0;
693
694 out:
695 vhost_vsock_free(vsock);
696 return ret;
697 }
698
vhost_vsock_flush(struct vhost_vsock * vsock)699 static void vhost_vsock_flush(struct vhost_vsock *vsock)
700 {
701 vhost_dev_flush(&vsock->dev);
702 }
703
vhost_vsock_reset_orphans(struct sock * sk)704 static void vhost_vsock_reset_orphans(struct sock *sk)
705 {
706 struct vsock_sock *vsk = vsock_sk(sk);
707
708 /* vmci_transport.c doesn't take sk_lock here either. At least we're
709 * under vsock_table_lock so the sock cannot disappear while we're
710 * executing.
711 */
712
713 rcu_read_lock();
714
715 /* If the peer is still valid, no need to reset connection */
716 if (vhost_vsock_get(vsk->remote_addr.svm_cid)) {
717 rcu_read_unlock();
718 return;
719 }
720
721 rcu_read_unlock();
722
723 /* If the close timeout is pending, let it expire. This avoids races
724 * with the timeout callback.
725 */
726 if (vsk->close_work_scheduled)
727 return;
728
729 sock_set_flag(sk, SOCK_DONE);
730 vsk->peer_shutdown = SHUTDOWN_MASK;
731 sk->sk_state = SS_UNCONNECTED;
732 sk->sk_err = ECONNRESET;
733 sk_error_report(sk);
734 }
735
vhost_vsock_dev_release(struct inode * inode,struct file * file)736 static int vhost_vsock_dev_release(struct inode *inode, struct file *file)
737 {
738 struct vhost_vsock *vsock = file->private_data;
739
740 mutex_lock(&vhost_vsock_mutex);
741 if (vsock->guest_cid)
742 hash_del_rcu(&vsock->hash);
743 mutex_unlock(&vhost_vsock_mutex);
744
745 /* Wait for other CPUs to finish using vsock */
746 synchronize_rcu();
747
748 /* Iterating over all connections for all CIDs to find orphans is
749 * inefficient. Room for improvement here. */
750 vsock_for_each_connected_socket(&vhost_transport.transport,
751 vhost_vsock_reset_orphans);
752
753 /* Don't check the owner, because we are in the release path, so we
754 * need to stop the vsock device in any case.
755 * vhost_vsock_stop() can not fail in this case, so we don't need to
756 * check the return code.
757 */
758 vhost_vsock_stop(vsock, false);
759 vhost_vsock_flush(vsock);
760 vhost_dev_stop(&vsock->dev);
761
762 virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue);
763
764 vhost_dev_cleanup(&vsock->dev);
765 kfree(vsock->dev.vqs);
766 vhost_vsock_free(vsock);
767 return 0;
768 }
769
vhost_vsock_set_cid(struct vhost_vsock * vsock,u64 guest_cid)770 static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid)
771 {
772 struct vhost_vsock *other;
773
774 /* Refuse reserved CIDs */
775 if (guest_cid <= VMADDR_CID_HOST ||
776 guest_cid == U32_MAX)
777 return -EINVAL;
778
779 /* 64-bit CIDs are not yet supported */
780 if (guest_cid > U32_MAX)
781 return -EINVAL;
782
783 /* Refuse if CID is assigned to the guest->host transport (i.e. nested
784 * VM), to make the loopback work.
785 */
786 if (vsock_find_cid(guest_cid))
787 return -EADDRINUSE;
788
789 /* Refuse if CID is already in use */
790 mutex_lock(&vhost_vsock_mutex);
791 other = vhost_vsock_get(guest_cid);
792 if (other && other != vsock) {
793 mutex_unlock(&vhost_vsock_mutex);
794 return -EADDRINUSE;
795 }
796
797 if (vsock->guest_cid)
798 hash_del_rcu(&vsock->hash);
799
800 vsock->guest_cid = guest_cid;
801 hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid);
802 mutex_unlock(&vhost_vsock_mutex);
803
804 return 0;
805 }
806
vhost_vsock_set_features(struct vhost_vsock * vsock,u64 features)807 static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
808 {
809 struct vhost_virtqueue *vq;
810 int i;
811
812 if (features & ~VHOST_VSOCK_FEATURES)
813 return -EOPNOTSUPP;
814
815 mutex_lock(&vsock->dev.mutex);
816 if ((features & (1 << VHOST_F_LOG_ALL)) &&
817 !vhost_log_access_ok(&vsock->dev)) {
818 goto err;
819 }
820
821 if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) {
822 if (vhost_init_device_iotlb(&vsock->dev))
823 goto err;
824 }
825
826 vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET);
827
828 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
829 vq = &vsock->vqs[i];
830 mutex_lock(&vq->mutex);
831 vq->acked_features = features;
832 mutex_unlock(&vq->mutex);
833 }
834 mutex_unlock(&vsock->dev.mutex);
835 return 0;
836
837 err:
838 mutex_unlock(&vsock->dev.mutex);
839 return -EFAULT;
840 }
841
vhost_vsock_dev_ioctl(struct file * f,unsigned int ioctl,unsigned long arg)842 static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl,
843 unsigned long arg)
844 {
845 struct vhost_vsock *vsock = f->private_data;
846 void __user *argp = (void __user *)arg;
847 u64 guest_cid;
848 u64 features;
849 int start;
850 int r;
851
852 switch (ioctl) {
853 case VHOST_VSOCK_SET_GUEST_CID:
854 if (copy_from_user(&guest_cid, argp, sizeof(guest_cid)))
855 return -EFAULT;
856 return vhost_vsock_set_cid(vsock, guest_cid);
857 case VHOST_VSOCK_SET_RUNNING:
858 if (copy_from_user(&start, argp, sizeof(start)))
859 return -EFAULT;
860 if (start)
861 return vhost_vsock_start(vsock);
862 else
863 return vhost_vsock_stop(vsock, true);
864 case VHOST_GET_FEATURES:
865 features = VHOST_VSOCK_FEATURES;
866 if (copy_to_user(argp, &features, sizeof(features)))
867 return -EFAULT;
868 return 0;
869 case VHOST_SET_FEATURES:
870 if (copy_from_user(&features, argp, sizeof(features)))
871 return -EFAULT;
872 return vhost_vsock_set_features(vsock, features);
873 case VHOST_GET_BACKEND_FEATURES:
874 features = VHOST_VSOCK_BACKEND_FEATURES;
875 if (copy_to_user(argp, &features, sizeof(features)))
876 return -EFAULT;
877 return 0;
878 case VHOST_SET_BACKEND_FEATURES:
879 if (copy_from_user(&features, argp, sizeof(features)))
880 return -EFAULT;
881 if (features & ~VHOST_VSOCK_BACKEND_FEATURES)
882 return -EOPNOTSUPP;
883 vhost_set_backend_features(&vsock->dev, features);
884 return 0;
885 default:
886 mutex_lock(&vsock->dev.mutex);
887 r = vhost_dev_ioctl(&vsock->dev, ioctl, argp);
888 if (r == -ENOIOCTLCMD)
889 r = vhost_vring_ioctl(&vsock->dev, ioctl, argp);
890 else
891 vhost_vsock_flush(vsock);
892 mutex_unlock(&vsock->dev.mutex);
893 return r;
894 }
895 }
896
vhost_vsock_chr_read_iter(struct kiocb * iocb,struct iov_iter * to)897 static ssize_t vhost_vsock_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
898 {
899 struct file *file = iocb->ki_filp;
900 struct vhost_vsock *vsock = file->private_data;
901 struct vhost_dev *dev = &vsock->dev;
902 int noblock = file->f_flags & O_NONBLOCK;
903
904 return vhost_chr_read_iter(dev, to, noblock);
905 }
906
vhost_vsock_chr_write_iter(struct kiocb * iocb,struct iov_iter * from)907 static ssize_t vhost_vsock_chr_write_iter(struct kiocb *iocb,
908 struct iov_iter *from)
909 {
910 struct file *file = iocb->ki_filp;
911 struct vhost_vsock *vsock = file->private_data;
912 struct vhost_dev *dev = &vsock->dev;
913
914 return vhost_chr_write_iter(dev, from);
915 }
916
vhost_vsock_chr_poll(struct file * file,poll_table * wait)917 static __poll_t vhost_vsock_chr_poll(struct file *file, poll_table *wait)
918 {
919 struct vhost_vsock *vsock = file->private_data;
920 struct vhost_dev *dev = &vsock->dev;
921
922 return vhost_chr_poll(file, dev, wait);
923 }
924
925 static const struct file_operations vhost_vsock_fops = {
926 .owner = THIS_MODULE,
927 .open = vhost_vsock_dev_open,
928 .release = vhost_vsock_dev_release,
929 .llseek = noop_llseek,
930 .unlocked_ioctl = vhost_vsock_dev_ioctl,
931 .compat_ioctl = compat_ptr_ioctl,
932 .read_iter = vhost_vsock_chr_read_iter,
933 .write_iter = vhost_vsock_chr_write_iter,
934 .poll = vhost_vsock_chr_poll,
935 };
936
937 static struct miscdevice vhost_vsock_misc = {
938 .minor = VHOST_VSOCK_MINOR,
939 .name = "vhost-vsock",
940 .fops = &vhost_vsock_fops,
941 };
942
vhost_vsock_init(void)943 static int __init vhost_vsock_init(void)
944 {
945 int ret;
946
947 ret = vsock_core_register(&vhost_transport.transport,
948 VSOCK_TRANSPORT_F_H2G);
949 if (ret < 0)
950 return ret;
951
952 ret = misc_register(&vhost_vsock_misc);
953 if (ret) {
954 vsock_core_unregister(&vhost_transport.transport);
955 return ret;
956 }
957
958 return 0;
959 };
960
vhost_vsock_exit(void)961 static void __exit vhost_vsock_exit(void)
962 {
963 misc_deregister(&vhost_vsock_misc);
964 vsock_core_unregister(&vhost_transport.transport);
965 };
966
967 module_init(vhost_vsock_init);
968 module_exit(vhost_vsock_exit);
969 MODULE_LICENSE("GPL v2");
970 MODULE_AUTHOR("Asias He");
971 MODULE_DESCRIPTION("vhost transport for vsock ");
972 MODULE_ALIAS_MISCDEV(VHOST_VSOCK_MINOR);
973 MODULE_ALIAS("devname:vhost-vsock");
974