1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /*
30 * Implements the virtqueue interface as basically described
31 * in the original VirtIO paper.
32 */
33
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/malloc.h>
38 #include <sys/sdt.h>
39 #include <sys/sglist.h>
40 #include <vm/vm.h>
41 #include <vm/pmap.h>
42
43 #include <machine/cpu.h>
44 #include <machine/bus.h>
45 #include <machine/atomic.h>
46 #include <machine/resource.h>
47 #include <sys/bus.h>
48 #include <sys/rman.h>
49
50 #include <dev/virtio/virtio.h>
51 #include <dev/virtio/virtqueue.h>
52 #include <dev/virtio/virtio_ring.h>
53
54 #include "virtio_bus_if.h"
55
56 struct virtqueue {
57 device_t vq_dev;
58 uint16_t vq_queue_index;
59 uint16_t vq_nentries;
60 uint32_t vq_flags;
61 #define VIRTQUEUE_FLAG_MODERN 0x0001
62 #define VIRTQUEUE_FLAG_INDIRECT 0x0002
63 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0004
64
65 int vq_max_indirect_size;
66 bus_size_t vq_notify_offset;
67 virtqueue_intr_t *vq_intrhand;
68 void *vq_intrhand_arg;
69
70 struct vring vq_ring;
71 uint16_t vq_free_cnt;
72 uint16_t vq_queued_cnt;
73 /*
74 * Head of the free chain in the descriptor table. If
75 * there are no free descriptors, this will be set to
76 * VQ_RING_DESC_CHAIN_END.
77 */
78 uint16_t vq_desc_head_idx;
79 /*
80 * Last consumed descriptor in the used table,
81 * trails vq_ring.used->idx.
82 */
83 uint16_t vq_used_cons_idx;
84
85 void *vq_ring_mem;
86 int vq_indirect_mem_size;
87 int vq_alignment;
88 int vq_ring_size;
89 char vq_name[VIRTQUEUE_MAX_NAME_SZ];
90
91 struct vq_desc_extra {
92 void *cookie;
93 struct vring_desc *indirect;
94 vm_paddr_t indirect_paddr;
95 uint16_t ndescs;
96 } vq_descx[0];
97 };
98
99 /*
100 * The maximum virtqueue size is 2^15. Use that value as the end of
101 * descriptor chain terminator since it will never be a valid index
102 * in the descriptor table. This is used to verify we are correctly
103 * handling vq_free_cnt.
104 */
105 #define VQ_RING_DESC_CHAIN_END 32768
106
107 #define VQASSERT(_vq, _exp, _msg, ...) \
108 KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name, \
109 ##__VA_ARGS__))
110
111 #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx) \
112 VQASSERT((_vq), (_idx) < (_vq)->vq_nentries, \
113 "invalid ring index: %d, max: %d", (_idx), \
114 (_vq)->vq_nentries)
115
116 #define VQ_RING_ASSERT_CHAIN_TERM(_vq) \
117 VQASSERT((_vq), (_vq)->vq_desc_head_idx == \
118 VQ_RING_DESC_CHAIN_END, "full ring terminated " \
119 "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx)
120
121 static int virtqueue_init_indirect(struct virtqueue *vq, int);
122 static void virtqueue_free_indirect(struct virtqueue *vq);
123 static void virtqueue_init_indirect_list(struct virtqueue *,
124 struct vring_desc *);
125
126 static void vq_ring_init(struct virtqueue *);
127 static void vq_ring_update_avail(struct virtqueue *, uint16_t);
128 static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
129 struct vring_desc *, uint16_t, struct sglist *, int, int);
130 static bool vq_ring_use_indirect(struct virtqueue *, int);
131 static void vq_ring_enqueue_indirect(struct virtqueue *, void *,
132 struct sglist *, int, int);
133 static int vq_ring_enable_interrupt(struct virtqueue *, uint16_t);
134 static int vq_ring_must_notify_host(struct virtqueue *);
135 static void vq_ring_notify_host(struct virtqueue *);
136 static void vq_ring_free_chain(struct virtqueue *, uint16_t);
137
138 SDT_PROVIDER_DEFINE(virtqueue);
139 SDT_PROBE_DEFINE6(virtqueue, , enqueue_segments, entry, "struct virtqueue *",
140 "struct vring_desc *", "uint16_t", "struct sglist *", "int", "int");
141 SDT_PROBE_DEFINE1(virtqueue, , enqueue_segments, return, "uint16_t");
142
143 #define vq_modern(_vq) (((_vq)->vq_flags & VIRTQUEUE_FLAG_MODERN) != 0)
144 #define vq_htog16(_vq, _val) virtio_htog16(vq_modern(_vq), _val)
145 #define vq_htog32(_vq, _val) virtio_htog32(vq_modern(_vq), _val)
146 #define vq_htog64(_vq, _val) virtio_htog64(vq_modern(_vq), _val)
147 #define vq_gtoh16(_vq, _val) virtio_gtoh16(vq_modern(_vq), _val)
148 #define vq_gtoh32(_vq, _val) virtio_gtoh32(vq_modern(_vq), _val)
149 #define vq_gtoh64(_vq, _val) virtio_gtoh64(vq_modern(_vq), _val)
150
151 int
virtqueue_alloc(device_t dev,uint16_t queue,uint16_t size,bus_size_t notify_offset,int align,vm_paddr_t highaddr,struct vq_alloc_info * info,struct virtqueue ** vqp)152 virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size,
153 bus_size_t notify_offset, int align, vm_paddr_t highaddr,
154 struct vq_alloc_info *info, struct virtqueue **vqp)
155 {
156 struct virtqueue *vq;
157 int error;
158
159 *vqp = NULL;
160 error = 0;
161
162 if (size == 0) {
163 device_printf(dev,
164 "virtqueue %d (%s) does not exist (size is zero)\n",
165 queue, info->vqai_name);
166 return (ENODEV);
167 } else if (!powerof2(size)) {
168 device_printf(dev,
169 "virtqueue %d (%s) size is not a power of 2: %d\n",
170 queue, info->vqai_name, size);
171 return (ENXIO);
172 } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) {
173 device_printf(dev, "virtqueue %d (%s) requested too many "
174 "indirect descriptors: %d, max %d\n",
175 queue, info->vqai_name, info->vqai_maxindirsz,
176 VIRTIO_MAX_INDIRECT);
177 return (EINVAL);
178 }
179
180 vq = malloc(sizeof(struct virtqueue) +
181 size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO);
182 if (vq == NULL) {
183 device_printf(dev, "cannot allocate virtqueue\n");
184 return (ENOMEM);
185 }
186
187 vq->vq_dev = dev;
188 strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
189 vq->vq_queue_index = queue;
190 vq->vq_notify_offset = notify_offset;
191 vq->vq_alignment = align;
192 vq->vq_nentries = size;
193 vq->vq_free_cnt = size;
194 vq->vq_intrhand = info->vqai_intr;
195 vq->vq_intrhand_arg = info->vqai_intr_arg;
196
197 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_F_VERSION_1) != 0)
198 vq->vq_flags |= VIRTQUEUE_FLAG_MODERN;
199 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
200 vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
201
202 if (info->vqai_maxindirsz > 1) {
203 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
204 if (error)
205 goto fail;
206 }
207
208 vq->vq_ring_size = round_page(vring_size(size, align));
209 vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
210 M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
211 if (vq->vq_ring_mem == NULL) {
212 device_printf(dev,
213 "cannot allocate memory for virtqueue ring\n");
214 error = ENOMEM;
215 goto fail;
216 }
217
218 vq_ring_init(vq);
219 virtqueue_disable_intr(vq);
220
221 *vqp = vq;
222
223 fail:
224 if (error)
225 virtqueue_free(vq);
226
227 return (error);
228 }
229
230 static int
virtqueue_init_indirect(struct virtqueue * vq,int indirect_size)231 virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
232 {
233 device_t dev;
234 struct vq_desc_extra *dxp;
235 int i, size;
236
237 dev = vq->vq_dev;
238
239 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) {
240 /*
241 * Indirect descriptors requested by the driver but not
242 * negotiated. Return zero to keep the initialization
243 * going: we'll run fine without.
244 */
245 if (bootverbose)
246 device_printf(dev, "virtqueue %d (%s) requested "
247 "indirect descriptors but not negotiated\n",
248 vq->vq_queue_index, vq->vq_name);
249 return (0);
250 }
251
252 size = indirect_size * sizeof(struct vring_desc);
253 vq->vq_max_indirect_size = indirect_size;
254 vq->vq_indirect_mem_size = size;
255 vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
256
257 for (i = 0; i < vq->vq_nentries; i++) {
258 dxp = &vq->vq_descx[i];
259
260 dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT);
261 if (dxp->indirect == NULL) {
262 device_printf(dev, "cannot allocate indirect list\n");
263 return (ENOMEM);
264 }
265
266 dxp->indirect_paddr = vtophys(dxp->indirect);
267 virtqueue_init_indirect_list(vq, dxp->indirect);
268 }
269
270 return (0);
271 }
272
273 static void
virtqueue_free_indirect(struct virtqueue * vq)274 virtqueue_free_indirect(struct virtqueue *vq)
275 {
276 struct vq_desc_extra *dxp;
277 int i;
278
279 for (i = 0; i < vq->vq_nentries; i++) {
280 dxp = &vq->vq_descx[i];
281
282 if (dxp->indirect == NULL)
283 break;
284
285 free(dxp->indirect, M_DEVBUF);
286 dxp->indirect = NULL;
287 dxp->indirect_paddr = 0;
288 }
289
290 vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT;
291 vq->vq_indirect_mem_size = 0;
292 }
293
294 static void
virtqueue_init_indirect_list(struct virtqueue * vq,struct vring_desc * indirect)295 virtqueue_init_indirect_list(struct virtqueue *vq,
296 struct vring_desc *indirect)
297 {
298 int i;
299
300 bzero(indirect, vq->vq_indirect_mem_size);
301
302 for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
303 indirect[i].next = vq_gtoh16(vq, i + 1);
304 indirect[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);
305 }
306
307 int
virtqueue_reinit(struct virtqueue * vq,uint16_t size)308 virtqueue_reinit(struct virtqueue *vq, uint16_t size)
309 {
310 struct vq_desc_extra *dxp;
311 int i;
312
313 if (vq->vq_nentries != size) {
314 device_printf(vq->vq_dev,
315 "%s: '%s' changed size; old=%hu, new=%hu\n",
316 __func__, vq->vq_name, vq->vq_nentries, size);
317 return (EINVAL);
318 }
319
320 /* Warn if the virtqueue was not properly cleaned up. */
321 if (vq->vq_free_cnt != vq->vq_nentries) {
322 device_printf(vq->vq_dev,
323 "%s: warning '%s' virtqueue not empty, "
324 "leaking %d entries\n", __func__, vq->vq_name,
325 vq->vq_nentries - vq->vq_free_cnt);
326 }
327
328 vq->vq_desc_head_idx = 0;
329 vq->vq_used_cons_idx = 0;
330 vq->vq_queued_cnt = 0;
331 vq->vq_free_cnt = vq->vq_nentries;
332
333 /* To be safe, reset all our allocated memory. */
334 bzero(vq->vq_ring_mem, vq->vq_ring_size);
335 for (i = 0; i < vq->vq_nentries; i++) {
336 dxp = &vq->vq_descx[i];
337 dxp->cookie = NULL;
338 dxp->ndescs = 0;
339 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
340 virtqueue_init_indirect_list(vq, dxp->indirect);
341 }
342
343 vq_ring_init(vq);
344 virtqueue_disable_intr(vq);
345
346 return (0);
347 }
348
349 void
virtqueue_free(struct virtqueue * vq)350 virtqueue_free(struct virtqueue *vq)
351 {
352
353 if (vq->vq_free_cnt != vq->vq_nentries) {
354 device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, "
355 "leaking %d entries\n", vq->vq_name,
356 vq->vq_nentries - vq->vq_free_cnt);
357 }
358
359 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
360 virtqueue_free_indirect(vq);
361
362 if (vq->vq_ring_mem != NULL) {
363 free(vq->vq_ring_mem, M_DEVBUF);
364 vq->vq_ring_size = 0;
365 vq->vq_ring_mem = NULL;
366 }
367
368 free(vq, M_DEVBUF);
369 }
370
371 vm_paddr_t
virtqueue_paddr(struct virtqueue * vq)372 virtqueue_paddr(struct virtqueue *vq)
373 {
374
375 return (vtophys(vq->vq_ring_mem));
376 }
377
378 vm_paddr_t
virtqueue_desc_paddr(struct virtqueue * vq)379 virtqueue_desc_paddr(struct virtqueue *vq)
380 {
381
382 return (vtophys(vq->vq_ring.desc));
383 }
384
385 vm_paddr_t
virtqueue_avail_paddr(struct virtqueue * vq)386 virtqueue_avail_paddr(struct virtqueue *vq)
387 {
388
389 return (vtophys(vq->vq_ring.avail));
390 }
391
392 vm_paddr_t
virtqueue_used_paddr(struct virtqueue * vq)393 virtqueue_used_paddr(struct virtqueue *vq)
394 {
395
396 return (vtophys(vq->vq_ring.used));
397 }
398
399 uint16_t
virtqueue_index(struct virtqueue * vq)400 virtqueue_index(struct virtqueue *vq)
401 {
402
403 return (vq->vq_queue_index);
404 }
405
406 int
virtqueue_size(struct virtqueue * vq)407 virtqueue_size(struct virtqueue *vq)
408 {
409
410 return (vq->vq_nentries);
411 }
412
413 int
virtqueue_nfree(struct virtqueue * vq)414 virtqueue_nfree(struct virtqueue *vq)
415 {
416
417 return (vq->vq_free_cnt);
418 }
419
420 bool
virtqueue_empty(struct virtqueue * vq)421 virtqueue_empty(struct virtqueue *vq)
422 {
423
424 return (vq->vq_nentries == vq->vq_free_cnt);
425 }
426
427 bool
virtqueue_full(struct virtqueue * vq)428 virtqueue_full(struct virtqueue *vq)
429 {
430
431 return (vq->vq_free_cnt == 0);
432 }
433
434 void
virtqueue_notify(struct virtqueue * vq)435 virtqueue_notify(struct virtqueue *vq)
436 {
437
438 /* Ensure updated avail->idx is visible to host. */
439 mb();
440
441 if (vq_ring_must_notify_host(vq))
442 vq_ring_notify_host(vq);
443 vq->vq_queued_cnt = 0;
444 }
445
446 int
virtqueue_nused(struct virtqueue * vq)447 virtqueue_nused(struct virtqueue *vq)
448 {
449 uint16_t used_idx, nused;
450
451 used_idx = vq_htog16(vq, vq->vq_ring.used->idx);
452
453 nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
454 VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
455
456 return (nused);
457 }
458
459 int
virtqueue_intr_filter(struct virtqueue * vq)460 virtqueue_intr_filter(struct virtqueue *vq)
461 {
462
463 if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))
464 return (0);
465
466 virtqueue_disable_intr(vq);
467
468 return (1);
469 }
470
471 void
virtqueue_intr(struct virtqueue * vq)472 virtqueue_intr(struct virtqueue *vq)
473 {
474
475 vq->vq_intrhand(vq->vq_intrhand_arg);
476 }
477
478 int
virtqueue_enable_intr(struct virtqueue * vq)479 virtqueue_enable_intr(struct virtqueue *vq)
480 {
481
482 return (vq_ring_enable_interrupt(vq, 0));
483 }
484
485 int
virtqueue_postpone_intr(struct virtqueue * vq,vq_postpone_t hint)486 virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint)
487 {
488 uint16_t ndesc, avail_idx;
489
490 avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
491 ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);
492
493 switch (hint) {
494 case VQ_POSTPONE_SHORT:
495 ndesc = ndesc / 4;
496 break;
497 case VQ_POSTPONE_LONG:
498 ndesc = (ndesc * 3) / 4;
499 break;
500 case VQ_POSTPONE_EMPTIED:
501 break;
502 }
503
504 return (vq_ring_enable_interrupt(vq, ndesc));
505 }
506
507 /*
508 * Note this is only considered a hint to the host.
509 */
510 void
virtqueue_disable_intr(struct virtqueue * vq)511 virtqueue_disable_intr(struct virtqueue *vq)
512 {
513
514 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
515 vring_used_event(&vq->vq_ring) = vq_gtoh16(vq,
516 vq->vq_used_cons_idx - vq->vq_nentries - 1);
517 return;
518 }
519
520 vq->vq_ring.avail->flags |= vq_gtoh16(vq, VRING_AVAIL_F_NO_INTERRUPT);
521 }
522
523 int
virtqueue_enqueue(struct virtqueue * vq,void * cookie,struct sglist * sg,int readable,int writable)524 virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
525 int readable, int writable)
526 {
527 struct vq_desc_extra *dxp;
528 int needed;
529 uint16_t head_idx, idx;
530
531 needed = readable + writable;
532
533 VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
534 VQASSERT(vq, needed == sg->sg_nseg,
535 "segment count mismatch, %d, %d", needed, sg->sg_nseg);
536 VQASSERT(vq,
537 needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
538 "too many segments to enqueue: %d, %d/%d", needed,
539 vq->vq_nentries, vq->vq_max_indirect_size);
540
541 if (needed < 1)
542 return (EINVAL);
543 if (vq->vq_free_cnt == 0)
544 return (ENOSPC);
545
546 if (vq_ring_use_indirect(vq, needed)) {
547 vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
548 return (0);
549 } else if (vq->vq_free_cnt < needed)
550 return (EMSGSIZE);
551
552 head_idx = vq->vq_desc_head_idx;
553 VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
554 dxp = &vq->vq_descx[head_idx];
555
556 VQASSERT(vq, dxp->cookie == NULL,
557 "cookie already exists for index %d", head_idx);
558 dxp->cookie = cookie;
559 dxp->ndescs = needed;
560
561 idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
562 sg, readable, writable);
563
564 vq->vq_desc_head_idx = idx;
565 vq->vq_free_cnt -= needed;
566 if (vq->vq_free_cnt == 0)
567 VQ_RING_ASSERT_CHAIN_TERM(vq);
568 else
569 VQ_RING_ASSERT_VALID_IDX(vq, idx);
570
571 vq_ring_update_avail(vq, head_idx);
572
573 return (0);
574 }
575
576 void *
virtqueue_dequeue(struct virtqueue * vq,uint32_t * len)577 virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
578 {
579 struct vring_used_elem *uep;
580 void *cookie;
581 uint16_t used_idx, desc_idx;
582
583 if (vq->vq_used_cons_idx ==
584 vq_htog16(vq, atomic_load_16(&vq->vq_ring.used->idx)))
585 return (NULL);
586
587 used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
588 uep = &vq->vq_ring.used->ring[used_idx];
589
590 rmb();
591 desc_idx = (uint16_t) vq_htog32(vq, uep->id);
592 if (len != NULL)
593 *len = vq_htog32(vq, uep->len);
594
595 vq_ring_free_chain(vq, desc_idx);
596
597 cookie = vq->vq_descx[desc_idx].cookie;
598 VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx);
599 vq->vq_descx[desc_idx].cookie = NULL;
600
601 return (cookie);
602 }
603
604 void *
virtqueue_poll(struct virtqueue * vq,uint32_t * len)605 virtqueue_poll(struct virtqueue *vq, uint32_t *len)
606 {
607 void *cookie;
608
609 while ((cookie = virtqueue_dequeue(vq, len)) == NULL) {
610 cpu_spinwait();
611 }
612
613 return (cookie);
614 }
615
616 void *
virtqueue_drain(struct virtqueue * vq,int * last)617 virtqueue_drain(struct virtqueue *vq, int *last)
618 {
619 void *cookie;
620 int idx;
621
622 cookie = NULL;
623 idx = *last;
624
625 while (idx < vq->vq_nentries && cookie == NULL) {
626 if ((cookie = vq->vq_descx[idx].cookie) != NULL) {
627 vq->vq_descx[idx].cookie = NULL;
628 /* Free chain to keep free count consistent. */
629 vq_ring_free_chain(vq, idx);
630 }
631 idx++;
632 }
633
634 *last = idx;
635
636 return (cookie);
637 }
638
639 void
virtqueue_dump(struct virtqueue * vq)640 virtqueue_dump(struct virtqueue *vq)
641 {
642
643 if (vq == NULL)
644 return;
645
646 printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
647 "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
648 "used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
649 vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, virtqueue_nused(vq),
650 vq->vq_queued_cnt, vq->vq_desc_head_idx,
651 vq_htog16(vq, vq->vq_ring.avail->idx), vq->vq_used_cons_idx,
652 vq_htog16(vq, vq->vq_ring.used->idx),
653 vq_htog16(vq, vring_used_event(&vq->vq_ring)),
654 vq_htog16(vq, vq->vq_ring.avail->flags),
655 vq_htog16(vq, vq->vq_ring.used->flags));
656 }
657
658 static void
vq_ring_init(struct virtqueue * vq)659 vq_ring_init(struct virtqueue *vq)
660 {
661 struct vring *vr;
662 char *ring_mem;
663 int i, size;
664
665 ring_mem = vq->vq_ring_mem;
666 size = vq->vq_nentries;
667 vr = &vq->vq_ring;
668
669 vring_init(vr, size, ring_mem, vq->vq_alignment);
670
671 for (i = 0; i < size - 1; i++)
672 vr->desc[i].next = vq_gtoh16(vq, i + 1);
673 vr->desc[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);
674 }
675
676 static void
vq_ring_update_avail(struct virtqueue * vq,uint16_t desc_idx)677 vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
678 {
679 uint16_t avail_idx, avail_ring_idx;
680
681 /*
682 * Place the head of the descriptor chain into the next slot and make
683 * it usable to the host. The chain is made available now rather than
684 * deferring to virtqueue_notify() in the hopes that if the host is
685 * currently running on another CPU, we can keep it processing the new
686 * descriptor.
687 */
688 avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
689 avail_ring_idx = avail_idx & (vq->vq_nentries - 1);
690 vq->vq_ring.avail->ring[avail_ring_idx] = vq_gtoh16(vq, desc_idx);
691
692 wmb();
693 vq->vq_ring.avail->idx = vq_gtoh16(vq, avail_idx + 1);
694
695 /* Keep pending count until virtqueue_notify(). */
696 vq->vq_queued_cnt++;
697 }
698
699 static uint16_t
vq_ring_enqueue_segments(struct virtqueue * vq,struct vring_desc * desc,uint16_t head_idx,struct sglist * sg,int readable,int writable)700 vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
701 uint16_t head_idx, struct sglist *sg, int readable, int writable)
702 {
703 struct sglist_seg *seg;
704 struct vring_desc *dp;
705 int i, needed;
706 uint16_t idx;
707
708 SDT_PROBE6(virtqueue, , enqueue_segments, entry, vq, desc, head_idx,
709 sg, readable, writable);
710
711 needed = readable + writable;
712
713 for (i = 0, idx = head_idx, seg = sg->sg_segs;
714 i < needed;
715 i++, idx = vq_htog16(vq, dp->next), seg++) {
716 VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
717 "premature end of free desc chain");
718
719 dp = &desc[idx];
720 dp->addr = vq_gtoh64(vq, seg->ss_paddr);
721 dp->len = vq_gtoh32(vq, seg->ss_len);
722 dp->flags = 0;
723
724 if (i < needed - 1)
725 dp->flags |= vq_gtoh16(vq, VRING_DESC_F_NEXT);
726 if (i >= readable)
727 dp->flags |= vq_gtoh16(vq, VRING_DESC_F_WRITE);
728 }
729
730 SDT_PROBE1(virtqueue, , enqueue_segments, return, idx);
731 return (idx);
732 }
733
734 static bool
vq_ring_use_indirect(struct virtqueue * vq,int needed)735 vq_ring_use_indirect(struct virtqueue *vq, int needed)
736 {
737
738 if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0)
739 return (false);
740
741 if (vq->vq_max_indirect_size < needed)
742 return (false);
743
744 if (needed < 2)
745 return (false);
746
747 return (true);
748 }
749
750 static void
vq_ring_enqueue_indirect(struct virtqueue * vq,void * cookie,struct sglist * sg,int readable,int writable)751 vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
752 struct sglist *sg, int readable, int writable)
753 {
754 struct vring_desc *dp;
755 struct vq_desc_extra *dxp;
756 int needed;
757 uint16_t head_idx;
758
759 needed = readable + writable;
760 VQASSERT(vq, needed <= vq->vq_max_indirect_size,
761 "enqueuing too many indirect descriptors");
762
763 head_idx = vq->vq_desc_head_idx;
764 VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
765 dp = &vq->vq_ring.desc[head_idx];
766 dxp = &vq->vq_descx[head_idx];
767
768 VQASSERT(vq, dxp->cookie == NULL,
769 "cookie already exists for index %d", head_idx);
770 dxp->cookie = cookie;
771 dxp->ndescs = 1;
772
773 dp->addr = vq_gtoh64(vq, dxp->indirect_paddr);
774 dp->len = vq_gtoh32(vq, needed * sizeof(struct vring_desc));
775 dp->flags = vq_gtoh16(vq, VRING_DESC_F_INDIRECT);
776
777 vq_ring_enqueue_segments(vq, dxp->indirect, 0,
778 sg, readable, writable);
779
780 vq->vq_desc_head_idx = vq_htog16(vq, dp->next);
781 vq->vq_free_cnt--;
782 if (vq->vq_free_cnt == 0)
783 VQ_RING_ASSERT_CHAIN_TERM(vq);
784 else
785 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
786
787 vq_ring_update_avail(vq, head_idx);
788 }
789
790 static int
vq_ring_enable_interrupt(struct virtqueue * vq,uint16_t ndesc)791 vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc)
792 {
793
794 /*
795 * Enable interrupts, making sure we get the latest index of
796 * what's already been consumed.
797 */
798 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
799 vring_used_event(&vq->vq_ring) =
800 vq_gtoh16(vq, vq->vq_used_cons_idx + ndesc);
801 } else {
802 vq->vq_ring.avail->flags &=
803 vq_gtoh16(vq, ~VRING_AVAIL_F_NO_INTERRUPT);
804 }
805
806 mb();
807
808 /*
809 * Enough items may have already been consumed to meet our threshold
810 * since we last checked. Let our caller know so it processes the new
811 * entries.
812 */
813 if (virtqueue_nused(vq) > ndesc)
814 return (1);
815
816 return (0);
817 }
818
819 static int
vq_ring_must_notify_host(struct virtqueue * vq)820 vq_ring_must_notify_host(struct virtqueue *vq)
821 {
822 uint16_t new_idx, prev_idx, event_idx, flags;
823
824 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
825 new_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
826 prev_idx = new_idx - vq->vq_queued_cnt;
827 event_idx = vq_htog16(vq, vring_avail_event(&vq->vq_ring));
828
829 return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
830 }
831
832 flags = vq->vq_ring.used->flags;
833 return ((flags & vq_gtoh16(vq, VRING_USED_F_NO_NOTIFY)) == 0);
834 }
835
836 static void
vq_ring_notify_host(struct virtqueue * vq)837 vq_ring_notify_host(struct virtqueue *vq)
838 {
839
840 VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index,
841 vq->vq_notify_offset);
842 }
843
844 static void
vq_ring_free_chain(struct virtqueue * vq,uint16_t desc_idx)845 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
846 {
847 struct vring_desc *dp;
848 struct vq_desc_extra *dxp;
849
850 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
851 dp = &vq->vq_ring.desc[desc_idx];
852 dxp = &vq->vq_descx[desc_idx];
853
854 if (vq->vq_free_cnt == 0)
855 VQ_RING_ASSERT_CHAIN_TERM(vq);
856
857 vq->vq_free_cnt += dxp->ndescs;
858 dxp->ndescs--;
859
860 if ((dp->flags & vq_gtoh16(vq, VRING_DESC_F_INDIRECT)) == 0) {
861 while (dp->flags & vq_gtoh16(vq, VRING_DESC_F_NEXT)) {
862 uint16_t next_idx = vq_htog16(vq, dp->next);
863 VQ_RING_ASSERT_VALID_IDX(vq, next_idx);
864 dp = &vq->vq_ring.desc[next_idx];
865 dxp->ndescs--;
866 }
867 }
868
869 VQASSERT(vq, dxp->ndescs == 0,
870 "failed to free entire desc chain, remaining: %d", dxp->ndescs);
871
872 /*
873 * We must append the existing free chain, if any, to the end of
874 * newly freed chain. If the virtqueue was completely used, then
875 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
876 */
877 dp->next = vq_gtoh16(vq, vq->vq_desc_head_idx);
878 vq->vq_desc_head_idx = desc_idx;
879 }
880