1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * Implements the virtqueue interface as basically described 31 * in the original VirtIO paper. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/sglist.h> 42 #include <vm/vm.h> 43 #include <vm/pmap.h> 44 45 #include <machine/cpu.h> 46 #include <machine/bus.h> 47 #include <machine/atomic.h> 48 #include <machine/resource.h> 49 #include <sys/bus.h> 50 #include <sys/rman.h> 51 52 #include <dev/virtio/virtio.h> 53 #include <dev/virtio/virtqueue.h> 54 #include <dev/virtio/virtio_ring.h> 55 56 #include "virtio_bus_if.h" 57 58 struct virtqueue { 59 device_t vq_dev; 60 char vq_name[VIRTQUEUE_MAX_NAME_SZ]; 61 uint16_t vq_queue_index; 62 uint16_t vq_nentries; 63 uint32_t vq_flags; 64 #define VIRTQUEUE_FLAG_INDIRECT 0x0001 65 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0002 66 67 int vq_alignment; 68 int vq_ring_size; 69 void *vq_ring_mem; 70 int vq_max_indirect_size; 71 int vq_indirect_mem_size; 72 virtqueue_intr_t *vq_intrhand; 73 void *vq_intrhand_arg; 74 75 struct vring vq_ring; 76 uint16_t vq_free_cnt; 77 uint16_t vq_queued_cnt; 78 /* 79 * Head of the free chain in the descriptor table. If 80 * there are no free descriptors, this will be set to 81 * VQ_RING_DESC_CHAIN_END. 82 */ 83 uint16_t vq_desc_head_idx; 84 /* 85 * Last consumed descriptor in the used table, 86 * trails vq_ring.used->idx. 87 */ 88 uint16_t vq_used_cons_idx; 89 90 struct vq_desc_extra { 91 void *cookie; 92 struct vring_desc *indirect; 93 vm_paddr_t indirect_paddr; 94 uint16_t ndescs; 95 } vq_descx[0]; 96 }; 97 98 /* 99 * The maximum virtqueue size is 2^15. Use that value as the end of 100 * descriptor chain terminator since it will never be a valid index 101 * in the descriptor table. This is used to verify we are correctly 102 * handling vq_free_cnt. 103 */ 104 #define VQ_RING_DESC_CHAIN_END 32768 105 106 #define VQASSERT(_vq, _exp, _msg, ...) \ 107 KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name, \ 108 ##__VA_ARGS__)) 109 110 #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx) \ 111 VQASSERT((_vq), (_idx) < (_vq)->vq_nentries, \ 112 "invalid ring index: %d, max: %d", (_idx), \ 113 (_vq)->vq_nentries) 114 115 #define VQ_RING_ASSERT_CHAIN_TERM(_vq) \ 116 VQASSERT((_vq), (_vq)->vq_desc_head_idx == \ 117 VQ_RING_DESC_CHAIN_END, "full ring terminated " \ 118 "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx) 119 120 static int virtqueue_init_indirect(struct virtqueue *vq, int); 121 static void virtqueue_free_indirect(struct virtqueue *vq); 122 static void virtqueue_init_indirect_list(struct virtqueue *, 123 struct vring_desc *); 124 125 static void vq_ring_init(struct virtqueue *); 126 static void vq_ring_update_avail(struct virtqueue *, uint16_t); 127 static uint16_t vq_ring_enqueue_segments(struct virtqueue *, 128 struct vring_desc *, uint16_t, struct sglist *, int, int); 129 static int vq_ring_use_indirect(struct virtqueue *, int); 130 static void vq_ring_enqueue_indirect(struct virtqueue *, void *, 131 struct sglist *, int, int); 132 static int vq_ring_enable_interrupt(struct virtqueue *, uint16_t); 133 static int vq_ring_must_notify_host(struct virtqueue *); 134 static void vq_ring_notify_host(struct virtqueue *); 135 static void vq_ring_free_chain(struct virtqueue *, uint16_t); 136 137 uint64_t 138 virtqueue_filter_features(uint64_t features) 139 { 140 uint64_t mask; 141 142 mask = (1 << VIRTIO_TRANSPORT_F_START) - 1; 143 mask |= VIRTIO_RING_F_INDIRECT_DESC; 144 mask |= VIRTIO_RING_F_EVENT_IDX; 145 146 return (features & mask); 147 } 148 149 int 150 virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align, 151 vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp) 152 { 153 struct virtqueue *vq; 154 int error; 155 156 *vqp = NULL; 157 error = 0; 158 159 if (size == 0) { 160 device_printf(dev, 161 "virtqueue %d (%s) does not exist (size is zero)\n", 162 queue, info->vqai_name); 163 return (ENODEV); 164 } else if (!powerof2(size)) { 165 device_printf(dev, 166 "virtqueue %d (%s) size is not a power of 2: %d\n", 167 queue, info->vqai_name, size); 168 return (ENXIO); 169 } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) { 170 device_printf(dev, "virtqueue %d (%s) requested too many " 171 "indirect descriptors: %d, max %d\n", 172 queue, info->vqai_name, info->vqai_maxindirsz, 173 VIRTIO_MAX_INDIRECT); 174 return (EINVAL); 175 } 176 177 vq = malloc(sizeof(struct virtqueue) + 178 size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO); 179 if (vq == NULL) { 180 device_printf(dev, "cannot allocate virtqueue\n"); 181 return (ENOMEM); 182 } 183 184 vq->vq_dev = dev; 185 strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name)); 186 vq->vq_queue_index = queue; 187 vq->vq_alignment = align; 188 vq->vq_nentries = size; 189 vq->vq_free_cnt = size; 190 vq->vq_intrhand = info->vqai_intr; 191 vq->vq_intrhand_arg = info->vqai_intr_arg; 192 193 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0) 194 vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX; 195 196 if (info->vqai_maxindirsz > 1) { 197 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz); 198 if (error) 199 goto fail; 200 } 201 202 vq->vq_ring_size = round_page(vring_size(size, align)); 203 vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF, 204 M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0); 205 if (vq->vq_ring_mem == NULL) { 206 device_printf(dev, 207 "cannot allocate memory for virtqueue ring\n"); 208 error = ENOMEM; 209 goto fail; 210 } 211 212 vq_ring_init(vq); 213 virtqueue_disable_intr(vq); 214 215 *vqp = vq; 216 217 fail: 218 if (error) 219 virtqueue_free(vq); 220 221 return (error); 222 } 223 224 static int 225 virtqueue_init_indirect(struct virtqueue *vq, int indirect_size) 226 { 227 device_t dev; 228 struct vq_desc_extra *dxp; 229 int i, size; 230 231 dev = vq->vq_dev; 232 233 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) { 234 /* 235 * Indirect descriptors requested by the driver but not 236 * negotiated. Return zero to keep the initialization 237 * going: we'll run fine without. 238 */ 239 if (bootverbose) 240 device_printf(dev, "virtqueue %d (%s) requested " 241 "indirect descriptors but not negotiated\n", 242 vq->vq_queue_index, vq->vq_name); 243 return (0); 244 } 245 246 size = indirect_size * sizeof(struct vring_desc); 247 vq->vq_max_indirect_size = indirect_size; 248 vq->vq_indirect_mem_size = size; 249 vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT; 250 251 for (i = 0; i < vq->vq_nentries; i++) { 252 dxp = &vq->vq_descx[i]; 253 254 dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT); 255 if (dxp->indirect == NULL) { 256 device_printf(dev, "cannot allocate indirect list\n"); 257 return (ENOMEM); 258 } 259 260 dxp->indirect_paddr = vtophys(dxp->indirect); 261 virtqueue_init_indirect_list(vq, dxp->indirect); 262 } 263 264 return (0); 265 } 266 267 static void 268 virtqueue_free_indirect(struct virtqueue *vq) 269 { 270 struct vq_desc_extra *dxp; 271 int i; 272 273 for (i = 0; i < vq->vq_nentries; i++) { 274 dxp = &vq->vq_descx[i]; 275 276 if (dxp->indirect == NULL) 277 break; 278 279 free(dxp->indirect, M_DEVBUF); 280 dxp->indirect = NULL; 281 dxp->indirect_paddr = 0; 282 } 283 284 vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT; 285 vq->vq_indirect_mem_size = 0; 286 } 287 288 static void 289 virtqueue_init_indirect_list(struct virtqueue *vq, 290 struct vring_desc *indirect) 291 { 292 int i; 293 294 bzero(indirect, vq->vq_indirect_mem_size); 295 296 for (i = 0; i < vq->vq_max_indirect_size - 1; i++) 297 indirect[i].next = i + 1; 298 indirect[i].next = VQ_RING_DESC_CHAIN_END; 299 } 300 301 int 302 virtqueue_reinit(struct virtqueue *vq, uint16_t size) 303 { 304 struct vq_desc_extra *dxp; 305 int i; 306 307 if (vq->vq_nentries != size) { 308 device_printf(vq->vq_dev, 309 "%s: '%s' changed size; old=%hu, new=%hu\n", 310 __func__, vq->vq_name, vq->vq_nentries, size); 311 return (EINVAL); 312 } 313 314 /* Warn if the virtqueue was not properly cleaned up. */ 315 if (vq->vq_free_cnt != vq->vq_nentries) { 316 device_printf(vq->vq_dev, 317 "%s: warning '%s' virtqueue not empty, " 318 "leaking %d entries\n", __func__, vq->vq_name, 319 vq->vq_nentries - vq->vq_free_cnt); 320 } 321 322 vq->vq_desc_head_idx = 0; 323 vq->vq_used_cons_idx = 0; 324 vq->vq_queued_cnt = 0; 325 vq->vq_free_cnt = vq->vq_nentries; 326 327 /* To be safe, reset all our allocated memory. */ 328 bzero(vq->vq_ring_mem, vq->vq_ring_size); 329 for (i = 0; i < vq->vq_nentries; i++) { 330 dxp = &vq->vq_descx[i]; 331 dxp->cookie = NULL; 332 dxp->ndescs = 0; 333 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) 334 virtqueue_init_indirect_list(vq, dxp->indirect); 335 } 336 337 vq_ring_init(vq); 338 virtqueue_disable_intr(vq); 339 340 return (0); 341 } 342 343 void 344 virtqueue_free(struct virtqueue *vq) 345 { 346 347 if (vq->vq_free_cnt != vq->vq_nentries) { 348 device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, " 349 "leaking %d entries\n", vq->vq_name, 350 vq->vq_nentries - vq->vq_free_cnt); 351 } 352 353 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) 354 virtqueue_free_indirect(vq); 355 356 if (vq->vq_ring_mem != NULL) { 357 contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF); 358 vq->vq_ring_size = 0; 359 vq->vq_ring_mem = NULL; 360 } 361 362 free(vq, M_DEVBUF); 363 } 364 365 vm_paddr_t 366 virtqueue_paddr(struct virtqueue *vq) 367 { 368 369 return (vtophys(vq->vq_ring_mem)); 370 } 371 372 vm_paddr_t 373 virtqueue_desc_paddr(struct virtqueue *vq) 374 { 375 376 return (vtophys(vq->vq_ring.desc)); 377 } 378 379 vm_paddr_t 380 virtqueue_avail_paddr(struct virtqueue *vq) 381 { 382 383 return (vtophys(vq->vq_ring.avail)); 384 } 385 386 vm_paddr_t 387 virtqueue_used_paddr(struct virtqueue *vq) 388 { 389 390 return (vtophys(vq->vq_ring.used)); 391 } 392 393 uint16_t 394 virtqueue_index(struct virtqueue *vq) 395 { 396 return (vq->vq_queue_index); 397 } 398 399 int 400 virtqueue_size(struct virtqueue *vq) 401 { 402 403 return (vq->vq_nentries); 404 } 405 406 int 407 virtqueue_nfree(struct virtqueue *vq) 408 { 409 410 return (vq->vq_free_cnt); 411 } 412 413 int 414 virtqueue_empty(struct virtqueue *vq) 415 { 416 417 return (vq->vq_nentries == vq->vq_free_cnt); 418 } 419 420 int 421 virtqueue_full(struct virtqueue *vq) 422 { 423 424 return (vq->vq_free_cnt == 0); 425 } 426 427 void 428 virtqueue_notify(struct virtqueue *vq) 429 { 430 431 /* Ensure updated avail->idx is visible to host. */ 432 mb(); 433 434 if (vq_ring_must_notify_host(vq)) 435 vq_ring_notify_host(vq); 436 vq->vq_queued_cnt = 0; 437 } 438 439 int 440 virtqueue_nused(struct virtqueue *vq) 441 { 442 uint16_t used_idx, nused; 443 444 used_idx = vq->vq_ring.used->idx; 445 446 nused = (uint16_t)(used_idx - vq->vq_used_cons_idx); 447 VQASSERT(vq, nused <= vq->vq_nentries, "used more than available"); 448 449 return (nused); 450 } 451 452 int 453 virtqueue_intr_filter(struct virtqueue *vq) 454 { 455 456 if (vq->vq_used_cons_idx == vq->vq_ring.used->idx) 457 return (0); 458 459 virtqueue_disable_intr(vq); 460 461 return (1); 462 } 463 464 void 465 virtqueue_intr(struct virtqueue *vq) 466 { 467 468 vq->vq_intrhand(vq->vq_intrhand_arg); 469 } 470 471 int 472 virtqueue_enable_intr(struct virtqueue *vq) 473 { 474 475 return (vq_ring_enable_interrupt(vq, 0)); 476 } 477 478 int 479 virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint) 480 { 481 uint16_t ndesc, avail_idx; 482 483 avail_idx = vq->vq_ring.avail->idx; 484 ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx); 485 486 switch (hint) { 487 case VQ_POSTPONE_SHORT: 488 ndesc = ndesc / 4; 489 break; 490 case VQ_POSTPONE_LONG: 491 ndesc = (ndesc * 3) / 4; 492 break; 493 case VQ_POSTPONE_EMPTIED: 494 break; 495 } 496 497 return (vq_ring_enable_interrupt(vq, ndesc)); 498 } 499 500 /* 501 * Note this is only considered a hint to the host. 502 */ 503 void 504 virtqueue_disable_intr(struct virtqueue *vq) 505 { 506 507 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { 508 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx - 509 vq->vq_nentries - 1; 510 } else 511 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 512 } 513 514 int 515 virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg, 516 int readable, int writable) 517 { 518 struct vq_desc_extra *dxp; 519 int needed; 520 uint16_t head_idx, idx; 521 522 needed = readable + writable; 523 524 VQASSERT(vq, cookie != NULL, "enqueuing with no cookie"); 525 VQASSERT(vq, needed == sg->sg_nseg, 526 "segment count mismatch, %d, %d", needed, sg->sg_nseg); 527 VQASSERT(vq, 528 needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size, 529 "too many segments to enqueue: %d, %d/%d", needed, 530 vq->vq_nentries, vq->vq_max_indirect_size); 531 532 if (needed < 1) 533 return (EINVAL); 534 if (vq->vq_free_cnt == 0) 535 return (ENOSPC); 536 537 if (vq_ring_use_indirect(vq, needed)) { 538 vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable); 539 return (0); 540 } else if (vq->vq_free_cnt < needed) 541 return (EMSGSIZE); 542 543 head_idx = vq->vq_desc_head_idx; 544 VQ_RING_ASSERT_VALID_IDX(vq, head_idx); 545 dxp = &vq->vq_descx[head_idx]; 546 547 VQASSERT(vq, dxp->cookie == NULL, 548 "cookie already exists for index %d", head_idx); 549 dxp->cookie = cookie; 550 dxp->ndescs = needed; 551 552 idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx, 553 sg, readable, writable); 554 555 vq->vq_desc_head_idx = idx; 556 vq->vq_free_cnt -= needed; 557 if (vq->vq_free_cnt == 0) 558 VQ_RING_ASSERT_CHAIN_TERM(vq); 559 else 560 VQ_RING_ASSERT_VALID_IDX(vq, idx); 561 562 vq_ring_update_avail(vq, head_idx); 563 564 return (0); 565 } 566 567 void * 568 virtqueue_dequeue(struct virtqueue *vq, uint32_t *len) 569 { 570 struct vring_used_elem *uep; 571 void *cookie; 572 uint16_t used_idx, desc_idx; 573 574 if (vq->vq_used_cons_idx == vq->vq_ring.used->idx) 575 return (NULL); 576 577 used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1); 578 uep = &vq->vq_ring.used->ring[used_idx]; 579 580 rmb(); 581 desc_idx = (uint16_t) uep->id; 582 if (len != NULL) 583 *len = uep->len; 584 585 vq_ring_free_chain(vq, desc_idx); 586 587 cookie = vq->vq_descx[desc_idx].cookie; 588 VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx); 589 vq->vq_descx[desc_idx].cookie = NULL; 590 591 return (cookie); 592 } 593 594 void * 595 virtqueue_poll(struct virtqueue *vq, uint32_t *len) 596 { 597 void *cookie; 598 599 VIRTIO_BUS_POLL(vq->vq_dev); 600 while ((cookie = virtqueue_dequeue(vq, len)) == NULL) { 601 cpu_spinwait(); 602 VIRTIO_BUS_POLL(vq->vq_dev); 603 } 604 605 return (cookie); 606 } 607 608 void * 609 virtqueue_drain(struct virtqueue *vq, int *last) 610 { 611 void *cookie; 612 int idx; 613 614 cookie = NULL; 615 idx = *last; 616 617 while (idx < vq->vq_nentries && cookie == NULL) { 618 if ((cookie = vq->vq_descx[idx].cookie) != NULL) { 619 vq->vq_descx[idx].cookie = NULL; 620 /* Free chain to keep free count consistent. */ 621 vq_ring_free_chain(vq, idx); 622 } 623 idx++; 624 } 625 626 *last = idx; 627 628 return (cookie); 629 } 630 631 void 632 virtqueue_dump(struct virtqueue *vq) 633 { 634 635 if (vq == NULL) 636 return; 637 638 printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; " 639 "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; " 640 "used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n", 641 vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, 642 virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx, 643 vq->vq_ring.avail->idx, vq->vq_used_cons_idx, 644 vq->vq_ring.used->idx, 645 vring_used_event(&vq->vq_ring), 646 vq->vq_ring.avail->flags, 647 vq->vq_ring.used->flags); 648 } 649 650 static void 651 vq_ring_init(struct virtqueue *vq) 652 { 653 struct vring *vr; 654 char *ring_mem; 655 int i, size; 656 657 ring_mem = vq->vq_ring_mem; 658 size = vq->vq_nentries; 659 vr = &vq->vq_ring; 660 661 vring_init(vr, size, ring_mem, vq->vq_alignment); 662 663 for (i = 0; i < size - 1; i++) 664 vr->desc[i].next = i + 1; 665 vr->desc[i].next = VQ_RING_DESC_CHAIN_END; 666 } 667 668 static void 669 vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx) 670 { 671 uint16_t avail_idx; 672 673 /* 674 * Place the head of the descriptor chain into the next slot and make 675 * it usable to the host. The chain is made available now rather than 676 * deferring to virtqueue_notify() in the hopes that if the host is 677 * currently running on another CPU, we can keep it processing the new 678 * descriptor. 679 */ 680 avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1); 681 vq->vq_ring.avail->ring[avail_idx] = desc_idx; 682 683 wmb(); 684 vq->vq_ring.avail->idx++; 685 686 /* Keep pending count until virtqueue_notify(). */ 687 vq->vq_queued_cnt++; 688 } 689 690 static uint16_t 691 vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc, 692 uint16_t head_idx, struct sglist *sg, int readable, int writable) 693 { 694 struct sglist_seg *seg; 695 struct vring_desc *dp; 696 int i, needed; 697 uint16_t idx; 698 699 needed = readable + writable; 700 701 for (i = 0, idx = head_idx, seg = sg->sg_segs; 702 i < needed; 703 i++, idx = dp->next, seg++) { 704 VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END, 705 "premature end of free desc chain"); 706 707 dp = &desc[idx]; 708 dp->addr = seg->ss_paddr; 709 dp->len = seg->ss_len; 710 dp->flags = 0; 711 712 if (i < needed - 1) 713 dp->flags |= VRING_DESC_F_NEXT; 714 if (i >= readable) 715 dp->flags |= VRING_DESC_F_WRITE; 716 } 717 718 return (idx); 719 } 720 721 static int 722 vq_ring_use_indirect(struct virtqueue *vq, int needed) 723 { 724 725 if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0) 726 return (0); 727 728 if (vq->vq_max_indirect_size < needed) 729 return (0); 730 731 if (needed < 2) 732 return (0); 733 734 return (1); 735 } 736 737 static void 738 vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie, 739 struct sglist *sg, int readable, int writable) 740 { 741 struct vring_desc *dp; 742 struct vq_desc_extra *dxp; 743 int needed; 744 uint16_t head_idx; 745 746 needed = readable + writable; 747 VQASSERT(vq, needed <= vq->vq_max_indirect_size, 748 "enqueuing too many indirect descriptors"); 749 750 head_idx = vq->vq_desc_head_idx; 751 VQ_RING_ASSERT_VALID_IDX(vq, head_idx); 752 dp = &vq->vq_ring.desc[head_idx]; 753 dxp = &vq->vq_descx[head_idx]; 754 755 VQASSERT(vq, dxp->cookie == NULL, 756 "cookie already exists for index %d", head_idx); 757 dxp->cookie = cookie; 758 dxp->ndescs = 1; 759 760 dp->addr = dxp->indirect_paddr; 761 dp->len = needed * sizeof(struct vring_desc); 762 dp->flags = VRING_DESC_F_INDIRECT; 763 764 vq_ring_enqueue_segments(vq, dxp->indirect, 0, 765 sg, readable, writable); 766 767 vq->vq_desc_head_idx = dp->next; 768 vq->vq_free_cnt--; 769 if (vq->vq_free_cnt == 0) 770 VQ_RING_ASSERT_CHAIN_TERM(vq); 771 else 772 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx); 773 774 vq_ring_update_avail(vq, head_idx); 775 } 776 777 static int 778 vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc) 779 { 780 781 /* 782 * Enable interrupts, making sure we get the latest index of 783 * what's already been consumed. 784 */ 785 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) 786 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc; 787 else 788 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 789 790 mb(); 791 792 /* 793 * Enough items may have already been consumed to meet our threshold 794 * since we last checked. Let our caller know so it processes the new 795 * entries. 796 */ 797 if (virtqueue_nused(vq) > ndesc) 798 return (1); 799 800 return (0); 801 } 802 803 static int 804 vq_ring_must_notify_host(struct virtqueue *vq) 805 { 806 uint16_t new_idx, prev_idx, event_idx; 807 808 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { 809 new_idx = vq->vq_ring.avail->idx; 810 prev_idx = new_idx - vq->vq_queued_cnt; 811 event_idx = vring_avail_event(&vq->vq_ring); 812 813 return (vring_need_event(event_idx, new_idx, prev_idx) != 0); 814 } 815 816 return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0); 817 } 818 819 static void 820 vq_ring_notify_host(struct virtqueue *vq) 821 { 822 823 VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index); 824 } 825 826 static void 827 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 828 { 829 struct vring_desc *dp; 830 struct vq_desc_extra *dxp; 831 832 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx); 833 dp = &vq->vq_ring.desc[desc_idx]; 834 dxp = &vq->vq_descx[desc_idx]; 835 836 if (vq->vq_free_cnt == 0) 837 VQ_RING_ASSERT_CHAIN_TERM(vq); 838 839 vq->vq_free_cnt += dxp->ndescs; 840 dxp->ndescs--; 841 842 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 843 while (dp->flags & VRING_DESC_F_NEXT) { 844 VQ_RING_ASSERT_VALID_IDX(vq, dp->next); 845 dp = &vq->vq_ring.desc[dp->next]; 846 dxp->ndescs--; 847 } 848 } 849 850 VQASSERT(vq, dxp->ndescs == 0, 851 "failed to free entire desc chain, remaining: %d", dxp->ndescs); 852 853 /* 854 * We must append the existing free chain, if any, to the end of 855 * newly freed chain. If the virtqueue was completely used, then 856 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 857 */ 858 dp->next = vq->vq_desc_head_idx; 859 vq->vq_desc_head_idx = desc_idx; 860 } 861