1 /*- 2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 /* 28 * Implements the virtqueue interface as basically described 29 * in the original VirtIO paper. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/sglist.h> 40 #include <vm/vm.h> 41 #include <vm/pmap.h> 42 43 #include <machine/cpu.h> 44 #include <machine/bus.h> 45 #include <machine/atomic.h> 46 #include <machine/resource.h> 47 #include <sys/bus.h> 48 #include <sys/rman.h> 49 50 #include <dev/virtio/virtio.h> 51 #include <dev/virtio/virtqueue.h> 52 #include <dev/virtio/virtio_ring.h> 53 54 #include "virtio_bus_if.h" 55 56 struct virtqueue { 57 device_t vq_dev; 58 char vq_name[VIRTQUEUE_MAX_NAME_SZ]; 59 uint16_t vq_queue_index; 60 uint16_t vq_nentries; 61 uint32_t vq_flags; 62 #define VIRTQUEUE_FLAG_INDIRECT 0x0001 63 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0002 64 65 int vq_alignment; 66 int vq_ring_size; 67 void *vq_ring_mem; 68 int vq_max_indirect_size; 69 int vq_indirect_mem_size; 70 virtqueue_intr_t *vq_intrhand; 71 void *vq_intrhand_arg; 72 73 struct vring vq_ring; 74 uint16_t vq_free_cnt; 75 uint16_t vq_queued_cnt; 76 /* 77 * Head of the free chain in the descriptor table. If 78 * there are no free descriptors, this will be set to 79 * VQ_RING_DESC_CHAIN_END. 80 */ 81 uint16_t vq_desc_head_idx; 82 /* 83 * Last consumed descriptor in the used table, 84 * trails vq_ring.used->idx. 85 */ 86 uint16_t vq_used_cons_idx; 87 88 struct vq_desc_extra { 89 void *cookie; 90 struct vring_desc *indirect; 91 vm_paddr_t indirect_paddr; 92 uint16_t ndescs; 93 } vq_descx[0]; 94 }; 95 96 /* 97 * The maximum virtqueue size is 2^15. Use that value as the end of 98 * descriptor chain terminator since it will never be a valid index 99 * in the descriptor table. This is used to verify we are correctly 100 * handling vq_free_cnt. 101 */ 102 #define VQ_RING_DESC_CHAIN_END 32768 103 104 #define VQASSERT(_vq, _exp, _msg, ...) \ 105 KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name, \ 106 ##__VA_ARGS__)) 107 108 #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx) \ 109 VQASSERT((_vq), (_idx) < (_vq)->vq_nentries, \ 110 "invalid ring index: %d, max: %d", (_idx), \ 111 (_vq)->vq_nentries) 112 113 #define VQ_RING_ASSERT_CHAIN_TERM(_vq) \ 114 VQASSERT((_vq), (_vq)->vq_desc_head_idx == \ 115 VQ_RING_DESC_CHAIN_END, "full ring terminated " \ 116 "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx) 117 118 static int virtqueue_init_indirect(struct virtqueue *vq, int); 119 static void virtqueue_free_indirect(struct virtqueue *vq); 120 static void virtqueue_init_indirect_list(struct virtqueue *, 121 struct vring_desc *); 122 123 static void vq_ring_init(struct virtqueue *); 124 static void vq_ring_update_avail(struct virtqueue *, uint16_t); 125 static uint16_t vq_ring_enqueue_segments(struct virtqueue *, 126 struct vring_desc *, uint16_t, struct sglist *, int, int); 127 static int vq_ring_use_indirect(struct virtqueue *, int); 128 static void vq_ring_enqueue_indirect(struct virtqueue *, void *, 129 struct sglist *, int, int); 130 static int vq_ring_must_notify_host(struct virtqueue *); 131 static void vq_ring_notify_host(struct virtqueue *); 132 static void vq_ring_free_chain(struct virtqueue *, uint16_t); 133 134 uint64_t 135 virtqueue_filter_features(uint64_t features) 136 { 137 uint64_t mask; 138 139 mask = (1 << VIRTIO_TRANSPORT_F_START) - 1; 140 mask |= VIRTIO_RING_F_INDIRECT_DESC; 141 mask |= VIRTIO_RING_F_EVENT_IDX; 142 143 return (features & mask); 144 } 145 146 int 147 virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align, 148 vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp) 149 { 150 struct virtqueue *vq; 151 int error; 152 153 *vqp = NULL; 154 error = 0; 155 156 if (size == 0) { 157 device_printf(dev, 158 "virtqueue %d (%s) does not exist (size is zero)\n", 159 queue, info->vqai_name); 160 return (ENODEV); 161 } else if (!powerof2(size)) { 162 device_printf(dev, 163 "virtqueue %d (%s) size is not a power of 2: %d\n", 164 queue, info->vqai_name, size); 165 return (ENXIO); 166 } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) { 167 device_printf(dev, "virtqueue %d (%s) requested too many " 168 "indirect descriptors: %d, max %d\n", 169 queue, info->vqai_name, info->vqai_maxindirsz, 170 VIRTIO_MAX_INDIRECT); 171 return (EINVAL); 172 } 173 174 vq = malloc(sizeof(struct virtqueue) + 175 size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO); 176 if (vq == NULL) { 177 device_printf(dev, "cannot allocate virtqueue\n"); 178 return (ENOMEM); 179 } 180 181 vq->vq_dev = dev; 182 strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name)); 183 vq->vq_queue_index = queue; 184 vq->vq_alignment = align; 185 vq->vq_nentries = size; 186 vq->vq_free_cnt = size; 187 vq->vq_intrhand = info->vqai_intr; 188 vq->vq_intrhand_arg = info->vqai_intr_arg; 189 190 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0) 191 vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX; 192 193 if (info->vqai_maxindirsz > 1) { 194 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz); 195 if (error) 196 goto fail; 197 } 198 199 vq->vq_ring_size = round_page(vring_size(size, align)); 200 vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF, 201 M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0); 202 if (vq->vq_ring_mem == NULL) { 203 device_printf(dev, 204 "cannot allocate memory for virtqueue ring\n"); 205 error = ENOMEM; 206 goto fail; 207 } 208 209 vq_ring_init(vq); 210 virtqueue_disable_intr(vq); 211 212 *vqp = vq; 213 214 fail: 215 if (error) 216 virtqueue_free(vq); 217 218 return (error); 219 } 220 221 static int 222 virtqueue_init_indirect(struct virtqueue *vq, int indirect_size) 223 { 224 device_t dev; 225 struct vq_desc_extra *dxp; 226 int i, size; 227 228 dev = vq->vq_dev; 229 230 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) { 231 /* 232 * Indirect descriptors requested by the driver but not 233 * negotiated. Return zero to keep the initialization 234 * going: we'll run fine without. 235 */ 236 if (bootverbose) 237 device_printf(dev, "virtqueue %d (%s) requested " 238 "indirect descriptors but not negotiated\n", 239 vq->vq_queue_index, vq->vq_name); 240 return (0); 241 } 242 243 size = indirect_size * sizeof(struct vring_desc); 244 vq->vq_max_indirect_size = indirect_size; 245 vq->vq_indirect_mem_size = size; 246 vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT; 247 248 for (i = 0; i < vq->vq_nentries; i++) { 249 dxp = &vq->vq_descx[i]; 250 251 dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT); 252 if (dxp->indirect == NULL) { 253 device_printf(dev, "cannot allocate indirect list\n"); 254 return (ENOMEM); 255 } 256 257 dxp->indirect_paddr = vtophys(dxp->indirect); 258 virtqueue_init_indirect_list(vq, dxp->indirect); 259 } 260 261 return (0); 262 } 263 264 static void 265 virtqueue_free_indirect(struct virtqueue *vq) 266 { 267 struct vq_desc_extra *dxp; 268 int i; 269 270 for (i = 0; i < vq->vq_nentries; i++) { 271 dxp = &vq->vq_descx[i]; 272 273 if (dxp->indirect == NULL) 274 break; 275 276 free(dxp->indirect, M_DEVBUF); 277 dxp->indirect = NULL; 278 dxp->indirect_paddr = 0; 279 } 280 281 vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT; 282 vq->vq_indirect_mem_size = 0; 283 } 284 285 static void 286 virtqueue_init_indirect_list(struct virtqueue *vq, 287 struct vring_desc *indirect) 288 { 289 int i; 290 291 bzero(indirect, vq->vq_indirect_mem_size); 292 293 for (i = 0; i < vq->vq_max_indirect_size - 1; i++) 294 indirect[i].next = i + 1; 295 indirect[i].next = VQ_RING_DESC_CHAIN_END; 296 } 297 298 int 299 virtqueue_reinit(struct virtqueue *vq, uint16_t size) 300 { 301 struct vq_desc_extra *dxp; 302 int i; 303 304 if (vq->vq_nentries != size) { 305 device_printf(vq->vq_dev, 306 "%s: '%s' changed size; old=%hu, new=%hu\n", 307 __func__, vq->vq_name, vq->vq_nentries, size); 308 return (EINVAL); 309 } 310 311 /* Warn if the virtqueue was not properly cleaned up. */ 312 if (vq->vq_free_cnt != vq->vq_nentries) { 313 device_printf(vq->vq_dev, 314 "%s: warning, '%s' virtqueue not empty, " 315 "leaking %d entries\n", __func__, vq->vq_name, 316 vq->vq_nentries - vq->vq_free_cnt); 317 } 318 319 vq->vq_desc_head_idx = 0; 320 vq->vq_used_cons_idx = 0; 321 vq->vq_queued_cnt = 0; 322 vq->vq_free_cnt = vq->vq_nentries; 323 324 /* To be safe, reset all our allocated memory. */ 325 bzero(vq->vq_ring_mem, vq->vq_ring_size); 326 for (i = 0; i < vq->vq_nentries; i++) { 327 dxp = &vq->vq_descx[i]; 328 dxp->cookie = NULL; 329 dxp->ndescs = 0; 330 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) 331 virtqueue_init_indirect_list(vq, dxp->indirect); 332 } 333 334 vq_ring_init(vq); 335 virtqueue_disable_intr(vq); 336 337 return (0); 338 } 339 340 void 341 virtqueue_free(struct virtqueue *vq) 342 { 343 344 if (vq->vq_free_cnt != vq->vq_nentries) { 345 device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, " 346 "leaking %d entries\n", vq->vq_name, 347 vq->vq_nentries - vq->vq_free_cnt); 348 } 349 350 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) 351 virtqueue_free_indirect(vq); 352 353 if (vq->vq_ring_mem != NULL) { 354 contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF); 355 vq->vq_ring_size = 0; 356 vq->vq_ring_mem = NULL; 357 } 358 359 free(vq, M_DEVBUF); 360 } 361 362 vm_paddr_t 363 virtqueue_paddr(struct virtqueue *vq) 364 { 365 366 return (vtophys(vq->vq_ring_mem)); 367 } 368 369 int 370 virtqueue_size(struct virtqueue *vq) 371 { 372 373 return (vq->vq_nentries); 374 } 375 376 int 377 virtqueue_empty(struct virtqueue *vq) 378 { 379 380 return (vq->vq_nentries == vq->vq_free_cnt); 381 } 382 383 int 384 virtqueue_full(struct virtqueue *vq) 385 { 386 387 return (vq->vq_free_cnt == 0); 388 } 389 390 void 391 virtqueue_notify(struct virtqueue *vq) 392 { 393 /* Ensure updated avail->idx is visible to host. */ 394 mb(); 395 396 if (vq_ring_must_notify_host(vq)) 397 vq_ring_notify_host(vq); 398 vq->vq_queued_cnt = 0; 399 } 400 401 int 402 virtqueue_nused(struct virtqueue *vq) 403 { 404 uint16_t used_idx, nused; 405 406 used_idx = vq->vq_ring.used->idx; 407 408 nused = (uint16_t)(used_idx - vq->vq_used_cons_idx); 409 VQASSERT(vq, nused <= vq->vq_nentries, "used more than available"); 410 411 return (nused); 412 } 413 414 int 415 virtqueue_intr(struct virtqueue *vq) 416 { 417 418 if (vq->vq_intrhand == NULL || 419 vq->vq_used_cons_idx == vq->vq_ring.used->idx) 420 return (0); 421 422 vq->vq_intrhand(vq->vq_intrhand_arg); 423 424 return (1); 425 } 426 427 int 428 virtqueue_enable_intr(struct virtqueue *vq) 429 { 430 431 /* 432 * Enable interrupts, making sure we get the latest 433 * index of what's already been consumed. 434 */ 435 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 436 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) 437 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx; 438 else 439 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 440 441 mb(); 442 443 /* 444 * Additional items may have been consumed in the time between 445 * since we last checked and enabled interrupts above. Let our 446 * caller know so it processes the new entries. 447 */ 448 if (vq->vq_used_cons_idx != vq->vq_ring.used->idx) 449 return (1); 450 451 return (0); 452 } 453 454 int 455 virtqueue_postpone_intr(struct virtqueue *vq) 456 { 457 uint16_t ndesc; 458 459 /* 460 * Postpone until at least half of the available descriptors 461 * have been consumed. 462 * 463 * XXX Adaptive factor? (Linux uses 3/4) 464 */ 465 ndesc = (uint16_t)(vq->vq_ring.avail->idx - vq->vq_used_cons_idx) / 2; 466 467 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) 468 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc; 469 else 470 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 471 472 mb(); 473 474 /* 475 * Enough items may have already been consumed to meet our 476 * threshold since we last checked. Let our caller know so 477 * it processes the new entries. 478 */ 479 if (virtqueue_nused(vq) > ndesc) 480 return (1); 481 482 return (0); 483 } 484 485 void 486 virtqueue_disable_intr(struct virtqueue *vq) 487 { 488 489 /* 490 * Note this is only considered a hint to the host. 491 */ 492 if ((vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) == 0) 493 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 494 } 495 496 int 497 virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg, 498 int readable, int writable) 499 { 500 struct vq_desc_extra *dxp; 501 int needed; 502 uint16_t head_idx, idx; 503 504 needed = readable + writable; 505 506 VQASSERT(vq, cookie != NULL, "enqueuing with no cookie"); 507 VQASSERT(vq, needed == sg->sg_nseg, 508 "segment count mismatch, %d, %d", needed, sg->sg_nseg); 509 VQASSERT(vq, 510 needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size, 511 "too many segments to enqueue: %d, %d/%d", needed, 512 vq->vq_nentries, vq->vq_max_indirect_size); 513 514 if (needed < 1) 515 return (EINVAL); 516 if (vq->vq_free_cnt == 0) 517 return (ENOSPC); 518 519 if (vq_ring_use_indirect(vq, needed)) { 520 vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable); 521 return (0); 522 } else if (vq->vq_free_cnt < needed) 523 return (EMSGSIZE); 524 525 head_idx = vq->vq_desc_head_idx; 526 VQ_RING_ASSERT_VALID_IDX(vq, head_idx); 527 dxp = &vq->vq_descx[head_idx]; 528 529 VQASSERT(vq, dxp->cookie == NULL, 530 "cookie already exists for index %d", head_idx); 531 dxp->cookie = cookie; 532 dxp->ndescs = needed; 533 534 idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx, 535 sg, readable, writable); 536 537 vq->vq_desc_head_idx = idx; 538 vq->vq_free_cnt -= needed; 539 if (vq->vq_free_cnt == 0) 540 VQ_RING_ASSERT_CHAIN_TERM(vq); 541 else 542 VQ_RING_ASSERT_VALID_IDX(vq, idx); 543 544 vq_ring_update_avail(vq, head_idx); 545 546 return (0); 547 } 548 549 void * 550 virtqueue_dequeue(struct virtqueue *vq, uint32_t *len) 551 { 552 struct vring_used_elem *uep; 553 void *cookie; 554 uint16_t used_idx, desc_idx; 555 556 if (vq->vq_used_cons_idx == vq->vq_ring.used->idx) 557 return (NULL); 558 559 used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1); 560 uep = &vq->vq_ring.used->ring[used_idx]; 561 562 mb(); 563 desc_idx = (uint16_t) uep->id; 564 if (len != NULL) 565 *len = uep->len; 566 567 vq_ring_free_chain(vq, desc_idx); 568 569 cookie = vq->vq_descx[desc_idx].cookie; 570 VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx); 571 vq->vq_descx[desc_idx].cookie = NULL; 572 573 return (cookie); 574 } 575 576 void * 577 virtqueue_poll(struct virtqueue *vq, uint32_t *len) 578 { 579 void *cookie; 580 581 while ((cookie = virtqueue_dequeue(vq, len)) == NULL) 582 cpu_spinwait(); 583 584 return (cookie); 585 } 586 587 void * 588 virtqueue_drain(struct virtqueue *vq, int *last) 589 { 590 void *cookie; 591 int idx; 592 593 cookie = NULL; 594 idx = *last; 595 596 while (idx < vq->vq_nentries && cookie == NULL) { 597 if ((cookie = vq->vq_descx[idx].cookie) != NULL) { 598 vq->vq_descx[idx].cookie = NULL; 599 /* Free chain to keep free count consistent. */ 600 vq_ring_free_chain(vq, idx); 601 } 602 idx++; 603 } 604 605 *last = idx; 606 607 return (cookie); 608 } 609 610 void 611 virtqueue_dump(struct virtqueue *vq) 612 { 613 614 if (vq == NULL) 615 return; 616 617 printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; " 618 "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; " 619 "used.idx=%d; avail.flags=0x%x; used.flags=0x%x\n", 620 vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, 621 virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx, 622 vq->vq_ring.avail->idx, vq->vq_used_cons_idx, 623 vq->vq_ring.used->idx, vq->vq_ring.avail->flags, 624 vq->vq_ring.used->flags); 625 } 626 627 static void 628 vq_ring_init(struct virtqueue *vq) 629 { 630 struct vring *vr; 631 char *ring_mem; 632 int i, size; 633 634 ring_mem = vq->vq_ring_mem; 635 size = vq->vq_nentries; 636 vr = &vq->vq_ring; 637 638 vring_init(vr, size, ring_mem, vq->vq_alignment); 639 640 for (i = 0; i < size - 1; i++) 641 vr->desc[i].next = i + 1; 642 vr->desc[i].next = VQ_RING_DESC_CHAIN_END; 643 } 644 645 static void 646 vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx) 647 { 648 uint16_t avail_idx; 649 650 /* 651 * Place the head of the descriptor chain into the next slot and make 652 * it usable to the host. The chain is made available now rather than 653 * deferring to virtqueue_notify() in the hopes that if the host is 654 * currently running on another CPU, we can keep it processing the new 655 * descriptor. 656 */ 657 avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1); 658 vq->vq_ring.avail->ring[avail_idx] = desc_idx; 659 660 mb(); 661 vq->vq_ring.avail->idx++; 662 663 /* Keep pending count until virtqueue_notify(). */ 664 vq->vq_queued_cnt++; 665 } 666 667 static uint16_t 668 vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc, 669 uint16_t head_idx, struct sglist *sg, int readable, int writable) 670 { 671 struct sglist_seg *seg; 672 struct vring_desc *dp; 673 int i, needed; 674 uint16_t idx; 675 676 needed = readable + writable; 677 678 for (i = 0, idx = head_idx, seg = sg->sg_segs; 679 i < needed; 680 i++, idx = dp->next, seg++) { 681 VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END, 682 "premature end of free desc chain"); 683 684 dp = &desc[idx]; 685 dp->addr = seg->ss_paddr; 686 dp->len = seg->ss_len; 687 dp->flags = 0; 688 689 if (i < needed - 1) 690 dp->flags |= VRING_DESC_F_NEXT; 691 if (i >= readable) 692 dp->flags |= VRING_DESC_F_WRITE; 693 } 694 695 return (idx); 696 } 697 698 static int 699 vq_ring_use_indirect(struct virtqueue *vq, int needed) 700 { 701 702 if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0) 703 return (0); 704 705 if (vq->vq_max_indirect_size < needed) 706 return (0); 707 708 if (needed < 2) 709 return (0); 710 711 return (1); 712 } 713 714 static void 715 vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie, 716 struct sglist *sg, int readable, int writable) 717 { 718 struct vring_desc *dp; 719 struct vq_desc_extra *dxp; 720 int needed; 721 uint16_t head_idx; 722 723 needed = readable + writable; 724 VQASSERT(vq, needed <= vq->vq_max_indirect_size, 725 "enqueuing too many indirect descriptors"); 726 727 head_idx = vq->vq_desc_head_idx; 728 VQ_RING_ASSERT_VALID_IDX(vq, head_idx); 729 dp = &vq->vq_ring.desc[head_idx]; 730 dxp = &vq->vq_descx[head_idx]; 731 732 VQASSERT(vq, dxp->cookie == NULL, 733 "cookie already exists for index %d", head_idx); 734 dxp->cookie = cookie; 735 dxp->ndescs = 1; 736 737 dp->addr = dxp->indirect_paddr; 738 dp->len = needed * sizeof(struct vring_desc); 739 dp->flags = VRING_DESC_F_INDIRECT; 740 741 vq_ring_enqueue_segments(vq, dxp->indirect, 0, 742 sg, readable, writable); 743 744 vq->vq_desc_head_idx = dp->next; 745 vq->vq_free_cnt--; 746 if (vq->vq_free_cnt == 0) 747 VQ_RING_ASSERT_CHAIN_TERM(vq); 748 else 749 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx); 750 751 vq_ring_update_avail(vq, head_idx); 752 } 753 754 static int 755 vq_ring_must_notify_host(struct virtqueue *vq) 756 { 757 uint16_t new_idx, prev_idx, event_idx; 758 759 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { 760 new_idx = vq->vq_ring.avail->idx; 761 prev_idx = new_idx - vq->vq_queued_cnt; 762 event_idx = vring_avail_event(&vq->vq_ring); 763 764 return (vring_need_event(event_idx, new_idx, prev_idx) != 0); 765 } 766 767 return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0); 768 } 769 770 static void 771 vq_ring_notify_host(struct virtqueue *vq) 772 { 773 774 VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index); 775 } 776 777 static void 778 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 779 { 780 struct vring_desc *dp; 781 struct vq_desc_extra *dxp; 782 783 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx); 784 dp = &vq->vq_ring.desc[desc_idx]; 785 dxp = &vq->vq_descx[desc_idx]; 786 787 if (vq->vq_free_cnt == 0) 788 VQ_RING_ASSERT_CHAIN_TERM(vq); 789 790 vq->vq_free_cnt += dxp->ndescs; 791 dxp->ndescs--; 792 793 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 794 while (dp->flags & VRING_DESC_F_NEXT) { 795 VQ_RING_ASSERT_VALID_IDX(vq, dp->next); 796 dp = &vq->vq_ring.desc[dp->next]; 797 dxp->ndescs--; 798 } 799 } 800 VQASSERT(vq, dxp->ndescs == 0, "failed to free entire desc chain"); 801 802 /* 803 * We must append the existing free chain, if any, to the end of 804 * newly freed chain. If the virtqueue was completely used, then 805 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 806 */ 807 dp->next = vq->vq_desc_head_idx; 808 vq->vq_desc_head_idx = desc_idx; 809 } 810