1 /*- 2 * Copyright (c) 2009-2012,2016 Microsoft Corp. 3 * Copyright (c) 2012 NetApp Inc. 4 * Copyright (c) 2012 Citrix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 #include <sys/param.h> 31 #include <sys/lock.h> 32 #include <sys/mutex.h> 33 #include <sys/sysctl.h> 34 35 #include <dev/hyperv/vmbus/vmbus_reg.h> 36 #include <dev/hyperv/vmbus/vmbus_brvar.h> 37 38 /* Amount of space available for write */ 39 #define VMBUS_BR_WAVAIL(r, w, z) \ 40 (((w) >= (r)) ? ((z) - ((w) - (r))) : ((r) - (w))) 41 42 /* Increase bufing index */ 43 #define VMBUS_BR_IDXINC(idx, inc, sz) (((idx) + (inc)) % (sz)) 44 45 static int vmbus_br_sysctl_state(SYSCTL_HANDLER_ARGS); 46 static int vmbus_br_sysctl_state_bin(SYSCTL_HANDLER_ARGS); 47 static void vmbus_br_setup(struct vmbus_br *, void *, int); 48 49 static int 50 vmbus_br_sysctl_state(SYSCTL_HANDLER_ARGS) 51 { 52 const struct vmbus_br *br = arg1; 53 uint32_t rindex, windex, imask, psndsz, fvalue, ravail, wavail; 54 uint64_t intrcnt; 55 char state[256]; 56 57 intrcnt = br->vbr_intrcnt; 58 rindex = br->vbr_rindex; 59 windex = br->vbr_windex; 60 imask = br->vbr_imask; 61 psndsz = br->vbr_psndsz; 62 fvalue = br->vbr_fvalue; 63 wavail = VMBUS_BR_WAVAIL(rindex, windex, br->vbr_dsize); 64 ravail = br->vbr_dsize - wavail; 65 66 snprintf(state, sizeof(state), 67 "intrcnt:%ju rindex:%u windex:%u imask:%u psndsz:%u fvalue:%u " 68 "ravail:%u wavail:%u", 69 (uintmax_t)intrcnt, rindex, windex, imask, psndsz, fvalue, 70 ravail, wavail); 71 return sysctl_handle_string(oidp, state, sizeof(state), req); 72 } 73 74 /* 75 * Binary bufring states. 76 */ 77 static int 78 vmbus_br_sysctl_state_bin(SYSCTL_HANDLER_ARGS) 79 { 80 #define BR_STATE_RIDX 0 81 #define BR_STATE_WIDX 1 82 #define BR_STATE_IMSK 2 83 #define BR_STATE_PSSZ 3 84 #define BR_STATE_FVAL 4 85 #define BR_STATE_RSPC 5 86 #define BR_STATE_WSPC 6 87 #define BR_STATE_MAX 7 88 89 const struct vmbus_br *br = arg1; 90 uint32_t rindex, windex, wavail, state[BR_STATE_MAX]; 91 92 rindex = br->vbr_rindex; 93 windex = br->vbr_windex; 94 wavail = VMBUS_BR_WAVAIL(rindex, windex, br->vbr_dsize); 95 96 state[BR_STATE_RIDX] = rindex; 97 state[BR_STATE_WIDX] = windex; 98 state[BR_STATE_IMSK] = br->vbr_imask; 99 state[BR_STATE_PSSZ] = br->vbr_psndsz; 100 state[BR_STATE_FVAL] = br->vbr_fvalue; 101 state[BR_STATE_WSPC] = wavail; 102 state[BR_STATE_RSPC] = br->vbr_dsize - wavail; 103 104 return sysctl_handle_opaque(oidp, state, sizeof(state), req); 105 } 106 107 void 108 vmbus_br_sysctl_create(struct sysctl_ctx_list *ctx, struct sysctl_oid *br_tree, 109 struct vmbus_br *br, const char *name) 110 { 111 struct sysctl_oid *tree; 112 char desc[64]; 113 114 tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(br_tree), OID_AUTO, 115 name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 116 if (tree == NULL) 117 return; 118 119 snprintf(desc, sizeof(desc), "%s state", name); 120 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "state", 121 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 122 br, 0, vmbus_br_sysctl_state, "A", desc); 123 124 snprintf(desc, sizeof(desc), "%s binary state", name); 125 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "state_bin", 126 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 127 br, 0, vmbus_br_sysctl_state_bin, "IU", desc); 128 } 129 130 void 131 vmbus_rxbr_intr_mask(struct vmbus_rxbr *rbr) 132 { 133 rbr->rxbr_imask = 1; 134 mb(); 135 } 136 137 static __inline uint32_t 138 vmbus_rxbr_avail(const struct vmbus_rxbr *rbr) 139 { 140 uint32_t rindex, windex; 141 142 /* Get snapshot */ 143 rindex = atomic_load_acq_32(&rbr->rxbr_rindex); 144 windex = atomic_load_acq_32(&rbr->rxbr_windex); 145 146 return (rbr->rxbr_dsize - 147 VMBUS_BR_WAVAIL(rindex, windex, rbr->rxbr_dsize)); 148 } 149 150 uint32_t 151 vmbus_rxbr_available(const struct vmbus_rxbr *rbr) 152 { 153 return (vmbus_rxbr_avail(rbr)); 154 } 155 156 uint32_t 157 vmbus_rxbr_intr_unmask(struct vmbus_rxbr *rbr) 158 { 159 rbr->rxbr_imask = 0; 160 mb(); 161 162 /* 163 * Now check to see if the ring buffer is still empty. 164 * If it is not, we raced and we need to process new 165 * incoming channel packets. 166 */ 167 return vmbus_rxbr_avail(rbr); 168 } 169 170 static void 171 vmbus_br_setup(struct vmbus_br *br, void *buf, int blen) 172 { 173 br->vbr = buf; 174 br->vbr_dsize = blen - sizeof(struct vmbus_bufring); 175 } 176 177 void 178 vmbus_rxbr_init(struct vmbus_rxbr *rbr) 179 { 180 mtx_init(&rbr->rxbr_lock, "vmbus_rxbr", NULL, MTX_SPIN); 181 } 182 183 void 184 vmbus_rxbr_deinit(struct vmbus_rxbr *rbr) 185 { 186 mtx_destroy(&rbr->rxbr_lock); 187 } 188 189 void 190 vmbus_rxbr_setup(struct vmbus_rxbr *rbr, void *buf, int blen) 191 { 192 vmbus_br_setup(&rbr->rxbr, buf, blen); 193 } 194 195 static __inline boolean_t 196 vmbus_rxbr_need_signal(const struct vmbus_rxbr *rbr, uint32_t bytes_read) 197 { 198 uint32_t pending_snd_sz, canwrite_size; 199 200 /* No need to signal if host doesn't want us to */ 201 if (!rbr->rxbr_fpsndsz) 202 return false; 203 204 mb(); 205 206 pending_snd_sz = rbr->rxbr_psndsz; 207 /* No need to signal if host sets pending_snd_sz to 0 */ 208 if (!pending_snd_sz) 209 return false; 210 211 mb(); 212 213 canwrite_size = rbr->rxbr_dsize - vmbus_rxbr_avail(rbr); 214 215 /* No need to signal if br already has enough space before read */ 216 if (canwrite_size - bytes_read > pending_snd_sz) 217 return false; 218 219 /* 220 * No need to signal if still doesn't have enough space 221 * asked by host 222 */ 223 if (canwrite_size <= pending_snd_sz) 224 return false; 225 226 return true; 227 } 228 229 void 230 vmbus_txbr_init(struct vmbus_txbr *tbr) 231 { 232 mtx_init(&tbr->txbr_lock, "vmbus_txbr", NULL, MTX_SPIN); 233 } 234 235 void 236 vmbus_txbr_deinit(struct vmbus_txbr *tbr) 237 { 238 mtx_destroy(&tbr->txbr_lock); 239 } 240 241 void 242 vmbus_txbr_setup(struct vmbus_txbr *tbr, void *buf, int blen) 243 { 244 vmbus_br_setup(&tbr->txbr, buf, blen); 245 246 /* Set feature bit enabling flow control */ 247 tbr->txbr_fpsndsz = 1; 248 } 249 250 uint32_t 251 vmbus_txbr_get_imask(const struct vmbus_txbr *tbr) 252 { 253 mb(); 254 255 return(tbr->txbr_imask); 256 } 257 258 void 259 vmbus_txbr_set_pending_snd_sz(struct vmbus_txbr *tbr, uint32_t size) 260 { 261 tbr->txbr_psndsz = size; 262 } 263 264 /* 265 * When we write to the ring buffer, check if the host needs to be 266 * signaled. 267 * 268 * The contract: 269 * - The host guarantees that while it is draining the TX bufring, 270 * it will set the br_imask to indicate it does not need to be 271 * interrupted when new data are added. 272 * - The host guarantees that it will completely drain the TX bufring 273 * before exiting the read loop. Further, once the TX bufring is 274 * empty, it will clear the br_imask and re-check to see if new 275 * data have arrived. 276 */ 277 static __inline boolean_t 278 vmbus_txbr_need_signal(const struct vmbus_txbr *tbr, uint32_t old_windex) 279 { 280 mb(); 281 if (tbr->txbr_imask) 282 return (FALSE); 283 284 __compiler_membar(); 285 286 /* 287 * This is the only case we need to signal when the 288 * ring transitions from being empty to non-empty. 289 */ 290 if (old_windex == atomic_load_acq_32(&tbr->txbr_rindex)) 291 return (TRUE); 292 293 return (FALSE); 294 } 295 296 static __inline uint32_t 297 vmbus_txbr_avail(const struct vmbus_txbr *tbr) 298 { 299 uint32_t rindex, windex; 300 301 /* Get snapshot */ 302 rindex = atomic_load_acq_32(&tbr->txbr_rindex); 303 windex = atomic_load_acq_32(&tbr->txbr_windex); 304 305 return VMBUS_BR_WAVAIL(rindex, windex, tbr->txbr_dsize); 306 } 307 308 static __inline uint32_t 309 vmbus_txbr_copyto(const struct vmbus_txbr *tbr, uint32_t windex, 310 const void *src0, uint32_t cplen) 311 { 312 const uint8_t *src = src0; 313 uint8_t *br_data = tbr->txbr_data; 314 uint32_t br_dsize = tbr->txbr_dsize; 315 316 if (cplen > br_dsize - windex) { 317 uint32_t fraglen = br_dsize - windex; 318 319 /* Wrap-around detected */ 320 memcpy(br_data + windex, src, fraglen); 321 memcpy(br_data, src + fraglen, cplen - fraglen); 322 } else { 323 memcpy(br_data + windex, src, cplen); 324 } 325 return VMBUS_BR_IDXINC(windex, cplen, br_dsize); 326 } 327 328 static __inline uint32_t 329 vmbus_txbr_copyto_call(const struct vmbus_txbr *tbr, uint32_t windex, 330 uint32_t cplen, vmbus_br_copy_callback_t cb, void *cbarg, int *ret) 331 { 332 uint8_t *br_data = tbr->txbr_data; 333 uint32_t br_dsize = tbr->txbr_dsize; 334 int err = 0; 335 336 if (cplen > br_dsize - windex) { 337 uint32_t fraglen = br_dsize - windex; 338 339 /* Wrap-around detected */ 340 err = cb((void *)(br_data + windex), fraglen, cbarg); 341 if (!err) 342 err = cb((void *)br_data, cplen - fraglen, cbarg); 343 } else { 344 err = cb((void *)(br_data + windex), cplen, cbarg); 345 } 346 347 *ret = err; 348 349 return VMBUS_BR_IDXINC(windex, cplen, br_dsize); 350 } 351 352 uint32_t 353 vmbus_txbr_available(const struct vmbus_txbr *tbr) 354 { 355 return (vmbus_txbr_avail(tbr)); 356 } 357 358 /* 359 * NOTE: 360 * Not holding lock when calling user provided callback routine. 361 * Caller should hold lock to serialize ring buffer accesses. 362 */ 363 int 364 vmbus_txbr_write_call(struct vmbus_txbr *tbr, 365 const struct iovec iov[], int iovlen, 366 vmbus_br_copy_callback_t cb, void *cbarg, 367 boolean_t *need_sig) 368 { 369 uint32_t old_windex, windex, total; 370 uint64_t save_windex; 371 int i; 372 int cb_ret = 0; 373 374 total = 0; 375 for (i = 0; i < iovlen; i++) 376 total += iov[i].iov_len; 377 total += sizeof(save_windex); 378 379 380 /* 381 * NOTE: 382 * If this write is going to make br_windex same as br_rindex, 383 * i.e. the available space for write is same as the write size, 384 * we can't do it then, since br_windex == br_rindex means that 385 * the bufring is empty. 386 */ 387 if (vmbus_txbr_avail(tbr) <= total) { 388 return (EAGAIN); 389 } 390 391 /* Save br_windex for later use */ 392 old_windex = tbr->txbr_windex; 393 394 /* 395 * Copy the scattered channel packet to the TX bufring. 396 */ 397 windex = old_windex; 398 for (i = 0; i < iovlen; i++) { 399 if (iov[i].iov_base != NULL) { 400 windex = vmbus_txbr_copyto(tbr, windex, 401 iov[i].iov_base, iov[i].iov_len); 402 } else if (cb != NULL) { 403 windex = vmbus_txbr_copyto_call(tbr, windex, 404 iov[i].iov_len, cb, cbarg, &cb_ret); 405 /* 406 * If callback fails, return without updating 407 * write index. 408 */ 409 if (cb_ret) 410 return (cb_ret); 411 } 412 } 413 414 mtx_lock_spin(&tbr->txbr_lock); 415 416 /* 417 * Set the offset of the current channel packet. 418 */ 419 save_windex = ((uint64_t)old_windex) << 32; 420 windex = vmbus_txbr_copyto(tbr, windex, &save_windex, 421 sizeof(save_windex)); 422 423 /* 424 * Update the write index _after_ the channel packet 425 * is copied. 426 */ 427 __compiler_membar(); 428 atomic_store_rel_32(&tbr->txbr_windex, windex); 429 430 mtx_unlock_spin(&tbr->txbr_lock); 431 432 if (need_sig) 433 *need_sig = vmbus_txbr_need_signal(tbr, old_windex); 434 435 return (0); 436 } 437 438 /* 439 * Write scattered channel packet to TX bufring. 440 * 441 * The offset of this channel packet is written as a 64bits value 442 * immediately after this channel packet. 443 */ 444 int 445 vmbus_txbr_write(struct vmbus_txbr *tbr, const struct iovec iov[], int iovlen, 446 boolean_t *need_sig) 447 { 448 uint32_t old_windex, windex, total; 449 uint64_t save_windex; 450 int i; 451 452 total = 0; 453 for (i = 0; i < iovlen; i++) 454 total += iov[i].iov_len; 455 total += sizeof(save_windex); 456 457 mtx_lock_spin(&tbr->txbr_lock); 458 459 /* 460 * NOTE: 461 * If this write is going to make br_windex same as br_rindex, 462 * i.e. the available space for write is same as the write size, 463 * we can't do it then, since br_windex == br_rindex means that 464 * the bufring is empty. 465 */ 466 if (vmbus_txbr_avail(tbr) <= total) { 467 mtx_unlock_spin(&tbr->txbr_lock); 468 return (EAGAIN); 469 } 470 471 /* Save br_windex for later use */ 472 old_windex = atomic_load_acq_32(&tbr->txbr_windex); 473 474 /* 475 * Copy the scattered channel packet to the TX bufring. 476 */ 477 windex = old_windex; 478 for (i = 0; i < iovlen; i++) { 479 windex = vmbus_txbr_copyto(tbr, windex, 480 iov[i].iov_base, iov[i].iov_len); 481 } 482 483 /* 484 * Set the offset of the current channel packet. 485 */ 486 save_windex = ((uint64_t)old_windex) << 32; 487 windex = vmbus_txbr_copyto(tbr, windex, &save_windex, 488 sizeof(save_windex)); 489 490 /* 491 * Update the write index _after_ the channel packet 492 * is copied. 493 */ 494 __compiler_membar(); 495 atomic_store_rel_32(&tbr->txbr_windex, windex); 496 497 mtx_unlock_spin(&tbr->txbr_lock); 498 499 *need_sig = vmbus_txbr_need_signal(tbr, old_windex); 500 501 return (0); 502 } 503 504 static __inline uint32_t 505 vmbus_rxbr_copyfrom(const struct vmbus_rxbr *rbr, uint32_t rindex, 506 void *dst0, int cplen) 507 { 508 uint8_t *dst = dst0; 509 const uint8_t *br_data = rbr->rxbr_data; 510 uint32_t br_dsize = rbr->rxbr_dsize; 511 512 if (cplen > br_dsize - rindex) { 513 uint32_t fraglen = br_dsize - rindex; 514 515 /* Wrap-around detected. */ 516 memcpy(dst, br_data + rindex, fraglen); 517 memcpy(dst + fraglen, br_data, cplen - fraglen); 518 } else { 519 memcpy(dst, br_data + rindex, cplen); 520 } 521 return VMBUS_BR_IDXINC(rindex, cplen, br_dsize); 522 } 523 524 static __inline uint32_t 525 vmbus_rxbr_copyfrom_call(const struct vmbus_rxbr *rbr, uint32_t rindex, 526 int cplen, vmbus_br_copy_callback_t cb, void *cbarg) 527 { 528 uint8_t *br_data = rbr->rxbr_data; 529 uint32_t br_dsize = rbr->rxbr_dsize; 530 int error = 0; 531 532 if (cplen > br_dsize - rindex) { 533 uint32_t fraglen = br_dsize - rindex; 534 535 /* Wrap-around detected. */ 536 error = cb((void *)(br_data + rindex), fraglen, cbarg); 537 if (!error) 538 error = cb((void *)br_data, cplen - fraglen, cbarg); 539 } else { 540 error = cb((void *)(br_data + rindex), cplen, cbarg); 541 } 542 return (error); 543 } 544 545 int 546 vmbus_rxbr_peek(struct vmbus_rxbr *rbr, void *data, int dlen) 547 { 548 mtx_lock_spin(&rbr->rxbr_lock); 549 550 /* 551 * The requested data and the 64bits channel packet 552 * offset should be there at least. 553 */ 554 if (vmbus_rxbr_avail(rbr) < dlen + sizeof(uint64_t)) { 555 mtx_unlock_spin(&rbr->rxbr_lock); 556 return (EAGAIN); 557 } 558 vmbus_rxbr_copyfrom(rbr, 559 atomic_load_acq_32(&rbr->rxbr_rindex), data, dlen); 560 561 mtx_unlock_spin(&rbr->rxbr_lock); 562 563 return (0); 564 } 565 566 /* 567 * NOTE: 568 * We only hold spin lock to check the ring buffer space. It is 569 * released before calling user provided callback routine. 570 * Caller should hold lock to serialize ring buffer accesses. 571 */ 572 int 573 vmbus_rxbr_peek_call(struct vmbus_rxbr *rbr, int dlen, uint32_t skip, 574 vmbus_br_copy_callback_t cb, void *cbarg) 575 { 576 uint32_t rindex, br_dsize0 = rbr->rxbr_dsize; 577 int ret; 578 579 mtx_lock_spin(&rbr->rxbr_lock); 580 /* 581 * The requested data + skip and the 64bits channel packet 582 * offset should be there at least. 583 */ 584 if (vmbus_rxbr_avail(rbr) < skip + dlen + sizeof(uint64_t)) { 585 mtx_unlock_spin(&rbr->rxbr_lock); 586 return (EAGAIN); 587 } 588 589 rindex = VMBUS_BR_IDXINC(rbr->rxbr_rindex, skip, br_dsize0); 590 mtx_unlock_spin(&rbr->rxbr_lock); 591 592 ret = vmbus_rxbr_copyfrom_call(rbr, rindex, dlen, cb, cbarg); 593 594 return (ret); 595 } 596 597 /* 598 * NOTE: 599 * We assume idx_adv == sizeof(channel packet). 600 */ 601 int 602 vmbus_rxbr_idxadv_peek(struct vmbus_rxbr *rbr, void *data, int dlen, 603 uint32_t idx_adv, boolean_t *need_sig) 604 { 605 uint32_t rindex, br_dsize = rbr->rxbr_dsize; 606 607 mtx_lock_spin(&rbr->rxbr_lock); 608 /* 609 * Make sure it has enough data to read. 610 */ 611 if (vmbus_rxbr_avail(rbr) < idx_adv + sizeof(uint64_t) + dlen) { 612 mtx_unlock_spin(&rbr->rxbr_lock); 613 return (EAGAIN); 614 } 615 616 if (idx_adv > 0) { 617 /* 618 * Advance the read index first, including the channel's 64bit 619 * previous write offset. 620 */ 621 rindex = VMBUS_BR_IDXINC(rbr->rxbr_rindex, 622 idx_adv + sizeof(uint64_t), br_dsize); 623 __compiler_membar(); 624 atomic_store_rel_32(&rbr->rxbr_rindex, rindex); 625 } 626 627 vmbus_rxbr_copyfrom(rbr, 628 atomic_load_acq_32(&rbr->rxbr_rindex), data, dlen); 629 630 mtx_unlock_spin(&rbr->rxbr_lock); 631 632 if (need_sig) { 633 if (idx_adv > 0) 634 *need_sig = 635 vmbus_rxbr_need_signal(rbr, idx_adv + 636 sizeof(uint64_t)); 637 else 638 *need_sig = false; 639 } 640 641 return (0); 642 } 643 644 /* 645 * NOTE: 646 * Just update the RX rb index. 647 */ 648 int 649 vmbus_rxbr_idxadv(struct vmbus_rxbr *rbr, uint32_t idx_adv, 650 boolean_t *need_sig) 651 { 652 uint32_t rindex, br_dsize = rbr->rxbr_dsize; 653 654 mtx_lock_spin(&rbr->rxbr_lock); 655 /* 656 * Make sure it has enough space to advance. 657 */ 658 if (vmbus_rxbr_avail(rbr) < idx_adv + sizeof(uint64_t)) { 659 mtx_unlock_spin(&rbr->rxbr_lock); 660 return (EAGAIN); 661 } 662 663 /* 664 * Advance the read index, including the channel's 64bit 665 * previous write offset. 666 */ 667 rindex = VMBUS_BR_IDXINC(rbr->rxbr_rindex, 668 idx_adv + sizeof(uint64_t), br_dsize); 669 __compiler_membar(); 670 atomic_store_rel_32(&rbr->rxbr_rindex, rindex); 671 672 mtx_unlock_spin(&rbr->rxbr_lock); 673 674 if (need_sig) { 675 *need_sig = 676 vmbus_rxbr_need_signal(rbr, idx_adv + sizeof(uint64_t)); 677 } 678 679 return (0); 680 } 681 682 /* 683 * NOTE: 684 * We assume (dlen + skip) == sizeof(channel packet). 685 */ 686 int 687 vmbus_rxbr_read(struct vmbus_rxbr *rbr, void *data, int dlen, uint32_t skip, 688 boolean_t *need_sig) 689 { 690 uint32_t rindex, br_dsize = rbr->rxbr_dsize; 691 692 KASSERT(dlen + skip > 0, ("invalid dlen %d, offset %u", dlen, skip)); 693 694 mtx_lock_spin(&rbr->rxbr_lock); 695 696 if (vmbus_rxbr_avail(rbr) < dlen + skip + sizeof(uint64_t)) { 697 mtx_unlock_spin(&rbr->rxbr_lock); 698 return (EAGAIN); 699 } 700 701 /* 702 * Copy channel packet from RX bufring. 703 */ 704 rindex = VMBUS_BR_IDXINC(atomic_load_acq_32(&rbr->rxbr_rindex), 705 skip, br_dsize); 706 rindex = vmbus_rxbr_copyfrom(rbr, rindex, data, dlen); 707 708 /* 709 * Discard this channel packet's 64bits offset, which is useless to us. 710 */ 711 rindex = VMBUS_BR_IDXINC(rindex, sizeof(uint64_t), br_dsize); 712 713 /* 714 * Update the read index _after_ the channel packet is fetched. 715 */ 716 __compiler_membar(); 717 atomic_store_rel_32(&rbr->rxbr_rindex, rindex); 718 719 mtx_unlock_spin(&rbr->rxbr_lock); 720 721 if (need_sig) { 722 *need_sig = 723 vmbus_rxbr_need_signal(rbr, 724 dlen + skip + sizeof(uint64_t)); 725 } 726 727 return (0); 728 } 729