1 /* 2 * CDDL HEADER START 3 * 4 * Copyright(c) 2007-2009 Intel Corporation. All rights reserved. 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at: 10 * http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When using or redistributing this file, you may do so under the 15 * License only. No other modification of this header is permitted. 16 * 17 * If applicable, add the following below this CDDL HEADER, with the 18 * fields enclosed by brackets "[]" replaced with your own identifying 19 * information: Portions Copyright [yyyy] [name of copyright owner] 20 * 21 * CDDL HEADER END 22 */ 23 24 /* 25 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 26 * Use is subject to license terms. 27 */ 28 29 #include "igb_sw.h" 30 31 static boolean_t igb_tx(igb_tx_ring_t *, mblk_t *); 32 static int igb_tx_copy(igb_tx_ring_t *, tx_control_block_t *, mblk_t *, 33 uint32_t, boolean_t); 34 static int igb_tx_bind(igb_tx_ring_t *, tx_control_block_t *, mblk_t *, 35 uint32_t); 36 static int igb_tx_fill_ring(igb_tx_ring_t *, link_list_t *, tx_context_t *, 37 size_t); 38 static void igb_save_desc(tx_control_block_t *, uint64_t, size_t); 39 static tx_control_block_t *igb_get_free_list(igb_tx_ring_t *); 40 static int igb_get_tx_context(mblk_t *, tx_context_t *); 41 static boolean_t igb_check_tx_context(igb_tx_ring_t *, tx_context_t *); 42 static void igb_fill_tx_context(struct e1000_adv_tx_context_desc *, 43 tx_context_t *, uint32_t); 44 45 #ifndef IGB_DEBUG 46 #pragma inline(igb_save_desc) 47 #pragma inline(igb_get_tx_context) 48 #pragma inline(igb_check_tx_context) 49 #pragma inline(igb_fill_tx_context) 50 #endif 51 52 mblk_t * 53 igb_tx_ring_send(void *arg, mblk_t *mp) 54 { 55 igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg; 56 57 ASSERT(tx_ring != NULL); 58 59 if ((tx_ring->igb->igb_state & IGB_SUSPENDED) || 60 (tx_ring->igb->igb_state & IGB_ERROR) || 61 !(tx_ring->igb->igb_state & IGB_STARTED)) { 62 freemsg(mp); 63 return (NULL); 64 } 65 66 return ((igb_tx(tx_ring, mp)) ? NULL : mp); 67 } 68 69 /* 70 * igb_tx - Main transmit processing 71 * 72 * Called from igb_m_tx with an mblk ready to transmit. this 73 * routine sets up the transmit descriptors and sends data to 74 * the wire. 75 * 76 * One mblk can consist of several fragments, each fragment 77 * will be processed with different methods based on the size. 78 * For the fragments with size less than the bcopy threshold, 79 * they will be processed by using bcopy; otherwise, they will 80 * be processed by using DMA binding. 81 * 82 * To process the mblk, a tx control block is got from the 83 * free list. One tx control block contains one tx buffer, which 84 * is used to copy mblk fragments' data; and one tx DMA handle, 85 * which is used to bind a mblk fragment with DMA resource. 86 * 87 * Several small mblk fragments can be copied into one tx control 88 * block's buffer, and then the buffer will be transmitted with 89 * one tx descriptor. 90 * 91 * A large fragment only binds with one tx control block's DMA 92 * handle, and it can span several tx descriptors for transmitting. 93 * 94 * So to transmit a packet (mblk), several tx control blocks can 95 * be used. After the processing, those tx control blocks will 96 * be put to the work list. 97 */ 98 static boolean_t 99 igb_tx(igb_tx_ring_t *tx_ring, mblk_t *mp) 100 { 101 igb_t *igb = tx_ring->igb; 102 tx_type_t current_flag, next_flag; 103 uint32_t current_len, next_len; 104 uint32_t desc_total; 105 size_t mbsize; 106 int desc_num; 107 boolean_t copy_done, eop; 108 mblk_t *current_mp, *next_mp, *nmp; 109 tx_control_block_t *tcb; 110 tx_context_t tx_context, *ctx; 111 link_list_t pending_list; 112 mblk_t *hdr_new_mp = NULL; 113 mblk_t *hdr_previous_mp = NULL; 114 mblk_t *hdr_current_mp = NULL; 115 uint32_t hdr_frag_len; 116 uint32_t hdr_len, len; 117 uint32_t copy_thresh; 118 119 copy_thresh = igb->tx_copy_thresh; 120 121 /* Get the mblk size */ 122 mbsize = 0; 123 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) { 124 mbsize += MBLKL(nmp); 125 } 126 127 if (igb->tx_hcksum_enable) { 128 ctx = &tx_context; 129 /* 130 * Retrieve offloading context information from the mblk 131 * that will be used to decide whether/how to fill the 132 * context descriptor. 133 */ 134 if (igb_get_tx_context(mp, ctx) != TX_CXT_SUCCESS) { 135 freemsg(mp); 136 return (B_TRUE); 137 } 138 139 if ((ctx->lso_flag && 140 (mbsize > (ctx->mac_hdr_len + IGB_LSO_MAXLEN))) || 141 (!ctx->lso_flag && 142 (mbsize > (igb->max_frame_size - ETHERFCSL)))) { 143 freemsg(mp); 144 IGB_DEBUGLOG_0(igb, "igb_tx: packet oversize"); 145 return (B_TRUE); 146 } 147 } else { 148 ctx = NULL; 149 if (mbsize > (igb->max_frame_size - ETHERFCSL)) { 150 freemsg(mp); 151 IGB_DEBUGLOG_0(igb, "igb_tx: packet oversize"); 152 return (B_TRUE); 153 } 154 } 155 156 /* 157 * Check and recycle tx descriptors. 158 * The recycle threshold here should be selected carefully 159 */ 160 if (tx_ring->tbd_free < igb->tx_recycle_thresh) 161 tx_ring->tx_recycle(tx_ring); 162 163 /* 164 * After the recycling, if the tbd_free is less than the 165 * tx_overload_threshold, assert overload, return B_FALSE; 166 * and we need to re-schedule the tx again. 167 */ 168 if (tx_ring->tbd_free < igb->tx_overload_thresh) { 169 tx_ring->reschedule = B_TRUE; 170 IGB_DEBUG_STAT(tx_ring->stat_overload); 171 return (B_FALSE); 172 } 173 174 /* 175 * The software should guarantee LSO packet header(MAC+IP+TCP) 176 * to be within one descriptor - this is required by h/w. 177 * Here will reallocate and refill the header if 178 * the headers(MAC+IP+TCP) is physical memory non-contiguous. 179 */ 180 if (ctx && ctx->lso_flag) { 181 hdr_len = ctx->mac_hdr_len + ctx->ip_hdr_len + ctx->l4_hdr_len; 182 len = MBLKL(mp); 183 hdr_current_mp = mp; 184 while (len < hdr_len) { 185 hdr_previous_mp = hdr_current_mp; 186 hdr_current_mp = hdr_current_mp->b_cont; 187 len += MBLKL(hdr_current_mp); 188 } 189 /* 190 * If the header and the payload are in different mblks, 191 * we simply force the header to be copied into pre-allocated 192 * page-aligned buffer. 193 */ 194 if (len == hdr_len) 195 goto adjust_threshold; 196 197 hdr_frag_len = hdr_len - (len - MBLKL(hdr_current_mp)); 198 /* 199 * There are two cases we will reallocate 200 * a mblk for the last header fragment. 201 * 1. the header is in multiple mblks and 202 * the last fragment shares the same mblk 203 * with the payload 204 * 2. the header is in a single mblk shared 205 * with the payload but the header crosses 206 * a page. 207 */ 208 if ((hdr_current_mp != mp) || 209 (P2NPHASE((uintptr_t)hdr_current_mp->b_rptr, igb->page_size) 210 < hdr_len)) { 211 /* 212 * reallocate the mblk for the last header fragment, 213 * expect it to be copied into pre-allocated 214 * page-aligned buffer 215 */ 216 hdr_new_mp = allocb(hdr_frag_len, NULL); 217 if (!hdr_new_mp) { 218 return (B_FALSE); 219 } 220 221 /* link the new header fragment with the other parts */ 222 bcopy(hdr_current_mp->b_rptr, 223 hdr_new_mp->b_rptr, hdr_frag_len); 224 hdr_new_mp->b_wptr = hdr_new_mp->b_rptr + hdr_frag_len; 225 hdr_new_mp->b_cont = hdr_current_mp; 226 if (hdr_previous_mp) 227 hdr_previous_mp->b_cont = hdr_new_mp; 228 else 229 mp = hdr_new_mp; 230 hdr_current_mp->b_rptr += hdr_frag_len; 231 } 232 adjust_threshold: 233 /* 234 * adjust the bcopy threshhold to guarantee 235 * the header to use bcopy way 236 */ 237 if (copy_thresh < hdr_len) 238 copy_thresh = hdr_len; 239 } 240 241 /* 242 * The pending_list is a linked list that is used to save 243 * the tx control blocks that have packet data processed 244 * but have not put the data to the tx descriptor ring. 245 * It is used to reduce the lock contention of the tx_lock. 246 */ 247 LINK_LIST_INIT(&pending_list); 248 desc_num = 0; 249 desc_total = 0; 250 251 current_mp = mp; 252 current_len = MBLKL(current_mp); 253 /* 254 * Decide which method to use for the first fragment 255 */ 256 current_flag = (current_len <= copy_thresh) ? 257 USE_COPY : USE_DMA; 258 /* 259 * If the mblk includes several contiguous small fragments, 260 * they may be copied into one buffer. This flag is used to 261 * indicate whether there are pending fragments that need to 262 * be copied to the current tx buffer. 263 * 264 * If this flag is B_TRUE, it indicates that a new tx control 265 * block is needed to process the next fragment using either 266 * copy or DMA binding. 267 * 268 * Otherwise, it indicates that the next fragment will be 269 * copied to the current tx buffer that is maintained by the 270 * current tx control block. No new tx control block is needed. 271 */ 272 copy_done = B_TRUE; 273 while (current_mp) { 274 next_mp = current_mp->b_cont; 275 eop = (next_mp == NULL); /* Last fragment of the packet? */ 276 next_len = eop ? 0: MBLKL(next_mp); 277 278 /* 279 * When the current fragment is an empty fragment, if 280 * the next fragment will still be copied to the current 281 * tx buffer, we cannot skip this fragment here. Because 282 * the copy processing is pending for completion. We have 283 * to process this empty fragment in the tx_copy routine. 284 * 285 * If the copy processing is completed or a DMA binding 286 * processing is just completed, we can just skip this 287 * empty fragment. 288 */ 289 if ((current_len == 0) && (copy_done)) { 290 current_mp = next_mp; 291 current_len = next_len; 292 current_flag = (current_len <= copy_thresh) ? 293 USE_COPY : USE_DMA; 294 continue; 295 } 296 297 if (copy_done) { 298 /* 299 * Get a new tx control block from the free list 300 */ 301 tcb = igb_get_free_list(tx_ring); 302 303 if (tcb == NULL) { 304 IGB_DEBUG_STAT(tx_ring->stat_fail_no_tcb); 305 goto tx_failure; 306 } 307 308 /* 309 * Push the tx control block to the pending list 310 * to avoid using lock too early 311 */ 312 LIST_PUSH_TAIL(&pending_list, &tcb->link); 313 } 314 315 if (current_flag == USE_COPY) { 316 /* 317 * Check whether to use bcopy or DMA binding to process 318 * the next fragment, and if using bcopy, whether we 319 * need to continue copying the next fragment into the 320 * current tx buffer. 321 */ 322 ASSERT((tcb->tx_buf.len + current_len) <= 323 tcb->tx_buf.size); 324 325 if (eop) { 326 /* 327 * This is the last fragment of the packet, so 328 * the copy processing will be completed with 329 * this fragment. 330 */ 331 next_flag = USE_NONE; 332 copy_done = B_TRUE; 333 } else if ((tcb->tx_buf.len + current_len + next_len) > 334 tcb->tx_buf.size) { 335 /* 336 * If the next fragment is too large to be 337 * copied to the current tx buffer, we need 338 * to complete the current copy processing. 339 */ 340 next_flag = (next_len > copy_thresh) ? 341 USE_DMA: USE_COPY; 342 copy_done = B_TRUE; 343 } else if (next_len > copy_thresh) { 344 /* 345 * The next fragment needs to be processed with 346 * DMA binding. So the copy prcessing will be 347 * completed with the current fragment. 348 */ 349 next_flag = USE_DMA; 350 copy_done = B_TRUE; 351 } else { 352 /* 353 * Continue to copy the next fragment to the 354 * current tx buffer. 355 */ 356 next_flag = USE_COPY; 357 copy_done = B_FALSE; 358 } 359 360 desc_num = igb_tx_copy(tx_ring, tcb, current_mp, 361 current_len, copy_done); 362 } else { 363 /* 364 * Check whether to use bcopy or DMA binding to process 365 * the next fragment. 366 */ 367 next_flag = (next_len > copy_thresh) ? 368 USE_DMA: USE_COPY; 369 ASSERT(copy_done == B_TRUE); 370 371 desc_num = igb_tx_bind(tx_ring, tcb, current_mp, 372 current_len); 373 } 374 375 if (desc_num > 0) 376 desc_total += desc_num; 377 else if (desc_num < 0) 378 goto tx_failure; 379 380 current_mp = next_mp; 381 current_len = next_len; 382 current_flag = next_flag; 383 } 384 385 /* 386 * Attach the mblk to the last tx control block 387 */ 388 ASSERT(tcb); 389 ASSERT(tcb->mp == NULL); 390 tcb->mp = mp; 391 392 /* 393 * Before fill the tx descriptor ring with the data, we need to 394 * ensure there are adequate free descriptors for transmit 395 * (including one context descriptor). 396 */ 397 if (tx_ring->tbd_free < (desc_total + 1)) { 398 tx_ring->tx_recycle(tx_ring); 399 } 400 401 mutex_enter(&tx_ring->tx_lock); 402 403 /* 404 * If the number of free tx descriptors is not enough for transmit 405 * then return failure. 406 * 407 * Note: we must put this check under the mutex protection to 408 * ensure the correctness when multiple threads access it in 409 * parallel. 410 */ 411 if (tx_ring->tbd_free < (desc_total + 1)) { 412 IGB_DEBUG_STAT(tx_ring->stat_fail_no_tbd); 413 mutex_exit(&tx_ring->tx_lock); 414 goto tx_failure; 415 } 416 417 desc_num = igb_tx_fill_ring(tx_ring, &pending_list, ctx, mbsize); 418 419 ASSERT((desc_num == desc_total) || (desc_num == (desc_total + 1))); 420 421 mutex_exit(&tx_ring->tx_lock); 422 423 return (B_TRUE); 424 425 tx_failure: 426 /* 427 * If new mblk has been allocted for the last header 428 * fragment of a LSO packet, we should restore the 429 * modified mp. 430 */ 431 if (hdr_new_mp) { 432 hdr_new_mp->b_cont = NULL; 433 freeb(hdr_new_mp); 434 hdr_current_mp->b_rptr -= hdr_frag_len; 435 if (hdr_previous_mp) 436 hdr_previous_mp->b_cont = hdr_current_mp; 437 else 438 mp = hdr_current_mp; 439 } 440 441 /* 442 * Discard the mblk and free the used resources 443 */ 444 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 445 while (tcb) { 446 tcb->mp = NULL; 447 448 igb_free_tcb(tcb); 449 450 tcb = (tx_control_block_t *) 451 LIST_GET_NEXT(&pending_list, &tcb->link); 452 } 453 454 /* 455 * Return the tx control blocks in the pending list to the free list. 456 */ 457 igb_put_free_list(tx_ring, &pending_list); 458 459 /* Transmit failed, do not drop the mblk, rechedule the transmit */ 460 tx_ring->reschedule = B_TRUE; 461 462 return (B_FALSE); 463 } 464 465 /* 466 * igb_tx_copy 467 * 468 * Copy the mblk fragment to the pre-allocated tx buffer 469 */ 470 static int 471 igb_tx_copy(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp, 472 uint32_t len, boolean_t copy_done) 473 { 474 dma_buffer_t *tx_buf; 475 uint32_t desc_num; 476 _NOTE(ARGUNUSED(tx_ring)); 477 478 tx_buf = &tcb->tx_buf; 479 480 /* 481 * Copy the packet data of the mblk fragment into the 482 * pre-allocated tx buffer, which is maintained by the 483 * tx control block. 484 * 485 * Several mblk fragments can be copied into one tx buffer. 486 * The destination address of the current copied fragment in 487 * the tx buffer is next to the end of the previous copied 488 * fragment. 489 */ 490 if (len > 0) { 491 bcopy(mp->b_rptr, tx_buf->address + tx_buf->len, len); 492 493 tx_buf->len += len; 494 tcb->frag_num++; 495 } 496 497 desc_num = 0; 498 499 /* 500 * If it is the last fragment copied to the current tx buffer, 501 * in other words, if there's no remaining fragment or the remaining 502 * fragment requires a new tx control block to process, we need to 503 * complete the current copy processing by syncing up the current 504 * DMA buffer and saving the descriptor data. 505 */ 506 if (copy_done) { 507 /* 508 * Sync the DMA buffer of the packet data 509 */ 510 DMA_SYNC(tx_buf, DDI_DMA_SYNC_FORDEV); 511 512 tcb->tx_type = USE_COPY; 513 514 /* 515 * Save the address and length to the private data structure 516 * of the tx control block, which will be used to fill the 517 * tx descriptor ring after all the fragments are processed. 518 */ 519 igb_save_desc(tcb, tx_buf->dma_address, tx_buf->len); 520 desc_num++; 521 } 522 523 return (desc_num); 524 } 525 526 /* 527 * igb_tx_bind 528 * 529 * Bind the mblk fragment with DMA 530 */ 531 static int 532 igb_tx_bind(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp, 533 uint32_t len) 534 { 535 int status, i; 536 ddi_dma_cookie_t dma_cookie; 537 uint_t ncookies; 538 int desc_num; 539 540 /* 541 * Use DMA binding to process the mblk fragment 542 */ 543 status = ddi_dma_addr_bind_handle(tcb->tx_dma_handle, NULL, 544 (caddr_t)mp->b_rptr, len, 545 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, 546 0, &dma_cookie, &ncookies); 547 548 if (status != DDI_DMA_MAPPED) { 549 IGB_DEBUG_STAT(tx_ring->stat_fail_dma_bind); 550 return (-1); 551 } 552 553 tcb->frag_num++; 554 tcb->tx_type = USE_DMA; 555 /* 556 * Each fragment can span several cookies. One cookie will have 557 * one tx descriptor to transmit. 558 */ 559 desc_num = 0; 560 for (i = ncookies; i > 0; i--) { 561 /* 562 * Save the address and length to the private data structure 563 * of the tx control block, which will be used to fill the 564 * tx descriptor ring after all the fragments are processed. 565 */ 566 igb_save_desc(tcb, 567 dma_cookie.dmac_laddress, 568 dma_cookie.dmac_size); 569 570 desc_num++; 571 572 if (i > 1) 573 ddi_dma_nextcookie(tcb->tx_dma_handle, &dma_cookie); 574 } 575 576 return (desc_num); 577 } 578 579 /* 580 * igb_get_tx_context 581 * 582 * Get the tx context information from the mblk 583 */ 584 static int 585 igb_get_tx_context(mblk_t *mp, tx_context_t *ctx) 586 { 587 uint32_t start; 588 uint32_t flags; 589 uint32_t lso_flag; 590 uint32_t mss; 591 uint32_t len; 592 uint32_t size; 593 uint32_t offset; 594 unsigned char *pos; 595 ushort_t etype; 596 uint32_t mac_hdr_len; 597 uint32_t l4_proto; 598 uint32_t l4_hdr_len; 599 600 ASSERT(mp != NULL); 601 602 hcksum_retrieve(mp, NULL, NULL, &start, NULL, NULL, NULL, &flags); 603 bzero(ctx, sizeof (tx_context_t)); 604 605 ctx->hcksum_flags = flags; 606 607 if (flags == 0) 608 return (TX_CXT_SUCCESS); 609 610 lso_info_get(mp, &mss, &lso_flag); 611 ctx->mss = mss; 612 ctx->lso_flag = (lso_flag == HW_LSO); 613 614 /* 615 * LSO relies on tx h/w checksum, so here the packet will be 616 * dropped if the h/w checksum flags are not set. 617 */ 618 if (ctx->lso_flag) { 619 if (!((ctx->hcksum_flags & HCK_PARTIALCKSUM) && 620 (ctx->hcksum_flags & HCK_IPV4_HDRCKSUM))) { 621 IGB_DEBUGLOG_0(NULL, "igb_tx: h/w " 622 "checksum flags are not set for LSO"); 623 return (TX_CXT_E_LSO_CSUM); 624 } 625 } 626 627 etype = 0; 628 mac_hdr_len = 0; 629 l4_proto = 0; 630 631 /* 632 * Firstly get the position of the ether_type/ether_tpid. 633 * Here we don't assume the ether (VLAN) header is fully included 634 * in one mblk fragment, so we go thourgh the fragments to parse 635 * the ether type. 636 */ 637 size = len = MBLKL(mp); 638 offset = offsetof(struct ether_header, ether_type); 639 while (size <= offset) { 640 mp = mp->b_cont; 641 ASSERT(mp != NULL); 642 len = MBLKL(mp); 643 size += len; 644 } 645 pos = mp->b_rptr + offset + len - size; 646 647 etype = ntohs(*(ushort_t *)(uintptr_t)pos); 648 if (etype == ETHERTYPE_VLAN) { 649 /* 650 * Get the position of the ether_type in VLAN header 651 */ 652 offset = offsetof(struct ether_vlan_header, ether_type); 653 while (size <= offset) { 654 mp = mp->b_cont; 655 ASSERT(mp != NULL); 656 len = MBLKL(mp); 657 size += len; 658 } 659 pos = mp->b_rptr + offset + len - size; 660 661 etype = ntohs(*(ushort_t *)(uintptr_t)pos); 662 mac_hdr_len = sizeof (struct ether_vlan_header); 663 } else { 664 mac_hdr_len = sizeof (struct ether_header); 665 } 666 667 /* 668 * Here we assume the IP(V6) header is fully included in one 669 * mblk fragment. 670 */ 671 switch (etype) { 672 case ETHERTYPE_IP: 673 offset = mac_hdr_len; 674 while (size <= offset) { 675 mp = mp->b_cont; 676 ASSERT(mp != NULL); 677 len = MBLKL(mp); 678 size += len; 679 } 680 pos = mp->b_rptr + offset + len - size; 681 682 if (ctx->lso_flag) { 683 *((uint16_t *)(uintptr_t)(pos + offsetof(ipha_t, 684 ipha_length))) = 0; 685 686 /* 687 * To utilize igb LSO, here need to fill 688 * the tcp checksum field of the packet with the 689 * following pseudo-header checksum: 690 * (ip_source_addr, ip_destination_addr, l4_proto) 691 * and also need to fill the ip header checksum 692 * with zero. Currently the tcp/ip stack has done 693 * these. 694 */ 695 } 696 697 l4_proto = *(uint8_t *)(pos + offsetof(ipha_t, ipha_protocol)); 698 break; 699 case ETHERTYPE_IPV6: 700 offset = offsetof(ip6_t, ip6_nxt) + mac_hdr_len; 701 while (size <= offset) { 702 mp = mp->b_cont; 703 ASSERT(mp != NULL); 704 len = MBLKL(mp); 705 size += len; 706 } 707 pos = mp->b_rptr + offset + len - size; 708 709 l4_proto = *(uint8_t *)pos; 710 break; 711 default: 712 /* Unrecoverable error */ 713 IGB_DEBUGLOG_0(NULL, "Ethernet type field error with " 714 "tx hcksum flag set"); 715 return (TX_CXT_E_ETHER_TYPE); 716 } 717 718 if (ctx->lso_flag) { 719 offset = mac_hdr_len + start; 720 while (size <= offset) { 721 mp = mp->b_cont; 722 ASSERT(mp != NULL); 723 len = MBLKL(mp); 724 size += len; 725 } 726 pos = mp->b_rptr + offset + len - size; 727 728 l4_hdr_len = TCP_HDR_LENGTH((tcph_t *)pos); 729 } else { 730 /* 731 * l4 header length is only required for LSO 732 */ 733 l4_hdr_len = 0; 734 } 735 736 ctx->mac_hdr_len = mac_hdr_len; 737 ctx->ip_hdr_len = start; 738 ctx->l4_proto = l4_proto; 739 ctx->l4_hdr_len = l4_hdr_len; 740 741 return (TX_CXT_SUCCESS); 742 } 743 744 /* 745 * igb_check_tx_context 746 * 747 * Check if a new context descriptor is needed 748 */ 749 static boolean_t 750 igb_check_tx_context(igb_tx_ring_t *tx_ring, tx_context_t *ctx) 751 { 752 tx_context_t *last; 753 754 if (ctx == NULL) 755 return (B_FALSE); 756 757 /* 758 * Compare the context data retrieved from the mblk and the 759 * stored context data of the last context descriptor. The data 760 * need to be checked are: 761 * hcksum_flags 762 * l4_proto 763 * mss (only check for LSO) 764 * l4_hdr_len (only check for LSO) 765 * ip_hdr_len 766 * mac_hdr_len 767 * Either one of the above data is changed, a new context descriptor 768 * will be needed. 769 */ 770 last = &tx_ring->tx_context; 771 772 if (ctx->hcksum_flags != 0) { 773 if ((ctx->hcksum_flags != last->hcksum_flags) || 774 (ctx->l4_proto != last->l4_proto) || 775 (ctx->lso_flag && ((ctx->mss != last->mss) || 776 (ctx->l4_hdr_len != last->l4_hdr_len))) || 777 (ctx->ip_hdr_len != last->ip_hdr_len) || 778 (ctx->mac_hdr_len != last->mac_hdr_len)) { 779 return (B_TRUE); 780 } 781 } 782 783 return (B_FALSE); 784 } 785 786 /* 787 * igb_fill_tx_context 788 * 789 * Fill the context descriptor with hardware checksum informations 790 */ 791 static void 792 igb_fill_tx_context(struct e1000_adv_tx_context_desc *ctx_tbd, 793 tx_context_t *ctx, uint32_t ring_index) 794 { 795 /* 796 * Fill the context descriptor with the checksum 797 * context information we've got 798 */ 799 ctx_tbd->vlan_macip_lens = ctx->ip_hdr_len; 800 ctx_tbd->vlan_macip_lens |= ctx->mac_hdr_len << 801 E1000_ADVTXD_MACLEN_SHIFT; 802 803 ctx_tbd->type_tucmd_mlhl = 804 E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 805 806 if (ctx->hcksum_flags & HCK_IPV4_HDRCKSUM) 807 ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 808 809 if (ctx->hcksum_flags & HCK_PARTIALCKSUM) { 810 switch (ctx->l4_proto) { 811 case IPPROTO_TCP: 812 ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 813 break; 814 case IPPROTO_UDP: 815 /* 816 * We don't have to explicitly set: 817 * ctx_tbd->type_tucmd_mlhl |= 818 * E1000_ADVTXD_TUCMD_L4T_UDP; 819 * Because E1000_ADVTXD_TUCMD_L4T_UDP == 0b 820 */ 821 break; 822 default: 823 /* Unrecoverable error */ 824 IGB_DEBUGLOG_0(NULL, "L4 type error with tx hcksum"); 825 break; 826 } 827 } 828 829 ctx_tbd->seqnum_seed = 0; 830 ctx_tbd->mss_l4len_idx = ring_index << 4; 831 if (ctx->lso_flag) { 832 ctx_tbd->mss_l4len_idx |= 833 (ctx->l4_hdr_len << E1000_ADVTXD_L4LEN_SHIFT) | 834 (ctx->mss << E1000_ADVTXD_MSS_SHIFT); 835 } 836 } 837 838 /* 839 * igb_tx_fill_ring 840 * 841 * Fill the tx descriptor ring with the data 842 */ 843 static int 844 igb_tx_fill_ring(igb_tx_ring_t *tx_ring, link_list_t *pending_list, 845 tx_context_t *ctx, size_t mbsize) 846 { 847 struct e1000_hw *hw = &tx_ring->igb->hw; 848 boolean_t load_context; 849 uint32_t index, tcb_index, desc_num; 850 union e1000_adv_tx_desc *tbd, *first_tbd; 851 tx_control_block_t *tcb, *first_tcb; 852 uint32_t hcksum_flags; 853 int i; 854 igb_t *igb = tx_ring->igb; 855 856 ASSERT(mutex_owned(&tx_ring->tx_lock)); 857 858 tbd = NULL; 859 first_tbd = NULL; 860 first_tcb = NULL; 861 desc_num = 0; 862 hcksum_flags = 0; 863 load_context = B_FALSE; 864 865 /* 866 * Get the index of the first tx descriptor that will be filled, 867 * and the index of the first work list item that will be attached 868 * with the first used tx control block in the pending list. 869 * Note: the two indexes are the same. 870 */ 871 index = tx_ring->tbd_tail; 872 tcb_index = tx_ring->tbd_tail; 873 874 if (ctx != NULL) { 875 hcksum_flags = ctx->hcksum_flags; 876 877 /* 878 * Check if a new context descriptor is needed for this packet 879 */ 880 load_context = igb_check_tx_context(tx_ring, ctx); 881 if (load_context) { 882 first_tcb = (tx_control_block_t *) 883 LIST_GET_HEAD(pending_list); 884 tbd = &tx_ring->tbd_ring[index]; 885 886 /* 887 * Fill the context descriptor with the 888 * hardware checksum offload informations. 889 */ 890 igb_fill_tx_context( 891 (struct e1000_adv_tx_context_desc *)tbd, 892 ctx, tx_ring->index); 893 894 index = NEXT_INDEX(index, 1, tx_ring->ring_size); 895 desc_num++; 896 897 /* 898 * Store the checksum context data if 899 * a new context descriptor is added 900 */ 901 tx_ring->tx_context = *ctx; 902 } 903 } 904 905 first_tbd = &tx_ring->tbd_ring[index]; 906 907 /* 908 * Fill tx data descriptors with the data saved in the pending list. 909 * The tx control blocks in the pending list are added to the work list 910 * at the same time. 911 * 912 * The work list is strictly 1:1 corresponding to the descriptor ring. 913 * One item of the work list corresponds to one tx descriptor. Because 914 * one tx control block can span multiple tx descriptors, the tx 915 * control block will be added to the first work list item that 916 * corresponds to the first tx descriptor generated from that tx 917 * control block. 918 */ 919 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 920 while (tcb != NULL) { 921 922 for (i = 0; i < tcb->desc_num; i++) { 923 tbd = &tx_ring->tbd_ring[index]; 924 925 tbd->read.buffer_addr = tcb->desc[i].address; 926 tbd->read.cmd_type_len = tcb->desc[i].length; 927 928 tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_RS | 929 E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_DATA | 930 E1000_ADVTXD_DCMD_IFCS; 931 932 tbd->read.olinfo_status = 0; 933 934 index = NEXT_INDEX(index, 1, tx_ring->ring_size); 935 desc_num++; 936 } 937 938 if (first_tcb != NULL) { 939 /* 940 * Count the checksum context descriptor for 941 * the first tx control block. 942 */ 943 first_tcb->desc_num++; 944 first_tcb = NULL; 945 } 946 947 /* 948 * Add the tx control block to the work list 949 */ 950 ASSERT(tx_ring->work_list[tcb_index] == NULL); 951 tx_ring->work_list[tcb_index] = tcb; 952 953 tcb_index = index; 954 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 955 } 956 957 /* 958 * The Insert Ethernet CRC (IFCS) bit and the checksum fields are only 959 * valid in the first descriptor of the packet. 960 * 82576 also requires the payload length setting even without LSO 961 */ 962 ASSERT(first_tbd != NULL); 963 first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_IFCS; 964 if (ctx != NULL && ctx->lso_flag) { 965 first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_TSE; 966 first_tbd->read.olinfo_status |= 967 (mbsize - ctx->mac_hdr_len - ctx->ip_hdr_len 968 - ctx->l4_hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT; 969 } else { 970 if (hw->mac.type >= e1000_82576) { 971 first_tbd->read.olinfo_status |= 972 (mbsize << E1000_ADVTXD_PAYLEN_SHIFT); 973 } 974 } 975 976 /* Set hardware checksum bits */ 977 if (hcksum_flags != 0) { 978 if (hcksum_flags & HCK_IPV4_HDRCKSUM) 979 first_tbd->read.olinfo_status |= 980 E1000_TXD_POPTS_IXSM << 8; 981 if (hcksum_flags & HCK_PARTIALCKSUM) 982 first_tbd->read.olinfo_status |= 983 E1000_TXD_POPTS_TXSM << 8; 984 first_tbd->read.olinfo_status |= tx_ring->index << 4; 985 } 986 987 /* 988 * The last descriptor of packet needs End Of Packet (EOP), 989 * and Report Status (RS) bits set 990 */ 991 ASSERT(tbd != NULL); 992 tbd->read.cmd_type_len |= 993 E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS; 994 995 IGB_DEBUG_STAT(tx_ring->stat_pkt_cnt); 996 997 /* 998 * Sync the DMA buffer of the tx descriptor ring 999 */ 1000 DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORDEV); 1001 1002 /* 1003 * Update the number of the free tx descriptors. 1004 * The mutual exclusion between the transmission and the recycling 1005 * (for the tx descriptor ring and the work list) is implemented 1006 * with the atomic operation on the number of the free tx descriptors. 1007 * 1008 * Note: we should always decrement the counter tbd_free before 1009 * advancing the hardware TDT pointer to avoid the race condition - 1010 * before the counter tbd_free is decremented, the transmit of the 1011 * tx descriptors has done and the counter tbd_free is increased by 1012 * the tx recycling. 1013 */ 1014 i = igb_atomic_reserve(&tx_ring->tbd_free, desc_num); 1015 ASSERT(i >= 0); 1016 1017 tx_ring->tbd_tail = index; 1018 1019 /* 1020 * Advance the hardware TDT pointer of the tx descriptor ring 1021 */ 1022 E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), index); 1023 1024 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 1025 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 1026 atomic_or_32(&igb->igb_state, IGB_ERROR); 1027 } 1028 1029 return (desc_num); 1030 } 1031 1032 /* 1033 * igb_save_desc 1034 * 1035 * Save the address/length pair to the private array 1036 * of the tx control block. The address/length pairs 1037 * will be filled into the tx descriptor ring later. 1038 */ 1039 static void 1040 igb_save_desc(tx_control_block_t *tcb, uint64_t address, size_t length) 1041 { 1042 sw_desc_t *desc; 1043 1044 desc = &tcb->desc[tcb->desc_num]; 1045 desc->address = address; 1046 desc->length = length; 1047 1048 tcb->desc_num++; 1049 } 1050 1051 /* 1052 * igb_tx_recycle_legacy 1053 * 1054 * Recycle the tx descriptors and tx control blocks. 1055 * 1056 * The work list is traversed to check if the corresponding 1057 * tx descriptors have been transmitted. If so, the resources 1058 * bound to the tx control blocks will be freed, and those 1059 * tx control blocks will be returned to the free list. 1060 */ 1061 uint32_t 1062 igb_tx_recycle_legacy(igb_tx_ring_t *tx_ring) 1063 { 1064 uint32_t index, last_index; 1065 int desc_num; 1066 boolean_t desc_done; 1067 tx_control_block_t *tcb; 1068 link_list_t pending_list; 1069 igb_t *igb = tx_ring->igb; 1070 1071 /* 1072 * The mutex_tryenter() is used to avoid unnecessary 1073 * lock contention. 1074 */ 1075 if (mutex_tryenter(&tx_ring->recycle_lock) == 0) 1076 return (0); 1077 1078 ASSERT(tx_ring->tbd_free <= tx_ring->ring_size); 1079 1080 if (tx_ring->tbd_free == tx_ring->ring_size) { 1081 tx_ring->recycle_fail = 0; 1082 tx_ring->stall_watchdog = 0; 1083 mutex_exit(&tx_ring->recycle_lock); 1084 return (0); 1085 } 1086 1087 /* 1088 * Sync the DMA buffer of the tx descriptor ring 1089 */ 1090 DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL); 1091 1092 if (igb_check_dma_handle( 1093 tx_ring->tbd_area.dma_handle) != DDI_FM_OK) { 1094 mutex_exit(&tx_ring->recycle_lock); 1095 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 1096 atomic_or_32(&igb->igb_state, IGB_ERROR); 1097 return (0); 1098 } 1099 1100 LINK_LIST_INIT(&pending_list); 1101 desc_num = 0; 1102 index = tx_ring->tbd_head; /* Index of next tbd/tcb to recycle */ 1103 1104 tcb = tx_ring->work_list[index]; 1105 ASSERT(tcb != NULL); 1106 1107 desc_done = B_TRUE; 1108 while (desc_done && (tcb != NULL)) { 1109 1110 /* 1111 * Get the last tx descriptor of the tx control block. 1112 * If the last tx descriptor is done, it is done with 1113 * all the tx descriptors of the tx control block. 1114 * Then the tx control block and all the corresponding 1115 * tx descriptors can be recycled. 1116 */ 1117 last_index = NEXT_INDEX(index, tcb->desc_num - 1, 1118 tx_ring->ring_size); 1119 1120 /* 1121 * Check if the Descriptor Done bit is set 1122 */ 1123 desc_done = tx_ring->tbd_ring[last_index].wb.status & 1124 E1000_TXD_STAT_DD; 1125 if (desc_done) { 1126 /* 1127 * Strip off the tx control block from the work list, 1128 * and add it to the pending list. 1129 */ 1130 tx_ring->work_list[index] = NULL; 1131 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1132 1133 /* 1134 * Count the total number of the tx descriptors recycled 1135 */ 1136 desc_num += tcb->desc_num; 1137 1138 /* 1139 * Advance the index of the tx descriptor ring 1140 */ 1141 index = NEXT_INDEX(last_index, 1, tx_ring->ring_size); 1142 1143 tcb = tx_ring->work_list[index]; 1144 } 1145 } 1146 1147 /* 1148 * If no tx descriptors are recycled, no need to do more processing 1149 */ 1150 if (desc_num == 0) { 1151 tx_ring->recycle_fail++; 1152 mutex_exit(&tx_ring->recycle_lock); 1153 return (0); 1154 } 1155 1156 tx_ring->recycle_fail = 0; 1157 tx_ring->stall_watchdog = 0; 1158 1159 /* 1160 * Update the head index of the tx descriptor ring 1161 */ 1162 tx_ring->tbd_head = index; 1163 1164 /* 1165 * Update the number of the free tx descriptors with atomic operations 1166 */ 1167 atomic_add_32(&tx_ring->tbd_free, desc_num); 1168 1169 mutex_exit(&tx_ring->recycle_lock); 1170 1171 /* 1172 * Free the resources used by the tx control blocks 1173 * in the pending list 1174 */ 1175 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 1176 while (tcb != NULL) { 1177 /* 1178 * Release the resources occupied by the tx control block 1179 */ 1180 igb_free_tcb(tcb); 1181 1182 tcb = (tx_control_block_t *) 1183 LIST_GET_NEXT(&pending_list, &tcb->link); 1184 } 1185 1186 /* 1187 * Add the tx control blocks in the pending list to the free list. 1188 */ 1189 igb_put_free_list(tx_ring, &pending_list); 1190 1191 return (desc_num); 1192 } 1193 1194 /* 1195 * igb_tx_recycle_head_wb 1196 * 1197 * Check the head write-back, and recycle all the transmitted 1198 * tx descriptors and tx control blocks. 1199 */ 1200 uint32_t 1201 igb_tx_recycle_head_wb(igb_tx_ring_t *tx_ring) 1202 { 1203 uint32_t index; 1204 uint32_t head_wb; 1205 int desc_num; 1206 tx_control_block_t *tcb; 1207 link_list_t pending_list; 1208 igb_t *igb = tx_ring->igb; 1209 1210 /* 1211 * The mutex_tryenter() is used to avoid unnecessary 1212 * lock contention. 1213 */ 1214 if (mutex_tryenter(&tx_ring->recycle_lock) == 0) 1215 return (0); 1216 1217 ASSERT(tx_ring->tbd_free <= tx_ring->ring_size); 1218 1219 if (tx_ring->tbd_free == tx_ring->ring_size) { 1220 tx_ring->recycle_fail = 0; 1221 tx_ring->stall_watchdog = 0; 1222 mutex_exit(&tx_ring->recycle_lock); 1223 return (0); 1224 } 1225 1226 /* 1227 * Sync the DMA buffer of the tx descriptor ring 1228 * 1229 * Note: For head write-back mode, the tx descriptors will not 1230 * be written back, but the head write-back value is stored at 1231 * the last extra tbd at the end of the DMA area, we still need 1232 * to sync the head write-back value for kernel. 1233 * 1234 * DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL); 1235 */ 1236 (void) ddi_dma_sync(tx_ring->tbd_area.dma_handle, 1237 sizeof (union e1000_adv_tx_desc) * tx_ring->ring_size, 1238 sizeof (uint32_t), 1239 DDI_DMA_SYNC_FORKERNEL); 1240 1241 if (igb_check_dma_handle( 1242 tx_ring->tbd_area.dma_handle) != DDI_FM_OK) { 1243 mutex_exit(&tx_ring->recycle_lock); 1244 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 1245 atomic_or_32(&igb->igb_state, IGB_ERROR); 1246 return (0); 1247 } 1248 1249 LINK_LIST_INIT(&pending_list); 1250 desc_num = 0; 1251 index = tx_ring->tbd_head; /* Next index to clean */ 1252 1253 /* 1254 * Get the value of head write-back 1255 */ 1256 head_wb = *tx_ring->tbd_head_wb; 1257 while (index != head_wb) { 1258 tcb = tx_ring->work_list[index]; 1259 ASSERT(tcb != NULL); 1260 1261 if (OFFSET(index, head_wb, tx_ring->ring_size) < 1262 tcb->desc_num) { 1263 /* 1264 * The current tx control block is not 1265 * completely transmitted, stop recycling 1266 */ 1267 break; 1268 } 1269 1270 /* 1271 * Strip off the tx control block from the work list, 1272 * and add it to the pending list. 1273 */ 1274 tx_ring->work_list[index] = NULL; 1275 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1276 1277 /* 1278 * Advance the index of the tx descriptor ring 1279 */ 1280 index = NEXT_INDEX(index, tcb->desc_num, tx_ring->ring_size); 1281 1282 /* 1283 * Count the total number of the tx descriptors recycled 1284 */ 1285 desc_num += tcb->desc_num; 1286 } 1287 1288 /* 1289 * If no tx descriptors are recycled, no need to do more processing 1290 */ 1291 if (desc_num == 0) { 1292 tx_ring->recycle_fail++; 1293 mutex_exit(&tx_ring->recycle_lock); 1294 return (0); 1295 } 1296 1297 tx_ring->recycle_fail = 0; 1298 tx_ring->stall_watchdog = 0; 1299 1300 /* 1301 * Update the head index of the tx descriptor ring 1302 */ 1303 tx_ring->tbd_head = index; 1304 1305 /* 1306 * Update the number of the free tx descriptors with atomic operations 1307 */ 1308 atomic_add_32(&tx_ring->tbd_free, desc_num); 1309 1310 mutex_exit(&tx_ring->recycle_lock); 1311 1312 /* 1313 * Free the resources used by the tx control blocks 1314 * in the pending list 1315 */ 1316 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 1317 while (tcb) { 1318 /* 1319 * Release the resources occupied by the tx control block 1320 */ 1321 igb_free_tcb(tcb); 1322 1323 tcb = (tx_control_block_t *) 1324 LIST_GET_NEXT(&pending_list, &tcb->link); 1325 } 1326 1327 /* 1328 * Add the tx control blocks in the pending list to the free list. 1329 */ 1330 igb_put_free_list(tx_ring, &pending_list); 1331 1332 return (desc_num); 1333 } 1334 1335 /* 1336 * igb_free_tcb - free up the tx control block 1337 * 1338 * Free the resources of the tx control block, including 1339 * unbind the previously bound DMA handle, and reset other 1340 * control fields. 1341 */ 1342 void 1343 igb_free_tcb(tx_control_block_t *tcb) 1344 { 1345 switch (tcb->tx_type) { 1346 case USE_COPY: 1347 /* 1348 * Reset the buffer length that is used for copy 1349 */ 1350 tcb->tx_buf.len = 0; 1351 break; 1352 case USE_DMA: 1353 /* 1354 * Release the DMA resource that is used for 1355 * DMA binding. 1356 */ 1357 (void) ddi_dma_unbind_handle(tcb->tx_dma_handle); 1358 break; 1359 default: 1360 break; 1361 } 1362 1363 /* 1364 * Free the mblk 1365 */ 1366 if (tcb->mp != NULL) { 1367 freemsg(tcb->mp); 1368 tcb->mp = NULL; 1369 } 1370 1371 tcb->tx_type = USE_NONE; 1372 tcb->frag_num = 0; 1373 tcb->desc_num = 0; 1374 } 1375 1376 /* 1377 * igb_get_free_list - Get a free tx control block from the free list 1378 * 1379 * The atomic operation on the number of the available tx control block 1380 * in the free list is used to keep this routine mutual exclusive with 1381 * the routine igb_put_check_list. 1382 */ 1383 static tx_control_block_t * 1384 igb_get_free_list(igb_tx_ring_t *tx_ring) 1385 { 1386 tx_control_block_t *tcb; 1387 1388 /* 1389 * Check and update the number of the free tx control block 1390 * in the free list. 1391 */ 1392 if (igb_atomic_reserve(&tx_ring->tcb_free, 1) < 0) 1393 return (NULL); 1394 1395 mutex_enter(&tx_ring->tcb_head_lock); 1396 1397 tcb = tx_ring->free_list[tx_ring->tcb_head]; 1398 ASSERT(tcb != NULL); 1399 tx_ring->free_list[tx_ring->tcb_head] = NULL; 1400 tx_ring->tcb_head = NEXT_INDEX(tx_ring->tcb_head, 1, 1401 tx_ring->free_list_size); 1402 1403 mutex_exit(&tx_ring->tcb_head_lock); 1404 1405 return (tcb); 1406 } 1407 1408 /* 1409 * igb_put_free_list 1410 * 1411 * Put a list of used tx control blocks back to the free list 1412 * 1413 * A mutex is used here to ensure the serialization. The mutual exclusion 1414 * between igb_get_free_list and igb_put_free_list is implemented with 1415 * the atomic operation on the counter tcb_free. 1416 */ 1417 void 1418 igb_put_free_list(igb_tx_ring_t *tx_ring, link_list_t *pending_list) 1419 { 1420 uint32_t index; 1421 int tcb_num; 1422 tx_control_block_t *tcb; 1423 1424 mutex_enter(&tx_ring->tcb_tail_lock); 1425 1426 index = tx_ring->tcb_tail; 1427 1428 tcb_num = 0; 1429 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 1430 while (tcb != NULL) { 1431 ASSERT(tx_ring->free_list[index] == NULL); 1432 tx_ring->free_list[index] = tcb; 1433 1434 tcb_num++; 1435 1436 index = NEXT_INDEX(index, 1, tx_ring->free_list_size); 1437 1438 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 1439 } 1440 1441 tx_ring->tcb_tail = index; 1442 1443 /* 1444 * Update the number of the free tx control block 1445 * in the free list. This operation must be placed 1446 * under the protection of the lock. 1447 */ 1448 atomic_add_32(&tx_ring->tcb_free, tcb_num); 1449 1450 mutex_exit(&tx_ring->tcb_tail_lock); 1451 } 1452