1 /* 2 * CDDL HEADER START 3 * 4 * Copyright(c) 2007-2009 Intel Corporation. All rights reserved. 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at: 10 * http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When using or redistributing this file, you may do so under the 15 * License only. No other modification of this header is permitted. 16 * 17 * If applicable, add the following below this CDDL HEADER, with the 18 * fields enclosed by brackets "[]" replaced with your own identifying 19 * information: Portions Copyright [yyyy] [name of copyright owner] 20 * 21 * CDDL HEADER END 22 */ 23 24 /* 25 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 26 * Use is subject to license terms. 27 */ 28 29 #include "igb_sw.h" 30 31 static boolean_t igb_tx(igb_tx_ring_t *, mblk_t *); 32 static int igb_tx_copy(igb_tx_ring_t *, tx_control_block_t *, mblk_t *, 33 uint32_t, boolean_t); 34 static int igb_tx_bind(igb_tx_ring_t *, tx_control_block_t *, mblk_t *, 35 uint32_t); 36 static int igb_tx_fill_ring(igb_tx_ring_t *, link_list_t *, tx_context_t *, 37 size_t); 38 static void igb_save_desc(tx_control_block_t *, uint64_t, size_t); 39 static tx_control_block_t *igb_get_free_list(igb_tx_ring_t *); 40 static int igb_get_tx_context(mblk_t *, tx_context_t *); 41 static boolean_t igb_check_tx_context(igb_tx_ring_t *, tx_context_t *); 42 static void igb_fill_tx_context(struct e1000_adv_tx_context_desc *, 43 tx_context_t *, uint32_t); 44 45 #ifndef IGB_DEBUG 46 #pragma inline(igb_save_desc) 47 #pragma inline(igb_get_tx_context) 48 #pragma inline(igb_check_tx_context) 49 #pragma inline(igb_fill_tx_context) 50 #endif 51 52 mblk_t * 53 igb_tx_ring_send(void *arg, mblk_t *mp) 54 { 55 igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg; 56 57 ASSERT(tx_ring != NULL); 58 59 if ((tx_ring->igb->igb_state & IGB_SUSPENDED) || 60 (tx_ring->igb->igb_state & IGB_ERROR) || 61 !(tx_ring->igb->igb_state & IGB_STARTED)) { 62 freemsg(mp); 63 return (NULL); 64 } 65 66 return ((igb_tx(tx_ring, mp)) ? NULL : mp); 67 } 68 69 /* 70 * igb_tx - Main transmit processing 71 * 72 * Called from igb_m_tx with an mblk ready to transmit. this 73 * routine sets up the transmit descriptors and sends data to 74 * the wire. 75 * 76 * One mblk can consist of several fragments, each fragment 77 * will be processed with different methods based on the size. 78 * For the fragments with size less than the bcopy threshold, 79 * they will be processed by using bcopy; otherwise, they will 80 * be processed by using DMA binding. 81 * 82 * To process the mblk, a tx control block is got from the 83 * free list. One tx control block contains one tx buffer, which 84 * is used to copy mblk fragments' data; and one tx DMA handle, 85 * which is used to bind a mblk fragment with DMA resource. 86 * 87 * Several small mblk fragments can be copied into one tx control 88 * block's buffer, and then the buffer will be transmitted with 89 * one tx descriptor. 90 * 91 * A large fragment only binds with one tx control block's DMA 92 * handle, and it can span several tx descriptors for transmitting. 93 * 94 * So to transmit a packet (mblk), several tx control blocks can 95 * be used. After the processing, those tx control blocks will 96 * be put to the work list. 97 */ 98 static boolean_t 99 igb_tx(igb_tx_ring_t *tx_ring, mblk_t *mp) 100 { 101 igb_t *igb = tx_ring->igb; 102 tx_type_t current_flag, next_flag; 103 uint32_t current_len, next_len; 104 uint32_t desc_total; 105 size_t mbsize; 106 int desc_num; 107 boolean_t copy_done, eop; 108 mblk_t *current_mp, *next_mp, *nmp; 109 tx_control_block_t *tcb; 110 tx_context_t tx_context, *ctx; 111 link_list_t pending_list; 112 mblk_t *hdr_new_mp = NULL; 113 mblk_t *hdr_previous_mp = NULL; 114 mblk_t *hdr_current_mp = NULL; 115 uint32_t hdr_frag_len; 116 uint32_t hdr_len, len; 117 uint32_t copy_thresh; 118 119 copy_thresh = igb->tx_copy_thresh; 120 121 /* Get the mblk size */ 122 mbsize = 0; 123 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) { 124 mbsize += MBLKL(nmp); 125 } 126 127 if (igb->tx_hcksum_enable) { 128 ctx = &tx_context; 129 /* 130 * Retrieve offloading context information from the mblk 131 * that will be used to decide whether/how to fill the 132 * context descriptor. 133 */ 134 if (igb_get_tx_context(mp, ctx) != TX_CXT_SUCCESS) { 135 freemsg(mp); 136 return (B_TRUE); 137 } 138 139 if ((ctx->lso_flag && 140 (mbsize > (ctx->mac_hdr_len + IGB_LSO_MAXLEN))) || 141 (!ctx->lso_flag && 142 (mbsize > (igb->max_frame_size - ETHERFCSL)))) { 143 freemsg(mp); 144 IGB_DEBUGLOG_0(igb, "igb_tx: packet oversize"); 145 return (B_TRUE); 146 } 147 } else { 148 ctx = NULL; 149 if (mbsize > (igb->max_frame_size - ETHERFCSL)) { 150 freemsg(mp); 151 IGB_DEBUGLOG_0(igb, "igb_tx: packet oversize"); 152 return (B_TRUE); 153 } 154 } 155 156 /* 157 * Check and recycle tx descriptors. 158 * The recycle threshold here should be selected carefully 159 */ 160 if (tx_ring->tbd_free < igb->tx_recycle_thresh) 161 tx_ring->tx_recycle(tx_ring); 162 163 /* 164 * After the recycling, if the tbd_free is less than the 165 * tx_overload_threshold, assert overload, return B_FALSE; 166 * and we need to re-schedule the tx again. 167 */ 168 if (tx_ring->tbd_free < igb->tx_overload_thresh) { 169 tx_ring->reschedule = B_TRUE; 170 IGB_DEBUG_STAT(tx_ring->stat_overload); 171 return (B_FALSE); 172 } 173 174 /* 175 * The software should guarantee LSO packet header(MAC+IP+TCP) 176 * to be within one descriptor - this is required by h/w. 177 * Here will reallocate and refill the header if 178 * the headers(MAC+IP+TCP) is physical memory non-contiguous. 179 */ 180 if (ctx && ctx->lso_flag) { 181 hdr_len = ctx->mac_hdr_len + ctx->ip_hdr_len + ctx->l4_hdr_len; 182 len = MBLKL(mp); 183 hdr_current_mp = mp; 184 while (len < hdr_len) { 185 hdr_previous_mp = hdr_current_mp; 186 hdr_current_mp = hdr_current_mp->b_cont; 187 len += MBLKL(hdr_current_mp); 188 } 189 /* 190 * If the header and the payload are in different mblks, 191 * we simply force the header to be copied into pre-allocated 192 * page-aligned buffer. 193 */ 194 if (len == hdr_len) 195 goto adjust_threshold; 196 197 hdr_frag_len = hdr_len - (len - MBLKL(hdr_current_mp)); 198 /* 199 * There are two cases we will reallocate 200 * a mblk for the last header fragment. 201 * 1. the header is in multiple mblks and 202 * the last fragment shares the same mblk 203 * with the payload 204 * 2. the header is in a single mblk shared 205 * with the payload but the header crosses 206 * a page. 207 */ 208 if ((hdr_current_mp != mp) || 209 (P2NPHASE((uintptr_t)hdr_current_mp->b_rptr, igb->page_size) 210 < hdr_len)) { 211 /* 212 * reallocate the mblk for the last header fragment, 213 * expect it to be copied into pre-allocated 214 * page-aligned buffer 215 */ 216 hdr_new_mp = allocb(hdr_frag_len, NULL); 217 if (!hdr_new_mp) { 218 return (B_FALSE); 219 } 220 221 /* link the new header fragment with the other parts */ 222 bcopy(hdr_current_mp->b_rptr, 223 hdr_new_mp->b_rptr, hdr_frag_len); 224 hdr_new_mp->b_wptr = hdr_new_mp->b_rptr + hdr_frag_len; 225 hdr_new_mp->b_cont = hdr_current_mp; 226 if (hdr_previous_mp) 227 hdr_previous_mp->b_cont = hdr_new_mp; 228 else 229 mp = hdr_new_mp; 230 hdr_current_mp->b_rptr += hdr_frag_len; 231 } 232 adjust_threshold: 233 /* 234 * adjust the bcopy threshhold to guarantee 235 * the header to use bcopy way 236 */ 237 if (copy_thresh < hdr_len) 238 copy_thresh = hdr_len; 239 } 240 241 /* 242 * The pending_list is a linked list that is used to save 243 * the tx control blocks that have packet data processed 244 * but have not put the data to the tx descriptor ring. 245 * It is used to reduce the lock contention of the tx_lock. 246 */ 247 LINK_LIST_INIT(&pending_list); 248 desc_num = 0; 249 desc_total = 0; 250 251 current_mp = mp; 252 current_len = MBLKL(current_mp); 253 /* 254 * Decide which method to use for the first fragment 255 */ 256 current_flag = (current_len <= copy_thresh) ? 257 USE_COPY : USE_DMA; 258 /* 259 * If the mblk includes several contiguous small fragments, 260 * they may be copied into one buffer. This flag is used to 261 * indicate whether there are pending fragments that need to 262 * be copied to the current tx buffer. 263 * 264 * If this flag is B_TRUE, it indicates that a new tx control 265 * block is needed to process the next fragment using either 266 * copy or DMA binding. 267 * 268 * Otherwise, it indicates that the next fragment will be 269 * copied to the current tx buffer that is maintained by the 270 * current tx control block. No new tx control block is needed. 271 */ 272 copy_done = B_TRUE; 273 while (current_mp) { 274 next_mp = current_mp->b_cont; 275 eop = (next_mp == NULL); /* Last fragment of the packet? */ 276 next_len = eop ? 0: MBLKL(next_mp); 277 278 /* 279 * When the current fragment is an empty fragment, if 280 * the next fragment will still be copied to the current 281 * tx buffer, we cannot skip this fragment here. Because 282 * the copy processing is pending for completion. We have 283 * to process this empty fragment in the tx_copy routine. 284 * 285 * If the copy processing is completed or a DMA binding 286 * processing is just completed, we can just skip this 287 * empty fragment. 288 */ 289 if ((current_len == 0) && (copy_done)) { 290 current_mp = next_mp; 291 current_len = next_len; 292 current_flag = (current_len <= copy_thresh) ? 293 USE_COPY : USE_DMA; 294 continue; 295 } 296 297 if (copy_done) { 298 /* 299 * Get a new tx control block from the free list 300 */ 301 tcb = igb_get_free_list(tx_ring); 302 303 if (tcb == NULL) { 304 IGB_DEBUG_STAT(tx_ring->stat_fail_no_tcb); 305 goto tx_failure; 306 } 307 308 /* 309 * Push the tx control block to the pending list 310 * to avoid using lock too early 311 */ 312 LIST_PUSH_TAIL(&pending_list, &tcb->link); 313 } 314 315 if (current_flag == USE_COPY) { 316 /* 317 * Check whether to use bcopy or DMA binding to process 318 * the next fragment, and if using bcopy, whether we 319 * need to continue copying the next fragment into the 320 * current tx buffer. 321 */ 322 ASSERT((tcb->tx_buf.len + current_len) <= 323 tcb->tx_buf.size); 324 325 if (eop) { 326 /* 327 * This is the last fragment of the packet, so 328 * the copy processing will be completed with 329 * this fragment. 330 */ 331 next_flag = USE_NONE; 332 copy_done = B_TRUE; 333 } else if ((tcb->tx_buf.len + current_len + next_len) > 334 tcb->tx_buf.size) { 335 /* 336 * If the next fragment is too large to be 337 * copied to the current tx buffer, we need 338 * to complete the current copy processing. 339 */ 340 next_flag = (next_len > copy_thresh) ? 341 USE_DMA: USE_COPY; 342 copy_done = B_TRUE; 343 } else if (next_len > copy_thresh) { 344 /* 345 * The next fragment needs to be processed with 346 * DMA binding. So the copy prcessing will be 347 * completed with the current fragment. 348 */ 349 next_flag = USE_DMA; 350 copy_done = B_TRUE; 351 } else { 352 /* 353 * Continue to copy the next fragment to the 354 * current tx buffer. 355 */ 356 next_flag = USE_COPY; 357 copy_done = B_FALSE; 358 } 359 360 desc_num = igb_tx_copy(tx_ring, tcb, current_mp, 361 current_len, copy_done); 362 } else { 363 /* 364 * Check whether to use bcopy or DMA binding to process 365 * the next fragment. 366 */ 367 next_flag = (next_len > copy_thresh) ? 368 USE_DMA: USE_COPY; 369 ASSERT(copy_done == B_TRUE); 370 371 desc_num = igb_tx_bind(tx_ring, tcb, current_mp, 372 current_len); 373 } 374 375 if (desc_num > 0) 376 desc_total += desc_num; 377 else if (desc_num < 0) 378 goto tx_failure; 379 380 current_mp = next_mp; 381 current_len = next_len; 382 current_flag = next_flag; 383 } 384 385 /* 386 * Attach the mblk to the last tx control block 387 */ 388 ASSERT(tcb); 389 ASSERT(tcb->mp == NULL); 390 tcb->mp = mp; 391 392 /* 393 * Before fill the tx descriptor ring with the data, we need to 394 * ensure there are adequate free descriptors for transmit 395 * (including one context descriptor). 396 */ 397 if (tx_ring->tbd_free < (desc_total + 1)) { 398 tx_ring->tx_recycle(tx_ring); 399 } 400 401 mutex_enter(&tx_ring->tx_lock); 402 403 /* 404 * If the number of free tx descriptors is not enough for transmit 405 * then return failure. 406 * 407 * Note: we must put this check under the mutex protection to 408 * ensure the correctness when multiple threads access it in 409 * parallel. 410 */ 411 if (tx_ring->tbd_free < (desc_total + 1)) { 412 IGB_DEBUG_STAT(tx_ring->stat_fail_no_tbd); 413 mutex_exit(&tx_ring->tx_lock); 414 goto tx_failure; 415 } 416 417 desc_num = igb_tx_fill_ring(tx_ring, &pending_list, ctx, mbsize); 418 419 ASSERT((desc_num == desc_total) || (desc_num == (desc_total + 1))); 420 421 mutex_exit(&tx_ring->tx_lock); 422 423 return (B_TRUE); 424 425 tx_failure: 426 /* 427 * If new mblk has been allocted for the last header 428 * fragment of a LSO packet, we should restore the 429 * modified mp. 430 */ 431 if (hdr_new_mp) { 432 hdr_new_mp->b_cont = NULL; 433 freeb(hdr_new_mp); 434 hdr_current_mp->b_rptr -= hdr_frag_len; 435 if (hdr_previous_mp) 436 hdr_previous_mp->b_cont = hdr_current_mp; 437 else 438 mp = hdr_current_mp; 439 } 440 441 /* 442 * Discard the mblk and free the used resources 443 */ 444 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 445 while (tcb) { 446 tcb->mp = NULL; 447 448 igb_free_tcb(tcb); 449 450 tcb = (tx_control_block_t *) 451 LIST_GET_NEXT(&pending_list, &tcb->link); 452 } 453 454 /* 455 * Return the tx control blocks in the pending list to the free list. 456 */ 457 igb_put_free_list(tx_ring, &pending_list); 458 459 /* Transmit failed, do not drop the mblk, rechedule the transmit */ 460 tx_ring->reschedule = B_TRUE; 461 462 return (B_FALSE); 463 } 464 465 /* 466 * igb_tx_copy 467 * 468 * Copy the mblk fragment to the pre-allocated tx buffer 469 */ 470 static int 471 igb_tx_copy(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp, 472 uint32_t len, boolean_t copy_done) 473 { 474 dma_buffer_t *tx_buf; 475 uint32_t desc_num; 476 _NOTE(ARGUNUSED(tx_ring)); 477 478 tx_buf = &tcb->tx_buf; 479 480 /* 481 * Copy the packet data of the mblk fragment into the 482 * pre-allocated tx buffer, which is maintained by the 483 * tx control block. 484 * 485 * Several mblk fragments can be copied into one tx buffer. 486 * The destination address of the current copied fragment in 487 * the tx buffer is next to the end of the previous copied 488 * fragment. 489 */ 490 if (len > 0) { 491 bcopy(mp->b_rptr, tx_buf->address + tx_buf->len, len); 492 493 tx_buf->len += len; 494 tcb->frag_num++; 495 } 496 497 desc_num = 0; 498 499 /* 500 * If it is the last fragment copied to the current tx buffer, 501 * in other words, if there's no remaining fragment or the remaining 502 * fragment requires a new tx control block to process, we need to 503 * complete the current copy processing by syncing up the current 504 * DMA buffer and saving the descriptor data. 505 */ 506 if (copy_done) { 507 /* 508 * Sync the DMA buffer of the packet data 509 */ 510 DMA_SYNC(tx_buf, DDI_DMA_SYNC_FORDEV); 511 512 tcb->tx_type = USE_COPY; 513 514 /* 515 * Save the address and length to the private data structure 516 * of the tx control block, which will be used to fill the 517 * tx descriptor ring after all the fragments are processed. 518 */ 519 igb_save_desc(tcb, tx_buf->dma_address, tx_buf->len); 520 desc_num++; 521 } 522 523 return (desc_num); 524 } 525 526 /* 527 * igb_tx_bind 528 * 529 * Bind the mblk fragment with DMA 530 */ 531 static int 532 igb_tx_bind(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp, 533 uint32_t len) 534 { 535 int status, i; 536 ddi_dma_cookie_t dma_cookie; 537 uint_t ncookies; 538 int desc_num; 539 540 /* 541 * Use DMA binding to process the mblk fragment 542 */ 543 status = ddi_dma_addr_bind_handle(tcb->tx_dma_handle, NULL, 544 (caddr_t)mp->b_rptr, len, 545 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, 546 0, &dma_cookie, &ncookies); 547 548 if (status != DDI_DMA_MAPPED) { 549 IGB_DEBUG_STAT(tx_ring->stat_fail_dma_bind); 550 return (-1); 551 } 552 553 tcb->frag_num++; 554 tcb->tx_type = USE_DMA; 555 /* 556 * Each fragment can span several cookies. One cookie will have 557 * one tx descriptor to transmit. 558 */ 559 desc_num = 0; 560 for (i = ncookies; i > 0; i--) { 561 /* 562 * Save the address and length to the private data structure 563 * of the tx control block, which will be used to fill the 564 * tx descriptor ring after all the fragments are processed. 565 */ 566 igb_save_desc(tcb, 567 dma_cookie.dmac_laddress, 568 dma_cookie.dmac_size); 569 570 desc_num++; 571 572 if (i > 1) 573 ddi_dma_nextcookie(tcb->tx_dma_handle, &dma_cookie); 574 } 575 576 return (desc_num); 577 } 578 579 /* 580 * igb_get_tx_context 581 * 582 * Get the tx context information from the mblk 583 */ 584 static int 585 igb_get_tx_context(mblk_t *mp, tx_context_t *ctx) 586 { 587 uint32_t start; 588 uint32_t flags; 589 uint32_t lso_flag; 590 uint32_t mss; 591 uint32_t len; 592 uint32_t size; 593 uint32_t offset; 594 unsigned char *pos; 595 ushort_t etype; 596 uint32_t mac_hdr_len; 597 uint32_t l4_proto; 598 uint32_t l4_hdr_len; 599 600 ASSERT(mp != NULL); 601 602 hcksum_retrieve(mp, NULL, NULL, &start, NULL, NULL, NULL, &flags); 603 bzero(ctx, sizeof (tx_context_t)); 604 605 ctx->hcksum_flags = flags; 606 607 if (flags == 0) 608 return (TX_CXT_SUCCESS); 609 610 lso_info_get(mp, &mss, &lso_flag); 611 ctx->mss = mss; 612 ctx->lso_flag = (lso_flag == HW_LSO); 613 614 /* 615 * LSO relies on tx h/w checksum, so here the packet will be 616 * dropped if the h/w checksum flags are not set. 617 */ 618 if (ctx->lso_flag) { 619 if (!((ctx->hcksum_flags & HCK_PARTIALCKSUM) && 620 (ctx->hcksum_flags & HCK_IPV4_HDRCKSUM))) { 621 IGB_DEBUGLOG_0(NULL, "igb_tx: h/w " 622 "checksum flags are not set for LSO"); 623 return (TX_CXT_E_LSO_CSUM); 624 } 625 } 626 627 etype = 0; 628 mac_hdr_len = 0; 629 l4_proto = 0; 630 631 /* 632 * Firstly get the position of the ether_type/ether_tpid. 633 * Here we don't assume the ether (VLAN) header is fully included 634 * in one mblk fragment, so we go thourgh the fragments to parse 635 * the ether type. 636 */ 637 size = len = MBLKL(mp); 638 offset = offsetof(struct ether_header, ether_type); 639 while (size <= offset) { 640 mp = mp->b_cont; 641 ASSERT(mp != NULL); 642 len = MBLKL(mp); 643 size += len; 644 } 645 pos = mp->b_rptr + offset + len - size; 646 647 etype = ntohs(*(ushort_t *)(uintptr_t)pos); 648 if (etype == ETHERTYPE_VLAN) { 649 /* 650 * Get the position of the ether_type in VLAN header 651 */ 652 offset = offsetof(struct ether_vlan_header, ether_type); 653 while (size <= offset) { 654 mp = mp->b_cont; 655 ASSERT(mp != NULL); 656 len = MBLKL(mp); 657 size += len; 658 } 659 pos = mp->b_rptr + offset + len - size; 660 661 etype = ntohs(*(ushort_t *)(uintptr_t)pos); 662 mac_hdr_len = sizeof (struct ether_vlan_header); 663 } else { 664 mac_hdr_len = sizeof (struct ether_header); 665 } 666 667 /* 668 * Here we assume the IP(V6) header is fully included in one 669 * mblk fragment. 670 */ 671 switch (etype) { 672 case ETHERTYPE_IP: 673 offset = mac_hdr_len; 674 while (size <= offset) { 675 mp = mp->b_cont; 676 ASSERT(mp != NULL); 677 len = MBLKL(mp); 678 size += len; 679 } 680 pos = mp->b_rptr + offset + len - size; 681 682 if (ctx->lso_flag) { 683 *((uint16_t *)(uintptr_t)(pos + offsetof(ipha_t, 684 ipha_length))) = 0; 685 686 /* 687 * To utilize igb LSO, here need to fill 688 * the tcp checksum field of the packet with the 689 * following pseudo-header checksum: 690 * (ip_source_addr, ip_destination_addr, l4_proto) 691 * and also need to fill the ip header checksum 692 * with zero. Currently the tcp/ip stack has done 693 * these. 694 */ 695 } 696 697 l4_proto = *(uint8_t *)(pos + offsetof(ipha_t, ipha_protocol)); 698 break; 699 case ETHERTYPE_IPV6: 700 offset = offsetof(ip6_t, ip6_nxt) + mac_hdr_len; 701 while (size <= offset) { 702 mp = mp->b_cont; 703 ASSERT(mp != NULL); 704 len = MBLKL(mp); 705 size += len; 706 } 707 pos = mp->b_rptr + offset + len - size; 708 709 l4_proto = *(uint8_t *)pos; 710 break; 711 default: 712 /* Unrecoverable error */ 713 IGB_DEBUGLOG_0(NULL, "Ethernet type field error with " 714 "tx hcksum flag set"); 715 return (TX_CXT_E_ETHER_TYPE); 716 } 717 718 if (ctx->lso_flag) { 719 offset = mac_hdr_len + start; 720 while (size <= offset) { 721 mp = mp->b_cont; 722 ASSERT(mp != NULL); 723 len = MBLKL(mp); 724 size += len; 725 } 726 pos = mp->b_rptr + offset + len - size; 727 728 l4_hdr_len = TCP_HDR_LENGTH((tcph_t *)pos); 729 } else { 730 /* 731 * l4 header length is only required for LSO 732 */ 733 l4_hdr_len = 0; 734 } 735 736 ctx->mac_hdr_len = mac_hdr_len; 737 ctx->ip_hdr_len = start; 738 ctx->l4_proto = l4_proto; 739 ctx->l4_hdr_len = l4_hdr_len; 740 741 return (TX_CXT_SUCCESS); 742 } 743 744 /* 745 * igb_check_tx_context 746 * 747 * Check if a new context descriptor is needed 748 */ 749 static boolean_t 750 igb_check_tx_context(igb_tx_ring_t *tx_ring, tx_context_t *ctx) 751 { 752 tx_context_t *last; 753 754 if (ctx == NULL) 755 return (B_FALSE); 756 757 /* 758 * Compare the context data retrieved from the mblk and the 759 * stored context data of the last context descriptor. The data 760 * need to be checked are: 761 * hcksum_flags 762 * l4_proto 763 * mss (only check for LSO) 764 * l4_hdr_len (only check for LSO) 765 * ip_hdr_len 766 * mac_hdr_len 767 * Either one of the above data is changed, a new context descriptor 768 * will be needed. 769 */ 770 last = &tx_ring->tx_context; 771 772 if (ctx->hcksum_flags != 0) { 773 if ((ctx->hcksum_flags != last->hcksum_flags) || 774 (ctx->l4_proto != last->l4_proto) || 775 (ctx->lso_flag && ((ctx->mss != last->mss) || 776 (ctx->l4_hdr_len != last->l4_hdr_len))) || 777 (ctx->ip_hdr_len != last->ip_hdr_len) || 778 (ctx->mac_hdr_len != last->mac_hdr_len)) { 779 return (B_TRUE); 780 } 781 } 782 783 return (B_FALSE); 784 } 785 786 /* 787 * igb_fill_tx_context 788 * 789 * Fill the context descriptor with hardware checksum informations 790 */ 791 static void 792 igb_fill_tx_context(struct e1000_adv_tx_context_desc *ctx_tbd, 793 tx_context_t *ctx, uint32_t ring_index) 794 { 795 /* 796 * Fill the context descriptor with the checksum 797 * context information we've got 798 */ 799 ctx_tbd->vlan_macip_lens = ctx->ip_hdr_len; 800 ctx_tbd->vlan_macip_lens |= ctx->mac_hdr_len << 801 E1000_ADVTXD_MACLEN_SHIFT; 802 803 ctx_tbd->type_tucmd_mlhl = 804 E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 805 806 if (ctx->hcksum_flags & HCK_IPV4_HDRCKSUM) 807 ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 808 809 if (ctx->hcksum_flags & HCK_PARTIALCKSUM) { 810 switch (ctx->l4_proto) { 811 case IPPROTO_TCP: 812 ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 813 break; 814 case IPPROTO_UDP: 815 /* 816 * We don't have to explicitly set: 817 * ctx_tbd->type_tucmd_mlhl |= 818 * E1000_ADVTXD_TUCMD_L4T_UDP; 819 * Because E1000_ADVTXD_TUCMD_L4T_UDP == 0b 820 */ 821 break; 822 default: 823 /* Unrecoverable error */ 824 IGB_DEBUGLOG_0(NULL, "L4 type error with tx hcksum"); 825 break; 826 } 827 } 828 829 ctx_tbd->seqnum_seed = 0; 830 ctx_tbd->mss_l4len_idx = ring_index << 4; 831 if (ctx->lso_flag) { 832 ctx_tbd->mss_l4len_idx |= 833 (ctx->l4_hdr_len << E1000_ADVTXD_L4LEN_SHIFT) | 834 (ctx->mss << E1000_ADVTXD_MSS_SHIFT); 835 } 836 } 837 838 /* 839 * igb_tx_fill_ring 840 * 841 * Fill the tx descriptor ring with the data 842 */ 843 static int 844 igb_tx_fill_ring(igb_tx_ring_t *tx_ring, link_list_t *pending_list, 845 tx_context_t *ctx, size_t mbsize) 846 { 847 struct e1000_hw *hw = &tx_ring->igb->hw; 848 boolean_t load_context; 849 uint32_t index, tcb_index, desc_num; 850 union e1000_adv_tx_desc *tbd, *first_tbd; 851 tx_control_block_t *tcb, *first_tcb; 852 uint32_t hcksum_flags; 853 int i; 854 igb_t *igb = tx_ring->igb; 855 856 ASSERT(mutex_owned(&tx_ring->tx_lock)); 857 858 tbd = NULL; 859 first_tbd = NULL; 860 first_tcb = NULL; 861 desc_num = 0; 862 hcksum_flags = 0; 863 load_context = B_FALSE; 864 865 /* 866 * Get the index of the first tx descriptor that will be filled, 867 * and the index of the first work list item that will be attached 868 * with the first used tx control block in the pending list. 869 * Note: the two indexes are the same. 870 */ 871 index = tx_ring->tbd_tail; 872 tcb_index = tx_ring->tbd_tail; 873 874 if (ctx != NULL) { 875 hcksum_flags = ctx->hcksum_flags; 876 877 /* 878 * Check if a new context descriptor is needed for this packet 879 */ 880 load_context = igb_check_tx_context(tx_ring, ctx); 881 if (load_context) { 882 first_tcb = (tx_control_block_t *) 883 LIST_GET_HEAD(pending_list); 884 tbd = &tx_ring->tbd_ring[index]; 885 886 /* 887 * Fill the context descriptor with the 888 * hardware checksum offload informations. 889 */ 890 igb_fill_tx_context( 891 (struct e1000_adv_tx_context_desc *)tbd, 892 ctx, tx_ring->index); 893 894 index = NEXT_INDEX(index, 1, tx_ring->ring_size); 895 desc_num++; 896 897 /* 898 * Store the checksum context data if 899 * a new context descriptor is added 900 */ 901 tx_ring->tx_context = *ctx; 902 } 903 } 904 905 first_tbd = &tx_ring->tbd_ring[index]; 906 907 /* 908 * Fill tx data descriptors with the data saved in the pending list. 909 * The tx control blocks in the pending list are added to the work list 910 * at the same time. 911 * 912 * The work list is strictly 1:1 corresponding to the descriptor ring. 913 * One item of the work list corresponds to one tx descriptor. Because 914 * one tx control block can span multiple tx descriptors, the tx 915 * control block will be added to the first work list item that 916 * corresponds to the first tx descriptor generated from that tx 917 * control block. 918 */ 919 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 920 while (tcb != NULL) { 921 922 for (i = 0; i < tcb->desc_num; i++) { 923 tbd = &tx_ring->tbd_ring[index]; 924 925 tbd->read.buffer_addr = tcb->desc[i].address; 926 tbd->read.cmd_type_len = tcb->desc[i].length; 927 928 tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_RS | 929 E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_DATA | 930 E1000_ADVTXD_DCMD_IFCS; 931 932 tbd->read.olinfo_status = 0; 933 934 index = NEXT_INDEX(index, 1, tx_ring->ring_size); 935 desc_num++; 936 } 937 938 if (first_tcb != NULL) { 939 /* 940 * Count the checksum context descriptor for 941 * the first tx control block. 942 */ 943 first_tcb->desc_num++; 944 first_tcb = NULL; 945 } 946 947 /* 948 * Add the tx control block to the work list 949 */ 950 ASSERT(tx_ring->work_list[tcb_index] == NULL); 951 tx_ring->work_list[tcb_index] = tcb; 952 953 tcb_index = index; 954 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 955 } 956 957 /* 958 * The Insert Ethernet CRC (IFCS) bit and the checksum fields are only 959 * valid in the first descriptor of the packet. 960 * 82576 also requires the payload length setting even without LSO 961 */ 962 ASSERT(first_tbd != NULL); 963 first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_IFCS; 964 if (ctx != NULL && ctx->lso_flag) { 965 first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_TSE; 966 first_tbd->read.olinfo_status |= 967 (mbsize - ctx->mac_hdr_len - ctx->ip_hdr_len 968 - ctx->l4_hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT; 969 } else { 970 if (hw->mac.type >= e1000_82576) { 971 first_tbd->read.olinfo_status |= 972 (mbsize << E1000_ADVTXD_PAYLEN_SHIFT); 973 } 974 } 975 976 /* Set hardware checksum bits */ 977 if (hcksum_flags != 0) { 978 if (hcksum_flags & HCK_IPV4_HDRCKSUM) 979 first_tbd->read.olinfo_status |= 980 E1000_TXD_POPTS_IXSM << 8; 981 if (hcksum_flags & HCK_PARTIALCKSUM) 982 first_tbd->read.olinfo_status |= 983 E1000_TXD_POPTS_TXSM << 8; 984 first_tbd->read.olinfo_status |= tx_ring->index << 4; 985 } 986 987 /* 988 * The last descriptor of packet needs End Of Packet (EOP), 989 * and Report Status (RS) bits set 990 */ 991 ASSERT(tbd != NULL); 992 tbd->read.cmd_type_len |= 993 E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS; 994 995 IGB_DEBUG_STAT(tx_ring->stat_pkt_cnt); 996 997 /* 998 * Sync the DMA buffer of the tx descriptor ring 999 */ 1000 DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORDEV); 1001 1002 /* 1003 * Update the number of the free tx descriptors. 1004 * The mutual exclusion between the transmission and the recycling 1005 * (for the tx descriptor ring and the work list) is implemented 1006 * with the atomic operation on the number of the free tx descriptors. 1007 * 1008 * Note: we should always decrement the counter tbd_free before 1009 * advancing the hardware TDT pointer to avoid the race condition - 1010 * before the counter tbd_free is decremented, the transmit of the 1011 * tx descriptors has done and the counter tbd_free is increased by 1012 * the tx recycling. 1013 */ 1014 i = igb_atomic_reserve(&tx_ring->tbd_free, desc_num); 1015 ASSERT(i >= 0); 1016 1017 tx_ring->tbd_tail = index; 1018 1019 /* 1020 * Advance the hardware TDT pointer of the tx descriptor ring 1021 */ 1022 E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), index); 1023 1024 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 1025 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 1026 atomic_or_32(&igb->igb_state, IGB_ERROR); 1027 } 1028 1029 return (desc_num); 1030 } 1031 1032 /* 1033 * igb_save_desc 1034 * 1035 * Save the address/length pair to the private array 1036 * of the tx control block. The address/length pairs 1037 * will be filled into the tx descriptor ring later. 1038 */ 1039 static void 1040 igb_save_desc(tx_control_block_t *tcb, uint64_t address, size_t length) 1041 { 1042 sw_desc_t *desc; 1043 1044 desc = &tcb->desc[tcb->desc_num]; 1045 desc->address = address; 1046 desc->length = length; 1047 1048 tcb->desc_num++; 1049 } 1050 1051 /* 1052 * igb_tx_recycle_legacy 1053 * 1054 * Recycle the tx descriptors and tx control blocks. 1055 * 1056 * The work list is traversed to check if the corresponding 1057 * tx descriptors have been transmitted. If so, the resources 1058 * bound to the tx control blocks will be freed, and those 1059 * tx control blocks will be returned to the free list. 1060 */ 1061 uint32_t 1062 igb_tx_recycle_legacy(igb_tx_ring_t *tx_ring) 1063 { 1064 uint32_t index, last_index; 1065 int desc_num; 1066 boolean_t desc_done; 1067 tx_control_block_t *tcb; 1068 link_list_t pending_list; 1069 igb_t *igb = tx_ring->igb; 1070 1071 /* 1072 * The mutex_tryenter() is used to avoid unnecessary 1073 * lock contention. 1074 */ 1075 if (mutex_tryenter(&tx_ring->recycle_lock) == 0) 1076 return (0); 1077 1078 ASSERT(tx_ring->tbd_free <= tx_ring->ring_size); 1079 1080 if (tx_ring->tbd_free == tx_ring->ring_size) { 1081 tx_ring->recycle_fail = 0; 1082 tx_ring->stall_watchdog = 0; 1083 mutex_exit(&tx_ring->recycle_lock); 1084 return (0); 1085 } 1086 1087 /* 1088 * Sync the DMA buffer of the tx descriptor ring 1089 */ 1090 DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL); 1091 1092 if (igb_check_dma_handle( 1093 tx_ring->tbd_area.dma_handle) != DDI_FM_OK) { 1094 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 1095 atomic_or_32(&igb->igb_state, IGB_ERROR); 1096 return (0); 1097 } 1098 1099 LINK_LIST_INIT(&pending_list); 1100 desc_num = 0; 1101 index = tx_ring->tbd_head; /* Index of next tbd/tcb to recycle */ 1102 1103 tcb = tx_ring->work_list[index]; 1104 ASSERT(tcb != NULL); 1105 1106 desc_done = B_TRUE; 1107 while (desc_done && (tcb != NULL)) { 1108 1109 /* 1110 * Get the last tx descriptor of the tx control block. 1111 * If the last tx descriptor is done, it is done with 1112 * all the tx descriptors of the tx control block. 1113 * Then the tx control block and all the corresponding 1114 * tx descriptors can be recycled. 1115 */ 1116 last_index = NEXT_INDEX(index, tcb->desc_num - 1, 1117 tx_ring->ring_size); 1118 1119 /* 1120 * Check if the Descriptor Done bit is set 1121 */ 1122 desc_done = tx_ring->tbd_ring[last_index].wb.status & 1123 E1000_TXD_STAT_DD; 1124 if (desc_done) { 1125 /* 1126 * Strip off the tx control block from the work list, 1127 * and add it to the pending list. 1128 */ 1129 tx_ring->work_list[index] = NULL; 1130 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1131 1132 /* 1133 * Count the total number of the tx descriptors recycled 1134 */ 1135 desc_num += tcb->desc_num; 1136 1137 /* 1138 * Advance the index of the tx descriptor ring 1139 */ 1140 index = NEXT_INDEX(last_index, 1, tx_ring->ring_size); 1141 1142 tcb = tx_ring->work_list[index]; 1143 } 1144 } 1145 1146 /* 1147 * If no tx descriptors are recycled, no need to do more processing 1148 */ 1149 if (desc_num == 0) { 1150 tx_ring->recycle_fail++; 1151 mutex_exit(&tx_ring->recycle_lock); 1152 return (0); 1153 } 1154 1155 tx_ring->recycle_fail = 0; 1156 tx_ring->stall_watchdog = 0; 1157 1158 /* 1159 * Update the head index of the tx descriptor ring 1160 */ 1161 tx_ring->tbd_head = index; 1162 1163 /* 1164 * Update the number of the free tx descriptors with atomic operations 1165 */ 1166 atomic_add_32(&tx_ring->tbd_free, desc_num); 1167 1168 mutex_exit(&tx_ring->recycle_lock); 1169 1170 /* 1171 * Free the resources used by the tx control blocks 1172 * in the pending list 1173 */ 1174 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 1175 while (tcb != NULL) { 1176 /* 1177 * Release the resources occupied by the tx control block 1178 */ 1179 igb_free_tcb(tcb); 1180 1181 tcb = (tx_control_block_t *) 1182 LIST_GET_NEXT(&pending_list, &tcb->link); 1183 } 1184 1185 /* 1186 * Add the tx control blocks in the pending list to the free list. 1187 */ 1188 igb_put_free_list(tx_ring, &pending_list); 1189 1190 return (desc_num); 1191 } 1192 1193 /* 1194 * igb_tx_recycle_head_wb 1195 * 1196 * Check the head write-back, and recycle all the transmitted 1197 * tx descriptors and tx control blocks. 1198 */ 1199 uint32_t 1200 igb_tx_recycle_head_wb(igb_tx_ring_t *tx_ring) 1201 { 1202 uint32_t index; 1203 uint32_t head_wb; 1204 int desc_num; 1205 tx_control_block_t *tcb; 1206 link_list_t pending_list; 1207 igb_t *igb = tx_ring->igb; 1208 1209 /* 1210 * The mutex_tryenter() is used to avoid unnecessary 1211 * lock contention. 1212 */ 1213 if (mutex_tryenter(&tx_ring->recycle_lock) == 0) 1214 return (0); 1215 1216 ASSERT(tx_ring->tbd_free <= tx_ring->ring_size); 1217 1218 if (tx_ring->tbd_free == tx_ring->ring_size) { 1219 tx_ring->recycle_fail = 0; 1220 tx_ring->stall_watchdog = 0; 1221 mutex_exit(&tx_ring->recycle_lock); 1222 return (0); 1223 } 1224 1225 /* 1226 * Sync the DMA buffer of the tx descriptor ring 1227 * 1228 * Note: For head write-back mode, the tx descriptors will not 1229 * be written back, but the head write-back value is stored at 1230 * the last extra tbd at the end of the DMA area, we still need 1231 * to sync the head write-back value for kernel. 1232 * 1233 * DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL); 1234 */ 1235 (void) ddi_dma_sync(tx_ring->tbd_area.dma_handle, 1236 sizeof (union e1000_adv_tx_desc) * tx_ring->ring_size, 1237 sizeof (uint32_t), 1238 DDI_DMA_SYNC_FORKERNEL); 1239 1240 if (igb_check_dma_handle( 1241 tx_ring->tbd_area.dma_handle) != DDI_FM_OK) { 1242 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 1243 atomic_or_32(&igb->igb_state, IGB_ERROR); 1244 return (0); 1245 } 1246 1247 LINK_LIST_INIT(&pending_list); 1248 desc_num = 0; 1249 index = tx_ring->tbd_head; /* Next index to clean */ 1250 1251 /* 1252 * Get the value of head write-back 1253 */ 1254 head_wb = *tx_ring->tbd_head_wb; 1255 while (index != head_wb) { 1256 tcb = tx_ring->work_list[index]; 1257 ASSERT(tcb != NULL); 1258 1259 if (OFFSET(index, head_wb, tx_ring->ring_size) < 1260 tcb->desc_num) { 1261 /* 1262 * The current tx control block is not 1263 * completely transmitted, stop recycling 1264 */ 1265 break; 1266 } 1267 1268 /* 1269 * Strip off the tx control block from the work list, 1270 * and add it to the pending list. 1271 */ 1272 tx_ring->work_list[index] = NULL; 1273 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1274 1275 /* 1276 * Advance the index of the tx descriptor ring 1277 */ 1278 index = NEXT_INDEX(index, tcb->desc_num, tx_ring->ring_size); 1279 1280 /* 1281 * Count the total number of the tx descriptors recycled 1282 */ 1283 desc_num += tcb->desc_num; 1284 } 1285 1286 /* 1287 * If no tx descriptors are recycled, no need to do more processing 1288 */ 1289 if (desc_num == 0) { 1290 tx_ring->recycle_fail++; 1291 mutex_exit(&tx_ring->recycle_lock); 1292 return (0); 1293 } 1294 1295 tx_ring->recycle_fail = 0; 1296 tx_ring->stall_watchdog = 0; 1297 1298 /* 1299 * Update the head index of the tx descriptor ring 1300 */ 1301 tx_ring->tbd_head = index; 1302 1303 /* 1304 * Update the number of the free tx descriptors with atomic operations 1305 */ 1306 atomic_add_32(&tx_ring->tbd_free, desc_num); 1307 1308 mutex_exit(&tx_ring->recycle_lock); 1309 1310 /* 1311 * Free the resources used by the tx control blocks 1312 * in the pending list 1313 */ 1314 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 1315 while (tcb) { 1316 /* 1317 * Release the resources occupied by the tx control block 1318 */ 1319 igb_free_tcb(tcb); 1320 1321 tcb = (tx_control_block_t *) 1322 LIST_GET_NEXT(&pending_list, &tcb->link); 1323 } 1324 1325 /* 1326 * Add the tx control blocks in the pending list to the free list. 1327 */ 1328 igb_put_free_list(tx_ring, &pending_list); 1329 1330 return (desc_num); 1331 } 1332 1333 /* 1334 * igb_free_tcb - free up the tx control block 1335 * 1336 * Free the resources of the tx control block, including 1337 * unbind the previously bound DMA handle, and reset other 1338 * control fields. 1339 */ 1340 void 1341 igb_free_tcb(tx_control_block_t *tcb) 1342 { 1343 switch (tcb->tx_type) { 1344 case USE_COPY: 1345 /* 1346 * Reset the buffer length that is used for copy 1347 */ 1348 tcb->tx_buf.len = 0; 1349 break; 1350 case USE_DMA: 1351 /* 1352 * Release the DMA resource that is used for 1353 * DMA binding. 1354 */ 1355 (void) ddi_dma_unbind_handle(tcb->tx_dma_handle); 1356 break; 1357 default: 1358 break; 1359 } 1360 1361 /* 1362 * Free the mblk 1363 */ 1364 if (tcb->mp != NULL) { 1365 freemsg(tcb->mp); 1366 tcb->mp = NULL; 1367 } 1368 1369 tcb->tx_type = USE_NONE; 1370 tcb->frag_num = 0; 1371 tcb->desc_num = 0; 1372 } 1373 1374 /* 1375 * igb_get_free_list - Get a free tx control block from the free list 1376 * 1377 * The atomic operation on the number of the available tx control block 1378 * in the free list is used to keep this routine mutual exclusive with 1379 * the routine igb_put_check_list. 1380 */ 1381 static tx_control_block_t * 1382 igb_get_free_list(igb_tx_ring_t *tx_ring) 1383 { 1384 tx_control_block_t *tcb; 1385 1386 /* 1387 * Check and update the number of the free tx control block 1388 * in the free list. 1389 */ 1390 if (igb_atomic_reserve(&tx_ring->tcb_free, 1) < 0) 1391 return (NULL); 1392 1393 mutex_enter(&tx_ring->tcb_head_lock); 1394 1395 tcb = tx_ring->free_list[tx_ring->tcb_head]; 1396 ASSERT(tcb != NULL); 1397 tx_ring->free_list[tx_ring->tcb_head] = NULL; 1398 tx_ring->tcb_head = NEXT_INDEX(tx_ring->tcb_head, 1, 1399 tx_ring->free_list_size); 1400 1401 mutex_exit(&tx_ring->tcb_head_lock); 1402 1403 return (tcb); 1404 } 1405 1406 /* 1407 * igb_put_free_list 1408 * 1409 * Put a list of used tx control blocks back to the free list 1410 * 1411 * A mutex is used here to ensure the serialization. The mutual exclusion 1412 * between igb_get_free_list and igb_put_free_list is implemented with 1413 * the atomic operation on the counter tcb_free. 1414 */ 1415 void 1416 igb_put_free_list(igb_tx_ring_t *tx_ring, link_list_t *pending_list) 1417 { 1418 uint32_t index; 1419 int tcb_num; 1420 tx_control_block_t *tcb; 1421 1422 mutex_enter(&tx_ring->tcb_tail_lock); 1423 1424 index = tx_ring->tcb_tail; 1425 1426 tcb_num = 0; 1427 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 1428 while (tcb != NULL) { 1429 ASSERT(tx_ring->free_list[index] == NULL); 1430 tx_ring->free_list[index] = tcb; 1431 1432 tcb_num++; 1433 1434 index = NEXT_INDEX(index, 1, tx_ring->free_list_size); 1435 1436 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 1437 } 1438 1439 tx_ring->tcb_tail = index; 1440 1441 /* 1442 * Update the number of the free tx control block 1443 * in the free list. This operation must be placed 1444 * under the protection of the lock. 1445 */ 1446 atomic_add_32(&tx_ring->tcb_free, tcb_num); 1447 1448 mutex_exit(&tx_ring->tcb_tail_lock); 1449 } 1450