1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved. 24 */ 25 26 /* 27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28 */ 29 30 #include "igb_sw.h" 31 32 static boolean_t igb_tx(igb_tx_ring_t *, mblk_t *); 33 static int igb_tx_copy(igb_tx_ring_t *, tx_control_block_t *, mblk_t *, 34 uint32_t, boolean_t); 35 static int igb_tx_bind(igb_tx_ring_t *, tx_control_block_t *, mblk_t *, 36 uint32_t); 37 static int igb_tx_fill_ring(igb_tx_ring_t *, link_list_t *, tx_context_t *, 38 size_t); 39 static void igb_save_desc(tx_control_block_t *, uint64_t, size_t); 40 static tx_control_block_t *igb_get_free_list(igb_tx_ring_t *); 41 static int igb_get_tx_context(mblk_t *, tx_context_t *); 42 static boolean_t igb_check_tx_context(igb_tx_ring_t *, tx_context_t *); 43 static void igb_fill_tx_context(struct e1000_adv_tx_context_desc *, 44 tx_context_t *, uint32_t); 45 46 #ifndef IGB_DEBUG 47 #pragma inline(igb_save_desc) 48 #pragma inline(igb_get_tx_context) 49 #pragma inline(igb_check_tx_context) 50 #pragma inline(igb_fill_tx_context) 51 #endif 52 53 mblk_t * 54 igb_tx_ring_send(void *arg, mblk_t *mp) 55 { 56 igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg; 57 igb_t *igb; 58 59 ASSERT(tx_ring != NULL); 60 61 igb = tx_ring->igb; 62 63 if ((igb->igb_state & IGB_SUSPENDED) || 64 (igb->igb_state & IGB_ERROR) || 65 !(igb->igb_state & IGB_STARTED) || 66 igb->link_state != LINK_STATE_UP) { 67 freemsg(mp); 68 return (NULL); 69 } 70 71 return ((igb_tx(tx_ring, mp)) ? NULL : mp); 72 } 73 74 /* 75 * igb_tx - Main transmit processing 76 * 77 * Called from igb_m_tx with an mblk ready to transmit. this 78 * routine sets up the transmit descriptors and sends data to 79 * the wire. 80 * 81 * One mblk can consist of several fragments, each fragment 82 * will be processed with different methods based on the size. 83 * For the fragments with size less than the bcopy threshold, 84 * they will be processed by using bcopy; otherwise, they will 85 * be processed by using DMA binding. 86 * 87 * To process the mblk, a tx control block is got from the 88 * free list. One tx control block contains one tx buffer, which 89 * is used to copy mblk fragments' data; and one tx DMA handle, 90 * which is used to bind a mblk fragment with DMA resource. 91 * 92 * Several small mblk fragments can be copied into one tx control 93 * block's buffer, and then the buffer will be transmitted with 94 * one tx descriptor. 95 * 96 * A large fragment only binds with one tx control block's DMA 97 * handle, and it can span several tx descriptors for transmitting. 98 * 99 * So to transmit a packet (mblk), several tx control blocks can 100 * be used. After the processing, those tx control blocks will 101 * be put to the work list. 102 */ 103 static boolean_t 104 igb_tx(igb_tx_ring_t *tx_ring, mblk_t *mp) 105 { 106 igb_t *igb = tx_ring->igb; 107 tx_type_t current_flag, next_flag; 108 uint32_t current_len, next_len; 109 uint32_t desc_total; 110 size_t mbsize; 111 int desc_num; 112 boolean_t copy_done, eop; 113 mblk_t *current_mp, *next_mp, *nmp; 114 tx_control_block_t *tcb; 115 tx_context_t tx_context, *ctx; 116 link_list_t pending_list; 117 mblk_t *hdr_new_mp = NULL; 118 mblk_t *hdr_previous_mp = NULL; 119 mblk_t *hdr_current_mp = NULL; 120 uint32_t hdr_frag_len; 121 uint32_t hdr_len, len; 122 uint32_t copy_thresh; 123 124 copy_thresh = igb->tx_copy_thresh; 125 126 /* Get the mblk size */ 127 mbsize = 0; 128 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) { 129 mbsize += MBLKL(nmp); 130 } 131 132 if (igb->tx_hcksum_enable) { 133 ctx = &tx_context; 134 /* 135 * Retrieve offloading context information from the mblk 136 * that will be used to decide whether/how to fill the 137 * context descriptor. 138 */ 139 if (igb_get_tx_context(mp, ctx) != TX_CXT_SUCCESS) { 140 freemsg(mp); 141 return (B_TRUE); 142 } 143 144 if ((ctx->lso_flag && 145 (mbsize > (ctx->mac_hdr_len + IGB_LSO_MAXLEN))) || 146 (!ctx->lso_flag && 147 (mbsize > (igb->max_frame_size - ETHERFCSL)))) { 148 freemsg(mp); 149 igb_log(igb, IGB_LOG_INFO, "igb_tx: packet oversize"); 150 return (B_TRUE); 151 } 152 } else { 153 ctx = NULL; 154 if (mbsize > (igb->max_frame_size - ETHERFCSL)) { 155 freemsg(mp); 156 igb_log(igb, IGB_LOG_INFO, "igb_tx: packet oversize"); 157 return (B_TRUE); 158 } 159 } 160 161 /* 162 * Check and recycle tx descriptors. 163 * The recycle threshold here should be selected carefully 164 */ 165 if (tx_ring->tbd_free < igb->tx_recycle_thresh) 166 tx_ring->tx_recycle(tx_ring); 167 168 /* 169 * After the recycling, if the tbd_free is less than the 170 * tx_overload_threshold, assert overload, return B_FALSE; 171 * and we need to re-schedule the tx again. 172 */ 173 if (tx_ring->tbd_free < igb->tx_overload_thresh) { 174 tx_ring->reschedule = B_TRUE; 175 IGB_DEBUG_STAT(tx_ring->stat_overload); 176 return (B_FALSE); 177 } 178 179 /* 180 * The software should guarantee LSO packet header(MAC+IP+TCP) 181 * to be within one descriptor - this is required by h/w. 182 * Here will reallocate and refill the header if 183 * the headers(MAC+IP+TCP) is physical memory non-contiguous. 184 */ 185 if (ctx && ctx->lso_flag) { 186 hdr_len = ctx->mac_hdr_len + ctx->ip_hdr_len + ctx->l4_hdr_len; 187 len = MBLKL(mp); 188 hdr_current_mp = mp; 189 while (len < hdr_len) { 190 hdr_previous_mp = hdr_current_mp; 191 hdr_current_mp = hdr_current_mp->b_cont; 192 len += MBLKL(hdr_current_mp); 193 } 194 /* 195 * If the header and the payload are in different mblks, 196 * we simply force the header to be copied into pre-allocated 197 * page-aligned buffer. 198 */ 199 if (len == hdr_len) 200 goto adjust_threshold; 201 202 hdr_frag_len = hdr_len - (len - MBLKL(hdr_current_mp)); 203 /* 204 * There are two cases we will reallocate 205 * a mblk for the last header fragment. 206 * 1. the header is in multiple mblks and 207 * the last fragment shares the same mblk 208 * with the payload 209 * 2. the header is in a single mblk shared 210 * with the payload but the header crosses 211 * a page. 212 */ 213 if ((hdr_current_mp != mp) || 214 (P2NPHASE((uintptr_t)hdr_current_mp->b_rptr, igb->page_size) 215 < hdr_len)) { 216 /* 217 * reallocate the mblk for the last header fragment, 218 * expect it to be copied into pre-allocated 219 * page-aligned buffer 220 */ 221 hdr_new_mp = allocb(hdr_frag_len, 0); 222 if (!hdr_new_mp) { 223 return (B_FALSE); 224 } 225 226 /* link the new header fragment with the other parts */ 227 bcopy(hdr_current_mp->b_rptr, 228 hdr_new_mp->b_rptr, hdr_frag_len); 229 hdr_new_mp->b_wptr = hdr_new_mp->b_rptr + hdr_frag_len; 230 hdr_new_mp->b_cont = hdr_current_mp; 231 if (hdr_previous_mp) 232 hdr_previous_mp->b_cont = hdr_new_mp; 233 else 234 mp = hdr_new_mp; 235 hdr_current_mp->b_rptr += hdr_frag_len; 236 } 237 adjust_threshold: 238 /* 239 * adjust the bcopy threshhold to guarantee 240 * the header to use bcopy way 241 */ 242 if (copy_thresh < hdr_len) 243 copy_thresh = hdr_len; 244 } 245 246 /* 247 * The pending_list is a linked list that is used to save 248 * the tx control blocks that have packet data processed 249 * but have not put the data to the tx descriptor ring. 250 * It is used to reduce the lock contention of the tx_lock. 251 */ 252 LINK_LIST_INIT(&pending_list); 253 desc_num = 0; 254 desc_total = 0; 255 256 current_mp = mp; 257 current_len = MBLKL(current_mp); 258 /* 259 * Decide which method to use for the first fragment 260 */ 261 current_flag = (current_len <= copy_thresh) ? 262 USE_COPY : USE_DMA; 263 /* 264 * If the mblk includes several contiguous small fragments, 265 * they may be copied into one buffer. This flag is used to 266 * indicate whether there are pending fragments that need to 267 * be copied to the current tx buffer. 268 * 269 * If this flag is B_TRUE, it indicates that a new tx control 270 * block is needed to process the next fragment using either 271 * copy or DMA binding. 272 * 273 * Otherwise, it indicates that the next fragment will be 274 * copied to the current tx buffer that is maintained by the 275 * current tx control block. No new tx control block is needed. 276 */ 277 copy_done = B_TRUE; 278 while (current_mp) { 279 next_mp = current_mp->b_cont; 280 eop = (next_mp == NULL); /* Last fragment of the packet? */ 281 next_len = eop ? 0: MBLKL(next_mp); 282 283 /* 284 * When the current fragment is an empty fragment, if 285 * the next fragment will still be copied to the current 286 * tx buffer, we cannot skip this fragment here. Because 287 * the copy processing is pending for completion. We have 288 * to process this empty fragment in the tx_copy routine. 289 * 290 * If the copy processing is completed or a DMA binding 291 * processing is just completed, we can just skip this 292 * empty fragment. 293 */ 294 if ((current_len == 0) && (copy_done)) { 295 current_mp = next_mp; 296 current_len = next_len; 297 current_flag = (current_len <= copy_thresh) ? 298 USE_COPY : USE_DMA; 299 continue; 300 } 301 302 if (copy_done) { 303 /* 304 * Get a new tx control block from the free list 305 */ 306 tcb = igb_get_free_list(tx_ring); 307 308 if (tcb == NULL) { 309 IGB_DEBUG_STAT(tx_ring->stat_fail_no_tcb); 310 goto tx_failure; 311 } 312 313 /* 314 * Push the tx control block to the pending list 315 * to avoid using lock too early 316 */ 317 LIST_PUSH_TAIL(&pending_list, &tcb->link); 318 } 319 320 if (current_flag == USE_COPY) { 321 /* 322 * Check whether to use bcopy or DMA binding to process 323 * the next fragment, and if using bcopy, whether we 324 * need to continue copying the next fragment into the 325 * current tx buffer. 326 */ 327 ASSERT((tcb->tx_buf.len + current_len) <= 328 tcb->tx_buf.size); 329 330 if (eop) { 331 /* 332 * This is the last fragment of the packet, so 333 * the copy processing will be completed with 334 * this fragment. 335 */ 336 next_flag = USE_NONE; 337 copy_done = B_TRUE; 338 } else if ((tcb->tx_buf.len + current_len + next_len) > 339 tcb->tx_buf.size) { 340 /* 341 * If the next fragment is too large to be 342 * copied to the current tx buffer, we need 343 * to complete the current copy processing. 344 */ 345 next_flag = (next_len > copy_thresh) ? 346 USE_DMA: USE_COPY; 347 copy_done = B_TRUE; 348 } else if (next_len > copy_thresh) { 349 /* 350 * The next fragment needs to be processed with 351 * DMA binding. So the copy prcessing will be 352 * completed with the current fragment. 353 */ 354 next_flag = USE_DMA; 355 copy_done = B_TRUE; 356 } else { 357 /* 358 * Continue to copy the next fragment to the 359 * current tx buffer. 360 */ 361 next_flag = USE_COPY; 362 copy_done = B_FALSE; 363 } 364 365 desc_num = igb_tx_copy(tx_ring, tcb, current_mp, 366 current_len, copy_done); 367 } else { 368 /* 369 * Check whether to use bcopy or DMA binding to process 370 * the next fragment. 371 */ 372 next_flag = (next_len > copy_thresh) ? 373 USE_DMA: USE_COPY; 374 ASSERT(copy_done == B_TRUE); 375 376 desc_num = igb_tx_bind(tx_ring, tcb, current_mp, 377 current_len); 378 } 379 380 if (desc_num > 0) 381 desc_total += desc_num; 382 else if (desc_num < 0) 383 goto tx_failure; 384 385 current_mp = next_mp; 386 current_len = next_len; 387 current_flag = next_flag; 388 } 389 390 /* 391 * Attach the mblk to the last tx control block 392 */ 393 ASSERT(tcb); 394 ASSERT(tcb->mp == NULL); 395 tcb->mp = mp; 396 397 /* 398 * Before fill the tx descriptor ring with the data, we need to 399 * ensure there are adequate free descriptors for transmit 400 * (including one context descriptor). 401 * Do not use up all the tx descriptors. 402 * Otherwise tx recycle will fail and cause false hang. 403 */ 404 if (tx_ring->tbd_free <= (desc_total + 1)) { 405 tx_ring->tx_recycle(tx_ring); 406 } 407 408 mutex_enter(&tx_ring->tx_lock); 409 410 /* 411 * If the number of free tx descriptors is not enough for transmit 412 * then return failure. 413 * 414 * Note: we must put this check under the mutex protection to 415 * ensure the correctness when multiple threads access it in 416 * parallel. 417 */ 418 if (tx_ring->tbd_free <= (desc_total + 1)) { 419 IGB_DEBUG_STAT(tx_ring->stat_fail_no_tbd); 420 mutex_exit(&tx_ring->tx_lock); 421 goto tx_failure; 422 } 423 424 desc_num = igb_tx_fill_ring(tx_ring, &pending_list, ctx, mbsize); 425 426 ASSERT((desc_num == desc_total) || (desc_num == (desc_total + 1))); 427 428 /* Update per-ring tx statistics */ 429 tx_ring->tx_pkts++; 430 tx_ring->tx_bytes += mbsize; 431 432 mutex_exit(&tx_ring->tx_lock); 433 434 return (B_TRUE); 435 436 tx_failure: 437 /* 438 * If new mblk has been allocted for the last header 439 * fragment of a LSO packet, we should restore the 440 * modified mp. 441 */ 442 if (hdr_new_mp) { 443 hdr_new_mp->b_cont = NULL; 444 freeb(hdr_new_mp); 445 hdr_current_mp->b_rptr -= hdr_frag_len; 446 if (hdr_previous_mp) 447 hdr_previous_mp->b_cont = hdr_current_mp; 448 else 449 mp = hdr_current_mp; 450 } 451 452 /* 453 * Discard the mblk and free the used resources 454 */ 455 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 456 while (tcb) { 457 tcb->mp = NULL; 458 459 igb_free_tcb(tcb); 460 461 tcb = (tx_control_block_t *) 462 LIST_GET_NEXT(&pending_list, &tcb->link); 463 } 464 465 /* 466 * Return the tx control blocks in the pending list to the free list. 467 */ 468 igb_put_free_list(tx_ring, &pending_list); 469 470 /* Transmit failed, do not drop the mblk, rechedule the transmit */ 471 tx_ring->reschedule = B_TRUE; 472 473 return (B_FALSE); 474 } 475 476 /* 477 * igb_tx_copy 478 * 479 * Copy the mblk fragment to the pre-allocated tx buffer 480 */ 481 static int 482 igb_tx_copy(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp, 483 uint32_t len, boolean_t copy_done) 484 { 485 dma_buffer_t *tx_buf; 486 uint32_t desc_num; 487 _NOTE(ARGUNUSED(tx_ring)); 488 489 tx_buf = &tcb->tx_buf; 490 491 /* 492 * Copy the packet data of the mblk fragment into the 493 * pre-allocated tx buffer, which is maintained by the 494 * tx control block. 495 * 496 * Several mblk fragments can be copied into one tx buffer. 497 * The destination address of the current copied fragment in 498 * the tx buffer is next to the end of the previous copied 499 * fragment. 500 */ 501 if (len > 0) { 502 bcopy(mp->b_rptr, tx_buf->address + tx_buf->len, len); 503 504 tx_buf->len += len; 505 tcb->frag_num++; 506 } 507 508 desc_num = 0; 509 510 /* 511 * If it is the last fragment copied to the current tx buffer, 512 * in other words, if there's no remaining fragment or the remaining 513 * fragment requires a new tx control block to process, we need to 514 * complete the current copy processing by syncing up the current 515 * DMA buffer and saving the descriptor data. 516 */ 517 if (copy_done) { 518 /* 519 * Sync the DMA buffer of the packet data 520 */ 521 DMA_SYNC(tx_buf, DDI_DMA_SYNC_FORDEV); 522 523 tcb->tx_type = USE_COPY; 524 525 /* 526 * Save the address and length to the private data structure 527 * of the tx control block, which will be used to fill the 528 * tx descriptor ring after all the fragments are processed. 529 */ 530 igb_save_desc(tcb, tx_buf->dma_address, tx_buf->len); 531 desc_num++; 532 } 533 534 return (desc_num); 535 } 536 537 /* 538 * igb_tx_bind 539 * 540 * Bind the mblk fragment with DMA 541 */ 542 static int 543 igb_tx_bind(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp, 544 uint32_t len) 545 { 546 int status, i; 547 ddi_dma_cookie_t dma_cookie; 548 uint_t ncookies; 549 int desc_num; 550 551 /* 552 * Use DMA binding to process the mblk fragment 553 */ 554 status = ddi_dma_addr_bind_handle(tcb->tx_dma_handle, NULL, 555 (caddr_t)mp->b_rptr, len, 556 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, 557 0, &dma_cookie, &ncookies); 558 559 if (status != DDI_DMA_MAPPED) { 560 IGB_DEBUG_STAT(tx_ring->stat_fail_dma_bind); 561 return (-1); 562 } 563 564 tcb->frag_num++; 565 tcb->tx_type = USE_DMA; 566 /* 567 * Each fragment can span several cookies. One cookie will have 568 * one tx descriptor to transmit. 569 */ 570 desc_num = 0; 571 for (i = ncookies; i > 0; i--) { 572 /* 573 * Save the address and length to the private data structure 574 * of the tx control block, which will be used to fill the 575 * tx descriptor ring after all the fragments are processed. 576 */ 577 igb_save_desc(tcb, 578 dma_cookie.dmac_laddress, 579 dma_cookie.dmac_size); 580 581 desc_num++; 582 583 if (i > 1) 584 ddi_dma_nextcookie(tcb->tx_dma_handle, &dma_cookie); 585 } 586 587 return (desc_num); 588 } 589 590 /* 591 * igb_get_tx_context 592 * 593 * Get the tx context information from the mblk 594 */ 595 static int 596 igb_get_tx_context(mblk_t *mp, tx_context_t *ctx) 597 { 598 uint32_t start; 599 uint32_t flags; 600 uint32_t lso_flag; 601 uint32_t mss; 602 uint32_t len; 603 uint32_t size; 604 uint32_t offset; 605 unsigned char *pos; 606 ushort_t etype; 607 uint32_t mac_hdr_len; 608 uint32_t l4_proto; 609 uint32_t l4_hdr_len; 610 611 ASSERT(mp != NULL); 612 613 mac_hcksum_get(mp, &start, NULL, NULL, NULL, &flags); 614 bzero(ctx, sizeof (tx_context_t)); 615 616 ctx->hcksum_flags = flags; 617 618 if (flags == 0) 619 return (TX_CXT_SUCCESS); 620 621 mac_lso_get(mp, &mss, &lso_flag); 622 ctx->mss = mss; 623 ctx->lso_flag = (lso_flag == HW_LSO); 624 625 /* 626 * LSO relies on tx h/w checksum, so here the packet will be 627 * dropped if the h/w checksum flags are not set. 628 */ 629 if (ctx->lso_flag) { 630 if (!((ctx->hcksum_flags & HCK_PARTIALCKSUM) && 631 (ctx->hcksum_flags & HCK_IPV4_HDRCKSUM))) { 632 igb_log(NULL, IGB_LOG_INFO, "igb_tx: h/w " 633 "checksum flags are not set for LSO"); 634 return (TX_CXT_E_LSO_CSUM); 635 } 636 } 637 638 etype = 0; 639 mac_hdr_len = 0; 640 l4_proto = 0; 641 642 /* 643 * Firstly get the position of the ether_type/ether_tpid. 644 * Here we don't assume the ether (VLAN) header is fully included 645 * in one mblk fragment, so we go thourgh the fragments to parse 646 * the ether type. 647 */ 648 size = len = MBLKL(mp); 649 offset = offsetof(struct ether_header, ether_type); 650 while (size <= offset) { 651 mp = mp->b_cont; 652 ASSERT(mp != NULL); 653 len = MBLKL(mp); 654 size += len; 655 } 656 pos = mp->b_rptr + offset + len - size; 657 658 etype = ntohs(*(ushort_t *)(uintptr_t)pos); 659 if (etype == ETHERTYPE_VLAN) { 660 /* 661 * Get the position of the ether_type in VLAN header 662 */ 663 offset = offsetof(struct ether_vlan_header, ether_type); 664 while (size <= offset) { 665 mp = mp->b_cont; 666 ASSERT(mp != NULL); 667 len = MBLKL(mp); 668 size += len; 669 } 670 pos = mp->b_rptr + offset + len - size; 671 672 etype = ntohs(*(ushort_t *)(uintptr_t)pos); 673 mac_hdr_len = sizeof (struct ether_vlan_header); 674 } else { 675 mac_hdr_len = sizeof (struct ether_header); 676 } 677 678 /* 679 * Here we assume the IP(V6) header is fully included in one 680 * mblk fragment. 681 */ 682 switch (etype) { 683 case ETHERTYPE_IP: 684 offset = mac_hdr_len; 685 while (size <= offset) { 686 mp = mp->b_cont; 687 ASSERT(mp != NULL); 688 len = MBLKL(mp); 689 size += len; 690 } 691 pos = mp->b_rptr + offset + len - size; 692 693 if (ctx->lso_flag) { 694 *((uint16_t *)(uintptr_t)(pos + offsetof(ipha_t, 695 ipha_length))) = 0; 696 697 /* 698 * To utilize igb LSO, here need to fill 699 * the tcp checksum field of the packet with the 700 * following pseudo-header checksum: 701 * (ip_source_addr, ip_destination_addr, l4_proto) 702 * and also need to fill the ip header checksum 703 * with zero. Currently the tcp/ip stack has done 704 * these. 705 */ 706 } 707 708 l4_proto = *(uint8_t *)(pos + offsetof(ipha_t, ipha_protocol)); 709 break; 710 case ETHERTYPE_IPV6: 711 offset = offsetof(ip6_t, ip6_nxt) + mac_hdr_len; 712 while (size <= offset) { 713 mp = mp->b_cont; 714 ASSERT(mp != NULL); 715 len = MBLKL(mp); 716 size += len; 717 } 718 pos = mp->b_rptr + offset + len - size; 719 720 l4_proto = *(uint8_t *)pos; 721 break; 722 default: 723 /* Unrecoverable error */ 724 igb_log(NULL, IGB_LOG_INFO, "Ethernet type field error with " 725 "tx hcksum flag set"); 726 return (TX_CXT_E_ETHER_TYPE); 727 } 728 729 if (ctx->lso_flag) { 730 offset = mac_hdr_len + start; 731 while (size <= offset) { 732 mp = mp->b_cont; 733 ASSERT(mp != NULL); 734 len = MBLKL(mp); 735 size += len; 736 } 737 pos = mp->b_rptr + offset + len - size; 738 739 l4_hdr_len = TCP_HDR_LENGTH((tcph_t *)pos); 740 } else { 741 /* 742 * l4 header length is only required for LSO 743 */ 744 l4_hdr_len = 0; 745 } 746 747 ctx->mac_hdr_len = mac_hdr_len; 748 ctx->ip_hdr_len = start; 749 ctx->l4_proto = l4_proto; 750 ctx->l4_hdr_len = l4_hdr_len; 751 752 return (TX_CXT_SUCCESS); 753 } 754 755 /* 756 * igb_check_tx_context 757 * 758 * Check if a new context descriptor is needed 759 */ 760 static boolean_t 761 igb_check_tx_context(igb_tx_ring_t *tx_ring, tx_context_t *ctx) 762 { 763 tx_context_t *last; 764 765 if (ctx == NULL) 766 return (B_FALSE); 767 768 /* 769 * Compare the context data retrieved from the mblk and the 770 * stored context data of the last context descriptor. The data 771 * need to be checked are: 772 * hcksum_flags 773 * l4_proto 774 * mss (only check for LSO) 775 * l4_hdr_len (only check for LSO) 776 * ip_hdr_len 777 * mac_hdr_len 778 * Either one of the above data is changed, a new context descriptor 779 * will be needed. 780 */ 781 last = &tx_ring->tx_context; 782 783 if (ctx->hcksum_flags != 0) { 784 if ((ctx->hcksum_flags != last->hcksum_flags) || 785 (ctx->l4_proto != last->l4_proto) || 786 (ctx->lso_flag && ((ctx->mss != last->mss) || 787 (ctx->l4_hdr_len != last->l4_hdr_len))) || 788 (ctx->ip_hdr_len != last->ip_hdr_len) || 789 (ctx->mac_hdr_len != last->mac_hdr_len)) { 790 return (B_TRUE); 791 } 792 } 793 794 return (B_FALSE); 795 } 796 797 /* 798 * igb_fill_tx_context 799 * 800 * Fill the context descriptor with hardware checksum informations 801 */ 802 static void 803 igb_fill_tx_context(struct e1000_adv_tx_context_desc *ctx_tbd, 804 tx_context_t *ctx, uint32_t ring_index) 805 { 806 /* 807 * Fill the context descriptor with the checksum 808 * context information we've got 809 */ 810 ctx_tbd->vlan_macip_lens = ctx->ip_hdr_len; 811 ctx_tbd->vlan_macip_lens |= ctx->mac_hdr_len << 812 E1000_ADVTXD_MACLEN_SHIFT; 813 814 ctx_tbd->type_tucmd_mlhl = 815 E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 816 817 if (ctx->hcksum_flags & HCK_IPV4_HDRCKSUM) 818 ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 819 820 if (ctx->hcksum_flags & HCK_PARTIALCKSUM) { 821 switch (ctx->l4_proto) { 822 case IPPROTO_TCP: 823 ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 824 break; 825 case IPPROTO_UDP: 826 /* 827 * We don't have to explicitly set: 828 * ctx_tbd->type_tucmd_mlhl |= 829 * E1000_ADVTXD_TUCMD_L4T_UDP; 830 * Because E1000_ADVTXD_TUCMD_L4T_UDP == 0b 831 */ 832 break; 833 default: 834 /* Unrecoverable error */ 835 igb_log(NULL, IGB_LOG_INFO, 836 "L4 type error with tx hcksum"); 837 break; 838 } 839 } 840 841 ctx_tbd->seqnum_seed = 0; 842 ctx_tbd->mss_l4len_idx = ring_index << 4; 843 if (ctx->lso_flag) { 844 ctx_tbd->mss_l4len_idx |= 845 (ctx->l4_hdr_len << E1000_ADVTXD_L4LEN_SHIFT) | 846 (ctx->mss << E1000_ADVTXD_MSS_SHIFT); 847 } 848 } 849 850 /* 851 * igb_tx_fill_ring 852 * 853 * Fill the tx descriptor ring with the data 854 */ 855 static int 856 igb_tx_fill_ring(igb_tx_ring_t *tx_ring, link_list_t *pending_list, 857 tx_context_t *ctx, size_t mbsize) 858 { 859 struct e1000_hw *hw = &tx_ring->igb->hw; 860 boolean_t load_context; 861 uint32_t index, tcb_index, desc_num; 862 union e1000_adv_tx_desc *tbd, *first_tbd; 863 tx_control_block_t *tcb, *first_tcb; 864 uint32_t hcksum_flags; 865 int i; 866 igb_t *igb = tx_ring->igb; 867 868 ASSERT(mutex_owned(&tx_ring->tx_lock)); 869 870 tbd = NULL; 871 first_tbd = NULL; 872 first_tcb = NULL; 873 desc_num = 0; 874 hcksum_flags = 0; 875 load_context = B_FALSE; 876 877 /* 878 * Get the index of the first tx descriptor that will be filled, 879 * and the index of the first work list item that will be attached 880 * with the first used tx control block in the pending list. 881 * Note: the two indexes are the same. 882 */ 883 index = tx_ring->tbd_tail; 884 tcb_index = tx_ring->tbd_tail; 885 886 if (ctx != NULL) { 887 hcksum_flags = ctx->hcksum_flags; 888 889 /* 890 * Check if a new context descriptor is needed for this packet 891 */ 892 load_context = igb_check_tx_context(tx_ring, ctx); 893 if (load_context) { 894 tbd = &tx_ring->tbd_ring[index]; 895 896 /* 897 * Fill the context descriptor with the 898 * hardware checksum offload informations. 899 */ 900 igb_fill_tx_context( 901 (struct e1000_adv_tx_context_desc *)tbd, 902 ctx, tx_ring->index); 903 904 index = NEXT_INDEX(index, 1, tx_ring->ring_size); 905 desc_num++; 906 907 /* 908 * Store the checksum context data if 909 * a new context descriptor is added 910 */ 911 tx_ring->tx_context = *ctx; 912 } 913 } 914 915 first_tbd = &tx_ring->tbd_ring[index]; 916 917 /* 918 * Fill tx data descriptors with the data saved in the pending list. 919 * The tx control blocks in the pending list are added to the work list 920 * at the same time. 921 * 922 * The work list is strictly 1:1 corresponding to the descriptor ring. 923 * One item of the work list corresponds to one tx descriptor. Because 924 * one tx control block can span multiple tx descriptors, the tx 925 * control block will be added to the first work list item that 926 * corresponds to the first tx descriptor generated from that tx 927 * control block. 928 */ 929 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 930 first_tcb = tcb; 931 while (tcb != NULL) { 932 933 for (i = 0; i < tcb->desc_num; i++) { 934 tbd = &tx_ring->tbd_ring[index]; 935 936 tbd->read.buffer_addr = tcb->desc[i].address; 937 tbd->read.cmd_type_len = tcb->desc[i].length; 938 939 tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_RS | 940 E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_DATA | 941 E1000_ADVTXD_DCMD_IFCS; 942 943 tbd->read.olinfo_status = 0; 944 945 index = NEXT_INDEX(index, 1, tx_ring->ring_size); 946 desc_num++; 947 } 948 949 /* 950 * Add the tx control block to the work list 951 */ 952 ASSERT(tx_ring->work_list[tcb_index] == NULL); 953 tx_ring->work_list[tcb_index] = tcb; 954 955 tcb_index = index; 956 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 957 } 958 959 if (load_context) { 960 /* 961 * Count the checksum context descriptor for 962 * the first tx control block. 963 */ 964 first_tcb->desc_num++; 965 } 966 first_tcb->last_index = PREV_INDEX(index, 1, tx_ring->ring_size); 967 968 /* 969 * The Insert Ethernet CRC (IFCS) bit and the checksum fields are only 970 * valid in the first descriptor of the packet. 971 * 82576 also requires the payload length setting even without LSO 972 */ 973 ASSERT(first_tbd != NULL); 974 first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_IFCS; 975 if (ctx != NULL && ctx->lso_flag) { 976 first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_TSE; 977 first_tbd->read.olinfo_status |= 978 (mbsize - ctx->mac_hdr_len - ctx->ip_hdr_len 979 - ctx->l4_hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT; 980 } else { 981 if (hw->mac.type >= e1000_82576) { 982 first_tbd->read.olinfo_status |= 983 (mbsize << E1000_ADVTXD_PAYLEN_SHIFT); 984 } 985 } 986 987 /* Set hardware checksum bits */ 988 if (hcksum_flags != 0) { 989 if (hcksum_flags & HCK_IPV4_HDRCKSUM) 990 first_tbd->read.olinfo_status |= 991 E1000_TXD_POPTS_IXSM << 8; 992 if (hcksum_flags & HCK_PARTIALCKSUM) 993 first_tbd->read.olinfo_status |= 994 E1000_TXD_POPTS_TXSM << 8; 995 first_tbd->read.olinfo_status |= tx_ring->index << 4; 996 } 997 998 /* 999 * The last descriptor of packet needs End Of Packet (EOP), 1000 * and Report Status (RS) bits set 1001 */ 1002 ASSERT(tbd != NULL); 1003 tbd->read.cmd_type_len |= 1004 E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS; 1005 1006 IGB_DEBUG_STAT(tx_ring->stat_pkt_cnt); 1007 1008 /* 1009 * Sync the DMA buffer of the tx descriptor ring 1010 */ 1011 DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORDEV); 1012 1013 /* 1014 * Update the number of the free tx descriptors. 1015 * The mutual exclusion between the transmission and the recycling 1016 * (for the tx descriptor ring and the work list) is implemented 1017 * with the atomic operation on the number of the free tx descriptors. 1018 * 1019 * Note: we should always decrement the counter tbd_free before 1020 * advancing the hardware TDT pointer to avoid the race condition - 1021 * before the counter tbd_free is decremented, the transmit of the 1022 * tx descriptors has done and the counter tbd_free is increased by 1023 * the tx recycling. 1024 */ 1025 i = igb_atomic_reserve(&tx_ring->tbd_free, desc_num); 1026 ASSERT(i >= 0); 1027 1028 tx_ring->tbd_tail = index; 1029 1030 /* 1031 * Advance the hardware TDT pointer of the tx descriptor ring 1032 */ 1033 E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), index); 1034 1035 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 1036 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 1037 atomic_or_32(&igb->igb_state, IGB_ERROR); 1038 } 1039 1040 return (desc_num); 1041 } 1042 1043 /* 1044 * igb_save_desc 1045 * 1046 * Save the address/length pair to the private array 1047 * of the tx control block. The address/length pairs 1048 * will be filled into the tx descriptor ring later. 1049 */ 1050 static void 1051 igb_save_desc(tx_control_block_t *tcb, uint64_t address, size_t length) 1052 { 1053 sw_desc_t *desc; 1054 1055 desc = &tcb->desc[tcb->desc_num]; 1056 desc->address = address; 1057 desc->length = length; 1058 1059 tcb->desc_num++; 1060 } 1061 1062 /* 1063 * igb_tx_recycle_legacy 1064 * 1065 * Recycle the tx descriptors and tx control blocks. 1066 * 1067 * The work list is traversed to check if the corresponding 1068 * tx descriptors have been transmitted. If so, the resources 1069 * bound to the tx control blocks will be freed, and those 1070 * tx control blocks will be returned to the free list. 1071 */ 1072 uint32_t 1073 igb_tx_recycle_legacy(igb_tx_ring_t *tx_ring) 1074 { 1075 uint32_t index, last_index, next_index; 1076 int desc_num; 1077 boolean_t desc_done; 1078 tx_control_block_t *tcb; 1079 link_list_t pending_list; 1080 igb_t *igb = tx_ring->igb; 1081 1082 /* 1083 * The mutex_tryenter() is used to avoid unnecessary 1084 * lock contention. 1085 */ 1086 if (mutex_tryenter(&tx_ring->recycle_lock) == 0) 1087 return (0); 1088 1089 ASSERT(tx_ring->tbd_free <= tx_ring->ring_size); 1090 1091 if (tx_ring->tbd_free == tx_ring->ring_size) { 1092 tx_ring->recycle_fail = 0; 1093 tx_ring->stall_watchdog = 0; 1094 mutex_exit(&tx_ring->recycle_lock); 1095 return (0); 1096 } 1097 1098 /* 1099 * Sync the DMA buffer of the tx descriptor ring 1100 */ 1101 DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL); 1102 1103 if (igb_check_dma_handle( 1104 tx_ring->tbd_area.dma_handle) != DDI_FM_OK) { 1105 mutex_exit(&tx_ring->recycle_lock); 1106 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 1107 atomic_or_32(&igb->igb_state, IGB_ERROR); 1108 return (0); 1109 } 1110 1111 LINK_LIST_INIT(&pending_list); 1112 desc_num = 0; 1113 index = tx_ring->tbd_head; /* Index of next tbd/tcb to recycle */ 1114 1115 tcb = tx_ring->work_list[index]; 1116 ASSERT(tcb != NULL); 1117 1118 while (tcb != NULL) { 1119 1120 /* 1121 * Get the last tx descriptor of this packet. 1122 * If the last tx descriptor is done, then 1123 * we can recycle all descriptors of a packet 1124 * which usually includes several tx control blocks. 1125 * For some chips, LSO descriptors can not be recycled 1126 * unless the whole packet's transmission is done. 1127 * That's why packet level recycling is used here. 1128 */ 1129 last_index = tcb->last_index; 1130 /* 1131 * MAX_TX_RING_SIZE is used to judge whether 1132 * the index is a valid value or not. 1133 */ 1134 if (last_index == MAX_TX_RING_SIZE) 1135 break; 1136 1137 next_index = NEXT_INDEX(last_index, 1, tx_ring->ring_size); 1138 1139 /* 1140 * Check if the Descriptor Done bit is set 1141 */ 1142 desc_done = tx_ring->tbd_ring[last_index].wb.status & 1143 E1000_TXD_STAT_DD; 1144 if (desc_done) { 1145 while (tcb != NULL) { 1146 /* 1147 * Strip off the tx control block from the work 1148 * list, and add it to the pending list. 1149 */ 1150 tx_ring->work_list[index] = NULL; 1151 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1152 1153 /* 1154 * Count the total number of the tx descriptors 1155 * recycled. 1156 */ 1157 desc_num += tcb->desc_num; 1158 1159 /* 1160 * Advance the index of the tx descriptor ring 1161 */ 1162 index = NEXT_INDEX(index, tcb->desc_num, 1163 tx_ring->ring_size); 1164 1165 tcb = tx_ring->work_list[index]; 1166 if (index == next_index) 1167 break; 1168 } 1169 } else { 1170 break; 1171 } 1172 } 1173 1174 /* 1175 * If no tx descriptors are recycled, no need to do more processing 1176 */ 1177 if (desc_num == 0) { 1178 tx_ring->recycle_fail++; 1179 mutex_exit(&tx_ring->recycle_lock); 1180 return (0); 1181 } 1182 1183 tx_ring->recycle_fail = 0; 1184 tx_ring->stall_watchdog = 0; 1185 1186 /* 1187 * Update the head index of the tx descriptor ring 1188 */ 1189 tx_ring->tbd_head = index; 1190 1191 /* 1192 * Update the number of the free tx descriptors with atomic operations 1193 */ 1194 atomic_add_32(&tx_ring->tbd_free, desc_num); 1195 1196 mutex_exit(&tx_ring->recycle_lock); 1197 1198 /* 1199 * Free the resources used by the tx control blocks 1200 * in the pending list 1201 */ 1202 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 1203 while (tcb != NULL) { 1204 /* 1205 * Release the resources occupied by the tx control block 1206 */ 1207 igb_free_tcb(tcb); 1208 1209 tcb = (tx_control_block_t *) 1210 LIST_GET_NEXT(&pending_list, &tcb->link); 1211 } 1212 1213 /* 1214 * Add the tx control blocks in the pending list to the free list. 1215 */ 1216 igb_put_free_list(tx_ring, &pending_list); 1217 1218 return (desc_num); 1219 } 1220 1221 /* 1222 * igb_tx_recycle_head_wb 1223 * 1224 * Check the head write-back, and recycle all the transmitted 1225 * tx descriptors and tx control blocks. 1226 */ 1227 uint32_t 1228 igb_tx_recycle_head_wb(igb_tx_ring_t *tx_ring) 1229 { 1230 uint32_t index; 1231 uint32_t head_wb; 1232 int desc_num; 1233 tx_control_block_t *tcb; 1234 link_list_t pending_list; 1235 igb_t *igb = tx_ring->igb; 1236 1237 /* 1238 * The mutex_tryenter() is used to avoid unnecessary 1239 * lock contention. 1240 */ 1241 if (mutex_tryenter(&tx_ring->recycle_lock) == 0) 1242 return (0); 1243 1244 ASSERT(tx_ring->tbd_free <= tx_ring->ring_size); 1245 1246 if (tx_ring->tbd_free == tx_ring->ring_size) { 1247 tx_ring->recycle_fail = 0; 1248 tx_ring->stall_watchdog = 0; 1249 mutex_exit(&tx_ring->recycle_lock); 1250 return (0); 1251 } 1252 1253 /* 1254 * Sync the DMA buffer of the tx descriptor ring 1255 * 1256 * Note: For head write-back mode, the tx descriptors will not 1257 * be written back, but the head write-back value is stored at 1258 * the last extra tbd at the end of the DMA area, we still need 1259 * to sync the head write-back value for kernel. 1260 * 1261 * DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL); 1262 */ 1263 (void) ddi_dma_sync(tx_ring->tbd_area.dma_handle, 1264 sizeof (union e1000_adv_tx_desc) * tx_ring->ring_size, 1265 sizeof (uint32_t), 1266 DDI_DMA_SYNC_FORKERNEL); 1267 1268 if (igb_check_dma_handle( 1269 tx_ring->tbd_area.dma_handle) != DDI_FM_OK) { 1270 mutex_exit(&tx_ring->recycle_lock); 1271 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 1272 atomic_or_32(&igb->igb_state, IGB_ERROR); 1273 return (0); 1274 } 1275 1276 LINK_LIST_INIT(&pending_list); 1277 desc_num = 0; 1278 index = tx_ring->tbd_head; /* Next index to clean */ 1279 1280 /* 1281 * Get the value of head write-back 1282 */ 1283 head_wb = *tx_ring->tbd_head_wb; 1284 while (index != head_wb) { 1285 tcb = tx_ring->work_list[index]; 1286 ASSERT(tcb != NULL); 1287 1288 if (OFFSET(index, head_wb, tx_ring->ring_size) < 1289 tcb->desc_num) { 1290 /* 1291 * The current tx control block is not 1292 * completely transmitted, stop recycling 1293 */ 1294 break; 1295 } 1296 1297 /* 1298 * Strip off the tx control block from the work list, 1299 * and add it to the pending list. 1300 */ 1301 tx_ring->work_list[index] = NULL; 1302 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1303 1304 /* 1305 * Advance the index of the tx descriptor ring 1306 */ 1307 index = NEXT_INDEX(index, tcb->desc_num, tx_ring->ring_size); 1308 1309 /* 1310 * Count the total number of the tx descriptors recycled 1311 */ 1312 desc_num += tcb->desc_num; 1313 } 1314 1315 /* 1316 * If no tx descriptors are recycled, no need to do more processing 1317 */ 1318 if (desc_num == 0) { 1319 tx_ring->recycle_fail++; 1320 mutex_exit(&tx_ring->recycle_lock); 1321 return (0); 1322 } 1323 1324 tx_ring->recycle_fail = 0; 1325 tx_ring->stall_watchdog = 0; 1326 1327 /* 1328 * Update the head index of the tx descriptor ring 1329 */ 1330 tx_ring->tbd_head = index; 1331 1332 /* 1333 * Update the number of the free tx descriptors with atomic operations 1334 */ 1335 atomic_add_32(&tx_ring->tbd_free, desc_num); 1336 1337 mutex_exit(&tx_ring->recycle_lock); 1338 1339 /* 1340 * Free the resources used by the tx control blocks 1341 * in the pending list 1342 */ 1343 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 1344 while (tcb) { 1345 /* 1346 * Release the resources occupied by the tx control block 1347 */ 1348 igb_free_tcb(tcb); 1349 1350 tcb = (tx_control_block_t *) 1351 LIST_GET_NEXT(&pending_list, &tcb->link); 1352 } 1353 1354 /* 1355 * Add the tx control blocks in the pending list to the free list. 1356 */ 1357 igb_put_free_list(tx_ring, &pending_list); 1358 1359 return (desc_num); 1360 } 1361 1362 /* 1363 * igb_free_tcb - free up the tx control block 1364 * 1365 * Free the resources of the tx control block, including 1366 * unbind the previously bound DMA handle, and reset other 1367 * control fields. 1368 */ 1369 void 1370 igb_free_tcb(tx_control_block_t *tcb) 1371 { 1372 switch (tcb->tx_type) { 1373 case USE_COPY: 1374 /* 1375 * Reset the buffer length that is used for copy 1376 */ 1377 tcb->tx_buf.len = 0; 1378 break; 1379 case USE_DMA: 1380 /* 1381 * Release the DMA resource that is used for 1382 * DMA binding. 1383 */ 1384 (void) ddi_dma_unbind_handle(tcb->tx_dma_handle); 1385 break; 1386 default: 1387 break; 1388 } 1389 1390 /* 1391 * Free the mblk 1392 */ 1393 if (tcb->mp != NULL) { 1394 freemsg(tcb->mp); 1395 tcb->mp = NULL; 1396 } 1397 1398 tcb->tx_type = USE_NONE; 1399 tcb->last_index = MAX_TX_RING_SIZE; 1400 tcb->frag_num = 0; 1401 tcb->desc_num = 0; 1402 } 1403 1404 /* 1405 * igb_get_free_list - Get a free tx control block from the free list 1406 * 1407 * The atomic operation on the number of the available tx control block 1408 * in the free list is used to keep this routine mutual exclusive with 1409 * the routine igb_put_check_list. 1410 */ 1411 static tx_control_block_t * 1412 igb_get_free_list(igb_tx_ring_t *tx_ring) 1413 { 1414 tx_control_block_t *tcb; 1415 1416 /* 1417 * Check and update the number of the free tx control block 1418 * in the free list. 1419 */ 1420 if (igb_atomic_reserve(&tx_ring->tcb_free, 1) < 0) 1421 return (NULL); 1422 1423 mutex_enter(&tx_ring->tcb_head_lock); 1424 1425 tcb = tx_ring->free_list[tx_ring->tcb_head]; 1426 ASSERT(tcb != NULL); 1427 tx_ring->free_list[tx_ring->tcb_head] = NULL; 1428 tx_ring->tcb_head = NEXT_INDEX(tx_ring->tcb_head, 1, 1429 tx_ring->free_list_size); 1430 1431 mutex_exit(&tx_ring->tcb_head_lock); 1432 1433 return (tcb); 1434 } 1435 1436 /* 1437 * igb_put_free_list 1438 * 1439 * Put a list of used tx control blocks back to the free list 1440 * 1441 * A mutex is used here to ensure the serialization. The mutual exclusion 1442 * between igb_get_free_list and igb_put_free_list is implemented with 1443 * the atomic operation on the counter tcb_free. 1444 */ 1445 void 1446 igb_put_free_list(igb_tx_ring_t *tx_ring, link_list_t *pending_list) 1447 { 1448 uint32_t index; 1449 int tcb_num; 1450 tx_control_block_t *tcb; 1451 1452 mutex_enter(&tx_ring->tcb_tail_lock); 1453 1454 index = tx_ring->tcb_tail; 1455 1456 tcb_num = 0; 1457 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 1458 while (tcb != NULL) { 1459 ASSERT(tx_ring->free_list[index] == NULL); 1460 tx_ring->free_list[index] = tcb; 1461 1462 tcb_num++; 1463 1464 index = NEXT_INDEX(index, 1, tx_ring->free_list_size); 1465 1466 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 1467 } 1468 1469 tx_ring->tcb_tail = index; 1470 1471 /* 1472 * Update the number of the free tx control block 1473 * in the free list. This operation must be placed 1474 * under the protection of the lock. 1475 */ 1476 atomic_add_32(&tx_ring->tcb_free, tcb_num); 1477 1478 mutex_exit(&tx_ring->tcb_tail_lock); 1479 } 1480