1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved. 24 */ 25 26 /* 27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28 * Copyright 2012 Nexenta Systems, Inc. All rights reserved. 29 * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved. 30 * Copyright 2017 Joyent, Inc. 31 */ 32 33 #include "ixgbe_sw.h" 34 35 static int ixgbe_tx_copy(ixgbe_tx_ring_t *, tx_control_block_t *, mblk_t *, 36 uint32_t, boolean_t); 37 static int ixgbe_tx_bind(ixgbe_tx_ring_t *, tx_control_block_t *, mblk_t *, 38 uint32_t); 39 static int ixgbe_tx_fill_ring(ixgbe_tx_ring_t *, link_list_t *, 40 ixgbe_tx_context_t *, size_t); 41 static void ixgbe_save_desc(tx_control_block_t *, uint64_t, size_t); 42 static tx_control_block_t *ixgbe_get_free_list(ixgbe_tx_ring_t *); 43 44 static int ixgbe_get_context(mblk_t *, ixgbe_tx_context_t *); 45 static boolean_t ixgbe_check_context(ixgbe_tx_ring_t *, 46 ixgbe_tx_context_t *); 47 static void ixgbe_fill_context(struct ixgbe_adv_tx_context_desc *, 48 ixgbe_tx_context_t *); 49 50 #ifndef IXGBE_DEBUG 51 #pragma inline(ixgbe_save_desc) 52 #pragma inline(ixgbe_get_context) 53 #pragma inline(ixgbe_check_context) 54 #pragma inline(ixgbe_fill_context) 55 #endif 56 57 /* 58 * ixgbe_ring_tx 59 * 60 * To transmit one mblk through one specified ring. 61 * 62 * One mblk can consist of several fragments, each fragment 63 * will be processed with different methods based on the size. 64 * For the fragments with size less than the bcopy threshold, 65 * they will be processed by using bcopy; otherwise, they will 66 * be processed by using DMA binding. 67 * 68 * To process the mblk, a tx control block is got from the 69 * free list. One tx control block contains one tx buffer, which 70 * is used to copy mblk fragments' data; and one tx DMA handle, 71 * which is used to bind a mblk fragment with DMA resource. 72 * 73 * Several small mblk fragments can be copied into one tx control 74 * block's buffer, and then the buffer will be transmitted with 75 * one tx descriptor. 76 * 77 * A large fragment only binds with one tx control block's DMA 78 * handle, and it can span several tx descriptors for transmitting. 79 * 80 * So to transmit a packet (mblk), several tx control blocks can 81 * be used. After the processing, those tx control blocks will 82 * be put to the work list. 83 */ 84 mblk_t * 85 ixgbe_ring_tx(void *arg, mblk_t *mp) 86 { 87 ixgbe_tx_ring_t *tx_ring = (ixgbe_tx_ring_t *)arg; 88 ixgbe_t *ixgbe = tx_ring->ixgbe; 89 tx_type_t current_flag, next_flag; 90 uint32_t current_len, next_len; 91 uint32_t desc_total; 92 size_t mbsize; 93 int desc_num; 94 boolean_t copy_done, eop; 95 mblk_t *current_mp, *next_mp, *nmp, *pull_mp = NULL; 96 tx_control_block_t *tcb; 97 ixgbe_tx_context_t tx_context, *ctx; 98 link_list_t pending_list; 99 uint32_t len, hdr_frag_len, hdr_len; 100 uint32_t copy_thresh; 101 mblk_t *hdr_new_mp = NULL; 102 mblk_t *hdr_pre_mp = NULL; 103 mblk_t *hdr_nmp = NULL; 104 105 ASSERT(mp->b_next == NULL); 106 107 if ((ixgbe->ixgbe_state & IXGBE_SUSPENDED) || 108 (ixgbe->ixgbe_state & IXGBE_ERROR) || 109 (ixgbe->ixgbe_state & IXGBE_OVERTEMP) || 110 !(ixgbe->ixgbe_state & IXGBE_STARTED) || 111 ixgbe->link_state != LINK_STATE_UP) { 112 freemsg(mp); 113 return (NULL); 114 } 115 116 copy_thresh = ixgbe->tx_copy_thresh; 117 118 /* Get the mblk size */ 119 mbsize = 0; 120 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) { 121 mbsize += MBLKL(nmp); 122 } 123 124 if (ixgbe->tx_hcksum_enable) { 125 /* 126 * Retrieve checksum context information from the mblk 127 * that will be used to decide whether/how to fill the 128 * context descriptor. 129 */ 130 ctx = &tx_context; 131 if (ixgbe_get_context(mp, ctx) < 0) { 132 freemsg(mp); 133 return (NULL); 134 } 135 136 /* 137 * If the mblk size exceeds the max size ixgbe could 138 * process, then discard this mblk, and return NULL. 139 */ 140 if ((ctx->lso_flag && 141 ((mbsize - ctx->mac_hdr_len) > IXGBE_LSO_MAXLEN)) || 142 (!ctx->lso_flag && 143 (mbsize > (ixgbe->max_frame_size - ETHERFCSL)))) { 144 freemsg(mp); 145 IXGBE_DEBUGLOG_0(ixgbe, "ixgbe_tx: packet oversize"); 146 return (NULL); 147 } 148 } else { 149 ctx = NULL; 150 } 151 152 /* 153 * Check and recycle tx descriptors. 154 * The recycle threshold here should be selected carefully 155 */ 156 if (tx_ring->tbd_free < ixgbe->tx_recycle_thresh) { 157 tx_ring->tx_recycle(tx_ring); 158 } 159 160 /* 161 * After the recycling, if the tbd_free is less than the 162 * overload_threshold, assert overload, return mp; 163 * and we need to re-schedule the tx again. 164 */ 165 if (tx_ring->tbd_free < ixgbe->tx_overload_thresh) { 166 tx_ring->reschedule = B_TRUE; 167 tx_ring->stat_overload++; 168 return (mp); 169 } 170 171 /* 172 * The pending_list is a linked list that is used to save 173 * the tx control blocks that have packet data processed 174 * but have not put the data to the tx descriptor ring. 175 * It is used to reduce the lock contention of the tx_lock. 176 */ 177 LINK_LIST_INIT(&pending_list); 178 desc_num = 0; 179 desc_total = 0; 180 181 /* 182 * The software should guarantee LSO packet header(MAC+IP+TCP) 183 * to be within one descriptor. Here we reallocate and refill the 184 * the header if it's physical memory non-contiguous. 185 */ 186 if ((ctx != NULL) && ctx->lso_flag) { 187 /* find the last fragment of the header */ 188 len = MBLKL(mp); 189 ASSERT(len > 0); 190 hdr_nmp = mp; 191 hdr_len = ctx->ip_hdr_len + ctx->mac_hdr_len + ctx->l4_hdr_len; 192 while (len < hdr_len) { 193 hdr_pre_mp = hdr_nmp; 194 hdr_nmp = hdr_nmp->b_cont; 195 len += MBLKL(hdr_nmp); 196 } 197 /* 198 * If the header and the payload are in different mblks, 199 * we simply force the header to be copied into pre-allocated 200 * page-aligned buffer. 201 */ 202 if (len == hdr_len) 203 goto adjust_threshold; 204 205 hdr_frag_len = hdr_len - (len - MBLKL(hdr_nmp)); 206 /* 207 * There are two cases we need to reallocate a mblk for the 208 * last header fragment: 209 * 1. the header is in multiple mblks and the last fragment 210 * share the same mblk with the payload 211 * 2. the header is in a single mblk shared with the payload 212 * and the header is physical memory non-contiguous 213 */ 214 if ((hdr_nmp != mp) || 215 (P2NPHASE((uintptr_t)hdr_nmp->b_rptr, ixgbe->sys_page_size) 216 < hdr_len)) { 217 tx_ring->stat_lso_header_fail++; 218 /* 219 * reallocate the mblk for the last header fragment, 220 * expect to bcopy into pre-allocated page-aligned 221 * buffer 222 */ 223 hdr_new_mp = allocb(hdr_frag_len, NULL); 224 if (!hdr_new_mp) 225 return (mp); 226 bcopy(hdr_nmp->b_rptr, hdr_new_mp->b_rptr, 227 hdr_frag_len); 228 /* link the new header fragment with the other parts */ 229 hdr_new_mp->b_wptr = hdr_new_mp->b_rptr + hdr_frag_len; 230 hdr_new_mp->b_cont = hdr_nmp; 231 if (hdr_pre_mp) 232 hdr_pre_mp->b_cont = hdr_new_mp; 233 else 234 mp = hdr_new_mp; 235 hdr_nmp->b_rptr += hdr_frag_len; 236 } 237 adjust_threshold: 238 /* 239 * adjust the bcopy threshhold to guarantee 240 * the header to use bcopy way 241 */ 242 if (copy_thresh < hdr_len) 243 copy_thresh = hdr_len; 244 } 245 246 current_mp = mp; 247 current_len = MBLKL(current_mp); 248 /* 249 * Decide which method to use for the first fragment 250 */ 251 current_flag = (current_len <= copy_thresh) ? 252 USE_COPY : USE_DMA; 253 /* 254 * If the mblk includes several contiguous small fragments, 255 * they may be copied into one buffer. This flag is used to 256 * indicate whether there are pending fragments that need to 257 * be copied to the current tx buffer. 258 * 259 * If this flag is B_TRUE, it indicates that a new tx control 260 * block is needed to process the next fragment using either 261 * copy or DMA binding. 262 * 263 * Otherwise, it indicates that the next fragment will be 264 * copied to the current tx buffer that is maintained by the 265 * current tx control block. No new tx control block is needed. 266 */ 267 copy_done = B_TRUE; 268 while (current_mp) { 269 next_mp = current_mp->b_cont; 270 eop = (next_mp == NULL); /* Last fragment of the packet? */ 271 next_len = eop ? 0: MBLKL(next_mp); 272 273 /* 274 * When the current fragment is an empty fragment, if 275 * the next fragment will still be copied to the current 276 * tx buffer, we cannot skip this fragment here. Because 277 * the copy processing is pending for completion. We have 278 * to process this empty fragment in the tx_copy routine. 279 * 280 * If the copy processing is completed or a DMA binding 281 * processing is just completed, we can just skip this 282 * empty fragment. 283 */ 284 if ((current_len == 0) && (copy_done)) { 285 current_mp = next_mp; 286 current_len = next_len; 287 current_flag = (current_len <= copy_thresh) ? 288 USE_COPY : USE_DMA; 289 continue; 290 } 291 292 if (copy_done) { 293 /* 294 * Get a new tx control block from the free list 295 */ 296 tcb = ixgbe_get_free_list(tx_ring); 297 298 if (tcb == NULL) { 299 tx_ring->stat_fail_no_tcb++; 300 goto tx_failure; 301 } 302 303 /* 304 * Push the tx control block to the pending list 305 * to avoid using lock too early 306 */ 307 LIST_PUSH_TAIL(&pending_list, &tcb->link); 308 } 309 310 if (current_flag == USE_COPY) { 311 /* 312 * Check whether to use bcopy or DMA binding to process 313 * the next fragment, and if using bcopy, whether we 314 * need to continue copying the next fragment into the 315 * current tx buffer. 316 */ 317 ASSERT((tcb->tx_buf.len + current_len) <= 318 tcb->tx_buf.size); 319 320 if (eop) { 321 /* 322 * This is the last fragment of the packet, so 323 * the copy processing will be completed with 324 * this fragment. 325 */ 326 next_flag = USE_NONE; 327 copy_done = B_TRUE; 328 } else if ((tcb->tx_buf.len + current_len + next_len) > 329 tcb->tx_buf.size) { 330 /* 331 * If the next fragment is too large to be 332 * copied to the current tx buffer, we need 333 * to complete the current copy processing. 334 */ 335 next_flag = (next_len > copy_thresh) ? 336 USE_DMA: USE_COPY; 337 copy_done = B_TRUE; 338 } else if (next_len > copy_thresh) { 339 /* 340 * The next fragment needs to be processed with 341 * DMA binding. So the copy prcessing will be 342 * completed with the current fragment. 343 */ 344 next_flag = USE_DMA; 345 copy_done = B_TRUE; 346 } else { 347 /* 348 * Continue to copy the next fragment to the 349 * current tx buffer. 350 */ 351 next_flag = USE_COPY; 352 copy_done = B_FALSE; 353 } 354 355 desc_num = ixgbe_tx_copy(tx_ring, tcb, current_mp, 356 current_len, copy_done); 357 } else { 358 /* 359 * Check whether to use bcopy or DMA binding to process 360 * the next fragment. 361 */ 362 next_flag = (next_len > copy_thresh) ? 363 USE_DMA: USE_COPY; 364 ASSERT(copy_done == B_TRUE); 365 366 desc_num = ixgbe_tx_bind(tx_ring, tcb, current_mp, 367 current_len); 368 } 369 370 if (desc_num > 0) 371 desc_total += desc_num; 372 else if (desc_num < 0) 373 goto tx_failure; 374 375 current_mp = next_mp; 376 current_len = next_len; 377 current_flag = next_flag; 378 } 379 380 /* 381 * Attach the mblk to the last tx control block 382 */ 383 ASSERT(tcb); 384 ASSERT(tcb->mp == NULL); 385 tcb->mp = mp; 386 387 /* 388 * 82598/82599 chipset has a limitation that no more than 32 tx 389 * descriptors can be transmited out at one time. 390 * 391 * Here is a workaround for it: pull up the mblk then send it 392 * out with bind way. By doing so, no more than MAX_COOKIE (18) 393 * descriptors is needed. 394 */ 395 if (desc_total + 1 > IXGBE_TX_DESC_LIMIT) { 396 tx_ring->stat_break_tbd_limit++; 397 398 /* 399 * Discard the mblk and free the used resources 400 */ 401 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 402 while (tcb) { 403 tcb->mp = NULL; 404 ixgbe_free_tcb(tcb); 405 tcb = (tx_control_block_t *) 406 LIST_GET_NEXT(&pending_list, &tcb->link); 407 } 408 409 /* 410 * Return the tx control blocks in the pending list to 411 * the free list. 412 */ 413 ixgbe_put_free_list(tx_ring, &pending_list); 414 415 /* 416 * pull up the mblk and send it out with bind way 417 */ 418 if ((pull_mp = msgpullup(mp, -1)) == NULL) { 419 tx_ring->reschedule = B_TRUE; 420 421 /* 422 * If new mblk has been allocted for the last header 423 * fragment of a LSO packet, we should restore the 424 * modified mp. 425 */ 426 if (hdr_new_mp) { 427 hdr_new_mp->b_cont = NULL; 428 freeb(hdr_new_mp); 429 hdr_nmp->b_rptr -= hdr_frag_len; 430 if (hdr_pre_mp) 431 hdr_pre_mp->b_cont = hdr_nmp; 432 else 433 mp = hdr_nmp; 434 } 435 return (mp); 436 } 437 438 LINK_LIST_INIT(&pending_list); 439 desc_total = 0; 440 441 /* 442 * if the packet is a LSO packet, we simply 443 * transmit the header in one descriptor using the copy way 444 */ 445 if ((ctx != NULL) && ctx->lso_flag) { 446 hdr_len = ctx->ip_hdr_len + ctx->mac_hdr_len + 447 ctx->l4_hdr_len; 448 449 tcb = ixgbe_get_free_list(tx_ring); 450 if (tcb == NULL) { 451 tx_ring->stat_fail_no_tcb++; 452 goto tx_failure; 453 } 454 desc_num = ixgbe_tx_copy(tx_ring, tcb, pull_mp, 455 hdr_len, B_TRUE); 456 LIST_PUSH_TAIL(&pending_list, &tcb->link); 457 desc_total += desc_num; 458 459 pull_mp->b_rptr += hdr_len; 460 } 461 462 tcb = ixgbe_get_free_list(tx_ring); 463 if (tcb == NULL) { 464 tx_ring->stat_fail_no_tcb++; 465 goto tx_failure; 466 } 467 if ((ctx != NULL) && ctx->lso_flag) { 468 desc_num = ixgbe_tx_bind(tx_ring, tcb, pull_mp, 469 mbsize - hdr_len); 470 } else { 471 desc_num = ixgbe_tx_bind(tx_ring, tcb, pull_mp, 472 mbsize); 473 } 474 if (desc_num < 0) { 475 goto tx_failure; 476 } 477 LIST_PUSH_TAIL(&pending_list, &tcb->link); 478 479 desc_total += desc_num; 480 tcb->mp = pull_mp; 481 } 482 483 /* 484 * Before fill the tx descriptor ring with the data, we need to 485 * ensure there are adequate free descriptors for transmit 486 * (including one context descriptor). 487 * Do not use up all the tx descriptors. 488 * Otherwise tx recycle will fail and cause false hang. 489 */ 490 if (tx_ring->tbd_free <= (desc_total + 1)) { 491 tx_ring->tx_recycle(tx_ring); 492 } 493 494 mutex_enter(&tx_ring->tx_lock); 495 /* 496 * If the number of free tx descriptors is not enough for transmit 497 * then return mp. 498 * 499 * Note: we must put this check under the mutex protection to 500 * ensure the correctness when multiple threads access it in 501 * parallel. 502 */ 503 if (tx_ring->tbd_free <= (desc_total + 1)) { 504 tx_ring->stat_fail_no_tbd++; 505 mutex_exit(&tx_ring->tx_lock); 506 goto tx_failure; 507 } 508 509 desc_num = ixgbe_tx_fill_ring(tx_ring, &pending_list, ctx, 510 mbsize); 511 512 ASSERT((desc_num == desc_total) || (desc_num == (desc_total + 1))); 513 514 tx_ring->stat_obytes += mbsize; 515 tx_ring->stat_opackets ++; 516 517 mutex_exit(&tx_ring->tx_lock); 518 519 /* 520 * now that the transmission succeeds, need to free the original 521 * mp if we used the pulling up mblk for transmission. 522 */ 523 if (pull_mp) { 524 freemsg(mp); 525 } 526 527 return (NULL); 528 529 tx_failure: 530 /* 531 * If transmission fails, need to free the pulling up mblk. 532 */ 533 if (pull_mp) { 534 freemsg(pull_mp); 535 } 536 537 /* 538 * If new mblk has been allocted for the last header 539 * fragment of a LSO packet, we should restore the 540 * modified mp. 541 */ 542 if (hdr_new_mp) { 543 hdr_new_mp->b_cont = NULL; 544 freeb(hdr_new_mp); 545 hdr_nmp->b_rptr -= hdr_frag_len; 546 if (hdr_pre_mp) 547 hdr_pre_mp->b_cont = hdr_nmp; 548 else 549 mp = hdr_nmp; 550 } 551 /* 552 * Discard the mblk and free the used resources 553 */ 554 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 555 while (tcb) { 556 tcb->mp = NULL; 557 558 ixgbe_free_tcb(tcb); 559 560 tcb = (tx_control_block_t *) 561 LIST_GET_NEXT(&pending_list, &tcb->link); 562 } 563 564 /* 565 * Return the tx control blocks in the pending list to the free list. 566 */ 567 ixgbe_put_free_list(tx_ring, &pending_list); 568 569 /* Transmit failed, do not drop the mblk, rechedule the transmit */ 570 tx_ring->reschedule = B_TRUE; 571 572 return (mp); 573 } 574 575 /* 576 * ixgbe_tx_copy 577 * 578 * Copy the mblk fragment to the pre-allocated tx buffer 579 */ 580 static int 581 ixgbe_tx_copy(ixgbe_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp, 582 uint32_t len, boolean_t copy_done) 583 { 584 dma_buffer_t *tx_buf; 585 uint32_t desc_num; 586 _NOTE(ARGUNUSED(tx_ring)); 587 588 tx_buf = &tcb->tx_buf; 589 590 /* 591 * Copy the packet data of the mblk fragment into the 592 * pre-allocated tx buffer, which is maintained by the 593 * tx control block. 594 * 595 * Several mblk fragments can be copied into one tx buffer. 596 * The destination address of the current copied fragment in 597 * the tx buffer is next to the end of the previous copied 598 * fragment. 599 */ 600 if (len > 0) { 601 bcopy(mp->b_rptr, tx_buf->address + tx_buf->len, len); 602 603 tx_buf->len += len; 604 tcb->frag_num++; 605 } 606 607 desc_num = 0; 608 609 /* 610 * If it is the last fragment copied to the current tx buffer, 611 * in other words, if there's no remaining fragment or the remaining 612 * fragment requires a new tx control block to process, we need to 613 * complete the current copy processing by syncing up the current 614 * DMA buffer and saving the descriptor data. 615 */ 616 if (copy_done) { 617 /* 618 * Sync the DMA buffer of the packet data 619 */ 620 DMA_SYNC(tx_buf, DDI_DMA_SYNC_FORDEV); 621 622 tcb->tx_type = USE_COPY; 623 624 /* 625 * Save the address and length to the private data structure 626 * of the tx control block, which will be used to fill the 627 * tx descriptor ring after all the fragments are processed. 628 */ 629 ixgbe_save_desc(tcb, tx_buf->dma_address, tx_buf->len); 630 desc_num++; 631 } 632 633 return (desc_num); 634 } 635 636 /* 637 * ixgbe_tx_bind 638 * 639 * Bind the mblk fragment with DMA 640 */ 641 static int 642 ixgbe_tx_bind(ixgbe_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp, 643 uint32_t len) 644 { 645 int status, i; 646 ddi_dma_cookie_t dma_cookie; 647 uint_t ncookies; 648 int desc_num; 649 650 /* 651 * Use DMA binding to process the mblk fragment 652 */ 653 status = ddi_dma_addr_bind_handle(tcb->tx_dma_handle, NULL, 654 (caddr_t)mp->b_rptr, len, 655 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, 656 0, &dma_cookie, &ncookies); 657 658 if (status != DDI_DMA_MAPPED) { 659 tx_ring->stat_fail_dma_bind++; 660 return (-1); 661 } 662 663 tcb->frag_num++; 664 tcb->tx_type = USE_DMA; 665 /* 666 * Each fragment can span several cookies. One cookie will have 667 * one tx descriptor to transmit. 668 */ 669 desc_num = 0; 670 for (i = ncookies; i > 0; i--) { 671 /* 672 * Save the address and length to the private data structure 673 * of the tx control block, which will be used to fill the 674 * tx descriptor ring after all the fragments are processed. 675 */ 676 ixgbe_save_desc(tcb, 677 dma_cookie.dmac_laddress, 678 dma_cookie.dmac_size); 679 680 desc_num++; 681 682 if (i > 1) 683 ddi_dma_nextcookie(tcb->tx_dma_handle, &dma_cookie); 684 } 685 686 return (desc_num); 687 } 688 689 /* 690 * ixgbe_get_context 691 * 692 * Get the context information from the mblk 693 */ 694 static int 695 ixgbe_get_context(mblk_t *mp, ixgbe_tx_context_t *ctx) 696 { 697 uint32_t start; 698 uint32_t hckflags; 699 uint32_t lsoflags; 700 uint32_t mss; 701 uint32_t len; 702 uint32_t size; 703 uint32_t offset; 704 unsigned char *pos; 705 ushort_t etype; 706 uint32_t mac_hdr_len; 707 uint32_t l4_proto; 708 uint32_t l4_hdr_len; 709 710 ASSERT(mp != NULL); 711 712 mac_hcksum_get(mp, &start, NULL, NULL, NULL, &hckflags); 713 bzero(ctx, sizeof (ixgbe_tx_context_t)); 714 715 if (hckflags == 0) { 716 return (0); 717 } 718 719 ctx->hcksum_flags = hckflags; 720 721 mac_lso_get(mp, &mss, &lsoflags); 722 ctx->mss = mss; 723 ctx->lso_flag = (lsoflags == HW_LSO); 724 725 /* 726 * LSO relies on tx h/w checksum, so here will drop the package 727 * if h/w checksum flag is not declared. 728 */ 729 if (ctx->lso_flag) { 730 if (!((ctx->hcksum_flags & HCK_PARTIALCKSUM) && 731 (ctx->hcksum_flags & HCK_IPV4_HDRCKSUM))) { 732 IXGBE_DEBUGLOG_0(NULL, "ixgbe_tx: h/w " 733 "checksum flags are not specified when doing LSO"); 734 return (-1); 735 } 736 } 737 738 etype = 0; 739 mac_hdr_len = 0; 740 l4_proto = 0; 741 742 /* 743 * Firstly get the position of the ether_type/ether_tpid. 744 * Here we don't assume the ether (VLAN) header is fully included 745 * in one mblk fragment, so we go thourgh the fragments to parse 746 * the ether type. 747 */ 748 size = len = MBLKL(mp); 749 offset = offsetof(struct ether_header, ether_type); 750 while (size <= offset) { 751 mp = mp->b_cont; 752 ASSERT(mp != NULL); 753 len = MBLKL(mp); 754 size += len; 755 } 756 pos = mp->b_rptr + offset + len - size; 757 758 etype = ntohs(*(ushort_t *)(uintptr_t)pos); 759 if (etype == ETHERTYPE_VLAN) { 760 /* 761 * Get the position of the ether_type in VLAN header 762 */ 763 offset = offsetof(struct ether_vlan_header, ether_type); 764 while (size <= offset) { 765 mp = mp->b_cont; 766 ASSERT(mp != NULL); 767 len = MBLKL(mp); 768 size += len; 769 } 770 pos = mp->b_rptr + offset + len - size; 771 772 etype = ntohs(*(ushort_t *)(uintptr_t)pos); 773 mac_hdr_len = sizeof (struct ether_vlan_header); 774 } else { 775 mac_hdr_len = sizeof (struct ether_header); 776 } 777 778 /* 779 * Here we don't assume the IP(V6) header is fully included in 780 * one mblk fragment. 781 */ 782 switch (etype) { 783 case ETHERTYPE_IP: 784 if (ctx->lso_flag) { 785 offset = offsetof(ipha_t, ipha_length) + mac_hdr_len; 786 while (size <= offset) { 787 mp = mp->b_cont; 788 ASSERT(mp != NULL); 789 len = MBLKL(mp); 790 size += len; 791 } 792 pos = mp->b_rptr + offset + len - size; 793 *((uint16_t *)(uintptr_t)(pos)) = 0; 794 795 offset = offsetof(ipha_t, ipha_hdr_checksum) + 796 mac_hdr_len; 797 while (size <= offset) { 798 mp = mp->b_cont; 799 ASSERT(mp != NULL); 800 len = MBLKL(mp); 801 size += len; 802 } 803 pos = mp->b_rptr + offset + len - size; 804 *((uint16_t *)(uintptr_t)(pos)) = 0; 805 806 /* 807 * To perform ixgbe LSO, here also need to fill 808 * the tcp checksum field of the packet with the 809 * following pseudo-header checksum: 810 * (ip_source_addr, ip_destination_addr, l4_proto) 811 * Currently the tcp/ip stack has done it. 812 */ 813 } 814 815 offset = offsetof(ipha_t, ipha_protocol) + mac_hdr_len; 816 while (size <= offset) { 817 mp = mp->b_cont; 818 ASSERT(mp != NULL); 819 len = MBLKL(mp); 820 size += len; 821 } 822 pos = mp->b_rptr + offset + len - size; 823 824 l4_proto = *(uint8_t *)pos; 825 break; 826 case ETHERTYPE_IPV6: 827 offset = offsetof(ip6_t, ip6_nxt) + mac_hdr_len; 828 while (size <= offset) { 829 mp = mp->b_cont; 830 ASSERT(mp != NULL); 831 len = MBLKL(mp); 832 size += len; 833 } 834 pos = mp->b_rptr + offset + len - size; 835 836 l4_proto = *(uint8_t *)pos; 837 break; 838 default: 839 /* Unrecoverable error */ 840 IXGBE_DEBUGLOG_0(NULL, "Ether type error with tx hcksum"); 841 return (-2); 842 } 843 844 if (ctx->lso_flag) { 845 offset = mac_hdr_len + start; 846 while (size <= offset) { 847 mp = mp->b_cont; 848 ASSERT(mp != NULL); 849 len = MBLKL(mp); 850 size += len; 851 } 852 pos = mp->b_rptr + offset + len - size; 853 854 l4_hdr_len = TCP_HDR_LENGTH((tcph_t *)pos); 855 } else { 856 /* 857 * l4 header length is only required for LSO 858 */ 859 l4_hdr_len = 0; 860 } 861 862 ctx->mac_hdr_len = mac_hdr_len; 863 ctx->ip_hdr_len = start; 864 ctx->l4_proto = l4_proto; 865 ctx->l4_hdr_len = l4_hdr_len; 866 867 return (0); 868 } 869 870 /* 871 * ixgbe_check_context 872 * 873 * Check if a new context descriptor is needed 874 */ 875 static boolean_t 876 ixgbe_check_context(ixgbe_tx_ring_t *tx_ring, ixgbe_tx_context_t *ctx) 877 { 878 ixgbe_tx_context_t *last; 879 880 if (ctx == NULL) 881 return (B_FALSE); 882 883 /* 884 * Compare the context data retrieved from the mblk and the 885 * stored data of the last context descriptor. The data need 886 * to be checked are: 887 * hcksum_flags 888 * l4_proto 889 * mac_hdr_len 890 * ip_hdr_len 891 * lso_flag 892 * mss (only checked for LSO) 893 * l4_hr_len (only checked for LSO) 894 * Either one of the above data is changed, a new context descriptor 895 * will be needed. 896 */ 897 last = &tx_ring->tx_context; 898 899 if ((ctx->hcksum_flags != last->hcksum_flags) || 900 (ctx->l4_proto != last->l4_proto) || 901 (ctx->mac_hdr_len != last->mac_hdr_len) || 902 (ctx->ip_hdr_len != last->ip_hdr_len) || 903 (ctx->lso_flag != last->lso_flag) || 904 (ctx->lso_flag && ((ctx->mss != last->mss) || 905 (ctx->l4_hdr_len != last->l4_hdr_len)))) { 906 return (B_TRUE); 907 } 908 909 return (B_FALSE); 910 } 911 912 /* 913 * ixgbe_fill_context 914 * 915 * Fill the context descriptor with hardware checksum informations 916 */ 917 static void 918 ixgbe_fill_context(struct ixgbe_adv_tx_context_desc *ctx_tbd, 919 ixgbe_tx_context_t *ctx) 920 { 921 /* 922 * Fill the context descriptor with the checksum 923 * context information we've got. 924 */ 925 ctx_tbd->vlan_macip_lens = ctx->ip_hdr_len; 926 ctx_tbd->vlan_macip_lens |= ctx->mac_hdr_len << 927 IXGBE_ADVTXD_MACLEN_SHIFT; 928 929 ctx_tbd->type_tucmd_mlhl = 930 IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 931 932 if (ctx->hcksum_flags & HCK_IPV4_HDRCKSUM) 933 ctx_tbd->type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 934 935 if (ctx->hcksum_flags & HCK_PARTIALCKSUM) { 936 switch (ctx->l4_proto) { 937 case IPPROTO_TCP: 938 ctx_tbd->type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 939 break; 940 case IPPROTO_UDP: 941 /* 942 * We don't have to explicitly set: 943 * ctx_tbd->type_tucmd_mlhl |= 944 * IXGBE_ADVTXD_TUCMD_L4T_UDP; 945 * Because IXGBE_ADVTXD_TUCMD_L4T_UDP == 0b 946 */ 947 break; 948 default: 949 /* Unrecoverable error */ 950 IXGBE_DEBUGLOG_0(NULL, "L4 type error with tx hcksum"); 951 break; 952 } 953 } 954 955 ctx_tbd->seqnum_seed = 0; 956 957 if (ctx->lso_flag) { 958 ctx_tbd->mss_l4len_idx = 959 (ctx->l4_hdr_len << IXGBE_ADVTXD_L4LEN_SHIFT) | 960 (ctx->mss << IXGBE_ADVTXD_MSS_SHIFT); 961 } else { 962 ctx_tbd->mss_l4len_idx = 0; 963 } 964 } 965 966 /* 967 * ixgbe_tx_fill_ring 968 * 969 * Fill the tx descriptor ring with the data 970 */ 971 static int 972 ixgbe_tx_fill_ring(ixgbe_tx_ring_t *tx_ring, link_list_t *pending_list, 973 ixgbe_tx_context_t *ctx, size_t mbsize) 974 { 975 struct ixgbe_hw *hw = &tx_ring->ixgbe->hw; 976 boolean_t load_context; 977 uint32_t index, tcb_index, desc_num; 978 union ixgbe_adv_tx_desc *tbd, *first_tbd; 979 tx_control_block_t *tcb, *first_tcb; 980 uint32_t hcksum_flags; 981 int i; 982 983 ASSERT(mutex_owned(&tx_ring->tx_lock)); 984 985 tbd = NULL; 986 first_tbd = NULL; 987 first_tcb = NULL; 988 desc_num = 0; 989 hcksum_flags = 0; 990 load_context = B_FALSE; 991 992 /* 993 * Get the index of the first tx descriptor that will be filled, 994 * and the index of the first work list item that will be attached 995 * with the first used tx control block in the pending list. 996 * Note: the two indexes are the same. 997 */ 998 index = tx_ring->tbd_tail; 999 tcb_index = tx_ring->tbd_tail; 1000 1001 if (ctx != NULL) { 1002 hcksum_flags = ctx->hcksum_flags; 1003 1004 /* 1005 * Check if a new context descriptor is needed for this packet 1006 */ 1007 load_context = ixgbe_check_context(tx_ring, ctx); 1008 1009 if (load_context) { 1010 tbd = &tx_ring->tbd_ring[index]; 1011 1012 /* 1013 * Fill the context descriptor with the 1014 * hardware checksum offload informations. 1015 */ 1016 ixgbe_fill_context( 1017 (struct ixgbe_adv_tx_context_desc *)tbd, ctx); 1018 1019 index = NEXT_INDEX(index, 1, tx_ring->ring_size); 1020 desc_num++; 1021 1022 /* 1023 * Store the checksum context data if 1024 * a new context descriptor is added 1025 */ 1026 tx_ring->tx_context = *ctx; 1027 } 1028 } 1029 1030 first_tbd = &tx_ring->tbd_ring[index]; 1031 1032 /* 1033 * Fill tx data descriptors with the data saved in the pending list. 1034 * The tx control blocks in the pending list are added to the work list 1035 * at the same time. 1036 * 1037 * The work list is strictly 1:1 corresponding to the descriptor ring. 1038 * One item of the work list corresponds to one tx descriptor. Because 1039 * one tx control block can span multiple tx descriptors, the tx 1040 * control block will be added to the first work list item that 1041 * corresponds to the first tx descriptor generated from that tx 1042 * control block. 1043 */ 1044 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 1045 first_tcb = tcb; 1046 while (tcb != NULL) { 1047 1048 for (i = 0; i < tcb->desc_num; i++) { 1049 tbd = &tx_ring->tbd_ring[index]; 1050 1051 tbd->read.buffer_addr = tcb->desc[i].address; 1052 tbd->read.cmd_type_len = tcb->desc[i].length; 1053 1054 tbd->read.cmd_type_len |= IXGBE_ADVTXD_DCMD_DEXT 1055 | IXGBE_ADVTXD_DTYP_DATA; 1056 1057 tbd->read.olinfo_status = 0; 1058 1059 index = NEXT_INDEX(index, 1, tx_ring->ring_size); 1060 desc_num++; 1061 } 1062 1063 /* 1064 * Add the tx control block to the work list 1065 */ 1066 ASSERT(tx_ring->work_list[tcb_index] == NULL); 1067 tx_ring->work_list[tcb_index] = tcb; 1068 1069 tcb_index = index; 1070 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 1071 } 1072 1073 if (load_context) { 1074 /* 1075 * Count the context descriptor for 1076 * the first tx control block. 1077 */ 1078 first_tcb->desc_num++; 1079 } 1080 first_tcb->last_index = PREV_INDEX(index, 1, tx_ring->ring_size); 1081 1082 /* 1083 * The Insert Ethernet CRC (IFCS) bit and the checksum fields are only 1084 * valid in the first descriptor of the packet. 1085 * Setting paylen in every first_tbd for all parts. 1086 * 82599, X540 and X550 require the packet length in paylen field 1087 * with or without LSO and 82598 will ignore it in non-LSO mode. 1088 */ 1089 ASSERT(first_tbd != NULL); 1090 first_tbd->read.cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS; 1091 1092 switch (hw->mac.type) { 1093 case ixgbe_mac_82598EB: 1094 if (ctx != NULL && ctx->lso_flag) { 1095 first_tbd->read.cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1096 first_tbd->read.olinfo_status |= 1097 (mbsize - ctx->mac_hdr_len - ctx->ip_hdr_len 1098 - ctx->l4_hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT; 1099 } 1100 break; 1101 1102 case ixgbe_mac_82599EB: 1103 case ixgbe_mac_X540: 1104 case ixgbe_mac_X550: 1105 case ixgbe_mac_X550EM_x: 1106 if (ctx != NULL && ctx->lso_flag) { 1107 first_tbd->read.cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1108 first_tbd->read.olinfo_status |= 1109 (mbsize - ctx->mac_hdr_len - ctx->ip_hdr_len 1110 - ctx->l4_hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT; 1111 } else { 1112 first_tbd->read.olinfo_status |= 1113 (mbsize << IXGBE_ADVTXD_PAYLEN_SHIFT); 1114 } 1115 break; 1116 1117 default: 1118 break; 1119 } 1120 1121 /* Set hardware checksum bits */ 1122 if (hcksum_flags != 0) { 1123 if (hcksum_flags & HCK_IPV4_HDRCKSUM) 1124 first_tbd->read.olinfo_status |= 1125 IXGBE_ADVTXD_POPTS_IXSM; 1126 if (hcksum_flags & HCK_PARTIALCKSUM) 1127 first_tbd->read.olinfo_status |= 1128 IXGBE_ADVTXD_POPTS_TXSM; 1129 } 1130 1131 /* 1132 * The last descriptor of packet needs End Of Packet (EOP), 1133 * and Report Status (RS) bits set 1134 */ 1135 ASSERT(tbd != NULL); 1136 tbd->read.cmd_type_len |= 1137 IXGBE_ADVTXD_DCMD_EOP | IXGBE_ADVTXD_DCMD_RS; 1138 1139 /* 1140 * Sync the DMA buffer of the tx descriptor ring 1141 */ 1142 DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORDEV); 1143 1144 /* 1145 * Update the number of the free tx descriptors. 1146 * The mutual exclusion between the transmission and the recycling 1147 * (for the tx descriptor ring and the work list) is implemented 1148 * with the atomic operation on the number of the free tx descriptors. 1149 * 1150 * Note: we should always decrement the counter tbd_free before 1151 * advancing the hardware TDT pointer to avoid the race condition - 1152 * before the counter tbd_free is decremented, the transmit of the 1153 * tx descriptors has done and the counter tbd_free is increased by 1154 * the tx recycling. 1155 */ 1156 i = ixgbe_atomic_reserve(&tx_ring->tbd_free, desc_num); 1157 ASSERT(i >= 0); 1158 1159 tx_ring->tbd_tail = index; 1160 1161 /* 1162 * Advance the hardware TDT pointer of the tx descriptor ring 1163 */ 1164 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), index); 1165 1166 if (ixgbe_check_acc_handle(tx_ring->ixgbe->osdep.reg_handle) != 1167 DDI_FM_OK) { 1168 ddi_fm_service_impact(tx_ring->ixgbe->dip, 1169 DDI_SERVICE_DEGRADED); 1170 atomic_or_32(&tx_ring->ixgbe->ixgbe_state, IXGBE_ERROR); 1171 } 1172 1173 return (desc_num); 1174 } 1175 1176 /* 1177 * ixgbe_save_desc 1178 * 1179 * Save the address/length pair to the private array 1180 * of the tx control block. The address/length pairs 1181 * will be filled into the tx descriptor ring later. 1182 */ 1183 static void 1184 ixgbe_save_desc(tx_control_block_t *tcb, uint64_t address, size_t length) 1185 { 1186 sw_desc_t *desc; 1187 1188 desc = &tcb->desc[tcb->desc_num]; 1189 desc->address = address; 1190 desc->length = length; 1191 1192 tcb->desc_num++; 1193 } 1194 1195 /* 1196 * ixgbe_tx_recycle_legacy 1197 * 1198 * Recycle the tx descriptors and tx control blocks. 1199 * 1200 * The work list is traversed to check if the corresponding 1201 * tx descriptors have been transmitted. If so, the resources 1202 * bound to the tx control blocks will be freed, and those 1203 * tx control blocks will be returned to the free list. 1204 */ 1205 uint32_t 1206 ixgbe_tx_recycle_legacy(ixgbe_tx_ring_t *tx_ring) 1207 { 1208 uint32_t index, last_index, prev_index; 1209 int desc_num; 1210 boolean_t desc_done; 1211 tx_control_block_t *tcb; 1212 link_list_t pending_list; 1213 ixgbe_t *ixgbe = tx_ring->ixgbe; 1214 1215 mutex_enter(&tx_ring->recycle_lock); 1216 1217 ASSERT(tx_ring->tbd_free <= tx_ring->ring_size); 1218 1219 if (tx_ring->tbd_free == tx_ring->ring_size) { 1220 tx_ring->recycle_fail = 0; 1221 tx_ring->stall_watchdog = 0; 1222 if (tx_ring->reschedule) { 1223 tx_ring->reschedule = B_FALSE; 1224 mac_tx_ring_update(ixgbe->mac_hdl, 1225 tx_ring->ring_handle); 1226 } 1227 mutex_exit(&tx_ring->recycle_lock); 1228 return (0); 1229 } 1230 1231 /* 1232 * Sync the DMA buffer of the tx descriptor ring 1233 */ 1234 DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL); 1235 1236 if (ixgbe_check_dma_handle(tx_ring->tbd_area.dma_handle) != DDI_FM_OK) { 1237 mutex_exit(&tx_ring->recycle_lock); 1238 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 1239 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 1240 return (0); 1241 } 1242 1243 LINK_LIST_INIT(&pending_list); 1244 desc_num = 0; 1245 index = tx_ring->tbd_head; /* Index of next tbd/tcb to recycle */ 1246 1247 tcb = tx_ring->work_list[index]; 1248 ASSERT(tcb != NULL); 1249 1250 while (tcb != NULL) { 1251 /* 1252 * Get the last tx descriptor of this packet. 1253 * If the last tx descriptor is done, then 1254 * we can recycle all descriptors of a packet 1255 * which usually includes several tx control blocks. 1256 * For 82599, LSO descriptors can not be recycled 1257 * unless the whole packet's transmission is done. 1258 * That's why packet level recycling is used here. 1259 * For 82598, there's not such limit. 1260 */ 1261 last_index = tcb->last_index; 1262 /* 1263 * MAX_TX_RING_SIZE is used to judge whether 1264 * the index is a valid value or not. 1265 */ 1266 if (last_index == MAX_TX_RING_SIZE) 1267 break; 1268 1269 /* 1270 * Check if the Descriptor Done bit is set 1271 */ 1272 desc_done = tx_ring->tbd_ring[last_index].wb.status & 1273 IXGBE_TXD_STAT_DD; 1274 if (desc_done) { 1275 /* 1276 * recycle all descriptors of the packet 1277 */ 1278 while (tcb != NULL) { 1279 /* 1280 * Strip off the tx control block from 1281 * the work list, and add it to the 1282 * pending list. 1283 */ 1284 tx_ring->work_list[index] = NULL; 1285 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1286 1287 /* 1288 * Count the total number of the tx 1289 * descriptors recycled 1290 */ 1291 desc_num += tcb->desc_num; 1292 1293 index = NEXT_INDEX(index, tcb->desc_num, 1294 tx_ring->ring_size); 1295 1296 tcb = tx_ring->work_list[index]; 1297 1298 prev_index = PREV_INDEX(index, 1, 1299 tx_ring->ring_size); 1300 if (prev_index == last_index) 1301 break; 1302 } 1303 } else { 1304 break; 1305 } 1306 } 1307 1308 /* 1309 * If no tx descriptors are recycled, no need to do more processing 1310 */ 1311 if (desc_num == 0) { 1312 tx_ring->recycle_fail++; 1313 mutex_exit(&tx_ring->recycle_lock); 1314 return (0); 1315 } 1316 1317 tx_ring->recycle_fail = 0; 1318 tx_ring->stall_watchdog = 0; 1319 1320 /* 1321 * Update the head index of the tx descriptor ring 1322 */ 1323 tx_ring->tbd_head = index; 1324 1325 /* 1326 * Update the number of the free tx descriptors with atomic operations 1327 */ 1328 atomic_add_32(&tx_ring->tbd_free, desc_num); 1329 1330 if ((tx_ring->tbd_free >= ixgbe->tx_resched_thresh) && 1331 (tx_ring->reschedule)) { 1332 tx_ring->reschedule = B_FALSE; 1333 mac_tx_ring_update(ixgbe->mac_hdl, 1334 tx_ring->ring_handle); 1335 } 1336 mutex_exit(&tx_ring->recycle_lock); 1337 1338 /* 1339 * Free the resources used by the tx control blocks 1340 * in the pending list 1341 */ 1342 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 1343 while (tcb != NULL) { 1344 /* 1345 * Release the resources occupied by the tx control block 1346 */ 1347 ixgbe_free_tcb(tcb); 1348 1349 tcb = (tx_control_block_t *) 1350 LIST_GET_NEXT(&pending_list, &tcb->link); 1351 } 1352 1353 /* 1354 * Add the tx control blocks in the pending list to the free list. 1355 */ 1356 ixgbe_put_free_list(tx_ring, &pending_list); 1357 1358 return (desc_num); 1359 } 1360 1361 /* 1362 * ixgbe_tx_recycle_head_wb 1363 * 1364 * Check the head write-back, and recycle all the transmitted 1365 * tx descriptors and tx control blocks. 1366 */ 1367 uint32_t 1368 ixgbe_tx_recycle_head_wb(ixgbe_tx_ring_t *tx_ring) 1369 { 1370 uint32_t index; 1371 uint32_t head_wb; 1372 int desc_num; 1373 tx_control_block_t *tcb; 1374 link_list_t pending_list; 1375 ixgbe_t *ixgbe = tx_ring->ixgbe; 1376 1377 mutex_enter(&tx_ring->recycle_lock); 1378 1379 ASSERT(tx_ring->tbd_free <= tx_ring->ring_size); 1380 1381 if (tx_ring->tbd_free == tx_ring->ring_size) { 1382 tx_ring->recycle_fail = 0; 1383 tx_ring->stall_watchdog = 0; 1384 if (tx_ring->reschedule) { 1385 tx_ring->reschedule = B_FALSE; 1386 mac_tx_ring_update(ixgbe->mac_hdl, 1387 tx_ring->ring_handle); 1388 } 1389 mutex_exit(&tx_ring->recycle_lock); 1390 return (0); 1391 } 1392 1393 /* 1394 * Sync the DMA buffer of the tx descriptor ring 1395 * 1396 * Note: For head write-back mode, the tx descriptors will not 1397 * be written back, but the head write-back value is stored at 1398 * the last extra tbd at the end of the DMA area, we still need 1399 * to sync the head write-back value for kernel. 1400 * 1401 * DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL); 1402 */ 1403 (void) ddi_dma_sync(tx_ring->tbd_area.dma_handle, 1404 sizeof (union ixgbe_adv_tx_desc) * tx_ring->ring_size, 1405 sizeof (uint32_t), 1406 DDI_DMA_SYNC_FORKERNEL); 1407 1408 if (ixgbe_check_dma_handle(tx_ring->tbd_area.dma_handle) != DDI_FM_OK) { 1409 mutex_exit(&tx_ring->recycle_lock); 1410 ddi_fm_service_impact(ixgbe->dip, 1411 DDI_SERVICE_DEGRADED); 1412 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 1413 return (0); 1414 } 1415 1416 LINK_LIST_INIT(&pending_list); 1417 desc_num = 0; 1418 index = tx_ring->tbd_head; /* Next index to clean */ 1419 1420 /* 1421 * Get the value of head write-back 1422 */ 1423 head_wb = *tx_ring->tbd_head_wb; 1424 while (index != head_wb) { 1425 tcb = tx_ring->work_list[index]; 1426 ASSERT(tcb != NULL); 1427 1428 if (OFFSET(index, head_wb, tx_ring->ring_size) < 1429 tcb->desc_num) { 1430 /* 1431 * The current tx control block is not 1432 * completely transmitted, stop recycling 1433 */ 1434 break; 1435 } 1436 1437 /* 1438 * Strip off the tx control block from the work list, 1439 * and add it to the pending list. 1440 */ 1441 tx_ring->work_list[index] = NULL; 1442 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1443 1444 /* 1445 * Advance the index of the tx descriptor ring 1446 */ 1447 index = NEXT_INDEX(index, tcb->desc_num, tx_ring->ring_size); 1448 1449 /* 1450 * Count the total number of the tx descriptors recycled 1451 */ 1452 desc_num += tcb->desc_num; 1453 } 1454 1455 /* 1456 * If no tx descriptors are recycled, no need to do more processing 1457 */ 1458 if (desc_num == 0) { 1459 tx_ring->recycle_fail++; 1460 mutex_exit(&tx_ring->recycle_lock); 1461 return (0); 1462 } 1463 1464 tx_ring->recycle_fail = 0; 1465 tx_ring->stall_watchdog = 0; 1466 1467 /* 1468 * Update the head index of the tx descriptor ring 1469 */ 1470 tx_ring->tbd_head = index; 1471 1472 /* 1473 * Update the number of the free tx descriptors with atomic operations 1474 */ 1475 atomic_add_32(&tx_ring->tbd_free, desc_num); 1476 1477 if ((tx_ring->tbd_free >= ixgbe->tx_resched_thresh) && 1478 (tx_ring->reschedule)) { 1479 tx_ring->reschedule = B_FALSE; 1480 mac_tx_ring_update(ixgbe->mac_hdl, 1481 tx_ring->ring_handle); 1482 } 1483 mutex_exit(&tx_ring->recycle_lock); 1484 1485 /* 1486 * Free the resources used by the tx control blocks 1487 * in the pending list 1488 */ 1489 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list); 1490 while (tcb) { 1491 /* 1492 * Release the resources occupied by the tx control block 1493 */ 1494 ixgbe_free_tcb(tcb); 1495 1496 tcb = (tx_control_block_t *) 1497 LIST_GET_NEXT(&pending_list, &tcb->link); 1498 } 1499 1500 /* 1501 * Add the tx control blocks in the pending list to the free list. 1502 */ 1503 ixgbe_put_free_list(tx_ring, &pending_list); 1504 1505 return (desc_num); 1506 } 1507 1508 /* 1509 * ixgbe_free_tcb - free up the tx control block 1510 * 1511 * Free the resources of the tx control block, including 1512 * unbind the previously bound DMA handle, and reset other 1513 * control fields. 1514 */ 1515 void 1516 ixgbe_free_tcb(tx_control_block_t *tcb) 1517 { 1518 switch (tcb->tx_type) { 1519 case USE_COPY: 1520 /* 1521 * Reset the buffer length that is used for copy 1522 */ 1523 tcb->tx_buf.len = 0; 1524 break; 1525 case USE_DMA: 1526 /* 1527 * Release the DMA resource that is used for 1528 * DMA binding. 1529 */ 1530 (void) ddi_dma_unbind_handle(tcb->tx_dma_handle); 1531 break; 1532 default: 1533 break; 1534 } 1535 1536 /* 1537 * Free the mblk 1538 */ 1539 if (tcb->mp != NULL) { 1540 freemsg(tcb->mp); 1541 tcb->mp = NULL; 1542 } 1543 1544 tcb->tx_type = USE_NONE; 1545 tcb->last_index = MAX_TX_RING_SIZE; 1546 tcb->frag_num = 0; 1547 tcb->desc_num = 0; 1548 } 1549 1550 /* 1551 * ixgbe_get_free_list - Get a free tx control block from the free list 1552 * 1553 * The atomic operation on the number of the available tx control block 1554 * in the free list is used to keep this routine mutual exclusive with 1555 * the routine ixgbe_put_check_list. 1556 */ 1557 static tx_control_block_t * 1558 ixgbe_get_free_list(ixgbe_tx_ring_t *tx_ring) 1559 { 1560 tx_control_block_t *tcb; 1561 1562 /* 1563 * Check and update the number of the free tx control block 1564 * in the free list. 1565 */ 1566 if (ixgbe_atomic_reserve(&tx_ring->tcb_free, 1) < 0) 1567 return (NULL); 1568 1569 mutex_enter(&tx_ring->tcb_head_lock); 1570 1571 tcb = tx_ring->free_list[tx_ring->tcb_head]; 1572 ASSERT(tcb != NULL); 1573 tx_ring->free_list[tx_ring->tcb_head] = NULL; 1574 tx_ring->tcb_head = NEXT_INDEX(tx_ring->tcb_head, 1, 1575 tx_ring->free_list_size); 1576 1577 mutex_exit(&tx_ring->tcb_head_lock); 1578 1579 return (tcb); 1580 } 1581 1582 /* 1583 * ixgbe_put_free_list 1584 * 1585 * Put a list of used tx control blocks back to the free list 1586 * 1587 * A mutex is used here to ensure the serialization. The mutual exclusion 1588 * between ixgbe_get_free_list and ixgbe_put_free_list is implemented with 1589 * the atomic operation on the counter tcb_free. 1590 */ 1591 void 1592 ixgbe_put_free_list(ixgbe_tx_ring_t *tx_ring, link_list_t *pending_list) 1593 { 1594 uint32_t index; 1595 int tcb_num; 1596 tx_control_block_t *tcb; 1597 1598 mutex_enter(&tx_ring->tcb_tail_lock); 1599 1600 index = tx_ring->tcb_tail; 1601 1602 tcb_num = 0; 1603 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 1604 while (tcb != NULL) { 1605 ASSERT(tx_ring->free_list[index] == NULL); 1606 tx_ring->free_list[index] = tcb; 1607 1608 tcb_num++; 1609 1610 index = NEXT_INDEX(index, 1, tx_ring->free_list_size); 1611 1612 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list); 1613 } 1614 1615 tx_ring->tcb_tail = index; 1616 1617 /* 1618 * Update the number of the free tx control block 1619 * in the free list. This operation must be placed 1620 * under the protection of the lock. 1621 */ 1622 atomic_add_32(&tx_ring->tcb_free, tcb_num); 1623 1624 mutex_exit(&tx_ring->tcb_tail_lock); 1625 } 1626