1 /* 2 * This file is provided under a CDDLv1 license. When using or 3 * redistributing this file, you may do so under this license. 4 * In redistributing this file this license must be included 5 * and no other modification of this header file is permitted. 6 * 7 * CDDL LICENSE SUMMARY 8 * 9 * Copyright(c) 1999 - 2008 Intel Corporation. All rights reserved. 10 * 11 * The contents of this file are subject to the terms of Version 12 * 1.0 of the Common Development and Distribution License (the "License"). 13 * 14 * You should have received a copy of the License with this software. 15 * You can obtain a copy of the License at 16 * http://www.opensolaris.org/os/licensing. 17 * See the License for the specific language governing permissions 18 * and limitations under the License. 19 */ 20 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms of the CDDLv1. 24 */ 25 26 /* 27 * ********************************************************************** 28 * * 29 * Module Name: * 30 * e1000g_tx.c * 31 * * 32 * Abstract: * 33 * This file contains some routines that take care of Transmit, * 34 * make the hardware to send the data pointed by the packet out * 35 * on to the physical medium. * 36 * * 37 * ********************************************************************** 38 */ 39 40 #include "e1000g_sw.h" 41 #include "e1000g_debug.h" 42 43 static boolean_t e1000g_send(struct e1000g *, mblk_t *); 44 static int e1000g_tx_copy(e1000g_tx_ring_t *, 45 p_tx_sw_packet_t, mblk_t *, boolean_t); 46 static int e1000g_tx_bind(e1000g_tx_ring_t *, 47 p_tx_sw_packet_t, mblk_t *); 48 static boolean_t e1000g_retrieve_context(mblk_t *, context_data_t *, size_t); 49 static boolean_t e1000g_check_context(e1000g_tx_ring_t *, context_data_t *); 50 static int e1000g_fill_tx_ring(e1000g_tx_ring_t *, LIST_DESCRIBER *, 51 context_data_t *); 52 static void e1000g_fill_context_descriptor(context_data_t *, 53 struct e1000_context_desc *); 54 static int e1000g_fill_tx_desc(e1000g_tx_ring_t *, 55 p_tx_sw_packet_t, uint64_t, size_t); 56 static uint32_t e1000g_fill_82544_desc(uint64_t Address, size_t Length, 57 p_desc_array_t desc_array); 58 static int e1000g_tx_workaround_PCIX_82544(p_tx_sw_packet_t, uint64_t, size_t); 59 static int e1000g_tx_workaround_jumbo_82544(p_tx_sw_packet_t, uint64_t, size_t); 60 static void e1000g_82547_timeout(void *); 61 static void e1000g_82547_tx_move_tail(e1000g_tx_ring_t *); 62 static void e1000g_82547_tx_move_tail_work(e1000g_tx_ring_t *); 63 64 #ifndef E1000G_DEBUG 65 #pragma inline(e1000g_tx_copy) 66 #pragma inline(e1000g_tx_bind) 67 #pragma inline(e1000g_retrieve_context) 68 #pragma inline(e1000g_check_context) 69 #pragma inline(e1000g_fill_tx_ring) 70 #pragma inline(e1000g_fill_context_descriptor) 71 #pragma inline(e1000g_fill_tx_desc) 72 #pragma inline(e1000g_fill_82544_desc) 73 #pragma inline(e1000g_tx_workaround_PCIX_82544) 74 #pragma inline(e1000g_tx_workaround_jumbo_82544) 75 #pragma inline(e1000g_free_tx_swpkt) 76 #endif 77 78 /* 79 * e1000g_free_tx_swpkt - free up the tx sw packet 80 * 81 * Unbind the previously bound DMA handle for a given 82 * transmit sw packet. And reset the sw packet data. 83 */ 84 void 85 e1000g_free_tx_swpkt(register p_tx_sw_packet_t packet) 86 { 87 switch (packet->data_transfer_type) { 88 case USE_BCOPY: 89 packet->tx_buf->len = 0; 90 break; 91 #ifdef __sparc 92 case USE_DVMA: 93 dvma_unload(packet->tx_dma_handle, 0, -1); 94 break; 95 #endif 96 case USE_DMA: 97 (void) ddi_dma_unbind_handle(packet->tx_dma_handle); 98 break; 99 default: 100 break; 101 } 102 103 /* 104 * The mblk has been stripped off the sw packet 105 * and will be freed in a triggered soft intr. 106 */ 107 ASSERT(packet->mp == NULL); 108 109 packet->data_transfer_type = USE_NONE; 110 packet->num_mblk_frag = 0; 111 packet->num_desc = 0; 112 } 113 114 mblk_t * 115 e1000g_m_tx(void *arg, mblk_t *mp) 116 { 117 struct e1000g *Adapter = (struct e1000g *)arg; 118 mblk_t *next; 119 120 rw_enter(&Adapter->chip_lock, RW_READER); 121 122 if ((Adapter->chip_state != E1000G_START) || 123 (Adapter->link_state != LINK_STATE_UP)) { 124 freemsgchain(mp); 125 mp = NULL; 126 } 127 128 while (mp != NULL) { 129 next = mp->b_next; 130 mp->b_next = NULL; 131 132 if (!e1000g_send(Adapter, mp)) { 133 mp->b_next = next; 134 break; 135 } 136 137 mp = next; 138 } 139 140 rw_exit(&Adapter->chip_lock); 141 return (mp); 142 } 143 144 /* 145 * e1000g_send - send packets onto the wire 146 * 147 * Called from e1000g_m_tx with an mblk ready to send. this 148 * routine sets up the transmit descriptors and sends data to 149 * the wire. It also pushes the just transmitted packet to 150 * the used tx sw packet list. 151 */ 152 static boolean_t 153 e1000g_send(struct e1000g *Adapter, mblk_t *mp) 154 { 155 p_tx_sw_packet_t packet; 156 LIST_DESCRIBER pending_list; 157 size_t len; 158 size_t msg_size; 159 uint32_t frag_count; 160 int desc_count; 161 uint32_t desc_total; 162 uint32_t bcopy_thresh; 163 uint32_t hdr_frag_len; 164 boolean_t tx_undersize_flag; 165 mblk_t *nmp; 166 mblk_t *tmp; 167 mblk_t *new_mp; 168 mblk_t *pre_mp; 169 e1000g_tx_ring_t *tx_ring; 170 context_data_t cur_context; 171 172 tx_ring = Adapter->tx_ring; 173 bcopy_thresh = Adapter->tx_bcopy_thresh; 174 175 /* Get the total size and frags number of the message */ 176 tx_undersize_flag = B_FALSE; 177 frag_count = 0; 178 msg_size = 0; 179 for (nmp = mp; nmp; nmp = nmp->b_cont) { 180 frag_count++; 181 msg_size += MBLKL(nmp); 182 } 183 184 /* retrieve and compute information for context descriptor */ 185 if (!e1000g_retrieve_context(mp, &cur_context, msg_size)) { 186 freemsg(mp); 187 return (B_TRUE); 188 } 189 190 /* 191 * Make sure the packet is less than the allowed size 192 */ 193 if (!cur_context.lso_flag && 194 (msg_size > Adapter->max_frame_size - ETHERFCSL)) { 195 /* 196 * For the over size packet, we'll just drop it. 197 * So we return B_TRUE here. 198 */ 199 E1000G_DEBUGLOG_1(Adapter, E1000G_WARN_LEVEL, 200 "Tx packet out of bound. length = %d \n", msg_size); 201 E1000G_STAT(tx_ring->stat_over_size); 202 freemsg(mp); 203 return (B_TRUE); 204 } 205 206 /* 207 * Check and reclaim tx descriptors. 208 * This low water mark check should be done all the time as 209 * Transmit interrupt delay can produce Transmit interrupts little 210 * late and that may cause few problems related to reaping Tx 211 * Descriptors... As you may run short of them before getting any 212 * transmit interrupt... 213 */ 214 if (tx_ring->resched_needed || 215 (tx_ring->tbd_avail < Adapter->tx_recycle_thresh)) { 216 (void) e1000g_recycle(tx_ring); 217 E1000G_DEBUG_STAT(tx_ring->stat_recycle); 218 219 if (tx_ring->tbd_avail < DEFAULT_TX_NO_RESOURCE) { 220 E1000G_DEBUG_STAT(tx_ring->stat_lack_desc); 221 goto tx_no_resource; 222 } 223 } 224 225 /* 226 * If the message size is less than the minimum ethernet packet size, 227 * we'll use bcopy to send it, and padd it to 60 bytes later. 228 */ 229 if (msg_size < ETHERMIN) { 230 E1000G_DEBUG_STAT(tx_ring->stat_under_size); 231 tx_undersize_flag = B_TRUE; 232 } 233 234 /* Initialize variables */ 235 desc_count = 1; /* The initial value should be greater than 0 */ 236 desc_total = 0; 237 QUEUE_INIT_LIST(&pending_list); 238 239 /* Process each mblk fragment and fill tx descriptors */ 240 /* 241 * The software should guarantee LSO packet header(MAC+IP+TCP) 242 * to be within one descriptor. Here we reallocate and refill the 243 * the header if it's physical memory non-contiguous. 244 */ 245 if (cur_context.lso_flag) { 246 /* find the last fragment of the header */ 247 len = MBLKL(mp); 248 ASSERT(len > 0); 249 nmp = mp; 250 pre_mp = NULL; 251 while (len < cur_context.hdr_len) { 252 pre_mp = nmp; 253 nmp = nmp->b_cont; 254 len += MBLKL(nmp); 255 } 256 /* 257 * If the header and the payload are in different mblks, 258 * we simply force the header to be copied into pre-allocated 259 * page-aligned buffer. 260 */ 261 if (len == cur_context.hdr_len) 262 goto adjust_threshold; 263 264 hdr_frag_len = cur_context.hdr_len - (len - MBLKL(nmp)); 265 /* 266 * There are two cases we need to reallocate a mblk for the 267 * last header fragment: 268 * 1. the header is in multiple mblks and the last fragment 269 * share the same mblk with the payload 270 * 2. the header is in a single mblk shared with the payload 271 * and the header is physical memory non-contiguous 272 */ 273 if ((nmp != mp) || 274 (P2NPHASE((uintptr_t)nmp->b_rptr, Adapter->sys_page_sz) 275 < len)) { 276 E1000G_DEBUG_STAT(tx_ring->stat_lso_header_fail); 277 /* 278 * reallocate the mblk for the last header fragment, 279 * expect to bcopy into pre-allocated page-aligned 280 * buffer 281 */ 282 new_mp = allocb(hdr_frag_len, NULL); 283 if (!new_mp) 284 return (B_FALSE); 285 bcopy(nmp->b_rptr, new_mp->b_rptr, hdr_frag_len); 286 /* link the new header fragment with the other parts */ 287 new_mp->b_wptr = new_mp->b_rptr + hdr_frag_len; 288 new_mp->b_cont = nmp; 289 if (pre_mp) 290 pre_mp->b_cont = new_mp; 291 nmp->b_rptr += hdr_frag_len; 292 if (hdr_frag_len == cur_context.hdr_len) 293 mp = new_mp; 294 frag_count ++; 295 } 296 adjust_threshold: 297 /* 298 * adjust the bcopy threshhold to guarantee 299 * the header to use bcopy way 300 */ 301 if (bcopy_thresh < cur_context.hdr_len) 302 bcopy_thresh = cur_context.hdr_len; 303 } 304 305 packet = NULL; 306 nmp = mp; 307 while (nmp) { 308 tmp = nmp->b_cont; 309 310 len = MBLKL(nmp); 311 /* Check zero length mblks */ 312 if (len == 0) { 313 E1000G_DEBUG_STAT(tx_ring->stat_empty_frags); 314 /* 315 * If there're no packet buffers have been used, 316 * or we just completed processing a buffer, then 317 * skip the empty mblk fragment. 318 * Otherwise, there's still a pending buffer that 319 * needs to be processed (tx_copy). 320 */ 321 if (desc_count > 0) { 322 nmp = tmp; 323 continue; 324 } 325 } 326 327 /* 328 * Get a new TxSwPacket to process mblk buffers. 329 */ 330 if (desc_count > 0) { 331 mutex_enter(&tx_ring->freelist_lock); 332 packet = (p_tx_sw_packet_t) 333 QUEUE_POP_HEAD(&tx_ring->free_list); 334 mutex_exit(&tx_ring->freelist_lock); 335 336 if (packet == NULL) { 337 E1000G_DEBUGLOG_0(Adapter, E1000G_INFO_LEVEL, 338 "No Tx SwPacket available\n"); 339 E1000G_STAT(tx_ring->stat_no_swpkt); 340 goto tx_send_failed; 341 } 342 QUEUE_PUSH_TAIL(&pending_list, &packet->Link); 343 } 344 345 ASSERT(packet); 346 /* 347 * If the size of the fragment is less than the tx_bcopy_thresh 348 * we'll use bcopy; Otherwise, we'll use DMA binding. 349 */ 350 if ((len <= bcopy_thresh) || tx_undersize_flag) { 351 desc_count = 352 e1000g_tx_copy(tx_ring, packet, nmp, 353 tx_undersize_flag); 354 E1000G_DEBUG_STAT(tx_ring->stat_copy); 355 } else { 356 desc_count = 357 e1000g_tx_bind(tx_ring, packet, nmp); 358 E1000G_DEBUG_STAT(tx_ring->stat_bind); 359 } 360 361 if (desc_count > 0) 362 desc_total += desc_count; 363 else if (desc_count < 0) 364 goto tx_send_failed; 365 366 nmp = tmp; 367 } 368 369 /* Assign the message to the last sw packet */ 370 ASSERT(packet); 371 ASSERT(packet->mp == NULL); 372 packet->mp = mp; 373 374 /* Try to recycle the tx descriptors again */ 375 if (tx_ring->tbd_avail < (desc_total + 2)) { 376 E1000G_DEBUG_STAT(tx_ring->stat_recycle_retry); 377 (void) e1000g_recycle(tx_ring); 378 } 379 380 mutex_enter(&tx_ring->tx_lock); 381 382 /* 383 * If the number of available tx descriptors is not enough for transmit 384 * (one redundant descriptor and one hw checksum context descriptor are 385 * included), then return failure. 386 */ 387 if (tx_ring->tbd_avail < (desc_total + 2)) { 388 E1000G_DEBUGLOG_0(Adapter, E1000G_INFO_LEVEL, 389 "No Enough Tx descriptors\n"); 390 E1000G_STAT(tx_ring->stat_no_desc); 391 mutex_exit(&tx_ring->tx_lock); 392 goto tx_send_failed; 393 } 394 395 desc_count = e1000g_fill_tx_ring(tx_ring, &pending_list, &cur_context); 396 397 mutex_exit(&tx_ring->tx_lock); 398 399 ASSERT(desc_count > 0); 400 401 /* Send successful */ 402 return (B_TRUE); 403 404 tx_send_failed: 405 /* 406 * Enable Transmit interrupts, so that the interrupt routine can 407 * call mac_tx_update() when transmit descriptors become available. 408 */ 409 tx_ring->resched_needed = B_TRUE; 410 if (!Adapter->tx_intr_enable) 411 e1000g_mask_tx_interrupt(Adapter); 412 413 /* Free pending TxSwPackets */ 414 packet = (p_tx_sw_packet_t)QUEUE_GET_HEAD(&pending_list); 415 while (packet) { 416 packet->mp = NULL; 417 e1000g_free_tx_swpkt(packet); 418 packet = (p_tx_sw_packet_t) 419 QUEUE_GET_NEXT(&pending_list, &packet->Link); 420 } 421 422 /* Return pending TxSwPackets to the "Free" list */ 423 mutex_enter(&tx_ring->freelist_lock); 424 QUEUE_APPEND(&tx_ring->free_list, &pending_list); 425 mutex_exit(&tx_ring->freelist_lock); 426 427 E1000G_STAT(tx_ring->stat_send_fail); 428 429 /* Message will be scheduled for re-transmit */ 430 return (B_FALSE); 431 432 tx_no_resource: 433 /* 434 * Enable Transmit interrupts, so that the interrupt routine can 435 * call mac_tx_update() when transmit descriptors become available. 436 */ 437 tx_ring->resched_needed = B_TRUE; 438 if (!Adapter->tx_intr_enable) 439 e1000g_mask_tx_interrupt(Adapter); 440 441 /* Message will be scheduled for re-transmit */ 442 return (B_FALSE); 443 } 444 445 static boolean_t 446 e1000g_retrieve_context(mblk_t *mp, context_data_t *cur_context, 447 size_t msg_size) 448 { 449 uintptr_t ip_start; 450 uintptr_t tcp_start; 451 mblk_t *nmp; 452 453 bzero(cur_context, sizeof (context_data_t)); 454 455 /* retrieve checksum info */ 456 hcksum_retrieve(mp, NULL, NULL, &cur_context->cksum_start, 457 &cur_context->cksum_stuff, NULL, NULL, &cur_context->cksum_flags); 458 /* retrieve ethernet header size */ 459 if (((struct ether_vlan_header *)(uintptr_t)mp->b_rptr)->ether_tpid == 460 htons(ETHERTYPE_VLAN)) 461 cur_context->ether_header_size = 462 sizeof (struct ether_vlan_header); 463 else 464 cur_context->ether_header_size = 465 sizeof (struct ether_header); 466 467 if (cur_context->cksum_flags & HW_LSO) { 468 if ((cur_context->mss = DB_LSOMSS(mp)) != 0) { 469 /* free the invaid packet */ 470 if (!((cur_context->cksum_flags & HCK_PARTIALCKSUM) && 471 (cur_context->cksum_flags & HCK_IPV4_HDRCKSUM))) { 472 return (B_FALSE); 473 } 474 cur_context->lso_flag = B_TRUE; 475 /* 476 * Some fields are cleared for the hardware to fill 477 * in. We don't assume Ethernet header, IP header and 478 * TCP header are always in the same mblk fragment, 479 * while we assume each header is always within one 480 * mblk fragment and Ethernet header is always in the 481 * first mblk fragment. 482 */ 483 nmp = mp; 484 ip_start = (uintptr_t)(nmp->b_rptr) 485 + cur_context->ether_header_size; 486 if (ip_start >= (uintptr_t)(nmp->b_wptr)) { 487 ip_start = (uintptr_t)nmp->b_cont->b_rptr 488 + (ip_start - (uintptr_t)(nmp->b_wptr)); 489 nmp = nmp->b_cont; 490 } 491 tcp_start = ip_start + 492 IPH_HDR_LENGTH((ipha_t *)ip_start); 493 if (tcp_start >= (uintptr_t)(nmp->b_wptr)) { 494 tcp_start = (uintptr_t)nmp->b_cont->b_rptr 495 + (tcp_start - (uintptr_t)(nmp->b_wptr)); 496 nmp = nmp->b_cont; 497 } 498 cur_context->hdr_len = cur_context->ether_header_size 499 + IPH_HDR_LENGTH((ipha_t *)ip_start) 500 + TCP_HDR_LENGTH((tcph_t *)tcp_start); 501 ((ipha_t *)ip_start)->ipha_length = 0; 502 ((ipha_t *)ip_start)->ipha_hdr_checksum = 0; 503 /* calculate the TCP packet payload length */ 504 cur_context->pay_len = msg_size - cur_context->hdr_len; 505 } 506 } 507 return (B_TRUE); 508 } 509 510 static boolean_t 511 e1000g_check_context(e1000g_tx_ring_t *tx_ring, context_data_t *cur_context) 512 { 513 boolean_t context_reload; 514 context_data_t *pre_context; 515 struct e1000g *Adapter; 516 517 context_reload = B_FALSE; 518 pre_context = &tx_ring->pre_context; 519 Adapter = tx_ring->adapter; 520 521 /* 522 * The following code determine if the context descriptor is 523 * needed to be reloaded. The sequence of the conditions is 524 * made by their possibilities of changing. 525 */ 526 /* 527 * workaround for 82546EB, context descriptor must be reloaded 528 * per LSO/hw_cksum packet if LSO is enabled. 529 */ 530 if (Adapter->lso_premature_issue && 531 Adapter->lso_enable && 532 (cur_context->cksum_flags != 0)) { 533 534 context_reload = B_TRUE; 535 } else if (cur_context->lso_flag) { 536 if ((cur_context->lso_flag != pre_context->lso_flag) || 537 (cur_context->cksum_flags != pre_context->cksum_flags) || 538 (cur_context->pay_len != pre_context->pay_len) || 539 (cur_context->mss != pre_context->mss) || 540 (cur_context->hdr_len != pre_context->hdr_len) || 541 (cur_context->cksum_stuff != pre_context->cksum_stuff) || 542 (cur_context->cksum_start != pre_context->cksum_start) || 543 (cur_context->ether_header_size != 544 pre_context->ether_header_size)) { 545 546 context_reload = B_TRUE; 547 } 548 } else if (cur_context->cksum_flags != 0) { 549 if ((cur_context->lso_flag != pre_context->lso_flag) || 550 (cur_context->cksum_flags != pre_context->cksum_flags) || 551 (cur_context->cksum_stuff != pre_context->cksum_stuff) || 552 (cur_context->cksum_start != pre_context->cksum_start) || 553 (cur_context->ether_header_size != 554 pre_context->ether_header_size)) { 555 556 context_reload = B_TRUE; 557 } 558 } 559 560 return (context_reload); 561 } 562 563 static int 564 e1000g_fill_tx_ring(e1000g_tx_ring_t *tx_ring, LIST_DESCRIBER *pending_list, 565 context_data_t *cur_context) 566 { 567 struct e1000g *Adapter; 568 struct e1000_hw *hw; 569 p_tx_sw_packet_t first_packet; 570 p_tx_sw_packet_t packet; 571 p_tx_sw_packet_t previous_packet; 572 boolean_t context_reload; 573 struct e1000_tx_desc *first_data_desc; 574 struct e1000_tx_desc *next_desc; 575 struct e1000_tx_desc *descriptor; 576 int desc_count; 577 boolean_t buff_overrun_flag; 578 int i; 579 580 Adapter = tx_ring->adapter; 581 hw = &Adapter->shared; 582 583 desc_count = 0; 584 first_packet = NULL; 585 first_data_desc = NULL; 586 descriptor = NULL; 587 first_packet = NULL; 588 packet = NULL; 589 buff_overrun_flag = B_FALSE; 590 591 next_desc = tx_ring->tbd_next; 592 593 /* Context descriptor reload check */ 594 context_reload = e1000g_check_context(tx_ring, cur_context); 595 596 if (context_reload) { 597 first_packet = (p_tx_sw_packet_t)QUEUE_GET_HEAD(pending_list); 598 599 descriptor = next_desc; 600 601 e1000g_fill_context_descriptor(cur_context, 602 (struct e1000_context_desc *)descriptor); 603 604 /* Check the wrap-around case */ 605 if (descriptor == tx_ring->tbd_last) 606 next_desc = tx_ring->tbd_first; 607 else 608 next_desc++; 609 610 desc_count++; 611 } 612 613 first_data_desc = next_desc; 614 615 packet = (p_tx_sw_packet_t)QUEUE_GET_HEAD(pending_list); 616 while (packet) { 617 ASSERT(packet->num_desc); 618 619 for (i = 0; i < packet->num_desc; i++) { 620 ASSERT(tx_ring->tbd_avail > 0); 621 622 descriptor = next_desc; 623 descriptor->buffer_addr = 624 packet->desc[i].address; 625 descriptor->lower.data = 626 packet->desc[i].length; 627 628 /* Zero out status */ 629 descriptor->upper.data = 0; 630 631 descriptor->lower.data |= 632 E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 633 /* must set RS on every outgoing descriptor */ 634 descriptor->lower.data |= 635 E1000_TXD_CMD_RS; 636 637 if (cur_context->lso_flag) 638 descriptor->lower.data |= E1000_TXD_CMD_TSE; 639 640 /* Check the wrap-around case */ 641 if (descriptor == tx_ring->tbd_last) 642 next_desc = tx_ring->tbd_first; 643 else 644 next_desc++; 645 646 desc_count++; 647 648 /* 649 * workaround for 82546EB errata 33, hang in PCI-X 650 * systems due to 2k Buffer Overrun during Transmit 651 * Operation. The workaround applies to all the Intel 652 * PCI-X chips. 653 */ 654 if (hw->bus.type == e1000_bus_type_pcix && 655 descriptor == first_data_desc && 656 ((descriptor->lower.data & E1000G_TBD_LENGTH_MASK) 657 > E1000_TX_BUFFER_OEVRRUN_THRESHOLD)) { 658 /* modified the first descriptor */ 659 descriptor->lower.data &= 660 ~E1000G_TBD_LENGTH_MASK; 661 descriptor->lower.flags.length = 662 E1000_TX_BUFFER_OEVRRUN_THRESHOLD; 663 664 /* insert a new descriptor */ 665 ASSERT(tx_ring->tbd_avail > 0); 666 next_desc->buffer_addr = 667 packet->desc[0].address + 668 E1000_TX_BUFFER_OEVRRUN_THRESHOLD; 669 next_desc->lower.data = 670 packet->desc[0].length - 671 E1000_TX_BUFFER_OEVRRUN_THRESHOLD; 672 673 /* Zero out status */ 674 next_desc->upper.data = 0; 675 676 next_desc->lower.data |= 677 E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 678 /* must set RS on every outgoing descriptor */ 679 next_desc->lower.data |= 680 E1000_TXD_CMD_RS; 681 682 if (cur_context->lso_flag) 683 next_desc->lower.data |= 684 E1000_TXD_CMD_TSE; 685 686 descriptor = next_desc; 687 688 /* Check the wrap-around case */ 689 if (next_desc == tx_ring->tbd_last) 690 next_desc = tx_ring->tbd_first; 691 else 692 next_desc++; 693 694 desc_count++; 695 buff_overrun_flag = B_TRUE; 696 } 697 } 698 699 if (buff_overrun_flag) { 700 packet->num_desc++; 701 buff_overrun_flag = B_FALSE; 702 } 703 704 if (first_packet != NULL) { 705 /* 706 * Count the checksum context descriptor for 707 * the first SwPacket. 708 */ 709 first_packet->num_desc++; 710 first_packet = NULL; 711 } 712 713 previous_packet = packet; 714 packet = (p_tx_sw_packet_t) 715 QUEUE_GET_NEXT(pending_list, &packet->Link); 716 } 717 718 /* 719 * workaround for 82546EB errata 21, LSO Premature Descriptor Write Back 720 */ 721 if (Adapter->lso_premature_issue && cur_context->lso_flag && 722 ((descriptor->lower.data & E1000G_TBD_LENGTH_MASK) > 8)) { 723 /* modified the previous descriptor */ 724 descriptor->lower.data -= 4; 725 726 /* insert a new descriptor */ 727 ASSERT(tx_ring->tbd_avail > 0); 728 /* the lower 20 bits of lower.data is the length field */ 729 next_desc->buffer_addr = 730 descriptor->buffer_addr + 731 (descriptor->lower.data & E1000G_TBD_LENGTH_MASK); 732 next_desc->lower.data = 4; 733 734 /* Zero out status */ 735 next_desc->upper.data = 0; 736 /* It must be part of a LSO packet */ 737 next_desc->lower.data |= 738 E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D | 739 E1000_TXD_CMD_RS | E1000_TXD_CMD_TSE; 740 741 descriptor = next_desc; 742 743 /* Check the wrap-around case */ 744 if (descriptor == tx_ring->tbd_last) 745 next_desc = tx_ring->tbd_first; 746 else 747 next_desc++; 748 749 desc_count++; 750 /* update the number of descriptors */ 751 previous_packet->num_desc++; 752 } 753 754 ASSERT(descriptor); 755 756 if (cur_context->cksum_flags) { 757 if (cur_context->cksum_flags & HCK_IPV4_HDRCKSUM) 758 ((struct e1000_data_desc *)first_data_desc)-> 759 upper.fields.popts |= E1000_TXD_POPTS_IXSM; 760 if (cur_context->cksum_flags & HCK_PARTIALCKSUM) 761 ((struct e1000_data_desc *)first_data_desc)-> 762 upper.fields.popts |= E1000_TXD_POPTS_TXSM; 763 } 764 765 /* 766 * Last Descriptor of Packet needs End Of Packet (EOP), Report 767 * Status (RS) set. 768 */ 769 if (Adapter->tx_intr_delay) { 770 descriptor->lower.data |= E1000_TXD_CMD_IDE | 771 E1000_TXD_CMD_EOP; 772 } else { 773 descriptor->lower.data |= E1000_TXD_CMD_EOP; 774 } 775 776 /* Set append Ethernet CRC (IFCS) bits */ 777 if (cur_context->lso_flag) { 778 first_data_desc->lower.data |= E1000_TXD_CMD_IFCS; 779 } else { 780 descriptor->lower.data |= E1000_TXD_CMD_IFCS; 781 } 782 783 /* 784 * Sync the Tx descriptors DMA buffer 785 */ 786 (void) ddi_dma_sync(tx_ring->tbd_dma_handle, 787 0, 0, DDI_DMA_SYNC_FORDEV); 788 789 tx_ring->tbd_next = next_desc; 790 791 /* 792 * Advance the Transmit Descriptor Tail (Tdt), this tells the 793 * FX1000 that this frame is available to transmit. 794 */ 795 if (hw->mac.type == e1000_82547) 796 e1000g_82547_tx_move_tail(tx_ring); 797 else 798 E1000_WRITE_REG(hw, E1000_TDT(0), 799 (uint32_t)(next_desc - tx_ring->tbd_first)); 800 801 if (e1000g_check_acc_handle(Adapter->osdep.reg_handle) != DDI_FM_OK) { 802 ddi_fm_service_impact(Adapter->dip, DDI_SERVICE_DEGRADED); 803 Adapter->chip_state = E1000G_ERROR; 804 } 805 806 /* Put the pending SwPackets to the "Used" list */ 807 mutex_enter(&tx_ring->usedlist_lock); 808 QUEUE_APPEND(&tx_ring->used_list, pending_list); 809 tx_ring->tbd_avail -= desc_count; 810 mutex_exit(&tx_ring->usedlist_lock); 811 812 /* update LSO related data */ 813 if (context_reload) 814 tx_ring->pre_context = *cur_context; 815 816 return (desc_count); 817 } 818 819 820 /* 821 * e1000g_tx_setup - setup tx data structures 822 * 823 * This routine initializes all of the transmit related 824 * structures. This includes the Transmit descriptors, 825 * and the tx_sw_packet structures. 826 */ 827 void 828 e1000g_tx_setup(struct e1000g *Adapter) 829 { 830 struct e1000_hw *hw; 831 p_tx_sw_packet_t packet; 832 uint32_t i; 833 uint32_t buf_high; 834 uint32_t buf_low; 835 uint32_t reg_tipg; 836 uint32_t reg_tctl; 837 int size; 838 e1000g_tx_ring_t *tx_ring; 839 840 hw = &Adapter->shared; 841 tx_ring = Adapter->tx_ring; 842 843 /* init the lists */ 844 /* 845 * Here we don't need to protect the lists using the 846 * usedlist_lock and freelist_lock, for they have 847 * been protected by the chip_lock. 848 */ 849 QUEUE_INIT_LIST(&tx_ring->used_list); 850 QUEUE_INIT_LIST(&tx_ring->free_list); 851 852 /* Go through and set up each SW_Packet */ 853 packet = tx_ring->packet_area; 854 for (i = 0; i < Adapter->tx_freelist_num; i++, packet++) { 855 /* Initialize this tx_sw_apcket area */ 856 e1000g_free_tx_swpkt(packet); 857 /* Add this tx_sw_packet to the free list */ 858 QUEUE_PUSH_TAIL(&tx_ring->free_list, 859 &packet->Link); 860 } 861 862 /* Setup TX descriptor pointers */ 863 tx_ring->tbd_next = tx_ring->tbd_first; 864 tx_ring->tbd_oldest = tx_ring->tbd_first; 865 866 /* 867 * Setup Hardware TX Registers 868 */ 869 /* Setup the Transmit Control Register (TCTL). */ 870 reg_tctl = E1000_READ_REG(hw, E1000_TCTL); 871 reg_tctl |= E1000_TCTL_PSP | E1000_TCTL_EN | 872 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT) | 873 (E1000_COLLISION_DISTANCE << E1000_COLD_SHIFT) | 874 E1000_TCTL_RTLC; 875 876 /* Enable the MULR bit */ 877 if (hw->bus.type == e1000_bus_type_pci_express) 878 reg_tctl |= E1000_TCTL_MULR; 879 880 E1000_WRITE_REG(hw, E1000_TCTL, reg_tctl); 881 882 /* Setup HW Base and Length of Tx descriptor area */ 883 size = (Adapter->tx_desc_num * sizeof (struct e1000_tx_desc)); 884 E1000_WRITE_REG(hw, E1000_TDLEN(0), size); 885 size = E1000_READ_REG(hw, E1000_TDLEN(0)); 886 887 buf_low = (uint32_t)tx_ring->tbd_dma_addr; 888 buf_high = (uint32_t)(tx_ring->tbd_dma_addr >> 32); 889 890 /* 891 * Write the highest location first and work backward to the lowest. 892 * This is necessary for some adapter types to 893 * prevent write combining from occurring. 894 */ 895 E1000_WRITE_REG(hw, E1000_TDBAH(0), buf_high); 896 E1000_WRITE_REG(hw, E1000_TDBAL(0), buf_low); 897 898 /* Setup our HW Tx Head & Tail descriptor pointers */ 899 E1000_WRITE_REG(hw, E1000_TDH(0), 0); 900 E1000_WRITE_REG(hw, E1000_TDT(0), 0); 901 902 /* Set the default values for the Tx Inter Packet Gap timer */ 903 if ((hw->mac.type == e1000_82542) && 904 ((hw->revision_id == E1000_REVISION_2) || 905 (hw->revision_id == E1000_REVISION_3))) { 906 reg_tipg = DEFAULT_82542_TIPG_IPGT; 907 reg_tipg |= 908 DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 909 reg_tipg |= 910 DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 911 } else if (hw->mac.type == e1000_80003es2lan) { 912 reg_tipg = DEFAULT_82543_TIPG_IPGR1; 913 reg_tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 914 E1000_TIPG_IPGR2_SHIFT; 915 } else { 916 if (hw->phy.media_type == e1000_media_type_fiber) 917 reg_tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 918 else 919 reg_tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 920 reg_tipg |= 921 DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 922 reg_tipg |= 923 DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 924 } 925 E1000_WRITE_REG(hw, E1000_TIPG, reg_tipg); 926 927 /* Setup Transmit Interrupt Delay Value */ 928 E1000_WRITE_REG(hw, E1000_TIDV, Adapter->tx_intr_delay); 929 E1000G_DEBUGLOG_1(Adapter, E1000G_INFO_LEVEL, 930 "E1000_TIDV: 0x%x\n", Adapter->tx_intr_delay); 931 932 if (hw->mac.type >= e1000_82540) { 933 E1000_WRITE_REG(&Adapter->shared, E1000_TADV, 934 Adapter->tx_intr_abs_delay); 935 E1000G_DEBUGLOG_1(Adapter, E1000G_INFO_LEVEL, 936 "E1000_TADV: 0x%x\n", Adapter->tx_intr_abs_delay); 937 } 938 939 tx_ring->tbd_avail = Adapter->tx_desc_num; 940 941 /* Initialize stored context information */ 942 bzero(&(tx_ring->pre_context), sizeof (context_data_t)); 943 } 944 945 /* 946 * e1000g_recycle - recycle the tx descriptors and tx sw packets 947 */ 948 int 949 e1000g_recycle(e1000g_tx_ring_t *tx_ring) 950 { 951 struct e1000g *Adapter; 952 LIST_DESCRIBER pending_list; 953 p_tx_sw_packet_t packet; 954 mblk_t *mp; 955 mblk_t *nmp; 956 struct e1000_tx_desc *descriptor; 957 int desc_count; 958 int is_intr; 959 960 /* 961 * This function will examine each TxSwPacket in the 'used' queue 962 * if the e1000g is done with it then the associated resources (Tx 963 * Descriptors) will be "freed" and the TxSwPacket will be 964 * returned to the 'free' queue. 965 */ 966 Adapter = tx_ring->adapter; 967 968 packet = (p_tx_sw_packet_t)QUEUE_GET_HEAD(&tx_ring->used_list); 969 if (packet == NULL) { 970 tx_ring->recycle_fail = 0; 971 tx_ring->stall_watchdog = 0; 972 return (0); 973 } 974 975 is_intr = servicing_interrupt(); 976 977 if (is_intr) 978 mutex_enter(&tx_ring->usedlist_lock); 979 else if (mutex_tryenter(&tx_ring->usedlist_lock) == 0) 980 return (0); 981 982 desc_count = 0; 983 QUEUE_INIT_LIST(&pending_list); 984 985 /* Sync the Tx descriptor DMA buffer */ 986 (void) ddi_dma_sync(tx_ring->tbd_dma_handle, 987 0, 0, DDI_DMA_SYNC_FORKERNEL); 988 if (e1000g_check_dma_handle( 989 tx_ring->tbd_dma_handle) != DDI_FM_OK) { 990 mutex_exit(&tx_ring->usedlist_lock); 991 ddi_fm_service_impact(Adapter->dip, DDI_SERVICE_DEGRADED); 992 Adapter->chip_state = E1000G_ERROR; 993 return (0); 994 } 995 996 /* 997 * While there are still TxSwPackets in the used queue check them 998 */ 999 while ((packet = 1000 (p_tx_sw_packet_t)QUEUE_GET_HEAD(&tx_ring->used_list)) != NULL) { 1001 1002 /* 1003 * Get hold of the next descriptor that the e1000g will 1004 * report status back to (this will be the last descriptor 1005 * of a given sw packet). We only want to free the 1006 * sw packet (and it resources) if the e1000g is done 1007 * with ALL of the descriptors. If the e1000g is done 1008 * with the last one then it is done with all of them. 1009 */ 1010 ASSERT(packet->num_desc); 1011 descriptor = tx_ring->tbd_oldest + (packet->num_desc - 1); 1012 1013 /* Check for wrap case */ 1014 if (descriptor > tx_ring->tbd_last) 1015 descriptor -= Adapter->tx_desc_num; 1016 1017 /* 1018 * If the descriptor done bit is set free TxSwPacket and 1019 * associated resources 1020 */ 1021 if (descriptor->upper.fields.status & E1000_TXD_STAT_DD) { 1022 QUEUE_POP_HEAD(&tx_ring->used_list); 1023 QUEUE_PUSH_TAIL(&pending_list, &packet->Link); 1024 1025 if (descriptor == tx_ring->tbd_last) 1026 tx_ring->tbd_oldest = 1027 tx_ring->tbd_first; 1028 else 1029 tx_ring->tbd_oldest = 1030 descriptor + 1; 1031 1032 desc_count += packet->num_desc; 1033 1034 if (is_intr && (desc_count >= Adapter->tx_recycle_num)) 1035 break; 1036 } else { 1037 /* 1038 * Found a sw packet that the e1000g is not done 1039 * with then there is no reason to check the rest 1040 * of the queue. 1041 */ 1042 break; 1043 } 1044 } 1045 1046 tx_ring->tbd_avail += desc_count; 1047 Adapter->tx_pkt_cnt += desc_count; 1048 1049 mutex_exit(&tx_ring->usedlist_lock); 1050 1051 if (desc_count == 0) { 1052 tx_ring->recycle_fail++; 1053 E1000G_DEBUG_STAT(tx_ring->stat_recycle_none); 1054 return (0); 1055 } 1056 1057 tx_ring->recycle_fail = 0; 1058 tx_ring->stall_watchdog = 0; 1059 1060 mp = NULL; 1061 nmp = NULL; 1062 packet = (p_tx_sw_packet_t)QUEUE_GET_HEAD(&pending_list); 1063 ASSERT(packet != NULL); 1064 while (packet != NULL) { 1065 if (packet->mp != NULL) { 1066 ASSERT(packet->mp->b_next == NULL); 1067 /* Assemble the message chain */ 1068 if (mp == NULL) { 1069 mp = packet->mp; 1070 nmp = packet->mp; 1071 } else { 1072 nmp->b_next = packet->mp; 1073 nmp = packet->mp; 1074 } 1075 /* Disconnect the message from the sw packet */ 1076 packet->mp = NULL; 1077 } 1078 1079 /* Free the TxSwPackets */ 1080 e1000g_free_tx_swpkt(packet); 1081 1082 packet = (p_tx_sw_packet_t) 1083 QUEUE_GET_NEXT(&pending_list, &packet->Link); 1084 } 1085 1086 /* Return the TxSwPackets back to the FreeList */ 1087 mutex_enter(&tx_ring->freelist_lock); 1088 QUEUE_APPEND(&tx_ring->free_list, &pending_list); 1089 mutex_exit(&tx_ring->freelist_lock); 1090 1091 if (mp != NULL) 1092 freemsgchain(mp); 1093 1094 return (desc_count); 1095 } 1096 /* 1097 * 82544 Coexistence issue workaround: 1098 * There are 2 issues. 1099 * 1. If a 32 bit split completion happens from P64H2 and another 1100 * agent drives a 64 bit request/split completion after ONLY 1101 * 1 idle clock (BRCM/Emulex/Adaptec fiber channel cards) then 1102 * 82544 has a problem where in to clock all the data in, it 1103 * looks at REQ64# signal and since it has changed so fast (i.e. 1 1104 * idle clock turn around), it will fail to clock all the data in. 1105 * Data coming from certain ending addresses has exposure to this issue. 1106 * 1107 * To detect this issue, following equation can be used... 1108 * SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 1109 * If SUM[3:0] is in between 1 to 4, we will have this issue. 1110 * 1111 * ROOT CAUSE: 1112 * The erratum involves the 82544 PCIX elasticity FIFO implementations as 1113 * 64-bit FIFO's and flushing of the final partial-bytes corresponding 1114 * to the end of a requested read burst. Under a specific burst condition 1115 * of ending-data alignment and 32-byte split-completions, the final 1116 * byte(s) of split-completion data require an extra clock cycle to flush 1117 * into 64-bit FIFO orientation. An incorrect logic dependency on the 1118 * REQ64# signal occurring during during this clock cycle may cause the 1119 * residual byte(s) to be lost, thereby rendering the internal DMA client 1120 * forever awaiting the final byte(s) for an outbound data-fetch. The 1121 * erratum is confirmed to *only* occur if certain subsequent external 1122 * 64-bit PCIX bus transactions occur immediately (minimum possible bus 1123 * turn- around) following the odd-aligned 32-bit split-completion 1124 * containing the final byte(s). Intel has confirmed that this has been 1125 * seen only with chipset/bridges which have the capability to provide 1126 * 32-bit split-completion data, and in the presence of newer PCIX bus 1127 * agents which fully-optimize the inter-transaction turn-around (zero 1128 * additional initiator latency when pre-granted bus ownership). 1129 * 1130 * This issue does not exist in PCI bus mode, when any agent is operating 1131 * in 32 bit only mode or on chipsets that do not do 32 bit split 1132 * completions for 64 bit read requests (Serverworks chipsets). P64H2 does 1133 * 32 bit split completions for any read request that has bit 2 set to 1 1134 * for the requested address and read request size is more than 8 bytes. 1135 * 1136 * 2. Another issue is related to 82544 driving DACs under the similar 1137 * scenario (32 bit split completion followed by 64 bit transaction with 1138 * only 1 cycle turnaround). This issue is still being root caused. We 1139 * think that both of these issues can be avoided if following workaround 1140 * is implemented. It seems DAC issues is related to ending addresses being 1141 * 0x9, 0xA, 0xB, 0xC and hence ending up at odd boundaries in elasticity 1142 * FIFO which does not get flushed due to REQ64# dependency. We will only 1143 * know the full story after it has been simulated successfully by HW team. 1144 * 1145 * WORKAROUND: 1146 * Make sure we do not have ending address as 1,2,3,4(Hang) or 9,a,b,c(DAC) 1147 */ 1148 static uint32_t 1149 e1000g_fill_82544_desc(uint64_t address, 1150 size_t length, p_desc_array_t desc_array) 1151 { 1152 /* 1153 * Since issue is sensitive to length and address. 1154 * Let us first check the address... 1155 */ 1156 uint32_t safe_terminator; 1157 1158 if (length <= 4) { 1159 desc_array->descriptor[0].address = address; 1160 desc_array->descriptor[0].length = (uint32_t)length; 1161 desc_array->elements = 1; 1162 return (desc_array->elements); 1163 } 1164 safe_terminator = 1165 (uint32_t)((((uint32_t)address & 0x7) + 1166 (length & 0xF)) & 0xF); 1167 /* 1168 * if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then 1169 * return 1170 */ 1171 if (safe_terminator == 0 || 1172 (safe_terminator > 4 && safe_terminator < 9) || 1173 (safe_terminator > 0xC && safe_terminator <= 0xF)) { 1174 desc_array->descriptor[0].address = address; 1175 desc_array->descriptor[0].length = (uint32_t)length; 1176 desc_array->elements = 1; 1177 return (desc_array->elements); 1178 } 1179 1180 desc_array->descriptor[0].address = address; 1181 desc_array->descriptor[0].length = length - 4; 1182 desc_array->descriptor[1].address = address + (length - 4); 1183 desc_array->descriptor[1].length = 4; 1184 desc_array->elements = 2; 1185 return (desc_array->elements); 1186 } 1187 1188 static int 1189 e1000g_tx_copy(e1000g_tx_ring_t *tx_ring, p_tx_sw_packet_t packet, 1190 mblk_t *mp, boolean_t tx_undersize_flag) 1191 { 1192 size_t len; 1193 size_t len1; 1194 dma_buffer_t *tx_buf; 1195 mblk_t *nmp; 1196 boolean_t finished; 1197 int desc_count; 1198 1199 desc_count = 0; 1200 tx_buf = packet->tx_buf; 1201 len = MBLKL(mp); 1202 1203 ASSERT((tx_buf->len + len) <= tx_buf->size); 1204 1205 if (len > 0) { 1206 bcopy(mp->b_rptr, 1207 tx_buf->address + tx_buf->len, 1208 len); 1209 tx_buf->len += len; 1210 1211 packet->num_mblk_frag++; 1212 } 1213 1214 nmp = mp->b_cont; 1215 if (nmp == NULL) { 1216 finished = B_TRUE; 1217 } else { 1218 len1 = MBLKL(nmp); 1219 if ((tx_buf->len + len1) > tx_buf->size) 1220 finished = B_TRUE; 1221 else if (tx_undersize_flag) 1222 finished = B_FALSE; 1223 else if (len1 > tx_ring->adapter->tx_bcopy_thresh) 1224 finished = B_TRUE; 1225 else 1226 finished = B_FALSE; 1227 } 1228 1229 if (finished) { 1230 E1000G_DEBUG_STAT_COND(tx_ring->stat_multi_copy, 1231 (tx_buf->len > len)); 1232 1233 /* 1234 * If the packet is smaller than 64 bytes, which is the 1235 * minimum ethernet packet size, pad the packet to make 1236 * it at least 60 bytes. The hardware will add 4 bytes 1237 * for CRC. 1238 */ 1239 if (tx_undersize_flag) { 1240 ASSERT(tx_buf->len < ETHERMIN); 1241 1242 bzero(tx_buf->address + tx_buf->len, 1243 ETHERMIN - tx_buf->len); 1244 tx_buf->len = ETHERMIN; 1245 } 1246 1247 #ifdef __sparc 1248 if (packet->dma_type == USE_DVMA) 1249 dvma_sync(tx_buf->dma_handle, 0, DDI_DMA_SYNC_FORDEV); 1250 else 1251 (void) ddi_dma_sync(tx_buf->dma_handle, 0, 1252 tx_buf->len, DDI_DMA_SYNC_FORDEV); 1253 #else 1254 (void) ddi_dma_sync(tx_buf->dma_handle, 0, 1255 tx_buf->len, DDI_DMA_SYNC_FORDEV); 1256 #endif 1257 1258 packet->data_transfer_type = USE_BCOPY; 1259 1260 desc_count = e1000g_fill_tx_desc(tx_ring, 1261 packet, 1262 tx_buf->dma_address, 1263 tx_buf->len); 1264 1265 if (desc_count <= 0) 1266 return (-1); 1267 } 1268 1269 return (desc_count); 1270 } 1271 1272 static int 1273 e1000g_tx_bind(e1000g_tx_ring_t *tx_ring, p_tx_sw_packet_t packet, mblk_t *mp) 1274 { 1275 int j; 1276 int mystat; 1277 size_t len; 1278 ddi_dma_cookie_t dma_cookie; 1279 uint_t ncookies; 1280 int desc_count; 1281 uint32_t desc_total; 1282 1283 desc_total = 0; 1284 len = MBLKL(mp); 1285 1286 /* 1287 * ddi_dma_addr_bind_handle() allocates DMA resources for a 1288 * memory object such that a device can perform DMA to or from 1289 * the object. DMA resources are allocated considering the 1290 * device's DMA attributes as expressed by ddi_dma_attr(9S) 1291 * (see ddi_dma_alloc_handle(9F)). 1292 * 1293 * ddi_dma_addr_bind_handle() fills in the first DMA cookie 1294 * pointed to by cookiep with the appropriate address, length, 1295 * and bus type. *ccountp is set to the number of DMA cookies 1296 * representing this DMA object. Subsequent DMA cookies must be 1297 * retrieved by calling ddi_dma_nextcookie(9F) the number of 1298 * times specified by *countp - 1. 1299 */ 1300 switch (packet->dma_type) { 1301 #ifdef __sparc 1302 case USE_DVMA: 1303 dvma_kaddr_load(packet->tx_dma_handle, 1304 (caddr_t)mp->b_rptr, len, 0, &dma_cookie); 1305 1306 dvma_sync(packet->tx_dma_handle, 0, 1307 DDI_DMA_SYNC_FORDEV); 1308 1309 ncookies = 1; 1310 packet->data_transfer_type = USE_DVMA; 1311 break; 1312 #endif 1313 case USE_DMA: 1314 if ((mystat = ddi_dma_addr_bind_handle( 1315 packet->tx_dma_handle, NULL, 1316 (caddr_t)mp->b_rptr, len, 1317 DDI_DMA_WRITE | DDI_DMA_STREAMING, 1318 DDI_DMA_DONTWAIT, 0, &dma_cookie, 1319 &ncookies)) != DDI_DMA_MAPPED) { 1320 1321 e1000g_log(tx_ring->adapter, CE_WARN, 1322 "Couldn't bind mblk buffer to Tx DMA handle: " 1323 "return: %X, Pkt: %X\n", 1324 mystat, packet); 1325 return (-1); 1326 } 1327 1328 /* 1329 * An implicit ddi_dma_sync() is done when the 1330 * ddi_dma_addr_bind_handle() is called. So we 1331 * don't need to explicitly call ddi_dma_sync() 1332 * here any more. 1333 */ 1334 ASSERT(ncookies); 1335 E1000G_DEBUG_STAT_COND(tx_ring->stat_multi_cookie, 1336 (ncookies > 1)); 1337 1338 /* 1339 * The data_transfer_type value must be set after the handle 1340 * has been bound, for it will be used in e1000g_free_tx_swpkt() 1341 * to decide whether we need to unbind the handle. 1342 */ 1343 packet->data_transfer_type = USE_DMA; 1344 break; 1345 default: 1346 ASSERT(B_FALSE); 1347 break; 1348 } 1349 1350 packet->num_mblk_frag++; 1351 1352 /* 1353 * Each address could span thru multpile cookie.. 1354 * Each cookie will have one descriptor 1355 */ 1356 for (j = ncookies; j != 0; j--) { 1357 1358 desc_count = e1000g_fill_tx_desc(tx_ring, 1359 packet, 1360 dma_cookie.dmac_laddress, 1361 dma_cookie.dmac_size); 1362 1363 if (desc_count <= 0) 1364 return (-1); 1365 1366 desc_total += desc_count; 1367 1368 /* 1369 * ddi_dma_nextcookie() retrieves subsequent DMA 1370 * cookies for a DMA object. 1371 * ddi_dma_nextcookie() fills in the 1372 * ddi_dma_cookie(9S) structure pointed to by 1373 * cookiep. The ddi_dma_cookie(9S) structure 1374 * must be allocated prior to calling 1375 * ddi_dma_nextcookie(). The DMA cookie count 1376 * returned by ddi_dma_buf_bind_handle(9F), 1377 * ddi_dma_addr_bind_handle(9F), or 1378 * ddi_dma_getwin(9F) indicates the number of DMA 1379 * cookies a DMA object consists of. If the 1380 * resulting cookie count, N, is larger than 1, 1381 * ddi_dma_nextcookie() must be called N-1 times 1382 * to retrieve all DMA cookies. 1383 */ 1384 if (j > 1) { 1385 ddi_dma_nextcookie(packet->tx_dma_handle, 1386 &dma_cookie); 1387 } 1388 } 1389 1390 return (desc_total); 1391 } 1392 1393 static void 1394 e1000g_fill_context_descriptor(context_data_t *cur_context, 1395 struct e1000_context_desc *context_desc) 1396 { 1397 if (cur_context->cksum_flags & HCK_IPV4_HDRCKSUM) { 1398 context_desc->lower_setup.ip_fields.ipcss = 1399 cur_context->ether_header_size; 1400 context_desc->lower_setup.ip_fields.ipcso = 1401 cur_context->ether_header_size + 1402 offsetof(struct ip, ip_sum); 1403 context_desc->lower_setup.ip_fields.ipcse = 1404 cur_context->ether_header_size + 1405 cur_context->cksum_start - 1; 1406 } else 1407 context_desc->lower_setup.ip_config = 0; 1408 1409 if (cur_context->cksum_flags & HCK_PARTIALCKSUM) { 1410 /* 1411 * The packet with same protocol has the following 1412 * stuff and start offset: 1413 * | Protocol | Stuff | Start | Checksum 1414 * | | Offset | Offset | Enable 1415 * | IPv4 + TCP | 0x24 | 0x14 | Yes 1416 * | IPv4 + UDP | 0x1A | 0x14 | Yes 1417 * | IPv6 + TCP | 0x20 | 0x10 | No 1418 * | IPv6 + UDP | 0x14 | 0x10 | No 1419 */ 1420 context_desc->upper_setup.tcp_fields.tucss = 1421 cur_context->cksum_start + cur_context->ether_header_size; 1422 context_desc->upper_setup.tcp_fields.tucso = 1423 cur_context->cksum_stuff + cur_context->ether_header_size; 1424 context_desc->upper_setup.tcp_fields.tucse = 0; 1425 } else 1426 context_desc->upper_setup.tcp_config = 0; 1427 1428 if (cur_context->lso_flag) { 1429 context_desc->tcp_seg_setup.fields.mss = cur_context->mss; 1430 context_desc->tcp_seg_setup.fields.hdr_len = 1431 cur_context->hdr_len; 1432 /* 1433 * workaround for 82546EB errata 23, status-writeback 1434 * reporting (RS) should not be set on context or 1435 * Null descriptors 1436 */ 1437 context_desc->cmd_and_length = E1000_TXD_CMD_DEXT 1438 | E1000_TXD_CMD_TSE | E1000_TXD_CMD_IP | E1000_TXD_CMD_TCP 1439 | E1000_TXD_DTYP_C | cur_context->pay_len; 1440 } else { 1441 context_desc->cmd_and_length = E1000_TXD_CMD_DEXT 1442 | E1000_TXD_DTYP_C; 1443 /* 1444 * Zero out the options for TCP Segmentation Offload 1445 */ 1446 context_desc->tcp_seg_setup.data = 0; 1447 } 1448 } 1449 1450 static int 1451 e1000g_fill_tx_desc(e1000g_tx_ring_t *tx_ring, 1452 p_tx_sw_packet_t packet, uint64_t address, size_t size) 1453 { 1454 struct e1000_hw *hw = &tx_ring->adapter->shared; 1455 p_sw_desc_t desc; 1456 1457 if (hw->mac.type == e1000_82544) { 1458 if (hw->bus.type == e1000_bus_type_pcix) 1459 return (e1000g_tx_workaround_PCIX_82544(packet, 1460 address, size)); 1461 1462 if (size > JUMBO_FRAG_LENGTH) 1463 return (e1000g_tx_workaround_jumbo_82544(packet, 1464 address, size)); 1465 } 1466 1467 ASSERT(packet->num_desc < MAX_TX_DESC_PER_PACKET); 1468 1469 desc = &packet->desc[packet->num_desc]; 1470 desc->address = address; 1471 desc->length = (uint32_t)size; 1472 1473 packet->num_desc++; 1474 1475 return (1); 1476 } 1477 1478 static int 1479 e1000g_tx_workaround_PCIX_82544(p_tx_sw_packet_t packet, 1480 uint64_t address, size_t size) 1481 { 1482 p_sw_desc_t desc; 1483 int desc_count; 1484 long size_left; 1485 size_t len; 1486 uint32_t counter; 1487 uint32_t array_elements; 1488 desc_array_t desc_array; 1489 1490 /* 1491 * Coexist Workaround for cordova: RP: 07/04/03 1492 * 1493 * RP: ERRATA: Workaround ISSUE: 1494 * 8kb_buffer_Lockup CONTROLLER: Cordova Breakup 1495 * Eachbuffer in to 8kb pieces until the 1496 * remainder is < 8kb 1497 */ 1498 size_left = size; 1499 desc_count = 0; 1500 1501 while (size_left > 0) { 1502 if (size_left > MAX_TX_BUF_SIZE) 1503 len = MAX_TX_BUF_SIZE; 1504 else 1505 len = size_left; 1506 1507 array_elements = e1000g_fill_82544_desc(address, 1508 len, &desc_array); 1509 1510 for (counter = 0; counter < array_elements; counter++) { 1511 ASSERT(packet->num_desc < MAX_TX_DESC_PER_PACKET); 1512 /* 1513 * Put in the buffer address 1514 */ 1515 desc = &packet->desc[packet->num_desc]; 1516 1517 desc->address = 1518 desc_array.descriptor[counter].address; 1519 desc->length = 1520 desc_array.descriptor[counter].length; 1521 1522 packet->num_desc++; 1523 desc_count++; 1524 } /* for */ 1525 1526 /* 1527 * Update the buffer address and length 1528 */ 1529 address += MAX_TX_BUF_SIZE; 1530 size_left -= MAX_TX_BUF_SIZE; 1531 } /* while */ 1532 1533 return (desc_count); 1534 } 1535 1536 static int 1537 e1000g_tx_workaround_jumbo_82544(p_tx_sw_packet_t packet, 1538 uint64_t address, size_t size) 1539 { 1540 p_sw_desc_t desc; 1541 int desc_count; 1542 long size_left; 1543 uint32_t offset; 1544 1545 /* 1546 * Workaround for Jumbo Frames on Cordova 1547 * PSD 06/01/2001 1548 */ 1549 size_left = size; 1550 desc_count = 0; 1551 offset = 0; 1552 while (size_left > 0) { 1553 ASSERT(packet->num_desc < MAX_TX_DESC_PER_PACKET); 1554 1555 desc = &packet->desc[packet->num_desc]; 1556 1557 desc->address = address + offset; 1558 1559 if (size_left > JUMBO_FRAG_LENGTH) 1560 desc->length = JUMBO_FRAG_LENGTH; 1561 else 1562 desc->length = (uint32_t)size_left; 1563 1564 packet->num_desc++; 1565 desc_count++; 1566 1567 offset += desc->length; 1568 size_left -= JUMBO_FRAG_LENGTH; 1569 } 1570 1571 return (desc_count); 1572 } 1573 1574 #pragma inline(e1000g_82547_tx_move_tail_work) 1575 1576 static void 1577 e1000g_82547_tx_move_tail_work(e1000g_tx_ring_t *tx_ring) 1578 { 1579 struct e1000_hw *hw; 1580 uint16_t hw_tdt; 1581 uint16_t sw_tdt; 1582 struct e1000_tx_desc *tx_desc; 1583 uint16_t length = 0; 1584 boolean_t eop = B_FALSE; 1585 struct e1000g *Adapter; 1586 1587 Adapter = tx_ring->adapter; 1588 hw = &Adapter->shared; 1589 1590 hw_tdt = E1000_READ_REG(hw, E1000_TDT(0)); 1591 sw_tdt = tx_ring->tbd_next - tx_ring->tbd_first; 1592 1593 while (hw_tdt != sw_tdt) { 1594 tx_desc = &(tx_ring->tbd_first[hw_tdt]); 1595 length += tx_desc->lower.flags.length; 1596 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP; 1597 if (++hw_tdt == Adapter->tx_desc_num) 1598 hw_tdt = 0; 1599 1600 if (eop) { 1601 if ((Adapter->link_duplex == HALF_DUPLEX) && 1602 (e1000_fifo_workaround_82547(hw, length) 1603 != E1000_SUCCESS)) { 1604 if (tx_ring->timer_enable_82547) { 1605 ASSERT(tx_ring->timer_id_82547 == 0); 1606 tx_ring->timer_id_82547 = 1607 timeout(e1000g_82547_timeout, 1608 (void *)tx_ring, 1609 drv_usectohz(10000)); 1610 } 1611 return; 1612 1613 } else { 1614 E1000_WRITE_REG(hw, E1000_TDT(0), hw_tdt); 1615 e1000_update_tx_fifo_head_82547(hw, length); 1616 length = 0; 1617 } 1618 } 1619 } 1620 } 1621 1622 static void 1623 e1000g_82547_timeout(void *arg) 1624 { 1625 e1000g_tx_ring_t *tx_ring; 1626 1627 tx_ring = (e1000g_tx_ring_t *)arg; 1628 1629 mutex_enter(&tx_ring->tx_lock); 1630 1631 tx_ring->timer_id_82547 = 0; 1632 e1000g_82547_tx_move_tail_work(tx_ring); 1633 1634 mutex_exit(&tx_ring->tx_lock); 1635 } 1636 1637 static void 1638 e1000g_82547_tx_move_tail(e1000g_tx_ring_t *tx_ring) 1639 { 1640 timeout_id_t tid; 1641 1642 ASSERT(MUTEX_HELD(&tx_ring->tx_lock)); 1643 1644 tid = tx_ring->timer_id_82547; 1645 tx_ring->timer_id_82547 = 0; 1646 if (tid != 0) { 1647 tx_ring->timer_enable_82547 = B_FALSE; 1648 mutex_exit(&tx_ring->tx_lock); 1649 1650 (void) untimeout(tid); 1651 1652 mutex_enter(&tx_ring->tx_lock); 1653 } 1654 tx_ring->timer_enable_82547 = B_TRUE; 1655 e1000g_82547_tx_move_tail_work(tx_ring); 1656 } 1657