1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/mac_provider.h> 27 #include <sys/nxge/nxge_impl.h> 28 #include <sys/nxge/nxge_hio.h> 29 #include <npi_tx_wr64.h> 30 31 /* Software LSO required header files */ 32 #include <netinet/tcp.h> 33 #include <inet/ip_impl.h> 34 #include <inet/tcp.h> 35 36 extern uint64_t mac_pkt_hash(uint_t, mblk_t *mp, uint8_t policy, 37 boolean_t is_outbound); 38 39 static mblk_t *nxge_lso_eliminate(mblk_t *); 40 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 41 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 42 static void nxge_hcksum_retrieve(mblk_t *, 43 uint32_t *, uint32_t *, uint32_t *, 44 uint32_t *, uint32_t *); 45 static uint32_t nxge_csgen(uint16_t *, int); 46 47 extern uint32_t nxge_reclaim_pending; 48 extern uint32_t nxge_bcopy_thresh; 49 extern uint32_t nxge_dvma_thresh; 50 extern uint32_t nxge_dma_stream_thresh; 51 extern uint32_t nxge_tx_minfree; 52 extern uint32_t nxge_tx_intr_thres; 53 extern uint32_t nxge_tx_max_gathers; 54 extern uint32_t nxge_tx_tiny_pack; 55 extern uint32_t nxge_tx_use_bcopy; 56 extern nxge_tx_mode_t nxge_tx_scheme; 57 uint32_t nxge_lso_kick_cnt = 2; 58 59 60 void 61 nxge_tx_ring_task(void *arg) 62 { 63 p_tx_ring_t ring = (p_tx_ring_t)arg; 64 65 ASSERT(ring->tx_ring_handle != NULL); 66 67 MUTEX_ENTER(&ring->lock); 68 (void) nxge_txdma_reclaim(ring->nxgep, ring, 0); 69 MUTEX_EXIT(&ring->lock); 70 71 if (!ring->tx_ring_offline) { 72 mac_tx_ring_update(ring->nxgep->mach, ring->tx_ring_handle); 73 } 74 } 75 76 static void 77 nxge_tx_ring_dispatch(p_tx_ring_t ring) 78 { 79 /* 80 * Kick the ring task to reclaim some buffers. 81 */ 82 (void) ddi_taskq_dispatch(ring->taskq, 83 nxge_tx_ring_task, (void *)ring, DDI_SLEEP); 84 } 85 86 mblk_t * 87 nxge_tx_ring_send(void *arg, mblk_t *mp) 88 { 89 p_nxge_ring_handle_t nrhp = (p_nxge_ring_handle_t)arg; 90 p_nxge_t nxgep; 91 p_tx_ring_t tx_ring_p; 92 int status, channel; 93 94 ASSERT(nrhp != NULL); 95 nxgep = nrhp->nxgep; 96 channel = nxgep->pt_config.hw_config.tdc.start + nrhp->index; 97 tx_ring_p = nxgep->tx_rings->rings[channel]; 98 99 /* 100 * We may be in a transition from offlined DMA to onlined 101 * DMA. 102 */ 103 if (tx_ring_p == NULL) { 104 ASSERT(tx_ring_p != NULL); 105 freemsg(mp); 106 return ((mblk_t *)NULL); 107 } 108 109 /* 110 * Valid DMA? 111 */ 112 ASSERT(nxgep == tx_ring_p->nxgep); 113 114 /* 115 * Make sure DMA is not offlined. 116 */ 117 if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline) { 118 ASSERT(!tx_ring_p->tx_ring_offline); 119 freemsg(mp); 120 return ((mblk_t *)NULL); 121 } 122 123 /* 124 * Transmit the packet. 125 */ 126 status = nxge_start(nxgep, tx_ring_p, mp); 127 if (status) { 128 nxge_tx_ring_dispatch(tx_ring_p); 129 return (mp); 130 } 131 132 return ((mblk_t *)NULL); 133 } 134 135 int 136 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 137 { 138 int dma_status, status = 0; 139 p_tx_desc_t tx_desc_ring_vp; 140 npi_handle_t npi_desc_handle; 141 nxge_os_dma_handle_t tx_desc_dma_handle; 142 p_tx_desc_t tx_desc_p; 143 p_tx_msg_t tx_msg_ring; 144 p_tx_msg_t tx_msg_p = NULL; 145 tx_desc_t tx_desc, *tmp_desc_p; 146 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 147 p_tx_pkt_header_t hdrp = NULL; 148 tx_pkt_hdr_all_t tmp_hdrp; 149 p_tx_pkt_hdr_all_t pkthdrp; 150 uint8_t npads = 0; 151 uint64_t dma_ioaddr; 152 uint32_t dma_flags; 153 int last_bidx; 154 uint8_t *b_rptr; 155 caddr_t kaddr; 156 uint32_t nmblks; 157 uint32_t ngathers; 158 uint32_t clen; 159 int len; 160 uint32_t pkt_len, pack_len, min_len; 161 uint32_t bcopy_thresh; 162 int i, cur_index, sop_index; 163 uint16_t tail_index; 164 boolean_t tail_wrap = B_FALSE; 165 nxge_dma_common_t desc_area; 166 nxge_os_dma_handle_t dma_handle; 167 ddi_dma_cookie_t dma_cookie; 168 npi_handle_t npi_handle; 169 p_mblk_t nmp; 170 p_mblk_t t_mp; 171 uint32_t ncookies; 172 boolean_t good_packet; 173 boolean_t mark_mode = B_FALSE; 174 p_nxge_stats_t statsp; 175 p_nxge_tx_ring_stats_t tdc_stats; 176 t_uscalar_t start_offset = 0; 177 t_uscalar_t stuff_offset = 0; 178 t_uscalar_t end_offset = 0; 179 t_uscalar_t value = 0; 180 t_uscalar_t cksum_flags = 0; 181 boolean_t cksum_on = B_FALSE; 182 uint32_t boff = 0; 183 uint64_t tot_xfer_len = 0; 184 boolean_t header_set = B_FALSE; 185 #ifdef NXGE_DEBUG 186 p_tx_desc_t tx_desc_ring_pp; 187 p_tx_desc_t tx_desc_pp; 188 tx_desc_t *save_desc_p; 189 int dump_len; 190 int sad_len; 191 uint64_t sad; 192 int xfer_len; 193 uint32_t msgsize; 194 #endif 195 p_mblk_t mp_chain = NULL; 196 boolean_t is_lso = B_FALSE; 197 boolean_t lso_again; 198 int cur_index_lso; 199 p_mblk_t nmp_lso_save = NULL; 200 uint32_t lso_ngathers; 201 boolean_t lso_tail_wrap = B_FALSE; 202 203 NXGE_DEBUG_MSG((nxgep, TX_CTL, 204 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 205 NXGE_DEBUG_MSG((nxgep, TX_CTL, 206 "==> nxge_start: Starting tdc %d desc pending %d", 207 tx_ring_p->tdc, tx_ring_p->descs_pending)); 208 209 statsp = nxgep->statsp; 210 211 if (!isLDOMguest(nxgep)) { 212 switch (nxgep->mac.portmode) { 213 default: 214 if (nxgep->statsp->port_stats.lb_mode == 215 nxge_lb_normal) { 216 if (!statsp->mac_stats.link_up) { 217 freemsg(mp); 218 NXGE_DEBUG_MSG((nxgep, TX_CTL, 219 "==> nxge_start: " 220 "link not up")); 221 goto nxge_start_fail1; 222 } 223 } 224 break; 225 case PORT_10G_FIBER: 226 /* 227 * For the following modes, check the link status 228 * before sending the packet out: 229 * nxge_lb_normal, 230 * nxge_lb_ext10g, 231 * nxge_lb_ext1000, 232 * nxge_lb_ext100, 233 * nxge_lb_ext10. 234 */ 235 if (nxgep->statsp->port_stats.lb_mode < 236 nxge_lb_phy10g) { 237 if (!statsp->mac_stats.link_up) { 238 freemsg(mp); 239 NXGE_DEBUG_MSG((nxgep, TX_CTL, 240 "==> nxge_start: " 241 "link not up")); 242 goto nxge_start_fail1; 243 } 244 } 245 break; 246 } 247 } 248 249 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 250 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 251 NXGE_DEBUG_MSG((nxgep, TX_CTL, 252 "==> nxge_start: hardware not initialized or stopped")); 253 freemsg(mp); 254 goto nxge_start_fail1; 255 } 256 257 if (nxgep->soft_lso_enable) { 258 mp_chain = nxge_lso_eliminate(mp); 259 NXGE_DEBUG_MSG((nxgep, TX_CTL, 260 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 261 mp, mp_chain)); 262 if (mp_chain == NULL) { 263 NXGE_ERROR_MSG((nxgep, TX_CTL, 264 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 265 mp_chain, mp)); 266 goto nxge_start_fail1; 267 } 268 if (mp_chain != mp) { 269 NXGE_DEBUG_MSG((nxgep, TX_CTL, 270 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 271 mp_chain, mp)); 272 is_lso = B_TRUE; 273 mp = mp_chain; 274 mp_chain = mp_chain->b_next; 275 mp->b_next = NULL; 276 } 277 } 278 279 mac_hcksum_get(mp, &start_offset, &stuff_offset, &end_offset, 280 &value, &cksum_flags); 281 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 282 start_offset += sizeof (ether_header_t); 283 stuff_offset += sizeof (ether_header_t); 284 } else { 285 start_offset += sizeof (struct ether_vlan_header); 286 stuff_offset += sizeof (struct ether_vlan_header); 287 } 288 289 if (cksum_flags & HCK_PARTIALCKSUM) { 290 NXGE_DEBUG_MSG((nxgep, TX_CTL, 291 "==> nxge_start: mp $%p len %d " 292 "cksum_flags 0x%x (partial checksum) ", 293 mp, MBLKL(mp), cksum_flags)); 294 cksum_on = B_TRUE; 295 } 296 297 pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 298 pkthdrp->reserved = 0; 299 tmp_hdrp.pkthdr.value = 0; 300 nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 301 0, 0, pkthdrp, 302 start_offset, stuff_offset); 303 304 lso_again = B_FALSE; 305 lso_ngathers = 0; 306 307 MUTEX_ENTER(&tx_ring_p->lock); 308 309 if (isLDOMservice(nxgep)) { 310 tx_ring_p->tx_ring_busy = B_TRUE; 311 if (tx_ring_p->tx_ring_offline) { 312 freemsg(mp); 313 tx_ring_p->tx_ring_busy = B_FALSE; 314 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 315 NXGE_TX_RING_OFFLINED); 316 MUTEX_EXIT(&tx_ring_p->lock); 317 return (status); 318 } 319 } 320 321 cur_index_lso = tx_ring_p->wr_index; 322 lso_tail_wrap = tx_ring_p->wr_index_wrap; 323 start_again: 324 ngathers = 0; 325 sop_index = tx_ring_p->wr_index; 326 #ifdef NXGE_DEBUG 327 if (tx_ring_p->descs_pending) { 328 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 329 "desc pending %d ", tx_ring_p->descs_pending)); 330 } 331 332 dump_len = (int)(MBLKL(mp)); 333 dump_len = (dump_len > 128) ? 128: dump_len; 334 335 NXGE_DEBUG_MSG((nxgep, TX_CTL, 336 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 337 "(Before header reserve: ORIGINAL LEN %d)", 338 tx_ring_p->tdc, 339 mp->b_rptr, 340 dump_len)); 341 342 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 343 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 344 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 345 #endif 346 347 tdc_stats = tx_ring_p->tdc_stats; 348 mark_mode = (tx_ring_p->descs_pending && 349 (((int)tx_ring_p->tx_ring_size - (int)tx_ring_p->descs_pending) < 350 (int)nxge_tx_minfree)); 351 352 NXGE_DEBUG_MSG((nxgep, TX_CTL, 353 "TX Descriptor ring is channel %d mark mode %d", 354 tx_ring_p->tdc, mark_mode)); 355 356 if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) { 357 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, 358 (nxge_tx_minfree + lso_ngathers))) { 359 NXGE_DEBUG_MSG((nxgep, TX_CTL, 360 "TX Descriptor ring is full: channel %d", 361 tx_ring_p->tdc)); 362 NXGE_DEBUG_MSG((nxgep, TX_CTL, 363 "TX Descriptor ring is full: channel %d", 364 tx_ring_p->tdc)); 365 if (is_lso) { 366 /* 367 * free the current mp and mp_chain if not FULL. 368 */ 369 tdc_stats->tx_no_desc++; 370 NXGE_DEBUG_MSG((nxgep, TX_CTL, 371 "LSO packet: TX Descriptor ring is full: " 372 "channel %d", 373 tx_ring_p->tdc)); 374 goto nxge_start_fail_lso; 375 } else { 376 (void) atomic_cas_32( 377 (uint32_t *)&tx_ring_p->queueing, 0, 1); 378 tdc_stats->tx_no_desc++; 379 380 if (isLDOMservice(nxgep)) { 381 tx_ring_p->tx_ring_busy = B_FALSE; 382 if (tx_ring_p->tx_ring_offline) { 383 (void) atomic_swap_32( 384 &tx_ring_p->tx_ring_offline, 385 NXGE_TX_RING_OFFLINED); 386 } 387 } 388 389 MUTEX_EXIT(&tx_ring_p->lock); 390 status = 1; 391 goto nxge_start_fail1; 392 } 393 } 394 } 395 396 nmp = mp; 397 i = sop_index = tx_ring_p->wr_index; 398 nmblks = 0; 399 ngathers = 0; 400 pkt_len = 0; 401 pack_len = 0; 402 clen = 0; 403 last_bidx = -1; 404 good_packet = B_TRUE; 405 406 desc_area = tx_ring_p->tdc_desc; 407 npi_handle = desc_area.npi_handle; 408 npi_desc_handle.regh = (nxge_os_acc_handle_t) 409 DMA_COMMON_ACC_HANDLE(desc_area); 410 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 411 tx_desc_dma_handle = (nxge_os_dma_handle_t) 412 DMA_COMMON_HANDLE(desc_area); 413 tx_msg_ring = tx_ring_p->tx_msg_ring; 414 415 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 416 sop_index, i)); 417 418 #ifdef NXGE_DEBUG 419 msgsize = msgdsize(nmp); 420 NXGE_DEBUG_MSG((nxgep, TX_CTL, 421 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 422 sop_index, i, msgsize)); 423 #endif 424 /* 425 * The first 16 bytes of the premapped buffer are reserved 426 * for header. No padding will be used. 427 */ 428 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 429 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 430 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 431 } else { 432 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 433 } 434 while (nmp) { 435 good_packet = B_TRUE; 436 b_rptr = nmp->b_rptr; 437 len = MBLKL(nmp); 438 if (len <= 0) { 439 nmp = nmp->b_cont; 440 continue; 441 } 442 nmblks++; 443 444 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 445 "len %d pkt_len %d pack_len %d", 446 nmblks, len, pkt_len, pack_len)); 447 /* 448 * Hardware limits the transfer length to 4K for NIU and 449 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 450 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 451 * If len is longer than the limit, then we break nmp into 452 * two chunks: Make the first chunk equal to the limit and 453 * the second chunk for the remaining data. If the second 454 * chunk is still larger than the limit, then it will be 455 * broken into two in the next pass. 456 */ 457 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 458 if ((t_mp = dupb(nmp)) != NULL) { 459 nmp->b_wptr = nmp->b_rptr + 460 (TX_MAX_TRANSFER_LENGTH 461 - TX_PKT_HEADER_SIZE); 462 t_mp->b_rptr = nmp->b_wptr; 463 t_mp->b_cont = nmp->b_cont; 464 nmp->b_cont = t_mp; 465 len = MBLKL(nmp); 466 } else { 467 if (is_lso) { 468 NXGE_DEBUG_MSG((nxgep, TX_CTL, 469 "LSO packet: dupb failed: " 470 "channel %d", 471 tx_ring_p->tdc)); 472 mp = nmp; 473 goto nxge_start_fail_lso; 474 } else { 475 good_packet = B_FALSE; 476 goto nxge_start_fail2; 477 } 478 } 479 } 480 tx_desc.value = 0; 481 tx_desc_p = &tx_desc_ring_vp[i]; 482 #ifdef NXGE_DEBUG 483 tx_desc_pp = &tx_desc_ring_pp[i]; 484 #endif 485 tx_msg_p = &tx_msg_ring[i]; 486 npi_desc_handle.regp = (uint64_t)tx_desc_p; 487 if (!header_set && 488 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 489 (len >= bcopy_thresh))) { 490 header_set = B_TRUE; 491 bcopy_thresh += TX_PKT_HEADER_SIZE; 492 boff = 0; 493 pack_len = 0; 494 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 495 hdrp = (p_tx_pkt_header_t)kaddr; 496 clen = pkt_len; 497 dma_handle = tx_msg_p->buf_dma_handle; 498 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 499 (void) ddi_dma_sync(dma_handle, 500 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 501 DDI_DMA_SYNC_FORDEV); 502 503 tx_msg_p->flags.dma_type = USE_BCOPY; 504 goto nxge_start_control_header_only; 505 } 506 507 pkt_len += len; 508 pack_len += len; 509 510 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 511 "desc entry %d " 512 "DESC IOADDR $%p " 513 "desc_vp $%p tx_desc_p $%p " 514 "desc_pp $%p tx_desc_pp $%p " 515 "len %d pkt_len %d pack_len %d", 516 i, 517 DMA_COMMON_IOADDR(desc_area), 518 tx_desc_ring_vp, tx_desc_p, 519 tx_desc_ring_pp, tx_desc_pp, 520 len, pkt_len, pack_len)); 521 522 if (len < bcopy_thresh) { 523 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 524 "USE BCOPY: ")); 525 if (nxge_tx_tiny_pack) { 526 uint32_t blst = 527 TXDMA_DESC_NEXT_INDEX(i, -1, 528 tx_ring_p->tx_wrap_mask); 529 NXGE_DEBUG_MSG((nxgep, TX_CTL, 530 "==> nxge_start(5): pack")); 531 if ((pack_len <= bcopy_thresh) && 532 (last_bidx == blst)) { 533 NXGE_DEBUG_MSG((nxgep, TX_CTL, 534 "==> nxge_start: pack(6) " 535 "(pkt_len %d pack_len %d)", 536 pkt_len, pack_len)); 537 i = blst; 538 tx_desc_p = &tx_desc_ring_vp[i]; 539 #ifdef NXGE_DEBUG 540 tx_desc_pp = &tx_desc_ring_pp[i]; 541 #endif 542 tx_msg_p = &tx_msg_ring[i]; 543 boff = pack_len - len; 544 ngathers--; 545 } else if (pack_len > bcopy_thresh && 546 header_set) { 547 pack_len = len; 548 boff = 0; 549 bcopy_thresh = nxge_bcopy_thresh; 550 NXGE_DEBUG_MSG((nxgep, TX_CTL, 551 "==> nxge_start(7): > max NEW " 552 "bcopy thresh %d " 553 "pkt_len %d pack_len %d(next)", 554 bcopy_thresh, 555 pkt_len, pack_len)); 556 } 557 last_bidx = i; 558 } 559 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 560 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 561 hdrp = (p_tx_pkt_header_t)kaddr; 562 header_set = B_TRUE; 563 NXGE_DEBUG_MSG((nxgep, TX_CTL, 564 "==> nxge_start(7_x2): " 565 "pkt_len %d pack_len %d (new hdrp $%p)", 566 pkt_len, pack_len, hdrp)); 567 } 568 tx_msg_p->flags.dma_type = USE_BCOPY; 569 kaddr += boff; 570 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 571 "USE BCOPY: before bcopy " 572 "DESC IOADDR $%p entry %d " 573 "bcopy packets %d " 574 "bcopy kaddr $%p " 575 "bcopy ioaddr (SAD) $%p " 576 "bcopy clen %d " 577 "bcopy boff %d", 578 DMA_COMMON_IOADDR(desc_area), i, 579 tdc_stats->tx_hdr_pkts, 580 kaddr, 581 dma_ioaddr, 582 clen, 583 boff)); 584 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 585 "1USE BCOPY: ")); 586 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 587 "2USE BCOPY: ")); 588 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 589 "last USE BCOPY: copy from b_rptr $%p " 590 "to KADDR $%p (len %d offset %d", 591 b_rptr, kaddr, len, boff)); 592 593 bcopy(b_rptr, kaddr, len); 594 595 #ifdef NXGE_DEBUG 596 dump_len = (len > 128) ? 128: len; 597 NXGE_DEBUG_MSG((nxgep, TX_CTL, 598 "==> nxge_start: dump packets " 599 "(After BCOPY len %d)" 600 "(b_rptr $%p): %s", len, nmp->b_rptr, 601 nxge_dump_packet((char *)nmp->b_rptr, 602 dump_len))); 603 #endif 604 605 dma_handle = tx_msg_p->buf_dma_handle; 606 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 607 (void) ddi_dma_sync(dma_handle, 608 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 609 DDI_DMA_SYNC_FORDEV); 610 clen = len + boff; 611 tdc_stats->tx_hdr_pkts++; 612 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 613 "USE BCOPY: " 614 "DESC IOADDR $%p entry %d " 615 "bcopy packets %d " 616 "bcopy kaddr $%p " 617 "bcopy ioaddr (SAD) $%p " 618 "bcopy clen %d " 619 "bcopy boff %d", 620 DMA_COMMON_IOADDR(desc_area), 621 i, 622 tdc_stats->tx_hdr_pkts, 623 kaddr, 624 dma_ioaddr, 625 clen, 626 boff)); 627 } else { 628 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 629 "USE DVMA: len %d", len)); 630 tx_msg_p->flags.dma_type = USE_DMA; 631 dma_flags = DDI_DMA_WRITE; 632 if (len < nxge_dma_stream_thresh) { 633 dma_flags |= DDI_DMA_CONSISTENT; 634 } else { 635 dma_flags |= DDI_DMA_STREAMING; 636 } 637 638 dma_handle = tx_msg_p->dma_handle; 639 dma_status = ddi_dma_addr_bind_handle(dma_handle, NULL, 640 (caddr_t)b_rptr, len, dma_flags, 641 DDI_DMA_DONTWAIT, NULL, 642 &dma_cookie, &ncookies); 643 if (dma_status == DDI_DMA_MAPPED) { 644 dma_ioaddr = dma_cookie.dmac_laddress; 645 len = (int)dma_cookie.dmac_size; 646 clen = (uint32_t)dma_cookie.dmac_size; 647 NXGE_DEBUG_MSG((nxgep, TX_CTL, 648 "==> nxge_start(12_1): " 649 "USE DVMA: len %d clen %d " 650 "ngathers %d", 651 len, clen, 652 ngathers)); 653 npi_desc_handle.regp = (uint64_t)tx_desc_p; 654 while (ncookies > 1) { 655 ngathers++; 656 /* 657 * this is the fix for multiple 658 * cookies, which are basically 659 * a descriptor entry, we don't set 660 * SOP bit as well as related fields 661 */ 662 663 (void) npi_txdma_desc_gather_set( 664 npi_desc_handle, 665 &tx_desc, 666 (ngathers -1), 667 mark_mode, 668 ngathers, 669 dma_ioaddr, 670 clen); 671 672 tx_msg_p->tx_msg_size = clen; 673 NXGE_DEBUG_MSG((nxgep, TX_CTL, 674 "==> nxge_start: DMA " 675 "ncookie %d " 676 "ngathers %d " 677 "dma_ioaddr $%p len %d" 678 "desc $%p descp $%p (%d)", 679 ncookies, 680 ngathers, 681 dma_ioaddr, clen, 682 *tx_desc_p, tx_desc_p, i)); 683 684 ddi_dma_nextcookie(dma_handle, 685 &dma_cookie); 686 dma_ioaddr = 687 dma_cookie.dmac_laddress; 688 689 len = (int)dma_cookie.dmac_size; 690 clen = (uint32_t)dma_cookie.dmac_size; 691 NXGE_DEBUG_MSG((nxgep, TX_CTL, 692 "==> nxge_start(12_2): " 693 "USE DVMA: len %d clen %d ", 694 len, clen)); 695 696 i = TXDMA_DESC_NEXT_INDEX(i, 1, 697 tx_ring_p->tx_wrap_mask); 698 tx_desc_p = &tx_desc_ring_vp[i]; 699 700 npi_desc_handle.regp = 701 (uint64_t)tx_desc_p; 702 tx_msg_p = &tx_msg_ring[i]; 703 tx_msg_p->flags.dma_type = USE_NONE; 704 tx_desc.value = 0; 705 706 ncookies--; 707 } 708 tdc_stats->tx_ddi_pkts++; 709 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 710 "DMA: ddi packets %d", 711 tdc_stats->tx_ddi_pkts)); 712 } else { 713 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 714 "dma mapping failed for %d " 715 "bytes addr $%p flags %x (%d)", 716 len, b_rptr, status, status)); 717 good_packet = B_FALSE; 718 tdc_stats->tx_dma_bind_fail++; 719 tx_msg_p->flags.dma_type = USE_NONE; 720 if (is_lso) { 721 mp = nmp; 722 goto nxge_start_fail_lso; 723 } else { 724 status = 1; 725 goto nxge_start_fail2; 726 } 727 } 728 } /* ddi dvma */ 729 730 if (is_lso) { 731 nmp_lso_save = nmp; 732 } 733 nmp = nmp->b_cont; 734 nxge_start_control_header_only: 735 npi_desc_handle.regp = (uint64_t)tx_desc_p; 736 ngathers++; 737 738 if (ngathers == 1) { 739 #ifdef NXGE_DEBUG 740 save_desc_p = &sop_tx_desc; 741 #endif 742 sop_tx_desc_p = &sop_tx_desc; 743 sop_tx_desc_p->value = 0; 744 sop_tx_desc_p->bits.hdw.tr_len = clen; 745 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 746 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 747 } else { 748 #ifdef NXGE_DEBUG 749 save_desc_p = &tx_desc; 750 #endif 751 tmp_desc_p = &tx_desc; 752 tmp_desc_p->value = 0; 753 tmp_desc_p->bits.hdw.tr_len = clen; 754 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 755 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 756 757 tx_desc_p->value = tmp_desc_p->value; 758 } 759 760 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 761 "Desc_entry %d ngathers %d " 762 "desc_vp $%p tx_desc_p $%p " 763 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 764 "dma_ioaddr (SAD) $%p mark %d", 765 i, ngathers, 766 tx_desc_ring_vp, tx_desc_p, 767 len, clen, pkt_len, pack_len, nmblks, 768 dma_ioaddr, mark_mode)); 769 770 #ifdef NXGE_DEBUG 771 npi_desc_handle.nxgep = nxgep; 772 npi_desc_handle.function.function = nxgep->function_num; 773 npi_desc_handle.function.instance = nxgep->instance; 774 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 775 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 776 TX_PKT_DESC_TR_LEN_SHIFT); 777 778 779 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 780 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 781 "mark %d sop %d\n", 782 save_desc_p->value, 783 sad, 784 save_desc_p->bits.hdw.tr_len, 785 xfer_len, 786 save_desc_p->bits.hdw.num_ptr, 787 save_desc_p->bits.hdw.mark, 788 save_desc_p->bits.hdw.sop)); 789 790 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 791 #endif 792 793 tx_msg_p->tx_msg_size = clen; 794 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 795 if (ngathers > nxge_tx_max_gathers) { 796 good_packet = B_FALSE; 797 mac_hcksum_get(mp, &start_offset, 798 &stuff_offset, &end_offset, &value, 799 &cksum_flags); 800 801 NXGE_DEBUG_MSG((NULL, TX_CTL, 802 "==> nxge_start(14): pull msg - " 803 "len %d pkt_len %d ngathers %d", 804 len, pkt_len, ngathers)); 805 806 /* 807 * Just give up on this packet. 808 */ 809 if (is_lso) { 810 mp = nmp_lso_save; 811 goto nxge_start_fail_lso; 812 } 813 status = 0; 814 goto nxge_start_fail2; 815 } 816 } /* while (nmp) */ 817 818 tx_msg_p->tx_message = mp; 819 tx_desc_p = &tx_desc_ring_vp[sop_index]; 820 npi_desc_handle.regp = (uint64_t)tx_desc_p; 821 822 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 823 pkthdrp->reserved = 0; 824 hdrp->value = 0; 825 bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 826 827 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 828 tdc_stats->tx_jumbo_pkts++; 829 } 830 831 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 832 if (pkt_len < min_len) { 833 /* Assume we use bcopy to premapped buffers */ 834 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 835 NXGE_DEBUG_MSG((NULL, TX_CTL, 836 "==> nxge_start(14-1): < (msg_min + 16)" 837 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 838 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 839 bzero((kaddr + pkt_len), (min_len - pkt_len)); 840 pkt_len = tx_msg_p->tx_msg_size = min_len; 841 842 sop_tx_desc_p->bits.hdw.tr_len = min_len; 843 844 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 845 tx_desc_p->value = sop_tx_desc_p->value; 846 847 NXGE_DEBUG_MSG((NULL, TX_CTL, 848 "==> nxge_start(14-2): < msg_min - " 849 "len %d pkt_len %d min_len %d ngathers %d", 850 len, pkt_len, min_len, ngathers)); 851 } 852 853 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 854 cksum_flags)); 855 { 856 uint64_t tmp_len; 857 858 /* pkt_len already includes 16 + paddings!! */ 859 /* Update the control header length */ 860 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 861 tmp_len = hdrp->value | 862 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 863 864 NXGE_DEBUG_MSG((nxgep, TX_CTL, 865 "==> nxge_start(15_x1): setting SOP " 866 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 867 "0x%llx hdrp->value 0x%llx", 868 tot_xfer_len, tot_xfer_len, pkt_len, 869 tmp_len, hdrp->value)); 870 #if defined(_BIG_ENDIAN) 871 hdrp->value = ddi_swap64(tmp_len); 872 #else 873 hdrp->value = tmp_len; 874 #endif 875 NXGE_DEBUG_MSG((nxgep, 876 TX_CTL, "==> nxge_start(15_x2): setting SOP " 877 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 878 "tmp_len 0x%llx hdrp->value 0x%llx", 879 tot_xfer_len, pkt_len, 880 tmp_len, hdrp->value)); 881 } 882 883 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 884 "wr_index %d " 885 "tot_xfer_len (%d) pkt_len %d npads %d", 886 sop_index, 887 tot_xfer_len, pkt_len, 888 npads)); 889 890 sop_tx_desc_p->bits.hdw.sop = 1; 891 sop_tx_desc_p->bits.hdw.mark = mark_mode; 892 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 893 894 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 895 896 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 897 898 #ifdef NXGE_DEBUG 899 npi_desc_handle.nxgep = nxgep; 900 npi_desc_handle.function.function = nxgep->function_num; 901 npi_desc_handle.function.instance = nxgep->instance; 902 903 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 904 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 905 save_desc_p->value, 906 sad, 907 save_desc_p->bits.hdw.tr_len, 908 xfer_len, 909 save_desc_p->bits.hdw.num_ptr, 910 save_desc_p->bits.hdw.mark, 911 save_desc_p->bits.hdw.sop)); 912 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 913 914 dump_len = (pkt_len > 128) ? 128: pkt_len; 915 NXGE_DEBUG_MSG((nxgep, TX_CTL, 916 "==> nxge_start: dump packets(17) (after sop set, len " 917 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 918 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 919 (char *)hdrp, 920 nxge_dump_packet((char *)hdrp, dump_len))); 921 NXGE_DEBUG_MSG((nxgep, TX_CTL, 922 "==> nxge_start(18): TX desc sync: sop_index %d", 923 sop_index)); 924 #endif 925 926 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 927 (void) ddi_dma_sync(tx_desc_dma_handle, 928 sop_index * sizeof (tx_desc_t), 929 ngathers * sizeof (tx_desc_t), 930 DDI_DMA_SYNC_FORDEV); 931 932 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 933 "cs_off = 0x%02X cs_s_off = 0x%02X " 934 "pkt_len %d ngathers %d sop_index %d\n", 935 stuff_offset, start_offset, 936 pkt_len, ngathers, sop_index)); 937 } else { /* more than one descriptor and wrap around */ 938 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 939 (void) ddi_dma_sync(tx_desc_dma_handle, 940 sop_index * sizeof (tx_desc_t), 941 nsdescs * sizeof (tx_desc_t), 942 DDI_DMA_SYNC_FORDEV); 943 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 944 "cs_off = 0x%02X cs_s_off = 0x%02X " 945 "pkt_len %d ngathers %d sop_index %d\n", 946 stuff_offset, start_offset, 947 pkt_len, ngathers, sop_index)); 948 949 (void) ddi_dma_sync(tx_desc_dma_handle, 950 0, 951 (ngathers - nsdescs) * sizeof (tx_desc_t), 952 DDI_DMA_SYNC_FORDEV); 953 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 954 "cs_off = 0x%02X cs_s_off = 0x%02X " 955 "pkt_len %d ngathers %d sop_index %d\n", 956 stuff_offset, start_offset, 957 pkt_len, ngathers, sop_index)); 958 } 959 960 tail_index = tx_ring_p->wr_index; 961 tail_wrap = tx_ring_p->wr_index_wrap; 962 963 tx_ring_p->wr_index = i; 964 if (tx_ring_p->wr_index <= tail_index) { 965 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 966 B_FALSE : B_TRUE); 967 } 968 969 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 970 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 971 tx_ring_p->tdc, 972 tx_ring_p->wr_index, 973 tx_ring_p->wr_index_wrap, 974 ngathers, 975 tx_ring_p->descs_pending)); 976 977 if (is_lso) { 978 lso_ngathers += ngathers; 979 if (mp_chain != NULL) { 980 mp = mp_chain; 981 mp_chain = mp_chain->b_next; 982 mp->b_next = NULL; 983 if (nxge_lso_kick_cnt == lso_ngathers) { 984 tx_ring_p->descs_pending += lso_ngathers; 985 { 986 tx_ring_kick_t kick; 987 988 kick.value = 0; 989 kick.bits.ldw.wrap = 990 tx_ring_p->wr_index_wrap; 991 kick.bits.ldw.tail = 992 (uint16_t)tx_ring_p->wr_index; 993 994 /* Kick the Transmit kick register */ 995 TXDMA_REG_WRITE64( 996 NXGE_DEV_NPI_HANDLE(nxgep), 997 TX_RING_KICK_REG, 998 (uint8_t)tx_ring_p->tdc, 999 kick.value); 1000 tdc_stats->tx_starts++; 1001 1002 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1003 "==> nxge_start: more LSO: " 1004 "LSO_CNT %d", 1005 lso_ngathers)); 1006 } 1007 lso_ngathers = 0; 1008 ngathers = 0; 1009 cur_index_lso = sop_index = tx_ring_p->wr_index; 1010 lso_tail_wrap = tx_ring_p->wr_index_wrap; 1011 } 1012 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1013 "==> nxge_start: lso again: " 1014 "lso_gathers %d ngathers %d cur_index_lso %d " 1015 "wr_index %d sop_index %d", 1016 lso_ngathers, ngathers, cur_index_lso, 1017 tx_ring_p->wr_index, sop_index)); 1018 1019 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1020 "==> nxge_start: next : count %d", 1021 lso_ngathers)); 1022 lso_again = B_TRUE; 1023 goto start_again; 1024 } 1025 ngathers = lso_ngathers; 1026 } 1027 1028 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 1029 1030 { 1031 tx_ring_kick_t kick; 1032 1033 kick.value = 0; 1034 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 1035 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 1036 1037 /* Kick start the Transmit kick register */ 1038 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 1039 TX_RING_KICK_REG, 1040 (uint8_t)tx_ring_p->tdc, 1041 kick.value); 1042 } 1043 1044 tx_ring_p->descs_pending += ngathers; 1045 tdc_stats->tx_starts++; 1046 1047 if (isLDOMservice(nxgep)) { 1048 tx_ring_p->tx_ring_busy = B_FALSE; 1049 if (tx_ring_p->tx_ring_offline) { 1050 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1051 NXGE_TX_RING_OFFLINED); 1052 } 1053 } 1054 1055 MUTEX_EXIT(&tx_ring_p->lock); 1056 1057 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1058 return (status); 1059 1060 nxge_start_fail_lso: 1061 status = 0; 1062 good_packet = B_FALSE; 1063 if (mp != NULL) 1064 freemsg(mp); 1065 if (mp_chain != NULL) 1066 freemsgchain(mp_chain); 1067 1068 if (!lso_again && !ngathers) { 1069 if (isLDOMservice(nxgep)) { 1070 tx_ring_p->tx_ring_busy = B_FALSE; 1071 if (tx_ring_p->tx_ring_offline) { 1072 (void) atomic_swap_32( 1073 &tx_ring_p->tx_ring_offline, 1074 NXGE_TX_RING_OFFLINED); 1075 } 1076 } 1077 1078 MUTEX_EXIT(&tx_ring_p->lock); 1079 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1080 "==> nxge_start: lso exit (nothing changed)")); 1081 goto nxge_start_fail1; 1082 } 1083 1084 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1085 "==> nxge_start (channel %d): before lso " 1086 "lso_gathers %d ngathers %d cur_index_lso %d " 1087 "wr_index %d sop_index %d lso_again %d", 1088 tx_ring_p->tdc, 1089 lso_ngathers, ngathers, cur_index_lso, 1090 tx_ring_p->wr_index, sop_index, lso_again)); 1091 1092 if (lso_again) { 1093 lso_ngathers += ngathers; 1094 ngathers = lso_ngathers; 1095 sop_index = cur_index_lso; 1096 tx_ring_p->wr_index = sop_index; 1097 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1098 } 1099 1100 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1101 "==> nxge_start (channel %d): after lso " 1102 "lso_gathers %d ngathers %d cur_index_lso %d " 1103 "wr_index %d sop_index %d lso_again %d", 1104 tx_ring_p->tdc, 1105 lso_ngathers, ngathers, cur_index_lso, 1106 tx_ring_p->wr_index, sop_index, lso_again)); 1107 1108 nxge_start_fail2: 1109 if (good_packet == B_FALSE) { 1110 cur_index = sop_index; 1111 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1112 for (i = 0; i < ngathers; i++) { 1113 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1114 npi_handle.regp = (uint64_t)tx_desc_p; 1115 tx_msg_p = &tx_msg_ring[cur_index]; 1116 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1117 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1118 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1119 "tx_desc_p = %X index = %d", 1120 tx_desc_p, tx_ring_p->rd_index)); 1121 (void) dvma_unload(tx_msg_p->dvma_handle, 1122 0, -1); 1123 tx_msg_p->dvma_handle = NULL; 1124 if (tx_ring_p->dvma_wr_index == 1125 tx_ring_p->dvma_wrap_mask) 1126 tx_ring_p->dvma_wr_index = 0; 1127 else 1128 tx_ring_p->dvma_wr_index++; 1129 tx_ring_p->dvma_pending--; 1130 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1131 if (ddi_dma_unbind_handle( 1132 tx_msg_p->dma_handle)) { 1133 cmn_err(CE_WARN, "!nxge_start: " 1134 "ddi_dma_unbind_handle failed"); 1135 } 1136 } 1137 tx_msg_p->flags.dma_type = USE_NONE; 1138 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1139 tx_ring_p->tx_wrap_mask); 1140 1141 } 1142 } 1143 1144 if (isLDOMservice(nxgep)) { 1145 tx_ring_p->tx_ring_busy = B_FALSE; 1146 if (tx_ring_p->tx_ring_offline) { 1147 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1148 NXGE_TX_RING_OFFLINED); 1149 } 1150 } 1151 1152 MUTEX_EXIT(&tx_ring_p->lock); 1153 1154 nxge_start_fail1: 1155 /* Add FMA to check the access handle nxge_hregh */ 1156 1157 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1158 return (status); 1159 } 1160 1161 /* Software LSO starts here */ 1162 static void 1163 nxge_hcksum_retrieve(mblk_t *mp, 1164 uint32_t *start, uint32_t *stuff, uint32_t *end, 1165 uint32_t *value, uint32_t *flags) 1166 { 1167 if (mp->b_datap->db_type == M_DATA) { 1168 if (flags != NULL) { 1169 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1170 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1171 HCK_FULLCKSUM_OK); 1172 if ((*flags & (HCK_PARTIALCKSUM | 1173 HCK_FULLCKSUM)) != 0) { 1174 if (value != NULL) 1175 *value = (uint32_t)DB_CKSUM16(mp); 1176 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1177 if (start != NULL) 1178 *start = 1179 (uint32_t)DB_CKSUMSTART(mp); 1180 if (stuff != NULL) 1181 *stuff = 1182 (uint32_t)DB_CKSUMSTUFF(mp); 1183 if (end != NULL) 1184 *end = 1185 (uint32_t)DB_CKSUMEND(mp); 1186 } 1187 } 1188 } 1189 } 1190 } 1191 1192 static void 1193 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1194 { 1195 ASSERT(DB_TYPE(mp) == M_DATA); 1196 1197 if (mss == NULL || flags == NULL) 1198 return; 1199 1200 *mss = 0; 1201 if (flags != NULL) { 1202 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1203 if (*flags != 0) { 1204 *mss = (uint32_t)DB_LSOMSS(mp); 1205 } 1206 NXGE_DEBUG_MSG((NULL, TX_CTL, 1207 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1208 *mss, *flags)); 1209 } 1210 1211 NXGE_DEBUG_MSG((NULL, TX_CTL, 1212 "<== nxge_lso_info_get: mss %d", *mss)); 1213 } 1214 1215 /* 1216 * Do Soft LSO on the oversized packet. 1217 * 1218 * 1. Create a chain of message for headers. 1219 * 2. Fill up header messages with proper information. 1220 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1221 * each new message with necessary adjustments. 1222 * * Unchange the ethernet header for DIX frames. (by default) 1223 * * IP Total Length field is updated to MSS or less(only for the last one). 1224 * * IP Identification value is incremented by one for each packet. 1225 * * TCP sequence Number is recalculated according to the payload length. 1226 * * Set FIN and/or PSH flags for the *last* packet if applied. 1227 * * TCP partial Checksum 1228 * 4. Update LSO information in the first message header. 1229 * 5. Release the original message header. 1230 */ 1231 static mblk_t * 1232 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1233 { 1234 uint32_t hckflags; 1235 int pktlen; 1236 int hdrlen; 1237 int segnum; 1238 int i; 1239 struct ether_vlan_header *evh; 1240 int ehlen, iphlen, tcphlen; 1241 struct ip *oiph, *niph; 1242 struct tcphdr *otcph, *ntcph; 1243 int available, len, left; 1244 uint16_t ip_id; 1245 uint32_t tcp_seq; 1246 #ifdef __sparc 1247 uint32_t tcp_seq_tmp; 1248 #endif 1249 mblk_t *datamp; 1250 uchar_t *rptr; 1251 mblk_t *nmp; 1252 mblk_t *cmp; 1253 mblk_t *mp_chain; 1254 boolean_t do_cleanup = B_FALSE; 1255 t_uscalar_t start_offset = 0; 1256 t_uscalar_t stuff_offset = 0; 1257 t_uscalar_t value = 0; 1258 uint16_t l4_len; 1259 ipaddr_t src, dst; 1260 uint32_t cksum, sum, l4cksum; 1261 1262 NXGE_DEBUG_MSG((NULL, TX_CTL, 1263 "==> nxge_do_softlso")); 1264 /* 1265 * check the length of LSO packet payload and calculate the number of 1266 * segments to be generated. 1267 */ 1268 pktlen = msgsize(mp); 1269 evh = (struct ether_vlan_header *)mp->b_rptr; 1270 1271 /* VLAN? */ 1272 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1273 ehlen = sizeof (struct ether_vlan_header); 1274 else 1275 ehlen = sizeof (struct ether_header); 1276 oiph = (struct ip *)(mp->b_rptr + ehlen); 1277 iphlen = oiph->ip_hl * 4; 1278 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1279 tcphlen = otcph->th_off * 4; 1280 1281 l4_len = pktlen - ehlen - iphlen; 1282 1283 NXGE_DEBUG_MSG((NULL, TX_CTL, 1284 "==> nxge_do_softlso: mss %d oiph $%p " 1285 "original ip_sum oiph->ip_sum 0x%x " 1286 "original tcp_sum otcph->th_sum 0x%x " 1287 "oiph->ip_len %d pktlen %d ehlen %d " 1288 "l4_len %d (0x%x) ip_len - iphlen %d ", 1289 mss, 1290 oiph, 1291 oiph->ip_sum, 1292 otcph->th_sum, 1293 ntohs(oiph->ip_len), pktlen, 1294 ehlen, 1295 l4_len, 1296 l4_len, 1297 ntohs(oiph->ip_len) - iphlen)); 1298 1299 /* IPv4 + TCP */ 1300 if (!(oiph->ip_v == IPV4_VERSION)) { 1301 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1302 "<== nxge_do_softlso: not IPV4 " 1303 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1304 ntohs(oiph->ip_len), pktlen, ehlen, 1305 tcphlen)); 1306 freemsg(mp); 1307 return (NULL); 1308 } 1309 1310 if (!(oiph->ip_p == IPPROTO_TCP)) { 1311 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1312 "<== nxge_do_softlso: not TCP " 1313 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1314 ntohs(oiph->ip_len), pktlen, ehlen, 1315 tcphlen)); 1316 freemsg(mp); 1317 return (NULL); 1318 } 1319 1320 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1321 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1322 "<== nxge_do_softlso: len not matched " 1323 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1324 ntohs(oiph->ip_len), pktlen, ehlen, 1325 tcphlen)); 1326 freemsg(mp); 1327 return (NULL); 1328 } 1329 1330 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1331 tcphlen = otcph->th_off * 4; 1332 1333 /* TCP flags can not include URG, RST, or SYN */ 1334 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1335 1336 hdrlen = ehlen + iphlen + tcphlen; 1337 1338 VERIFY(MBLKL(mp) >= hdrlen); 1339 1340 if (MBLKL(mp) > hdrlen) { 1341 datamp = mp; 1342 rptr = mp->b_rptr + hdrlen; 1343 } else { /* = */ 1344 datamp = mp->b_cont; 1345 rptr = datamp->b_rptr; 1346 } 1347 1348 NXGE_DEBUG_MSG((NULL, TX_CTL, 1349 "nxge_do_softlso: otcph $%p pktlen: %d, " 1350 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1351 "mblkl(mp): %d, mblkl(datamp): %d", 1352 otcph, 1353 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1354 (int)MBLKL(mp), (int)MBLKL(datamp))); 1355 1356 hckflags = 0; 1357 nxge_hcksum_retrieve(mp, 1358 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1359 1360 dst = oiph->ip_dst.s_addr; 1361 src = oiph->ip_src.s_addr; 1362 1363 cksum = (dst >> 16) + (dst & 0xFFFF) + 1364 (src >> 16) + (src & 0xFFFF); 1365 l4cksum = cksum + IP_TCP_CSUM_COMP; 1366 1367 sum = l4_len + l4cksum; 1368 sum = (sum & 0xFFFF) + (sum >> 16); 1369 1370 NXGE_DEBUG_MSG((NULL, TX_CTL, 1371 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1372 "hckflags 0x%x start_offset %d stuff_offset %d " 1373 "value (original) 0x%x th_sum 0x%x " 1374 "pktlen %d l4_len %d (0x%x) " 1375 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1376 dst, src, 1377 (sum & 0xffff), (~sum & 0xffff), 1378 hckflags, start_offset, stuff_offset, 1379 value, otcph->th_sum, 1380 pktlen, 1381 l4_len, 1382 l4_len, 1383 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1384 (int)MBLKL(datamp), 1385 nxge_dump_packet((char *)evh, 12))); 1386 1387 /* 1388 * Start to process. 1389 */ 1390 available = pktlen - hdrlen; 1391 segnum = (available - 1) / mss + 1; 1392 1393 NXGE_DEBUG_MSG((NULL, TX_CTL, 1394 "==> nxge_do_softlso: pktlen %d " 1395 "MBLKL(mp): %d, MBLKL(datamp): %d " 1396 "available %d mss %d segnum %d", 1397 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1398 available, 1399 mss, 1400 segnum)); 1401 1402 VERIFY(segnum >= 2); 1403 1404 /* 1405 * Try to pre-allocate all header messages 1406 */ 1407 mp_chain = NULL; 1408 for (i = 0; i < segnum; i++) { 1409 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1410 /* Clean up the mp_chain */ 1411 while (mp_chain != NULL) { 1412 nmp = mp_chain; 1413 mp_chain = mp_chain->b_next; 1414 freemsg(nmp); 1415 } 1416 NXGE_DEBUG_MSG((NULL, TX_CTL, 1417 "<== nxge_do_softlso: " 1418 "Could not allocate enough messages for headers!")); 1419 freemsg(mp); 1420 return (NULL); 1421 } 1422 nmp->b_next = mp_chain; 1423 mp_chain = nmp; 1424 1425 NXGE_DEBUG_MSG((NULL, TX_CTL, 1426 "==> nxge_do_softlso: " 1427 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1428 mp, nmp, mp_chain, mp_chain->b_next)); 1429 } 1430 1431 NXGE_DEBUG_MSG((NULL, TX_CTL, 1432 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1433 mp, nmp, mp_chain)); 1434 1435 /* 1436 * Associate payload with new packets 1437 */ 1438 cmp = mp_chain; 1439 left = available; 1440 while (cmp != NULL) { 1441 nmp = dupb(datamp); 1442 if (nmp == NULL) { 1443 do_cleanup = B_TRUE; 1444 NXGE_DEBUG_MSG((NULL, TX_CTL, 1445 "==>nxge_do_softlso: " 1446 "Can not dupb(datamp), have to do clean up")); 1447 goto cleanup_allocated_msgs; 1448 } 1449 1450 NXGE_DEBUG_MSG((NULL, TX_CTL, 1451 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1452 "dupb nmp $%p len %d left %d msd %d ", 1453 mp, cmp, nmp, len, left, mss)); 1454 1455 cmp->b_cont = nmp; 1456 nmp->b_rptr = rptr; 1457 len = (left < mss) ? left : mss; 1458 left -= len; 1459 1460 NXGE_DEBUG_MSG((NULL, TX_CTL, 1461 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1462 "dupb nmp $%p len %d left %d mss %d ", 1463 mp, cmp, nmp, len, left, mss)); 1464 NXGE_DEBUG_MSG((NULL, TX_CTL, 1465 "nxge_do_softlso: before available: %d, " 1466 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1467 available, left, len, segnum, (int)MBLKL(nmp))); 1468 1469 len -= MBLKL(nmp); 1470 NXGE_DEBUG_MSG((NULL, TX_CTL, 1471 "nxge_do_softlso: after available: %d, " 1472 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1473 available, left, len, segnum, (int)MBLKL(nmp))); 1474 1475 while (len > 0) { 1476 mblk_t *mmp = NULL; 1477 1478 NXGE_DEBUG_MSG((NULL, TX_CTL, 1479 "nxge_do_softlso: (4) len > 0 available: %d, " 1480 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1481 available, left, len, segnum, (int)MBLKL(nmp))); 1482 1483 if (datamp->b_cont != NULL) { 1484 datamp = datamp->b_cont; 1485 rptr = datamp->b_rptr; 1486 mmp = dupb(datamp); 1487 if (mmp == NULL) { 1488 do_cleanup = B_TRUE; 1489 NXGE_DEBUG_MSG((NULL, TX_CTL, 1490 "==> nxge_do_softlso: " 1491 "Can not dupb(datamp) (1), :" 1492 "have to do clean up")); 1493 NXGE_DEBUG_MSG((NULL, TX_CTL, 1494 "==> nxge_do_softlso: " 1495 "available: %d, left: %d, " 1496 "len: %d, MBLKL(nmp): %d", 1497 available, left, len, 1498 (int)MBLKL(nmp))); 1499 goto cleanup_allocated_msgs; 1500 } 1501 } else { 1502 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1503 "==> nxge_do_softlso: " 1504 "(1)available: %d, left: %d, " 1505 "len: %d, MBLKL(nmp): %d", 1506 available, left, len, 1507 (int)MBLKL(nmp))); 1508 cmn_err(CE_PANIC, 1509 "==> nxge_do_softlso: " 1510 "Pointers must have been corrupted!\n" 1511 "datamp: $%p, nmp: $%p, rptr: $%p", 1512 (void *)datamp, 1513 (void *)nmp, 1514 (void *)rptr); 1515 } 1516 nmp->b_cont = mmp; 1517 nmp = mmp; 1518 len -= MBLKL(nmp); 1519 } 1520 if (len < 0) { 1521 nmp->b_wptr += len; 1522 rptr = nmp->b_wptr; 1523 NXGE_DEBUG_MSG((NULL, TX_CTL, 1524 "(5) len < 0 (less than 0)" 1525 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1526 available, left, len, (int)MBLKL(nmp))); 1527 1528 } else if (len == 0) { 1529 if (datamp->b_cont != NULL) { 1530 NXGE_DEBUG_MSG((NULL, TX_CTL, 1531 "(5) len == 0" 1532 "available: %d, left: %d, len: %d, " 1533 "MBLKL(nmp): %d", 1534 available, left, len, (int)MBLKL(nmp))); 1535 datamp = datamp->b_cont; 1536 rptr = datamp->b_rptr; 1537 } else { 1538 NXGE_DEBUG_MSG((NULL, TX_CTL, 1539 "(6)available b_cont == NULL : %d, " 1540 "left: %d, len: %d, MBLKL(nmp): %d", 1541 available, left, len, (int)MBLKL(nmp))); 1542 1543 VERIFY(cmp->b_next == NULL); 1544 VERIFY(left == 0); 1545 break; /* Done! */ 1546 } 1547 } 1548 cmp = cmp->b_next; 1549 1550 NXGE_DEBUG_MSG((NULL, TX_CTL, 1551 "(7) do_softlso: " 1552 "next mp in mp_chain available len != 0 : %d, " 1553 "left: %d, len: %d, MBLKL(nmp): %d", 1554 available, left, len, (int)MBLKL(nmp))); 1555 } 1556 1557 /* 1558 * From now, start to fill up all headers for the first message 1559 * Hardware checksum flags need to be updated separately for FULLCKSUM 1560 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1561 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1562 * required fields need to be updated properly. 1563 */ 1564 nmp = mp_chain; 1565 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1566 nmp->b_wptr = nmp->b_rptr + hdrlen; 1567 niph = (struct ip *)(nmp->b_rptr + ehlen); 1568 niph->ip_len = htons(mss + iphlen + tcphlen); 1569 ip_id = ntohs(niph->ip_id); 1570 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1571 #ifdef __sparc 1572 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1573 tcp_seq = ntohl(tcp_seq_tmp); 1574 #else 1575 tcp_seq = ntohl(ntcph->th_seq); 1576 #endif 1577 1578 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1579 1580 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1581 DB_CKSUMSTART(nmp) = start_offset; 1582 DB_CKSUMSTUFF(nmp) = stuff_offset; 1583 1584 /* calculate IP checksum and TCP pseudo header checksum */ 1585 niph->ip_sum = 0; 1586 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1587 1588 l4_len = mss + tcphlen; 1589 sum = htons(l4_len) + l4cksum; 1590 sum = (sum & 0xFFFF) + (sum >> 16); 1591 ntcph->th_sum = (sum & 0xffff); 1592 1593 NXGE_DEBUG_MSG((NULL, TX_CTL, 1594 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1595 "mss %d pktlen %d l4_len %d (0x%x) " 1596 "MBLKL(mp): %d, MBLKL(datamp): %d " 1597 "ip_sum 0x%x " 1598 "th_sum 0x%x sum 0x%x ) " 1599 "dump first ip->tcp %s", 1600 nmp, mp_chain, 1601 mss, 1602 pktlen, 1603 l4_len, 1604 l4_len, 1605 (int)MBLKL(mp), (int)MBLKL(datamp), 1606 niph->ip_sum, 1607 ntcph->th_sum, 1608 sum, 1609 nxge_dump_packet((char *)niph, 52))); 1610 1611 cmp = nmp; 1612 while ((nmp = nmp->b_next)->b_next != NULL) { 1613 NXGE_DEBUG_MSG((NULL, TX_CTL, 1614 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1615 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1616 nmp->b_wptr = nmp->b_rptr + hdrlen; 1617 niph = (struct ip *)(nmp->b_rptr + ehlen); 1618 niph->ip_id = htons(++ip_id); 1619 niph->ip_len = htons(mss + iphlen + tcphlen); 1620 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1621 tcp_seq += mss; 1622 1623 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1624 1625 #ifdef __sparc 1626 tcp_seq_tmp = htonl(tcp_seq); 1627 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1628 #else 1629 ntcph->th_seq = htonl(tcp_seq); 1630 #endif 1631 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1632 DB_CKSUMSTART(nmp) = start_offset; 1633 DB_CKSUMSTUFF(nmp) = stuff_offset; 1634 1635 /* calculate IP checksum and TCP pseudo header checksum */ 1636 niph->ip_sum = 0; 1637 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1638 ntcph->th_sum = (sum & 0xffff); 1639 1640 NXGE_DEBUG_MSG((NULL, TX_CTL, 1641 "==> nxge_do_softlso: middle ip_sum 0x%x " 1642 "th_sum 0x%x " 1643 " mp $%p (mp_chain $%p) pktlen %d " 1644 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1645 niph->ip_sum, 1646 ntcph->th_sum, 1647 nmp, mp_chain, 1648 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1649 } 1650 1651 /* Last segment */ 1652 /* 1653 * Set FIN and/or PSH flags if present only in the last packet. 1654 * The ip_len could be different from prior packets. 1655 */ 1656 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1657 nmp->b_wptr = nmp->b_rptr + hdrlen; 1658 niph = (struct ip *)(nmp->b_rptr + ehlen); 1659 niph->ip_id = htons(++ip_id); 1660 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1661 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1662 tcp_seq += mss; 1663 #ifdef __sparc 1664 tcp_seq_tmp = htonl(tcp_seq); 1665 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1666 #else 1667 ntcph->th_seq = htonl(tcp_seq); 1668 #endif 1669 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1670 1671 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1672 DB_CKSUMSTART(nmp) = start_offset; 1673 DB_CKSUMSTUFF(nmp) = stuff_offset; 1674 1675 /* calculate IP checksum and TCP pseudo header checksum */ 1676 niph->ip_sum = 0; 1677 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1678 1679 l4_len = ntohs(niph->ip_len) - iphlen; 1680 sum = htons(l4_len) + l4cksum; 1681 sum = (sum & 0xFFFF) + (sum >> 16); 1682 ntcph->th_sum = (sum & 0xffff); 1683 1684 NXGE_DEBUG_MSG((NULL, TX_CTL, 1685 "==> nxge_do_softlso: last next " 1686 "niph->ip_sum 0x%x " 1687 "ntcph->th_sum 0x%x sum 0x%x " 1688 "dump last ip->tcp %s " 1689 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1690 "l4_len %d (0x%x) " 1691 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1692 niph->ip_sum, 1693 ntcph->th_sum, sum, 1694 nxge_dump_packet((char *)niph, 52), 1695 cmp, nmp, mp_chain, 1696 pktlen, pktlen, 1697 l4_len, 1698 l4_len, 1699 (int)MBLKL(mp), (int)MBLKL(datamp))); 1700 1701 cleanup_allocated_msgs: 1702 if (do_cleanup) { 1703 NXGE_DEBUG_MSG((NULL, TX_CTL, 1704 "==> nxge_do_softlso: " 1705 "Failed allocating messages, " 1706 "have to clean up and fail!")); 1707 while (mp_chain != NULL) { 1708 nmp = mp_chain; 1709 mp_chain = mp_chain->b_next; 1710 freemsg(nmp); 1711 } 1712 } 1713 /* 1714 * We're done here, so just free the original message and return the 1715 * new message chain, that could be NULL if failed, back to the caller. 1716 */ 1717 freemsg(mp); 1718 1719 NXGE_DEBUG_MSG((NULL, TX_CTL, 1720 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1721 return (mp_chain); 1722 } 1723 1724 /* 1725 * Will be called before NIC driver do further operation on the message. 1726 * The input message may include LSO information, if so, go to softlso logic 1727 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1728 * The return could be the same non-LSO message or a message chain for LSO case. 1729 * 1730 * The driver needs to call this function per packet and process the whole chain 1731 * if applied. 1732 */ 1733 static mblk_t * 1734 nxge_lso_eliminate(mblk_t *mp) 1735 { 1736 uint32_t lsoflags; 1737 uint32_t mss; 1738 1739 NXGE_DEBUG_MSG((NULL, TX_CTL, 1740 "==>nxge_lso_eliminate:")); 1741 nxge_lso_info_get(mp, &mss, &lsoflags); 1742 1743 if (lsoflags & HW_LSO) { 1744 mblk_t *nmp; 1745 1746 NXGE_DEBUG_MSG((NULL, TX_CTL, 1747 "==>nxge_lso_eliminate:" 1748 "HW_LSO:mss %d mp $%p", 1749 mss, mp)); 1750 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1751 NXGE_DEBUG_MSG((NULL, TX_CTL, 1752 "<== nxge_lso_eliminate: " 1753 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1754 nmp, mss, mp)); 1755 return (nmp); 1756 } else { 1757 NXGE_DEBUG_MSG((NULL, TX_CTL, 1758 "<== nxge_lso_eliminate_ " 1759 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1760 nmp, mss, mp)); 1761 return (NULL); 1762 } 1763 } 1764 1765 NXGE_DEBUG_MSG((NULL, TX_CTL, 1766 "<== nxge_lso_eliminate")); 1767 return (mp); 1768 } 1769 1770 static uint32_t 1771 nxge_csgen(uint16_t *adr, int len) 1772 { 1773 int i, odd; 1774 uint32_t sum = 0; 1775 uint32_t c = 0; 1776 1777 odd = len % 2; 1778 for (i = 0; i < (len / 2); i++) { 1779 sum += (adr[i] & 0xffff); 1780 } 1781 if (odd) { 1782 sum += adr[len / 2] & 0xff00; 1783 } 1784 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 1785 sum &= 0xffff; 1786 sum += c; 1787 } 1788 return (~sum & 0xffff); 1789 } 1790