1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/mac_provider.h> 27 #include <sys/nxge/nxge_impl.h> 28 #include <sys/nxge/nxge_hio.h> 29 #include <npi_tx_wr64.h> 30 31 /* Software LSO required header files */ 32 #include <netinet/tcp.h> 33 #include <inet/ip_impl.h> 34 #include <inet/tcp.h> 35 36 extern uint64_t mac_pkt_hash(uint_t, mblk_t *mp, uint8_t policy, 37 boolean_t is_outbound); 38 39 static mblk_t *nxge_lso_eliminate(mblk_t *); 40 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 41 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 42 static void nxge_hcksum_retrieve(mblk_t *, 43 uint32_t *, uint32_t *, uint32_t *, 44 uint32_t *, uint32_t *); 45 static uint32_t nxge_csgen(uint16_t *, int); 46 47 extern uint32_t nxge_reclaim_pending; 48 extern uint32_t nxge_bcopy_thresh; 49 extern uint32_t nxge_dvma_thresh; 50 extern uint32_t nxge_dma_stream_thresh; 51 extern uint32_t nxge_tx_minfree; 52 extern uint32_t nxge_tx_intr_thres; 53 extern uint32_t nxge_tx_max_gathers; 54 extern uint32_t nxge_tx_tiny_pack; 55 extern uint32_t nxge_tx_use_bcopy; 56 extern nxge_tx_mode_t nxge_tx_scheme; 57 uint32_t nxge_lso_kick_cnt = 2; 58 59 60 void 61 nxge_tx_ring_task(void *arg) 62 { 63 p_tx_ring_t ring = (p_tx_ring_t)arg; 64 65 MUTEX_ENTER(&ring->lock); 66 (void) nxge_txdma_reclaim(ring->nxgep, ring, 0); 67 MUTEX_EXIT(&ring->lock); 68 69 if (!isLDOMguest(ring->nxgep) && !ring->tx_ring_offline) 70 mac_tx_ring_update(ring->nxgep->mach, ring->tx_ring_handle); 71 #if defined(sun4v) 72 else { 73 nxge_hio_data_t *nhd = 74 (nxge_hio_data_t *)ring->nxgep->nxge_hw_p->hio; 75 nx_vio_fp_t *vio = &nhd->hio.vio; 76 77 /* Call back vnet. */ 78 if (vio->cb.vio_net_tx_update) { 79 (*vio->cb.vio_net_tx_update)(ring->nxgep->hio_vr->vhp); 80 } 81 } 82 #endif 83 } 84 85 static void 86 nxge_tx_ring_dispatch(p_tx_ring_t ring) 87 { 88 /* 89 * Kick the ring task to reclaim some buffers. 90 */ 91 (void) ddi_taskq_dispatch(ring->taskq, 92 nxge_tx_ring_task, (void *)ring, DDI_SLEEP); 93 } 94 95 mblk_t * 96 nxge_tx_ring_send(void *arg, mblk_t *mp) 97 { 98 p_nxge_ring_handle_t nrhp = (p_nxge_ring_handle_t)arg; 99 p_nxge_t nxgep; 100 p_tx_ring_t tx_ring_p; 101 int status, channel; 102 103 ASSERT(nrhp != NULL); 104 nxgep = nrhp->nxgep; 105 channel = nxgep->pt_config.hw_config.tdc.start + nrhp->index; 106 tx_ring_p = nxgep->tx_rings->rings[channel]; 107 108 ASSERT(nxgep == tx_ring_p->nxgep); 109 110 #ifdef DEBUG 111 if (isLDOMservice(nxgep)) { 112 ASSERT(!tx_ring_p->tx_ring_offline); 113 } 114 #endif 115 116 status = nxge_start(nxgep, tx_ring_p, mp); 117 if (status) { 118 nxge_tx_ring_dispatch(tx_ring_p); 119 return (mp); 120 } 121 122 return ((mblk_t *)NULL); 123 } 124 125 #if defined(sun4v) 126 127 /* 128 * Hashing policy for load balancing over the set of TX rings 129 * available to the driver. 130 */ 131 static uint8_t nxge_tx_hash_policy = MAC_PKT_HASH_L4; 132 133 /* 134 * nxge_m_tx() is needed for Hybrid I/O operation of the vnet in 135 * the guest domain. See CR 6778758 for long term solution. 136 * 137 * The guest domain driver will for now hash the packet 138 * to pick a DMA channel from the only group it has group 0. 139 */ 140 141 mblk_t * 142 nxge_m_tx(void *arg, mblk_t *mp) 143 { 144 p_nxge_t nxgep = (p_nxge_t)arg; 145 mblk_t *next; 146 uint64_t rindex; 147 p_tx_ring_t tx_ring_p; 148 int status; 149 150 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_m_tx")); 151 152 /* 153 * Hash to pick a ring from Group 0, the only TX group 154 * for a guest domain driver. 155 */ 156 rindex = mac_pkt_hash(DL_ETHER, mp, nxge_tx_hash_policy, B_TRUE); 157 rindex = rindex % nxgep->pt_config.tdc_grps[0].max_tdcs; 158 159 /* 160 * Get the ring handle. 161 */ 162 tx_ring_p = nxgep->tx_rings->rings[rindex]; 163 164 while (mp != NULL) { 165 next = mp->b_next; 166 mp->b_next = NULL; 167 168 status = nxge_start(nxgep, tx_ring_p, mp); 169 if (status != 0) { 170 mp->b_next = next; 171 nxge_tx_ring_dispatch(tx_ring_p); 172 return (mp); 173 } 174 175 mp = next; 176 } 177 178 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_m_tx")); 179 return ((mblk_t *)NULL); 180 } 181 182 #endif 183 184 int 185 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 186 { 187 int status = 0; 188 p_tx_desc_t tx_desc_ring_vp; 189 npi_handle_t npi_desc_handle; 190 nxge_os_dma_handle_t tx_desc_dma_handle; 191 p_tx_desc_t tx_desc_p; 192 p_tx_msg_t tx_msg_ring; 193 p_tx_msg_t tx_msg_p; 194 tx_desc_t tx_desc, *tmp_desc_p; 195 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 196 p_tx_pkt_header_t hdrp; 197 tx_pkt_hdr_all_t tmp_hdrp; 198 p_tx_pkt_hdr_all_t pkthdrp; 199 uint8_t npads = 0; 200 uint64_t dma_ioaddr; 201 uint32_t dma_flags; 202 int last_bidx; 203 uint8_t *b_rptr; 204 caddr_t kaddr; 205 uint32_t nmblks; 206 uint32_t ngathers; 207 uint32_t clen; 208 int len; 209 uint32_t pkt_len, pack_len, min_len; 210 uint32_t bcopy_thresh; 211 int i, cur_index, sop_index; 212 uint16_t tail_index; 213 boolean_t tail_wrap = B_FALSE; 214 nxge_dma_common_t desc_area; 215 nxge_os_dma_handle_t dma_handle; 216 ddi_dma_cookie_t dma_cookie; 217 npi_handle_t npi_handle; 218 p_mblk_t nmp; 219 p_mblk_t t_mp; 220 uint32_t ncookies; 221 boolean_t good_packet; 222 boolean_t mark_mode = B_FALSE; 223 p_nxge_stats_t statsp; 224 p_nxge_tx_ring_stats_t tdc_stats; 225 t_uscalar_t start_offset = 0; 226 t_uscalar_t stuff_offset = 0; 227 t_uscalar_t end_offset = 0; 228 t_uscalar_t value = 0; 229 t_uscalar_t cksum_flags = 0; 230 boolean_t cksum_on = B_FALSE; 231 uint32_t boff = 0; 232 uint64_t tot_xfer_len = 0; 233 boolean_t header_set = B_FALSE; 234 #ifdef NXGE_DEBUG 235 p_tx_desc_t tx_desc_ring_pp; 236 p_tx_desc_t tx_desc_pp; 237 tx_desc_t *save_desc_p; 238 int dump_len; 239 int sad_len; 240 uint64_t sad; 241 int xfer_len; 242 uint32_t msgsize; 243 #endif 244 p_mblk_t mp_chain = NULL; 245 boolean_t is_lso = B_FALSE; 246 boolean_t lso_again; 247 int cur_index_lso; 248 p_mblk_t nmp_lso_save; 249 uint32_t lso_ngathers; 250 boolean_t lso_tail_wrap = B_FALSE; 251 252 NXGE_DEBUG_MSG((nxgep, TX_CTL, 253 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 254 NXGE_DEBUG_MSG((nxgep, TX_CTL, 255 "==> nxge_start: Starting tdc %d desc pending %d", 256 tx_ring_p->tdc, tx_ring_p->descs_pending)); 257 258 statsp = nxgep->statsp; 259 260 if (!isLDOMguest(nxgep)) { 261 switch (nxgep->mac.portmode) { 262 default: 263 if (nxgep->statsp->port_stats.lb_mode == 264 nxge_lb_normal) { 265 if (!statsp->mac_stats.link_up) { 266 freemsg(mp); 267 NXGE_DEBUG_MSG((nxgep, TX_CTL, 268 "==> nxge_start: " 269 "link not up")); 270 goto nxge_start_fail1; 271 } 272 } 273 break; 274 case PORT_10G_FIBER: 275 /* 276 * For the following modes, check the link status 277 * before sending the packet out: 278 * nxge_lb_normal, 279 * nxge_lb_ext10g, 280 * nxge_lb_ext1000, 281 * nxge_lb_ext100, 282 * nxge_lb_ext10. 283 */ 284 if (nxgep->statsp->port_stats.lb_mode < 285 nxge_lb_phy10g) { 286 if (!statsp->mac_stats.link_up) { 287 freemsg(mp); 288 NXGE_DEBUG_MSG((nxgep, TX_CTL, 289 "==> nxge_start: " 290 "link not up")); 291 goto nxge_start_fail1; 292 } 293 } 294 break; 295 } 296 } 297 298 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 299 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 300 NXGE_DEBUG_MSG((nxgep, TX_CTL, 301 "==> nxge_start: hardware not initialized or stopped")); 302 freemsg(mp); 303 goto nxge_start_fail1; 304 } 305 306 if (nxgep->soft_lso_enable) { 307 mp_chain = nxge_lso_eliminate(mp); 308 NXGE_DEBUG_MSG((nxgep, TX_CTL, 309 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 310 mp, mp_chain)); 311 if (mp_chain == NULL) { 312 NXGE_ERROR_MSG((nxgep, TX_CTL, 313 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 314 mp_chain, mp)); 315 goto nxge_start_fail1; 316 } 317 if (mp_chain != mp) { 318 NXGE_DEBUG_MSG((nxgep, TX_CTL, 319 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 320 mp_chain, mp)); 321 is_lso = B_TRUE; 322 mp = mp_chain; 323 mp_chain = mp_chain->b_next; 324 mp->b_next = NULL; 325 } 326 } 327 328 hcksum_retrieve(mp, NULL, NULL, &start_offset, 329 &stuff_offset, &end_offset, &value, &cksum_flags); 330 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 331 start_offset += sizeof (ether_header_t); 332 stuff_offset += sizeof (ether_header_t); 333 } else { 334 start_offset += sizeof (struct ether_vlan_header); 335 stuff_offset += sizeof (struct ether_vlan_header); 336 } 337 338 if (cksum_flags & HCK_PARTIALCKSUM) { 339 NXGE_DEBUG_MSG((nxgep, TX_CTL, 340 "==> nxge_start: mp $%p len %d " 341 "cksum_flags 0x%x (partial checksum) ", 342 mp, MBLKL(mp), cksum_flags)); 343 cksum_on = B_TRUE; 344 } 345 346 pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 347 pkthdrp->reserved = 0; 348 tmp_hdrp.pkthdr.value = 0; 349 nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 350 0, 0, pkthdrp, 351 start_offset, stuff_offset); 352 353 lso_again = B_FALSE; 354 lso_ngathers = 0; 355 356 MUTEX_ENTER(&tx_ring_p->lock); 357 358 if (isLDOMservice(nxgep)) { 359 tx_ring_p->tx_ring_busy = B_TRUE; 360 if (tx_ring_p->tx_ring_offline) { 361 freemsg(mp); 362 tx_ring_p->tx_ring_busy = B_FALSE; 363 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 364 NXGE_TX_RING_OFFLINED); 365 MUTEX_EXIT(&tx_ring_p->lock); 366 return (status); 367 } 368 } 369 370 cur_index_lso = tx_ring_p->wr_index; 371 lso_tail_wrap = tx_ring_p->wr_index_wrap; 372 start_again: 373 ngathers = 0; 374 sop_index = tx_ring_p->wr_index; 375 #ifdef NXGE_DEBUG 376 if (tx_ring_p->descs_pending) { 377 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 378 "desc pending %d ", tx_ring_p->descs_pending)); 379 } 380 381 dump_len = (int)(MBLKL(mp)); 382 dump_len = (dump_len > 128) ? 128: dump_len; 383 384 NXGE_DEBUG_MSG((nxgep, TX_CTL, 385 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 386 "(Before header reserve: ORIGINAL LEN %d)", 387 tx_ring_p->tdc, 388 mp->b_rptr, 389 dump_len)); 390 391 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 392 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 393 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 394 #endif 395 396 tdc_stats = tx_ring_p->tdc_stats; 397 mark_mode = (tx_ring_p->descs_pending && 398 (((int)tx_ring_p->tx_ring_size - (int)tx_ring_p->descs_pending) < 399 (int)nxge_tx_minfree)); 400 401 NXGE_DEBUG_MSG((nxgep, TX_CTL, 402 "TX Descriptor ring is channel %d mark mode %d", 403 tx_ring_p->tdc, mark_mode)); 404 405 if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) { 406 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, 407 (nxge_tx_minfree + lso_ngathers))) { 408 NXGE_DEBUG_MSG((nxgep, TX_CTL, 409 "TX Descriptor ring is full: channel %d", 410 tx_ring_p->tdc)); 411 NXGE_DEBUG_MSG((nxgep, TX_CTL, 412 "TX Descriptor ring is full: channel %d", 413 tx_ring_p->tdc)); 414 if (is_lso) { 415 /* 416 * free the current mp and mp_chain if not FULL. 417 */ 418 tdc_stats->tx_no_desc++; 419 NXGE_DEBUG_MSG((nxgep, TX_CTL, 420 "LSO packet: TX Descriptor ring is full: " 421 "channel %d", 422 tx_ring_p->tdc)); 423 goto nxge_start_fail_lso; 424 } else { 425 (void) cas32((uint32_t *)&tx_ring_p->queueing, 426 0, 1); 427 tdc_stats->tx_no_desc++; 428 429 if (isLDOMservice(nxgep)) { 430 tx_ring_p->tx_ring_busy = B_FALSE; 431 if (tx_ring_p->tx_ring_offline) { 432 (void) atomic_swap_32( 433 &tx_ring_p->tx_ring_offline, 434 NXGE_TX_RING_OFFLINED); 435 } 436 } 437 438 MUTEX_EXIT(&tx_ring_p->lock); 439 status = 1; 440 goto nxge_start_fail1; 441 } 442 } 443 } 444 445 nmp = mp; 446 i = sop_index = tx_ring_p->wr_index; 447 nmblks = 0; 448 ngathers = 0; 449 pkt_len = 0; 450 pack_len = 0; 451 clen = 0; 452 last_bidx = -1; 453 good_packet = B_TRUE; 454 455 desc_area = tx_ring_p->tdc_desc; 456 npi_handle = desc_area.npi_handle; 457 npi_desc_handle.regh = (nxge_os_acc_handle_t) 458 DMA_COMMON_ACC_HANDLE(desc_area); 459 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 460 tx_desc_dma_handle = (nxge_os_dma_handle_t) 461 DMA_COMMON_HANDLE(desc_area); 462 tx_msg_ring = tx_ring_p->tx_msg_ring; 463 464 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 465 sop_index, i)); 466 467 #ifdef NXGE_DEBUG 468 msgsize = msgdsize(nmp); 469 NXGE_DEBUG_MSG((nxgep, TX_CTL, 470 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 471 sop_index, i, msgsize)); 472 #endif 473 /* 474 * The first 16 bytes of the premapped buffer are reserved 475 * for header. No padding will be used. 476 */ 477 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 478 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 479 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 480 } else { 481 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 482 } 483 while (nmp) { 484 good_packet = B_TRUE; 485 b_rptr = nmp->b_rptr; 486 len = MBLKL(nmp); 487 if (len <= 0) { 488 nmp = nmp->b_cont; 489 continue; 490 } 491 nmblks++; 492 493 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 494 "len %d pkt_len %d pack_len %d", 495 nmblks, len, pkt_len, pack_len)); 496 /* 497 * Hardware limits the transfer length to 4K for NIU and 498 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 499 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 500 * If len is longer than the limit, then we break nmp into 501 * two chunks: Make the first chunk equal to the limit and 502 * the second chunk for the remaining data. If the second 503 * chunk is still larger than the limit, then it will be 504 * broken into two in the next pass. 505 */ 506 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 507 if ((t_mp = dupb(nmp)) != NULL) { 508 nmp->b_wptr = nmp->b_rptr + 509 (TX_MAX_TRANSFER_LENGTH 510 - TX_PKT_HEADER_SIZE); 511 t_mp->b_rptr = nmp->b_wptr; 512 t_mp->b_cont = nmp->b_cont; 513 nmp->b_cont = t_mp; 514 len = MBLKL(nmp); 515 } else { 516 if (is_lso) { 517 NXGE_DEBUG_MSG((nxgep, TX_CTL, 518 "LSO packet: dupb failed: " 519 "channel %d", 520 tx_ring_p->tdc)); 521 mp = nmp; 522 goto nxge_start_fail_lso; 523 } else { 524 good_packet = B_FALSE; 525 goto nxge_start_fail2; 526 } 527 } 528 } 529 tx_desc.value = 0; 530 tx_desc_p = &tx_desc_ring_vp[i]; 531 #ifdef NXGE_DEBUG 532 tx_desc_pp = &tx_desc_ring_pp[i]; 533 #endif 534 tx_msg_p = &tx_msg_ring[i]; 535 #if defined(__i386) 536 npi_desc_handle.regp = (uint32_t)tx_desc_p; 537 #else 538 npi_desc_handle.regp = (uint64_t)tx_desc_p; 539 #endif 540 if (!header_set && 541 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 542 (len >= bcopy_thresh))) { 543 header_set = B_TRUE; 544 bcopy_thresh += TX_PKT_HEADER_SIZE; 545 boff = 0; 546 pack_len = 0; 547 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 548 hdrp = (p_tx_pkt_header_t)kaddr; 549 clen = pkt_len; 550 dma_handle = tx_msg_p->buf_dma_handle; 551 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 552 (void) ddi_dma_sync(dma_handle, 553 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 554 DDI_DMA_SYNC_FORDEV); 555 556 tx_msg_p->flags.dma_type = USE_BCOPY; 557 goto nxge_start_control_header_only; 558 } 559 560 pkt_len += len; 561 pack_len += len; 562 563 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 564 "desc entry %d " 565 "DESC IOADDR $%p " 566 "desc_vp $%p tx_desc_p $%p " 567 "desc_pp $%p tx_desc_pp $%p " 568 "len %d pkt_len %d pack_len %d", 569 i, 570 DMA_COMMON_IOADDR(desc_area), 571 tx_desc_ring_vp, tx_desc_p, 572 tx_desc_ring_pp, tx_desc_pp, 573 len, pkt_len, pack_len)); 574 575 if (len < bcopy_thresh) { 576 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 577 "USE BCOPY: ")); 578 if (nxge_tx_tiny_pack) { 579 uint32_t blst = 580 TXDMA_DESC_NEXT_INDEX(i, -1, 581 tx_ring_p->tx_wrap_mask); 582 NXGE_DEBUG_MSG((nxgep, TX_CTL, 583 "==> nxge_start(5): pack")); 584 if ((pack_len <= bcopy_thresh) && 585 (last_bidx == blst)) { 586 NXGE_DEBUG_MSG((nxgep, TX_CTL, 587 "==> nxge_start: pack(6) " 588 "(pkt_len %d pack_len %d)", 589 pkt_len, pack_len)); 590 i = blst; 591 tx_desc_p = &tx_desc_ring_vp[i]; 592 #ifdef NXGE_DEBUG 593 tx_desc_pp = &tx_desc_ring_pp[i]; 594 #endif 595 tx_msg_p = &tx_msg_ring[i]; 596 boff = pack_len - len; 597 ngathers--; 598 } else if (pack_len > bcopy_thresh && 599 header_set) { 600 pack_len = len; 601 boff = 0; 602 bcopy_thresh = nxge_bcopy_thresh; 603 NXGE_DEBUG_MSG((nxgep, TX_CTL, 604 "==> nxge_start(7): > max NEW " 605 "bcopy thresh %d " 606 "pkt_len %d pack_len %d(next)", 607 bcopy_thresh, 608 pkt_len, pack_len)); 609 } 610 last_bidx = i; 611 } 612 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 613 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 614 hdrp = (p_tx_pkt_header_t)kaddr; 615 header_set = B_TRUE; 616 NXGE_DEBUG_MSG((nxgep, TX_CTL, 617 "==> nxge_start(7_x2): " 618 "pkt_len %d pack_len %d (new hdrp $%p)", 619 pkt_len, pack_len, hdrp)); 620 } 621 tx_msg_p->flags.dma_type = USE_BCOPY; 622 kaddr += boff; 623 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 624 "USE BCOPY: before bcopy " 625 "DESC IOADDR $%p entry %d " 626 "bcopy packets %d " 627 "bcopy kaddr $%p " 628 "bcopy ioaddr (SAD) $%p " 629 "bcopy clen %d " 630 "bcopy boff %d", 631 DMA_COMMON_IOADDR(desc_area), i, 632 tdc_stats->tx_hdr_pkts, 633 kaddr, 634 dma_ioaddr, 635 clen, 636 boff)); 637 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 638 "1USE BCOPY: ")); 639 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 640 "2USE BCOPY: ")); 641 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 642 "last USE BCOPY: copy from b_rptr $%p " 643 "to KADDR $%p (len %d offset %d", 644 b_rptr, kaddr, len, boff)); 645 646 bcopy(b_rptr, kaddr, len); 647 648 #ifdef NXGE_DEBUG 649 dump_len = (len > 128) ? 128: len; 650 NXGE_DEBUG_MSG((nxgep, TX_CTL, 651 "==> nxge_start: dump packets " 652 "(After BCOPY len %d)" 653 "(b_rptr $%p): %s", len, nmp->b_rptr, 654 nxge_dump_packet((char *)nmp->b_rptr, 655 dump_len))); 656 #endif 657 658 dma_handle = tx_msg_p->buf_dma_handle; 659 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 660 (void) ddi_dma_sync(dma_handle, 661 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 662 DDI_DMA_SYNC_FORDEV); 663 clen = len + boff; 664 tdc_stats->tx_hdr_pkts++; 665 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 666 "USE BCOPY: " 667 "DESC IOADDR $%p entry %d " 668 "bcopy packets %d " 669 "bcopy kaddr $%p " 670 "bcopy ioaddr (SAD) $%p " 671 "bcopy clen %d " 672 "bcopy boff %d", 673 DMA_COMMON_IOADDR(desc_area), 674 i, 675 tdc_stats->tx_hdr_pkts, 676 kaddr, 677 dma_ioaddr, 678 clen, 679 boff)); 680 } else { 681 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 682 "USE DVMA: len %d", len)); 683 tx_msg_p->flags.dma_type = USE_DMA; 684 dma_flags = DDI_DMA_WRITE; 685 if (len < nxge_dma_stream_thresh) { 686 dma_flags |= DDI_DMA_CONSISTENT; 687 } else { 688 dma_flags |= DDI_DMA_STREAMING; 689 } 690 691 dma_handle = tx_msg_p->dma_handle; 692 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 693 (caddr_t)b_rptr, len, dma_flags, 694 DDI_DMA_DONTWAIT, NULL, 695 &dma_cookie, &ncookies); 696 if (status == DDI_DMA_MAPPED) { 697 dma_ioaddr = dma_cookie.dmac_laddress; 698 len = (int)dma_cookie.dmac_size; 699 clen = (uint32_t)dma_cookie.dmac_size; 700 NXGE_DEBUG_MSG((nxgep, TX_CTL, 701 "==> nxge_start(12_1): " 702 "USE DVMA: len %d clen %d " 703 "ngathers %d", 704 len, clen, 705 ngathers)); 706 #if defined(__i386) 707 npi_desc_handle.regp = (uint32_t)tx_desc_p; 708 #else 709 npi_desc_handle.regp = (uint64_t)tx_desc_p; 710 #endif 711 while (ncookies > 1) { 712 ngathers++; 713 /* 714 * this is the fix for multiple 715 * cookies, which are basically 716 * a descriptor entry, we don't set 717 * SOP bit as well as related fields 718 */ 719 720 (void) npi_txdma_desc_gather_set( 721 npi_desc_handle, 722 &tx_desc, 723 (ngathers -1), 724 mark_mode, 725 ngathers, 726 dma_ioaddr, 727 clen); 728 729 tx_msg_p->tx_msg_size = clen; 730 NXGE_DEBUG_MSG((nxgep, TX_CTL, 731 "==> nxge_start: DMA " 732 "ncookie %d " 733 "ngathers %d " 734 "dma_ioaddr $%p len %d" 735 "desc $%p descp $%p (%d)", 736 ncookies, 737 ngathers, 738 dma_ioaddr, clen, 739 *tx_desc_p, tx_desc_p, i)); 740 741 ddi_dma_nextcookie(dma_handle, 742 &dma_cookie); 743 dma_ioaddr = 744 dma_cookie.dmac_laddress; 745 746 len = (int)dma_cookie.dmac_size; 747 clen = (uint32_t)dma_cookie.dmac_size; 748 NXGE_DEBUG_MSG((nxgep, TX_CTL, 749 "==> nxge_start(12_2): " 750 "USE DVMA: len %d clen %d ", 751 len, clen)); 752 753 i = TXDMA_DESC_NEXT_INDEX(i, 1, 754 tx_ring_p->tx_wrap_mask); 755 tx_desc_p = &tx_desc_ring_vp[i]; 756 757 #if defined(__i386) 758 npi_desc_handle.regp = 759 (uint32_t)tx_desc_p; 760 #else 761 npi_desc_handle.regp = 762 (uint64_t)tx_desc_p; 763 #endif 764 tx_msg_p = &tx_msg_ring[i]; 765 tx_msg_p->flags.dma_type = USE_NONE; 766 tx_desc.value = 0; 767 768 ncookies--; 769 } 770 tdc_stats->tx_ddi_pkts++; 771 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 772 "DMA: ddi packets %d", 773 tdc_stats->tx_ddi_pkts)); 774 } else { 775 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 776 "dma mapping failed for %d " 777 "bytes addr $%p flags %x (%d)", 778 len, b_rptr, status, status)); 779 good_packet = B_FALSE; 780 tdc_stats->tx_dma_bind_fail++; 781 tx_msg_p->flags.dma_type = USE_NONE; 782 if (is_lso) { 783 mp = nmp; 784 goto nxge_start_fail_lso; 785 } else { 786 goto nxge_start_fail2; 787 } 788 } 789 } /* ddi dvma */ 790 791 if (is_lso) { 792 nmp_lso_save = nmp; 793 } 794 nmp = nmp->b_cont; 795 nxge_start_control_header_only: 796 #if defined(__i386) 797 npi_desc_handle.regp = (uint32_t)tx_desc_p; 798 #else 799 npi_desc_handle.regp = (uint64_t)tx_desc_p; 800 #endif 801 ngathers++; 802 803 if (ngathers == 1) { 804 #ifdef NXGE_DEBUG 805 save_desc_p = &sop_tx_desc; 806 #endif 807 sop_tx_desc_p = &sop_tx_desc; 808 sop_tx_desc_p->value = 0; 809 sop_tx_desc_p->bits.hdw.tr_len = clen; 810 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 811 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 812 } else { 813 #ifdef NXGE_DEBUG 814 save_desc_p = &tx_desc; 815 #endif 816 tmp_desc_p = &tx_desc; 817 tmp_desc_p->value = 0; 818 tmp_desc_p->bits.hdw.tr_len = clen; 819 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 820 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 821 822 tx_desc_p->value = tmp_desc_p->value; 823 } 824 825 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 826 "Desc_entry %d ngathers %d " 827 "desc_vp $%p tx_desc_p $%p " 828 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 829 "dma_ioaddr (SAD) $%p mark %d", 830 i, ngathers, 831 tx_desc_ring_vp, tx_desc_p, 832 len, clen, pkt_len, pack_len, nmblks, 833 dma_ioaddr, mark_mode)); 834 835 #ifdef NXGE_DEBUG 836 npi_desc_handle.nxgep = nxgep; 837 npi_desc_handle.function.function = nxgep->function_num; 838 npi_desc_handle.function.instance = nxgep->instance; 839 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 840 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 841 TX_PKT_DESC_TR_LEN_SHIFT); 842 843 844 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 845 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 846 "mark %d sop %d\n", 847 save_desc_p->value, 848 sad, 849 save_desc_p->bits.hdw.tr_len, 850 xfer_len, 851 save_desc_p->bits.hdw.num_ptr, 852 save_desc_p->bits.hdw.mark, 853 save_desc_p->bits.hdw.sop)); 854 855 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 856 #endif 857 858 tx_msg_p->tx_msg_size = clen; 859 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 860 if (ngathers > nxge_tx_max_gathers) { 861 good_packet = B_FALSE; 862 hcksum_retrieve(mp, NULL, NULL, &start_offset, 863 &stuff_offset, &end_offset, &value, 864 &cksum_flags); 865 866 NXGE_DEBUG_MSG((NULL, TX_CTL, 867 "==> nxge_start(14): pull msg - " 868 "len %d pkt_len %d ngathers %d", 869 len, pkt_len, ngathers)); 870 /* Pull all message blocks from b_cont */ 871 if (is_lso) { 872 mp = nmp_lso_save; 873 goto nxge_start_fail_lso; 874 } 875 if ((msgpullup(mp, -1)) == NULL) { 876 goto nxge_start_fail2; 877 } 878 goto nxge_start_fail2; 879 } 880 } /* while (nmp) */ 881 882 tx_msg_p->tx_message = mp; 883 tx_desc_p = &tx_desc_ring_vp[sop_index]; 884 #if defined(__i386) 885 npi_desc_handle.regp = (uint32_t)tx_desc_p; 886 #else 887 npi_desc_handle.regp = (uint64_t)tx_desc_p; 888 #endif 889 890 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 891 pkthdrp->reserved = 0; 892 hdrp->value = 0; 893 bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 894 895 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 896 tdc_stats->tx_jumbo_pkts++; 897 } 898 899 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 900 if (pkt_len < min_len) { 901 /* Assume we use bcopy to premapped buffers */ 902 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 903 NXGE_DEBUG_MSG((NULL, TX_CTL, 904 "==> nxge_start(14-1): < (msg_min + 16)" 905 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 906 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 907 bzero((kaddr + pkt_len), (min_len - pkt_len)); 908 pkt_len = tx_msg_p->tx_msg_size = min_len; 909 910 sop_tx_desc_p->bits.hdw.tr_len = min_len; 911 912 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 913 tx_desc_p->value = sop_tx_desc_p->value; 914 915 NXGE_DEBUG_MSG((NULL, TX_CTL, 916 "==> nxge_start(14-2): < msg_min - " 917 "len %d pkt_len %d min_len %d ngathers %d", 918 len, pkt_len, min_len, ngathers)); 919 } 920 921 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 922 cksum_flags)); 923 { 924 uint64_t tmp_len; 925 926 /* pkt_len already includes 16 + paddings!! */ 927 /* Update the control header length */ 928 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 929 tmp_len = hdrp->value | 930 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 931 932 NXGE_DEBUG_MSG((nxgep, TX_CTL, 933 "==> nxge_start(15_x1): setting SOP " 934 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 935 "0x%llx hdrp->value 0x%llx", 936 tot_xfer_len, tot_xfer_len, pkt_len, 937 tmp_len, hdrp->value)); 938 #if defined(_BIG_ENDIAN) 939 hdrp->value = ddi_swap64(tmp_len); 940 #else 941 hdrp->value = tmp_len; 942 #endif 943 NXGE_DEBUG_MSG((nxgep, 944 TX_CTL, "==> nxge_start(15_x2): setting SOP " 945 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 946 "tmp_len 0x%llx hdrp->value 0x%llx", 947 tot_xfer_len, pkt_len, 948 tmp_len, hdrp->value)); 949 } 950 951 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 952 "wr_index %d " 953 "tot_xfer_len (%d) pkt_len %d npads %d", 954 sop_index, 955 tot_xfer_len, pkt_len, 956 npads)); 957 958 sop_tx_desc_p->bits.hdw.sop = 1; 959 sop_tx_desc_p->bits.hdw.mark = mark_mode; 960 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 961 962 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 963 964 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 965 966 #ifdef NXGE_DEBUG 967 npi_desc_handle.nxgep = nxgep; 968 npi_desc_handle.function.function = nxgep->function_num; 969 npi_desc_handle.function.instance = nxgep->instance; 970 971 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 972 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 973 save_desc_p->value, 974 sad, 975 save_desc_p->bits.hdw.tr_len, 976 xfer_len, 977 save_desc_p->bits.hdw.num_ptr, 978 save_desc_p->bits.hdw.mark, 979 save_desc_p->bits.hdw.sop)); 980 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 981 982 dump_len = (pkt_len > 128) ? 128: pkt_len; 983 NXGE_DEBUG_MSG((nxgep, TX_CTL, 984 "==> nxge_start: dump packets(17) (after sop set, len " 985 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 986 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 987 (char *)hdrp, 988 nxge_dump_packet((char *)hdrp, dump_len))); 989 NXGE_DEBUG_MSG((nxgep, TX_CTL, 990 "==> nxge_start(18): TX desc sync: sop_index %d", 991 sop_index)); 992 #endif 993 994 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 995 (void) ddi_dma_sync(tx_desc_dma_handle, 996 sop_index * sizeof (tx_desc_t), 997 ngathers * sizeof (tx_desc_t), 998 DDI_DMA_SYNC_FORDEV); 999 1000 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 1001 "cs_off = 0x%02X cs_s_off = 0x%02X " 1002 "pkt_len %d ngathers %d sop_index %d\n", 1003 stuff_offset, start_offset, 1004 pkt_len, ngathers, sop_index)); 1005 } else { /* more than one descriptor and wrap around */ 1006 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 1007 (void) ddi_dma_sync(tx_desc_dma_handle, 1008 sop_index * sizeof (tx_desc_t), 1009 nsdescs * sizeof (tx_desc_t), 1010 DDI_DMA_SYNC_FORDEV); 1011 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 1012 "cs_off = 0x%02X cs_s_off = 0x%02X " 1013 "pkt_len %d ngathers %d sop_index %d\n", 1014 stuff_offset, start_offset, 1015 pkt_len, ngathers, sop_index)); 1016 1017 (void) ddi_dma_sync(tx_desc_dma_handle, 1018 0, 1019 (ngathers - nsdescs) * sizeof (tx_desc_t), 1020 DDI_DMA_SYNC_FORDEV); 1021 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 1022 "cs_off = 0x%02X cs_s_off = 0x%02X " 1023 "pkt_len %d ngathers %d sop_index %d\n", 1024 stuff_offset, start_offset, 1025 pkt_len, ngathers, sop_index)); 1026 } 1027 1028 tail_index = tx_ring_p->wr_index; 1029 tail_wrap = tx_ring_p->wr_index_wrap; 1030 1031 tx_ring_p->wr_index = i; 1032 if (tx_ring_p->wr_index <= tail_index) { 1033 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 1034 B_FALSE : B_TRUE); 1035 } 1036 1037 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 1038 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 1039 tx_ring_p->tdc, 1040 tx_ring_p->wr_index, 1041 tx_ring_p->wr_index_wrap, 1042 ngathers, 1043 tx_ring_p->descs_pending)); 1044 1045 if (is_lso) { 1046 lso_ngathers += ngathers; 1047 if (mp_chain != NULL) { 1048 mp = mp_chain; 1049 mp_chain = mp_chain->b_next; 1050 mp->b_next = NULL; 1051 if (nxge_lso_kick_cnt == lso_ngathers) { 1052 tx_ring_p->descs_pending += lso_ngathers; 1053 { 1054 tx_ring_kick_t kick; 1055 1056 kick.value = 0; 1057 kick.bits.ldw.wrap = 1058 tx_ring_p->wr_index_wrap; 1059 kick.bits.ldw.tail = 1060 (uint16_t)tx_ring_p->wr_index; 1061 1062 /* Kick the Transmit kick register */ 1063 TXDMA_REG_WRITE64( 1064 NXGE_DEV_NPI_HANDLE(nxgep), 1065 TX_RING_KICK_REG, 1066 (uint8_t)tx_ring_p->tdc, 1067 kick.value); 1068 tdc_stats->tx_starts++; 1069 1070 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1071 "==> nxge_start: more LSO: " 1072 "LSO_CNT %d", 1073 lso_ngathers)); 1074 } 1075 lso_ngathers = 0; 1076 ngathers = 0; 1077 cur_index_lso = sop_index = tx_ring_p->wr_index; 1078 lso_tail_wrap = tx_ring_p->wr_index_wrap; 1079 } 1080 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1081 "==> nxge_start: lso again: " 1082 "lso_gathers %d ngathers %d cur_index_lso %d " 1083 "wr_index %d sop_index %d", 1084 lso_ngathers, ngathers, cur_index_lso, 1085 tx_ring_p->wr_index, sop_index)); 1086 1087 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1088 "==> nxge_start: next : count %d", 1089 lso_ngathers)); 1090 lso_again = B_TRUE; 1091 goto start_again; 1092 } 1093 ngathers = lso_ngathers; 1094 } 1095 1096 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 1097 1098 { 1099 tx_ring_kick_t kick; 1100 1101 kick.value = 0; 1102 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 1103 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 1104 1105 /* Kick start the Transmit kick register */ 1106 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 1107 TX_RING_KICK_REG, 1108 (uint8_t)tx_ring_p->tdc, 1109 kick.value); 1110 } 1111 1112 tx_ring_p->descs_pending += ngathers; 1113 tdc_stats->tx_starts++; 1114 1115 if (isLDOMservice(nxgep)) { 1116 tx_ring_p->tx_ring_busy = B_FALSE; 1117 if (tx_ring_p->tx_ring_offline) { 1118 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1119 NXGE_TX_RING_OFFLINED); 1120 } 1121 } 1122 1123 MUTEX_EXIT(&tx_ring_p->lock); 1124 1125 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1126 return (status); 1127 1128 nxge_start_fail_lso: 1129 status = 0; 1130 good_packet = B_FALSE; 1131 if (mp != NULL) { 1132 freemsg(mp); 1133 } 1134 if (mp_chain != NULL) { 1135 freemsg(mp_chain); 1136 } 1137 if (!lso_again && !ngathers) { 1138 if (isLDOMservice(nxgep)) { 1139 tx_ring_p->tx_ring_busy = B_FALSE; 1140 if (tx_ring_p->tx_ring_offline) { 1141 (void) atomic_swap_32( 1142 &tx_ring_p->tx_ring_offline, 1143 NXGE_TX_RING_OFFLINED); 1144 } 1145 } 1146 1147 MUTEX_EXIT(&tx_ring_p->lock); 1148 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1149 "==> nxge_start: lso exit (nothing changed)")); 1150 goto nxge_start_fail1; 1151 } 1152 1153 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1154 "==> nxge_start (channel %d): before lso " 1155 "lso_gathers %d ngathers %d cur_index_lso %d " 1156 "wr_index %d sop_index %d lso_again %d", 1157 tx_ring_p->tdc, 1158 lso_ngathers, ngathers, cur_index_lso, 1159 tx_ring_p->wr_index, sop_index, lso_again)); 1160 1161 if (lso_again) { 1162 lso_ngathers += ngathers; 1163 ngathers = lso_ngathers; 1164 sop_index = cur_index_lso; 1165 tx_ring_p->wr_index = sop_index; 1166 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1167 } 1168 1169 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1170 "==> nxge_start (channel %d): after lso " 1171 "lso_gathers %d ngathers %d cur_index_lso %d " 1172 "wr_index %d sop_index %d lso_again %d", 1173 tx_ring_p->tdc, 1174 lso_ngathers, ngathers, cur_index_lso, 1175 tx_ring_p->wr_index, sop_index, lso_again)); 1176 1177 nxge_start_fail2: 1178 if (good_packet == B_FALSE) { 1179 cur_index = sop_index; 1180 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1181 for (i = 0; i < ngathers; i++) { 1182 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1183 #if defined(__i386) 1184 npi_handle.regp = (uint32_t)tx_desc_p; 1185 #else 1186 npi_handle.regp = (uint64_t)tx_desc_p; 1187 #endif 1188 tx_msg_p = &tx_msg_ring[cur_index]; 1189 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1190 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1191 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1192 "tx_desc_p = %X index = %d", 1193 tx_desc_p, tx_ring_p->rd_index)); 1194 (void) dvma_unload(tx_msg_p->dvma_handle, 1195 0, -1); 1196 tx_msg_p->dvma_handle = NULL; 1197 if (tx_ring_p->dvma_wr_index == 1198 tx_ring_p->dvma_wrap_mask) 1199 tx_ring_p->dvma_wr_index = 0; 1200 else 1201 tx_ring_p->dvma_wr_index++; 1202 tx_ring_p->dvma_pending--; 1203 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1204 if (ddi_dma_unbind_handle( 1205 tx_msg_p->dma_handle)) { 1206 cmn_err(CE_WARN, "!nxge_start: " 1207 "ddi_dma_unbind_handle failed"); 1208 } 1209 } 1210 tx_msg_p->flags.dma_type = USE_NONE; 1211 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1212 tx_ring_p->tx_wrap_mask); 1213 1214 } 1215 } 1216 1217 if (isLDOMservice(nxgep)) { 1218 tx_ring_p->tx_ring_busy = B_FALSE; 1219 if (tx_ring_p->tx_ring_offline) { 1220 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1221 NXGE_TX_RING_OFFLINED); 1222 } 1223 } 1224 1225 MUTEX_EXIT(&tx_ring_p->lock); 1226 1227 nxge_start_fail1: 1228 /* Add FMA to check the access handle nxge_hregh */ 1229 1230 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1231 return (status); 1232 } 1233 1234 /* Software LSO starts here */ 1235 static void 1236 nxge_hcksum_retrieve(mblk_t *mp, 1237 uint32_t *start, uint32_t *stuff, uint32_t *end, 1238 uint32_t *value, uint32_t *flags) 1239 { 1240 if (mp->b_datap->db_type == M_DATA) { 1241 if (flags != NULL) { 1242 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1243 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1244 HCK_FULLCKSUM_OK); 1245 if ((*flags & (HCK_PARTIALCKSUM | 1246 HCK_FULLCKSUM)) != 0) { 1247 if (value != NULL) 1248 *value = (uint32_t)DB_CKSUM16(mp); 1249 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1250 if (start != NULL) 1251 *start = 1252 (uint32_t)DB_CKSUMSTART(mp); 1253 if (stuff != NULL) 1254 *stuff = 1255 (uint32_t)DB_CKSUMSTUFF(mp); 1256 if (end != NULL) 1257 *end = 1258 (uint32_t)DB_CKSUMEND(mp); 1259 } 1260 } 1261 } 1262 } 1263 } 1264 1265 static void 1266 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1267 { 1268 ASSERT(DB_TYPE(mp) == M_DATA); 1269 1270 *mss = 0; 1271 if (flags != NULL) { 1272 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1273 if ((*flags != 0) && (mss != NULL)) { 1274 *mss = (uint32_t)DB_LSOMSS(mp); 1275 } 1276 NXGE_DEBUG_MSG((NULL, TX_CTL, 1277 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1278 *mss, *flags)); 1279 } 1280 1281 NXGE_DEBUG_MSG((NULL, TX_CTL, 1282 "<== nxge_lso_info_get: mss %d", *mss)); 1283 } 1284 1285 /* 1286 * Do Soft LSO on the oversized packet. 1287 * 1288 * 1. Create a chain of message for headers. 1289 * 2. Fill up header messages with proper information. 1290 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1291 * each new message with necessary adjustments. 1292 * * Unchange the ethernet header for DIX frames. (by default) 1293 * * IP Total Length field is updated to MSS or less(only for the last one). 1294 * * IP Identification value is incremented by one for each packet. 1295 * * TCP sequence Number is recalculated according to the payload length. 1296 * * Set FIN and/or PSH flags for the *last* packet if applied. 1297 * * TCP partial Checksum 1298 * 4. Update LSO information in the first message header. 1299 * 5. Release the original message header. 1300 */ 1301 static mblk_t * 1302 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1303 { 1304 uint32_t hckflags; 1305 int pktlen; 1306 int hdrlen; 1307 int segnum; 1308 int i; 1309 struct ether_vlan_header *evh; 1310 int ehlen, iphlen, tcphlen; 1311 struct ip *oiph, *niph; 1312 struct tcphdr *otcph, *ntcph; 1313 int available, len, left; 1314 uint16_t ip_id; 1315 uint32_t tcp_seq; 1316 #ifdef __sparc 1317 uint32_t tcp_seq_tmp; 1318 #endif 1319 mblk_t *datamp; 1320 uchar_t *rptr; 1321 mblk_t *nmp; 1322 mblk_t *cmp; 1323 mblk_t *mp_chain; 1324 boolean_t do_cleanup = B_FALSE; 1325 t_uscalar_t start_offset = 0; 1326 t_uscalar_t stuff_offset = 0; 1327 t_uscalar_t value = 0; 1328 uint16_t l4_len; 1329 ipaddr_t src, dst; 1330 uint32_t cksum, sum, l4cksum; 1331 1332 NXGE_DEBUG_MSG((NULL, TX_CTL, 1333 "==> nxge_do_softlso")); 1334 /* 1335 * check the length of LSO packet payload and calculate the number of 1336 * segments to be generated. 1337 */ 1338 pktlen = msgsize(mp); 1339 evh = (struct ether_vlan_header *)mp->b_rptr; 1340 1341 /* VLAN? */ 1342 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1343 ehlen = sizeof (struct ether_vlan_header); 1344 else 1345 ehlen = sizeof (struct ether_header); 1346 oiph = (struct ip *)(mp->b_rptr + ehlen); 1347 iphlen = oiph->ip_hl * 4; 1348 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1349 tcphlen = otcph->th_off * 4; 1350 1351 l4_len = pktlen - ehlen - iphlen; 1352 1353 NXGE_DEBUG_MSG((NULL, TX_CTL, 1354 "==> nxge_do_softlso: mss %d oiph $%p " 1355 "original ip_sum oiph->ip_sum 0x%x " 1356 "original tcp_sum otcph->th_sum 0x%x " 1357 "oiph->ip_len %d pktlen %d ehlen %d " 1358 "l4_len %d (0x%x) ip_len - iphlen %d ", 1359 mss, 1360 oiph, 1361 oiph->ip_sum, 1362 otcph->th_sum, 1363 ntohs(oiph->ip_len), pktlen, 1364 ehlen, 1365 l4_len, 1366 l4_len, 1367 ntohs(oiph->ip_len) - iphlen)); 1368 1369 /* IPv4 + TCP */ 1370 if (!(oiph->ip_v == IPV4_VERSION)) { 1371 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1372 "<== nxge_do_softlso: not IPV4 " 1373 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1374 ntohs(oiph->ip_len), pktlen, ehlen, 1375 tcphlen)); 1376 freemsg(mp); 1377 return (NULL); 1378 } 1379 1380 if (!(oiph->ip_p == IPPROTO_TCP)) { 1381 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1382 "<== nxge_do_softlso: not TCP " 1383 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1384 ntohs(oiph->ip_len), pktlen, ehlen, 1385 tcphlen)); 1386 freemsg(mp); 1387 return (NULL); 1388 } 1389 1390 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1391 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1392 "<== nxge_do_softlso: len not matched " 1393 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1394 ntohs(oiph->ip_len), pktlen, ehlen, 1395 tcphlen)); 1396 freemsg(mp); 1397 return (NULL); 1398 } 1399 1400 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1401 tcphlen = otcph->th_off * 4; 1402 1403 /* TCP flags can not include URG, RST, or SYN */ 1404 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1405 1406 hdrlen = ehlen + iphlen + tcphlen; 1407 1408 VERIFY(MBLKL(mp) >= hdrlen); 1409 1410 if (MBLKL(mp) > hdrlen) { 1411 datamp = mp; 1412 rptr = mp->b_rptr + hdrlen; 1413 } else { /* = */ 1414 datamp = mp->b_cont; 1415 rptr = datamp->b_rptr; 1416 } 1417 1418 NXGE_DEBUG_MSG((NULL, TX_CTL, 1419 "nxge_do_softlso: otcph $%p pktlen: %d, " 1420 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1421 "mblkl(mp): %d, mblkl(datamp): %d", 1422 otcph, 1423 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1424 (int)MBLKL(mp), (int)MBLKL(datamp))); 1425 1426 hckflags = 0; 1427 nxge_hcksum_retrieve(mp, 1428 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1429 1430 dst = oiph->ip_dst.s_addr; 1431 src = oiph->ip_src.s_addr; 1432 1433 cksum = (dst >> 16) + (dst & 0xFFFF) + 1434 (src >> 16) + (src & 0xFFFF); 1435 l4cksum = cksum + IP_TCP_CSUM_COMP; 1436 1437 sum = l4_len + l4cksum; 1438 sum = (sum & 0xFFFF) + (sum >> 16); 1439 1440 NXGE_DEBUG_MSG((NULL, TX_CTL, 1441 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1442 "hckflags 0x%x start_offset %d stuff_offset %d " 1443 "value (original) 0x%x th_sum 0x%x " 1444 "pktlen %d l4_len %d (0x%x) " 1445 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1446 dst, src, 1447 (sum & 0xffff), (~sum & 0xffff), 1448 hckflags, start_offset, stuff_offset, 1449 value, otcph->th_sum, 1450 pktlen, 1451 l4_len, 1452 l4_len, 1453 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1454 (int)MBLKL(datamp), 1455 nxge_dump_packet((char *)evh, 12))); 1456 1457 /* 1458 * Start to process. 1459 */ 1460 available = pktlen - hdrlen; 1461 segnum = (available - 1) / mss + 1; 1462 1463 NXGE_DEBUG_MSG((NULL, TX_CTL, 1464 "==> nxge_do_softlso: pktlen %d " 1465 "MBLKL(mp): %d, MBLKL(datamp): %d " 1466 "available %d mss %d segnum %d", 1467 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1468 available, 1469 mss, 1470 segnum)); 1471 1472 VERIFY(segnum >= 2); 1473 1474 /* 1475 * Try to pre-allocate all header messages 1476 */ 1477 mp_chain = NULL; 1478 for (i = 0; i < segnum; i++) { 1479 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1480 /* Clean up the mp_chain */ 1481 while (mp_chain != NULL) { 1482 nmp = mp_chain; 1483 mp_chain = mp_chain->b_next; 1484 freemsg(nmp); 1485 } 1486 NXGE_DEBUG_MSG((NULL, TX_CTL, 1487 "<== nxge_do_softlso: " 1488 "Could not allocate enough messages for headers!")); 1489 freemsg(mp); 1490 return (NULL); 1491 } 1492 nmp->b_next = mp_chain; 1493 mp_chain = nmp; 1494 1495 NXGE_DEBUG_MSG((NULL, TX_CTL, 1496 "==> nxge_do_softlso: " 1497 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1498 mp, nmp, mp_chain, mp_chain->b_next)); 1499 } 1500 1501 NXGE_DEBUG_MSG((NULL, TX_CTL, 1502 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1503 mp, nmp, mp_chain)); 1504 1505 /* 1506 * Associate payload with new packets 1507 */ 1508 cmp = mp_chain; 1509 left = available; 1510 while (cmp != NULL) { 1511 nmp = dupb(datamp); 1512 if (nmp == NULL) { 1513 do_cleanup = B_TRUE; 1514 NXGE_DEBUG_MSG((NULL, TX_CTL, 1515 "==>nxge_do_softlso: " 1516 "Can not dupb(datamp), have to do clean up")); 1517 goto cleanup_allocated_msgs; 1518 } 1519 1520 NXGE_DEBUG_MSG((NULL, TX_CTL, 1521 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1522 "dupb nmp $%p len %d left %d msd %d ", 1523 mp, cmp, nmp, len, left, mss)); 1524 1525 cmp->b_cont = nmp; 1526 nmp->b_rptr = rptr; 1527 len = (left < mss) ? left : mss; 1528 left -= len; 1529 1530 NXGE_DEBUG_MSG((NULL, TX_CTL, 1531 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1532 "dupb nmp $%p len %d left %d mss %d ", 1533 mp, cmp, nmp, len, left, mss)); 1534 NXGE_DEBUG_MSG((NULL, TX_CTL, 1535 "nxge_do_softlso: before available: %d, " 1536 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1537 available, left, len, segnum, (int)MBLKL(nmp))); 1538 1539 len -= MBLKL(nmp); 1540 NXGE_DEBUG_MSG((NULL, TX_CTL, 1541 "nxge_do_softlso: after available: %d, " 1542 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1543 available, left, len, segnum, (int)MBLKL(nmp))); 1544 1545 while (len > 0) { 1546 mblk_t *mmp = NULL; 1547 1548 NXGE_DEBUG_MSG((NULL, TX_CTL, 1549 "nxge_do_softlso: (4) len > 0 available: %d, " 1550 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1551 available, left, len, segnum, (int)MBLKL(nmp))); 1552 1553 if (datamp->b_cont != NULL) { 1554 datamp = datamp->b_cont; 1555 rptr = datamp->b_rptr; 1556 mmp = dupb(datamp); 1557 if (mmp == NULL) { 1558 do_cleanup = B_TRUE; 1559 NXGE_DEBUG_MSG((NULL, TX_CTL, 1560 "==> nxge_do_softlso: " 1561 "Can not dupb(datamp) (1), :" 1562 "have to do clean up")); 1563 NXGE_DEBUG_MSG((NULL, TX_CTL, 1564 "==> nxge_do_softlso: " 1565 "available: %d, left: %d, " 1566 "len: %d, MBLKL(nmp): %d", 1567 available, left, len, 1568 (int)MBLKL(nmp))); 1569 goto cleanup_allocated_msgs; 1570 } 1571 } else { 1572 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1573 "==> nxge_do_softlso: " 1574 "(1)available: %d, left: %d, " 1575 "len: %d, MBLKL(nmp): %d", 1576 available, left, len, 1577 (int)MBLKL(nmp))); 1578 cmn_err(CE_PANIC, 1579 "==> nxge_do_softlso: " 1580 "Pointers must have been corrupted!\n" 1581 "datamp: $%p, nmp: $%p, rptr: $%p", 1582 (void *)datamp, 1583 (void *)nmp, 1584 (void *)rptr); 1585 } 1586 nmp->b_cont = mmp; 1587 nmp = mmp; 1588 len -= MBLKL(nmp); 1589 } 1590 if (len < 0) { 1591 nmp->b_wptr += len; 1592 rptr = nmp->b_wptr; 1593 NXGE_DEBUG_MSG((NULL, TX_CTL, 1594 "(5) len < 0 (less than 0)" 1595 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1596 available, left, len, (int)MBLKL(nmp))); 1597 1598 } else if (len == 0) { 1599 if (datamp->b_cont != NULL) { 1600 NXGE_DEBUG_MSG((NULL, TX_CTL, 1601 "(5) len == 0" 1602 "available: %d, left: %d, len: %d, " 1603 "MBLKL(nmp): %d", 1604 available, left, len, (int)MBLKL(nmp))); 1605 datamp = datamp->b_cont; 1606 rptr = datamp->b_rptr; 1607 } else { 1608 NXGE_DEBUG_MSG((NULL, TX_CTL, 1609 "(6)available b_cont == NULL : %d, " 1610 "left: %d, len: %d, MBLKL(nmp): %d", 1611 available, left, len, (int)MBLKL(nmp))); 1612 1613 VERIFY(cmp->b_next == NULL); 1614 VERIFY(left == 0); 1615 break; /* Done! */ 1616 } 1617 } 1618 cmp = cmp->b_next; 1619 1620 NXGE_DEBUG_MSG((NULL, TX_CTL, 1621 "(7) do_softlso: " 1622 "next mp in mp_chain available len != 0 : %d, " 1623 "left: %d, len: %d, MBLKL(nmp): %d", 1624 available, left, len, (int)MBLKL(nmp))); 1625 } 1626 1627 /* 1628 * From now, start to fill up all headers for the first message 1629 * Hardware checksum flags need to be updated separately for FULLCKSUM 1630 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1631 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1632 * required fields need to be updated properly. 1633 */ 1634 nmp = mp_chain; 1635 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1636 nmp->b_wptr = nmp->b_rptr + hdrlen; 1637 niph = (struct ip *)(nmp->b_rptr + ehlen); 1638 niph->ip_len = htons(mss + iphlen + tcphlen); 1639 ip_id = ntohs(niph->ip_id); 1640 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1641 #ifdef __sparc 1642 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1643 tcp_seq = ntohl(tcp_seq_tmp); 1644 #else 1645 tcp_seq = ntohl(ntcph->th_seq); 1646 #endif 1647 1648 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1649 1650 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1651 DB_CKSUMSTART(nmp) = start_offset; 1652 DB_CKSUMSTUFF(nmp) = stuff_offset; 1653 1654 /* calculate IP checksum and TCP pseudo header checksum */ 1655 niph->ip_sum = 0; 1656 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1657 1658 l4_len = mss + tcphlen; 1659 sum = htons(l4_len) + l4cksum; 1660 sum = (sum & 0xFFFF) + (sum >> 16); 1661 ntcph->th_sum = (sum & 0xffff); 1662 1663 NXGE_DEBUG_MSG((NULL, TX_CTL, 1664 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1665 "mss %d pktlen %d l4_len %d (0x%x) " 1666 "MBLKL(mp): %d, MBLKL(datamp): %d " 1667 "ip_sum 0x%x " 1668 "th_sum 0x%x sum 0x%x ) " 1669 "dump first ip->tcp %s", 1670 nmp, mp_chain, 1671 mss, 1672 pktlen, 1673 l4_len, 1674 l4_len, 1675 (int)MBLKL(mp), (int)MBLKL(datamp), 1676 niph->ip_sum, 1677 ntcph->th_sum, 1678 sum, 1679 nxge_dump_packet((char *)niph, 52))); 1680 1681 cmp = nmp; 1682 while ((nmp = nmp->b_next)->b_next != NULL) { 1683 NXGE_DEBUG_MSG((NULL, TX_CTL, 1684 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1685 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1686 nmp->b_wptr = nmp->b_rptr + hdrlen; 1687 niph = (struct ip *)(nmp->b_rptr + ehlen); 1688 niph->ip_id = htons(++ip_id); 1689 niph->ip_len = htons(mss + iphlen + tcphlen); 1690 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1691 tcp_seq += mss; 1692 1693 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1694 1695 #ifdef __sparc 1696 tcp_seq_tmp = htonl(tcp_seq); 1697 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1698 #else 1699 ntcph->th_seq = htonl(tcp_seq); 1700 #endif 1701 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1702 DB_CKSUMSTART(nmp) = start_offset; 1703 DB_CKSUMSTUFF(nmp) = stuff_offset; 1704 1705 /* calculate IP checksum and TCP pseudo header checksum */ 1706 niph->ip_sum = 0; 1707 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1708 ntcph->th_sum = (sum & 0xffff); 1709 1710 NXGE_DEBUG_MSG((NULL, TX_CTL, 1711 "==> nxge_do_softlso: middle ip_sum 0x%x " 1712 "th_sum 0x%x " 1713 " mp $%p (mp_chain $%p) pktlen %d " 1714 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1715 niph->ip_sum, 1716 ntcph->th_sum, 1717 nmp, mp_chain, 1718 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1719 } 1720 1721 /* Last segment */ 1722 /* 1723 * Set FIN and/or PSH flags if present only in the last packet. 1724 * The ip_len could be different from prior packets. 1725 */ 1726 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1727 nmp->b_wptr = nmp->b_rptr + hdrlen; 1728 niph = (struct ip *)(nmp->b_rptr + ehlen); 1729 niph->ip_id = htons(++ip_id); 1730 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1731 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1732 tcp_seq += mss; 1733 #ifdef __sparc 1734 tcp_seq_tmp = htonl(tcp_seq); 1735 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1736 #else 1737 ntcph->th_seq = htonl(tcp_seq); 1738 #endif 1739 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1740 1741 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1742 DB_CKSUMSTART(nmp) = start_offset; 1743 DB_CKSUMSTUFF(nmp) = stuff_offset; 1744 1745 /* calculate IP checksum and TCP pseudo header checksum */ 1746 niph->ip_sum = 0; 1747 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1748 1749 l4_len = ntohs(niph->ip_len) - iphlen; 1750 sum = htons(l4_len) + l4cksum; 1751 sum = (sum & 0xFFFF) + (sum >> 16); 1752 ntcph->th_sum = (sum & 0xffff); 1753 1754 NXGE_DEBUG_MSG((NULL, TX_CTL, 1755 "==> nxge_do_softlso: last next " 1756 "niph->ip_sum 0x%x " 1757 "ntcph->th_sum 0x%x sum 0x%x " 1758 "dump last ip->tcp %s " 1759 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1760 "l4_len %d (0x%x) " 1761 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1762 niph->ip_sum, 1763 ntcph->th_sum, sum, 1764 nxge_dump_packet((char *)niph, 52), 1765 cmp, nmp, mp_chain, 1766 pktlen, pktlen, 1767 l4_len, 1768 l4_len, 1769 (int)MBLKL(mp), (int)MBLKL(datamp))); 1770 1771 cleanup_allocated_msgs: 1772 if (do_cleanup) { 1773 NXGE_DEBUG_MSG((NULL, TX_CTL, 1774 "==> nxge_do_softlso: " 1775 "Failed allocating messages, " 1776 "have to clean up and fail!")); 1777 while (mp_chain != NULL) { 1778 nmp = mp_chain; 1779 mp_chain = mp_chain->b_next; 1780 freemsg(nmp); 1781 } 1782 } 1783 /* 1784 * We're done here, so just free the original message and return the 1785 * new message chain, that could be NULL if failed, back to the caller. 1786 */ 1787 freemsg(mp); 1788 1789 NXGE_DEBUG_MSG((NULL, TX_CTL, 1790 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1791 return (mp_chain); 1792 } 1793 1794 /* 1795 * Will be called before NIC driver do further operation on the message. 1796 * The input message may include LSO information, if so, go to softlso logic 1797 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1798 * The return could be the same non-LSO message or a message chain for LSO case. 1799 * 1800 * The driver needs to call this function per packet and process the whole chain 1801 * if applied. 1802 */ 1803 static mblk_t * 1804 nxge_lso_eliminate(mblk_t *mp) 1805 { 1806 uint32_t lsoflags; 1807 uint32_t mss; 1808 1809 NXGE_DEBUG_MSG((NULL, TX_CTL, 1810 "==>nxge_lso_eliminate:")); 1811 nxge_lso_info_get(mp, &mss, &lsoflags); 1812 1813 if (lsoflags & HW_LSO) { 1814 mblk_t *nmp; 1815 1816 NXGE_DEBUG_MSG((NULL, TX_CTL, 1817 "==>nxge_lso_eliminate:" 1818 "HW_LSO:mss %d mp $%p", 1819 mss, mp)); 1820 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1821 NXGE_DEBUG_MSG((NULL, TX_CTL, 1822 "<== nxge_lso_eliminate: " 1823 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1824 nmp, mss, mp)); 1825 return (nmp); 1826 } else { 1827 NXGE_DEBUG_MSG((NULL, TX_CTL, 1828 "<== nxge_lso_eliminate_ " 1829 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1830 nmp, mss, mp)); 1831 return (NULL); 1832 } 1833 } 1834 1835 NXGE_DEBUG_MSG((NULL, TX_CTL, 1836 "<== nxge_lso_eliminate")); 1837 return (mp); 1838 } 1839 1840 static uint32_t 1841 nxge_csgen(uint16_t *adr, int len) 1842 { 1843 int i, odd; 1844 uint32_t sum = 0; 1845 uint32_t c = 0; 1846 1847 odd = len % 2; 1848 for (i = 0; i < (len / 2); i++) { 1849 sum += (adr[i] & 0xffff); 1850 } 1851 if (odd) { 1852 sum += adr[len / 2] & 0xff00; 1853 } 1854 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 1855 sum &= 0xffff; 1856 sum += c; 1857 } 1858 return (~sum & 0xffff); 1859 } 1860