1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/nxge/nxge_impl.h> 27 #include <sys/nxge/nxge_hio.h> 28 #include <npi_tx_wr64.h> 29 30 /* Software LSO required header files */ 31 #include <netinet/tcp.h> 32 #include <inet/ip_impl.h> 33 #include <inet/tcp.h> 34 35 static mblk_t *nxge_lso_eliminate(mblk_t *); 36 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 37 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 38 static void nxge_hcksum_retrieve(mblk_t *, 39 uint32_t *, uint32_t *, uint32_t *, 40 uint32_t *, uint32_t *); 41 static uint32_t nxge_csgen(uint16_t *, int); 42 43 extern uint32_t nxge_reclaim_pending; 44 extern uint32_t nxge_bcopy_thresh; 45 extern uint32_t nxge_dvma_thresh; 46 extern uint32_t nxge_dma_stream_thresh; 47 extern uint32_t nxge_tx_minfree; 48 extern uint32_t nxge_tx_intr_thres; 49 extern uint32_t nxge_tx_max_gathers; 50 extern uint32_t nxge_tx_tiny_pack; 51 extern uint32_t nxge_tx_use_bcopy; 52 extern nxge_tx_mode_t nxge_tx_scheme; 53 uint32_t nxge_lso_kick_cnt = 2; 54 55 56 void 57 nxge_tx_ring_task(void *arg) 58 { 59 p_tx_ring_t ring = (p_tx_ring_t)arg; 60 61 MUTEX_ENTER(&ring->lock); 62 (void) nxge_txdma_reclaim(ring->nxgep, ring, 0); 63 MUTEX_EXIT(&ring->lock); 64 65 if (!isLDOMguest(ring->nxgep) && !ring->tx_ring_offline) 66 mac_tx_ring_update(ring->nxgep->mach, ring->tx_ring_handle); 67 #if defined(sun4v) 68 else { 69 nxge_hio_data_t *nhd = 70 (nxge_hio_data_t *)ring->nxgep->nxge_hw_p->hio; 71 nx_vio_fp_t *vio = &nhd->hio.vio; 72 73 /* Call back vnet. */ 74 if (vio->cb.vio_net_tx_update) { 75 (*vio->cb.vio_net_tx_update)(ring->nxgep->hio_vr->vhp); 76 } 77 } 78 #endif 79 } 80 81 static void 82 nxge_tx_ring_dispatch(p_tx_ring_t ring) 83 { 84 /* 85 * Kick the ring task to reclaim some buffers. 86 */ 87 (void) ddi_taskq_dispatch(ring->taskq, 88 nxge_tx_ring_task, (void *)ring, DDI_SLEEP); 89 } 90 91 mblk_t * 92 nxge_tx_ring_send(void *arg, mblk_t *mp) 93 { 94 p_nxge_ring_handle_t nrhp = (p_nxge_ring_handle_t)arg; 95 p_nxge_t nxgep; 96 p_tx_ring_t tx_ring_p; 97 int status, channel; 98 99 ASSERT(nrhp != NULL); 100 nxgep = nrhp->nxgep; 101 channel = nxgep->pt_config.hw_config.tdc.start + nrhp->index; 102 tx_ring_p = nxgep->tx_rings->rings[channel]; 103 104 ASSERT(nxgep == tx_ring_p->nxgep); 105 106 #ifdef DEBUG 107 if (isLDOMservice(nxgep)) { 108 ASSERT(!tx_ring_p->tx_ring_offline); 109 } 110 #endif 111 112 status = nxge_start(nxgep, tx_ring_p, mp); 113 if (status) { 114 nxge_tx_ring_dispatch(tx_ring_p); 115 return (mp); 116 } 117 118 return ((mblk_t *)NULL); 119 } 120 121 #if defined(sun4v) 122 123 /* 124 * nxge_m_tx() is needed for Hybrid I/O operation of the vnet in 125 * the guest domain. See CR 6778758 for long term solution. 126 */ 127 128 mblk_t * 129 nxge_m_tx(void *arg, mblk_t *mp) 130 { 131 p_nxge_t nxgep = (p_nxge_t)arg; 132 mblk_t *next; 133 p_tx_ring_t tx_ring_p; 134 int status; 135 136 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_m_tx")); 137 138 /* 139 * Get the default ring handle. 140 */ 141 tx_ring_p = nxgep->tx_rings->rings[0]; 142 143 while (mp != NULL) { 144 next = mp->b_next; 145 mp->b_next = NULL; 146 147 status = nxge_start(nxgep, tx_ring_p, mp); 148 if (status != 0) { 149 mp->b_next = next; 150 nxge_tx_ring_dispatch(tx_ring_p); 151 return (mp); 152 } 153 154 mp = next; 155 } 156 157 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_m_tx")); 158 return ((mblk_t *)NULL); 159 } 160 161 #endif 162 163 int 164 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 165 { 166 int status = 0; 167 p_tx_desc_t tx_desc_ring_vp; 168 npi_handle_t npi_desc_handle; 169 nxge_os_dma_handle_t tx_desc_dma_handle; 170 p_tx_desc_t tx_desc_p; 171 p_tx_msg_t tx_msg_ring; 172 p_tx_msg_t tx_msg_p; 173 tx_desc_t tx_desc, *tmp_desc_p; 174 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 175 p_tx_pkt_header_t hdrp; 176 tx_pkt_header_t tmp_hdrp; 177 p_tx_pkt_hdr_all_t pkthdrp; 178 uint8_t npads = 0; 179 uint64_t dma_ioaddr; 180 uint32_t dma_flags; 181 int last_bidx; 182 uint8_t *b_rptr; 183 caddr_t kaddr; 184 uint32_t nmblks; 185 uint32_t ngathers; 186 uint32_t clen; 187 int len; 188 uint32_t pkt_len, pack_len, min_len; 189 uint32_t bcopy_thresh; 190 int i, cur_index, sop_index; 191 uint16_t tail_index; 192 boolean_t tail_wrap = B_FALSE; 193 nxge_dma_common_t desc_area; 194 nxge_os_dma_handle_t dma_handle; 195 ddi_dma_cookie_t dma_cookie; 196 npi_handle_t npi_handle; 197 p_mblk_t nmp; 198 p_mblk_t t_mp; 199 uint32_t ncookies; 200 boolean_t good_packet; 201 boolean_t mark_mode = B_FALSE; 202 p_nxge_stats_t statsp; 203 p_nxge_tx_ring_stats_t tdc_stats; 204 t_uscalar_t start_offset = 0; 205 t_uscalar_t stuff_offset = 0; 206 t_uscalar_t end_offset = 0; 207 t_uscalar_t value = 0; 208 t_uscalar_t cksum_flags = 0; 209 boolean_t cksum_on = B_FALSE; 210 uint32_t boff = 0; 211 uint64_t tot_xfer_len = 0; 212 boolean_t header_set = B_FALSE; 213 #ifdef NXGE_DEBUG 214 p_tx_desc_t tx_desc_ring_pp; 215 p_tx_desc_t tx_desc_pp; 216 tx_desc_t *save_desc_p; 217 int dump_len; 218 int sad_len; 219 uint64_t sad; 220 int xfer_len; 221 uint32_t msgsize; 222 #endif 223 p_mblk_t mp_chain = NULL; 224 boolean_t is_lso = B_FALSE; 225 boolean_t lso_again; 226 int cur_index_lso; 227 p_mblk_t nmp_lso_save; 228 uint32_t lso_ngathers; 229 boolean_t lso_tail_wrap = B_FALSE; 230 231 NXGE_DEBUG_MSG((nxgep, TX_CTL, 232 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 233 NXGE_DEBUG_MSG((nxgep, TX_CTL, 234 "==> nxge_start: Starting tdc %d desc pending %d", 235 tx_ring_p->tdc, tx_ring_p->descs_pending)); 236 237 statsp = nxgep->statsp; 238 239 if (!isLDOMguest(nxgep)) { 240 switch (nxgep->mac.portmode) { 241 default: 242 if (nxgep->statsp->port_stats.lb_mode == 243 nxge_lb_normal) { 244 if (!statsp->mac_stats.link_up) { 245 freemsg(mp); 246 NXGE_DEBUG_MSG((nxgep, TX_CTL, 247 "==> nxge_start: " 248 "link not up")); 249 goto nxge_start_fail1; 250 } 251 } 252 break; 253 case PORT_10G_FIBER: 254 /* 255 * For the following modes, check the link status 256 * before sending the packet out: 257 * nxge_lb_normal, 258 * nxge_lb_ext10g, 259 * nxge_lb_ext1000, 260 * nxge_lb_ext100, 261 * nxge_lb_ext10. 262 */ 263 if (nxgep->statsp->port_stats.lb_mode < 264 nxge_lb_phy10g) { 265 if (!statsp->mac_stats.link_up) { 266 freemsg(mp); 267 NXGE_DEBUG_MSG((nxgep, TX_CTL, 268 "==> nxge_start: " 269 "link not up")); 270 goto nxge_start_fail1; 271 } 272 } 273 break; 274 } 275 } 276 277 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 278 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 279 NXGE_DEBUG_MSG((nxgep, TX_CTL, 280 "==> nxge_start: hardware not initialized or stopped")); 281 freemsg(mp); 282 goto nxge_start_fail1; 283 } 284 285 if (nxgep->soft_lso_enable) { 286 mp_chain = nxge_lso_eliminate(mp); 287 NXGE_DEBUG_MSG((nxgep, TX_CTL, 288 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 289 mp, mp_chain)); 290 if (mp_chain == NULL) { 291 NXGE_ERROR_MSG((nxgep, TX_CTL, 292 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 293 mp_chain, mp)); 294 goto nxge_start_fail1; 295 } 296 if (mp_chain != mp) { 297 NXGE_DEBUG_MSG((nxgep, TX_CTL, 298 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 299 mp_chain, mp)); 300 is_lso = B_TRUE; 301 mp = mp_chain; 302 mp_chain = mp_chain->b_next; 303 mp->b_next = NULL; 304 } 305 } 306 307 hcksum_retrieve(mp, NULL, NULL, &start_offset, 308 &stuff_offset, &end_offset, &value, &cksum_flags); 309 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 310 start_offset += sizeof (ether_header_t); 311 stuff_offset += sizeof (ether_header_t); 312 } else { 313 start_offset += sizeof (struct ether_vlan_header); 314 stuff_offset += sizeof (struct ether_vlan_header); 315 } 316 317 if (cksum_flags & HCK_PARTIALCKSUM) { 318 NXGE_DEBUG_MSG((nxgep, TX_CTL, 319 "==> nxge_start: mp $%p len %d " 320 "cksum_flags 0x%x (partial checksum) ", 321 mp, MBLKL(mp), cksum_flags)); 322 cksum_on = B_TRUE; 323 } 324 325 pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 326 pkthdrp->reserved = 0; 327 tmp_hdrp.value = 0; 328 nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 329 0, 0, pkthdrp, 330 start_offset, stuff_offset); 331 332 lso_again = B_FALSE; 333 lso_ngathers = 0; 334 335 MUTEX_ENTER(&tx_ring_p->lock); 336 337 if (isLDOMservice(nxgep)) { 338 tx_ring_p->tx_ring_busy = B_TRUE; 339 if (tx_ring_p->tx_ring_offline) { 340 freemsg(mp); 341 tx_ring_p->tx_ring_busy = B_FALSE; 342 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 343 NXGE_TX_RING_OFFLINED); 344 MUTEX_EXIT(&tx_ring_p->lock); 345 return (status); 346 } 347 } 348 349 cur_index_lso = tx_ring_p->wr_index; 350 lso_tail_wrap = tx_ring_p->wr_index_wrap; 351 start_again: 352 ngathers = 0; 353 sop_index = tx_ring_p->wr_index; 354 #ifdef NXGE_DEBUG 355 if (tx_ring_p->descs_pending) { 356 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 357 "desc pending %d ", tx_ring_p->descs_pending)); 358 } 359 360 dump_len = (int)(MBLKL(mp)); 361 dump_len = (dump_len > 128) ? 128: dump_len; 362 363 NXGE_DEBUG_MSG((nxgep, TX_CTL, 364 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 365 "(Before header reserve: ORIGINAL LEN %d)", 366 tx_ring_p->tdc, 367 mp->b_rptr, 368 dump_len)); 369 370 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 371 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 372 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 373 #endif 374 375 tdc_stats = tx_ring_p->tdc_stats; 376 mark_mode = (tx_ring_p->descs_pending && 377 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 378 < nxge_tx_minfree)); 379 380 NXGE_DEBUG_MSG((nxgep, TX_CTL, 381 "TX Descriptor ring is channel %d mark mode %d", 382 tx_ring_p->tdc, mark_mode)); 383 384 if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) { 385 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, 386 (nxge_tx_minfree + lso_ngathers))) { 387 NXGE_DEBUG_MSG((nxgep, TX_CTL, 388 "TX Descriptor ring is full: channel %d", 389 tx_ring_p->tdc)); 390 NXGE_DEBUG_MSG((nxgep, TX_CTL, 391 "TX Descriptor ring is full: channel %d", 392 tx_ring_p->tdc)); 393 if (is_lso) { 394 /* 395 * free the current mp and mp_chain if not FULL. 396 */ 397 tdc_stats->tx_no_desc++; 398 NXGE_DEBUG_MSG((nxgep, TX_CTL, 399 "LSO packet: TX Descriptor ring is full: " 400 "channel %d", 401 tx_ring_p->tdc)); 402 goto nxge_start_fail_lso; 403 } else { 404 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 405 tdc_stats->tx_no_desc++; 406 407 if (isLDOMservice(nxgep)) { 408 tx_ring_p->tx_ring_busy = B_FALSE; 409 if (tx_ring_p->tx_ring_offline) { 410 (void) atomic_swap_32( 411 &tx_ring_p->tx_ring_offline, 412 NXGE_TX_RING_OFFLINED); 413 } 414 } 415 416 MUTEX_EXIT(&tx_ring_p->lock); 417 status = 1; 418 goto nxge_start_fail1; 419 } 420 } 421 } 422 423 nmp = mp; 424 i = sop_index = tx_ring_p->wr_index; 425 nmblks = 0; 426 ngathers = 0; 427 pkt_len = 0; 428 pack_len = 0; 429 clen = 0; 430 last_bidx = -1; 431 good_packet = B_TRUE; 432 433 desc_area = tx_ring_p->tdc_desc; 434 npi_handle = desc_area.npi_handle; 435 npi_desc_handle.regh = (nxge_os_acc_handle_t) 436 DMA_COMMON_ACC_HANDLE(desc_area); 437 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 438 tx_desc_dma_handle = (nxge_os_dma_handle_t) 439 DMA_COMMON_HANDLE(desc_area); 440 tx_msg_ring = tx_ring_p->tx_msg_ring; 441 442 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 443 sop_index, i)); 444 445 #ifdef NXGE_DEBUG 446 msgsize = msgdsize(nmp); 447 NXGE_DEBUG_MSG((nxgep, TX_CTL, 448 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 449 sop_index, i, msgsize)); 450 #endif 451 /* 452 * The first 16 bytes of the premapped buffer are reserved 453 * for header. No padding will be used. 454 */ 455 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 456 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 457 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 458 } else { 459 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 460 } 461 while (nmp) { 462 good_packet = B_TRUE; 463 b_rptr = nmp->b_rptr; 464 len = MBLKL(nmp); 465 if (len <= 0) { 466 nmp = nmp->b_cont; 467 continue; 468 } 469 nmblks++; 470 471 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 472 "len %d pkt_len %d pack_len %d", 473 nmblks, len, pkt_len, pack_len)); 474 /* 475 * Hardware limits the transfer length to 4K for NIU and 476 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 477 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 478 * If len is longer than the limit, then we break nmp into 479 * two chunks: Make the first chunk equal to the limit and 480 * the second chunk for the remaining data. If the second 481 * chunk is still larger than the limit, then it will be 482 * broken into two in the next pass. 483 */ 484 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 485 if ((t_mp = dupb(nmp)) != NULL) { 486 nmp->b_wptr = nmp->b_rptr + 487 (TX_MAX_TRANSFER_LENGTH 488 - TX_PKT_HEADER_SIZE); 489 t_mp->b_rptr = nmp->b_wptr; 490 t_mp->b_cont = nmp->b_cont; 491 nmp->b_cont = t_mp; 492 len = MBLKL(nmp); 493 } else { 494 if (is_lso) { 495 NXGE_DEBUG_MSG((nxgep, TX_CTL, 496 "LSO packet: dupb failed: " 497 "channel %d", 498 tx_ring_p->tdc)); 499 mp = nmp; 500 goto nxge_start_fail_lso; 501 } else { 502 good_packet = B_FALSE; 503 goto nxge_start_fail2; 504 } 505 } 506 } 507 tx_desc.value = 0; 508 tx_desc_p = &tx_desc_ring_vp[i]; 509 #ifdef NXGE_DEBUG 510 tx_desc_pp = &tx_desc_ring_pp[i]; 511 #endif 512 tx_msg_p = &tx_msg_ring[i]; 513 #if defined(__i386) 514 npi_desc_handle.regp = (uint32_t)tx_desc_p; 515 #else 516 npi_desc_handle.regp = (uint64_t)tx_desc_p; 517 #endif 518 if (!header_set && 519 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 520 (len >= bcopy_thresh))) { 521 header_set = B_TRUE; 522 bcopy_thresh += TX_PKT_HEADER_SIZE; 523 boff = 0; 524 pack_len = 0; 525 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 526 hdrp = (p_tx_pkt_header_t)kaddr; 527 clen = pkt_len; 528 dma_handle = tx_msg_p->buf_dma_handle; 529 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 530 (void) ddi_dma_sync(dma_handle, 531 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 532 DDI_DMA_SYNC_FORDEV); 533 534 tx_msg_p->flags.dma_type = USE_BCOPY; 535 goto nxge_start_control_header_only; 536 } 537 538 pkt_len += len; 539 pack_len += len; 540 541 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 542 "desc entry %d " 543 "DESC IOADDR $%p " 544 "desc_vp $%p tx_desc_p $%p " 545 "desc_pp $%p tx_desc_pp $%p " 546 "len %d pkt_len %d pack_len %d", 547 i, 548 DMA_COMMON_IOADDR(desc_area), 549 tx_desc_ring_vp, tx_desc_p, 550 tx_desc_ring_pp, tx_desc_pp, 551 len, pkt_len, pack_len)); 552 553 if (len < bcopy_thresh) { 554 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 555 "USE BCOPY: ")); 556 if (nxge_tx_tiny_pack) { 557 uint32_t blst = 558 TXDMA_DESC_NEXT_INDEX(i, -1, 559 tx_ring_p->tx_wrap_mask); 560 NXGE_DEBUG_MSG((nxgep, TX_CTL, 561 "==> nxge_start(5): pack")); 562 if ((pack_len <= bcopy_thresh) && 563 (last_bidx == blst)) { 564 NXGE_DEBUG_MSG((nxgep, TX_CTL, 565 "==> nxge_start: pack(6) " 566 "(pkt_len %d pack_len %d)", 567 pkt_len, pack_len)); 568 i = blst; 569 tx_desc_p = &tx_desc_ring_vp[i]; 570 #ifdef NXGE_DEBUG 571 tx_desc_pp = &tx_desc_ring_pp[i]; 572 #endif 573 tx_msg_p = &tx_msg_ring[i]; 574 boff = pack_len - len; 575 ngathers--; 576 } else if (pack_len > bcopy_thresh && 577 header_set) { 578 pack_len = len; 579 boff = 0; 580 bcopy_thresh = nxge_bcopy_thresh; 581 NXGE_DEBUG_MSG((nxgep, TX_CTL, 582 "==> nxge_start(7): > max NEW " 583 "bcopy thresh %d " 584 "pkt_len %d pack_len %d(next)", 585 bcopy_thresh, 586 pkt_len, pack_len)); 587 } 588 last_bidx = i; 589 } 590 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 591 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 592 hdrp = (p_tx_pkt_header_t)kaddr; 593 header_set = B_TRUE; 594 NXGE_DEBUG_MSG((nxgep, TX_CTL, 595 "==> nxge_start(7_x2): " 596 "pkt_len %d pack_len %d (new hdrp $%p)", 597 pkt_len, pack_len, hdrp)); 598 } 599 tx_msg_p->flags.dma_type = USE_BCOPY; 600 kaddr += boff; 601 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 602 "USE BCOPY: before bcopy " 603 "DESC IOADDR $%p entry %d " 604 "bcopy packets %d " 605 "bcopy kaddr $%p " 606 "bcopy ioaddr (SAD) $%p " 607 "bcopy clen %d " 608 "bcopy boff %d", 609 DMA_COMMON_IOADDR(desc_area), i, 610 tdc_stats->tx_hdr_pkts, 611 kaddr, 612 dma_ioaddr, 613 clen, 614 boff)); 615 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 616 "1USE BCOPY: ")); 617 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 618 "2USE BCOPY: ")); 619 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 620 "last USE BCOPY: copy from b_rptr $%p " 621 "to KADDR $%p (len %d offset %d", 622 b_rptr, kaddr, len, boff)); 623 624 bcopy(b_rptr, kaddr, len); 625 626 #ifdef NXGE_DEBUG 627 dump_len = (len > 128) ? 128: len; 628 NXGE_DEBUG_MSG((nxgep, TX_CTL, 629 "==> nxge_start: dump packets " 630 "(After BCOPY len %d)" 631 "(b_rptr $%p): %s", len, nmp->b_rptr, 632 nxge_dump_packet((char *)nmp->b_rptr, 633 dump_len))); 634 #endif 635 636 dma_handle = tx_msg_p->buf_dma_handle; 637 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 638 (void) ddi_dma_sync(dma_handle, 639 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 640 DDI_DMA_SYNC_FORDEV); 641 clen = len + boff; 642 tdc_stats->tx_hdr_pkts++; 643 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 644 "USE BCOPY: " 645 "DESC IOADDR $%p entry %d " 646 "bcopy packets %d " 647 "bcopy kaddr $%p " 648 "bcopy ioaddr (SAD) $%p " 649 "bcopy clen %d " 650 "bcopy boff %d", 651 DMA_COMMON_IOADDR(desc_area), 652 i, 653 tdc_stats->tx_hdr_pkts, 654 kaddr, 655 dma_ioaddr, 656 clen, 657 boff)); 658 } else { 659 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 660 "USE DVMA: len %d", len)); 661 tx_msg_p->flags.dma_type = USE_DMA; 662 dma_flags = DDI_DMA_WRITE; 663 if (len < nxge_dma_stream_thresh) { 664 dma_flags |= DDI_DMA_CONSISTENT; 665 } else { 666 dma_flags |= DDI_DMA_STREAMING; 667 } 668 669 dma_handle = tx_msg_p->dma_handle; 670 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 671 (caddr_t)b_rptr, len, dma_flags, 672 DDI_DMA_DONTWAIT, NULL, 673 &dma_cookie, &ncookies); 674 if (status == DDI_DMA_MAPPED) { 675 dma_ioaddr = dma_cookie.dmac_laddress; 676 len = (int)dma_cookie.dmac_size; 677 clen = (uint32_t)dma_cookie.dmac_size; 678 NXGE_DEBUG_MSG((nxgep, TX_CTL, 679 "==> nxge_start(12_1): " 680 "USE DVMA: len %d clen %d " 681 "ngathers %d", 682 len, clen, 683 ngathers)); 684 #if defined(__i386) 685 npi_desc_handle.regp = (uint32_t)tx_desc_p; 686 #else 687 npi_desc_handle.regp = (uint64_t)tx_desc_p; 688 #endif 689 while (ncookies > 1) { 690 ngathers++; 691 /* 692 * this is the fix for multiple 693 * cookies, which are basically 694 * a descriptor entry, we don't set 695 * SOP bit as well as related fields 696 */ 697 698 (void) npi_txdma_desc_gather_set( 699 npi_desc_handle, 700 &tx_desc, 701 (ngathers -1), 702 mark_mode, 703 ngathers, 704 dma_ioaddr, 705 clen); 706 707 tx_msg_p->tx_msg_size = clen; 708 NXGE_DEBUG_MSG((nxgep, TX_CTL, 709 "==> nxge_start: DMA " 710 "ncookie %d " 711 "ngathers %d " 712 "dma_ioaddr $%p len %d" 713 "desc $%p descp $%p (%d)", 714 ncookies, 715 ngathers, 716 dma_ioaddr, clen, 717 *tx_desc_p, tx_desc_p, i)); 718 719 ddi_dma_nextcookie(dma_handle, 720 &dma_cookie); 721 dma_ioaddr = 722 dma_cookie.dmac_laddress; 723 724 len = (int)dma_cookie.dmac_size; 725 clen = (uint32_t)dma_cookie.dmac_size; 726 NXGE_DEBUG_MSG((nxgep, TX_CTL, 727 "==> nxge_start(12_2): " 728 "USE DVMA: len %d clen %d ", 729 len, clen)); 730 731 i = TXDMA_DESC_NEXT_INDEX(i, 1, 732 tx_ring_p->tx_wrap_mask); 733 tx_desc_p = &tx_desc_ring_vp[i]; 734 735 #if defined(__i386) 736 npi_desc_handle.regp = 737 (uint32_t)tx_desc_p; 738 #else 739 npi_desc_handle.regp = 740 (uint64_t)tx_desc_p; 741 #endif 742 tx_msg_p = &tx_msg_ring[i]; 743 tx_msg_p->flags.dma_type = USE_NONE; 744 tx_desc.value = 0; 745 746 ncookies--; 747 } 748 tdc_stats->tx_ddi_pkts++; 749 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 750 "DMA: ddi packets %d", 751 tdc_stats->tx_ddi_pkts)); 752 } else { 753 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 754 "dma mapping failed for %d " 755 "bytes addr $%p flags %x (%d)", 756 len, b_rptr, status, status)); 757 good_packet = B_FALSE; 758 tdc_stats->tx_dma_bind_fail++; 759 tx_msg_p->flags.dma_type = USE_NONE; 760 if (is_lso) { 761 mp = nmp; 762 goto nxge_start_fail_lso; 763 } else { 764 goto nxge_start_fail2; 765 } 766 } 767 } /* ddi dvma */ 768 769 if (is_lso) { 770 nmp_lso_save = nmp; 771 } 772 nmp = nmp->b_cont; 773 nxge_start_control_header_only: 774 #if defined(__i386) 775 npi_desc_handle.regp = (uint32_t)tx_desc_p; 776 #else 777 npi_desc_handle.regp = (uint64_t)tx_desc_p; 778 #endif 779 ngathers++; 780 781 if (ngathers == 1) { 782 #ifdef NXGE_DEBUG 783 save_desc_p = &sop_tx_desc; 784 #endif 785 sop_tx_desc_p = &sop_tx_desc; 786 sop_tx_desc_p->value = 0; 787 sop_tx_desc_p->bits.hdw.tr_len = clen; 788 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 789 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 790 } else { 791 #ifdef NXGE_DEBUG 792 save_desc_p = &tx_desc; 793 #endif 794 tmp_desc_p = &tx_desc; 795 tmp_desc_p->value = 0; 796 tmp_desc_p->bits.hdw.tr_len = clen; 797 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 798 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 799 800 tx_desc_p->value = tmp_desc_p->value; 801 } 802 803 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 804 "Desc_entry %d ngathers %d " 805 "desc_vp $%p tx_desc_p $%p " 806 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 807 "dma_ioaddr (SAD) $%p mark %d", 808 i, ngathers, 809 tx_desc_ring_vp, tx_desc_p, 810 len, clen, pkt_len, pack_len, nmblks, 811 dma_ioaddr, mark_mode)); 812 813 #ifdef NXGE_DEBUG 814 npi_desc_handle.nxgep = nxgep; 815 npi_desc_handle.function.function = nxgep->function_num; 816 npi_desc_handle.function.instance = nxgep->instance; 817 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 818 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 819 TX_PKT_DESC_TR_LEN_SHIFT); 820 821 822 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 823 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 824 "mark %d sop %d\n", 825 save_desc_p->value, 826 sad, 827 save_desc_p->bits.hdw.tr_len, 828 xfer_len, 829 save_desc_p->bits.hdw.num_ptr, 830 save_desc_p->bits.hdw.mark, 831 save_desc_p->bits.hdw.sop)); 832 833 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 834 #endif 835 836 tx_msg_p->tx_msg_size = clen; 837 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 838 if (ngathers > nxge_tx_max_gathers) { 839 good_packet = B_FALSE; 840 hcksum_retrieve(mp, NULL, NULL, &start_offset, 841 &stuff_offset, &end_offset, &value, 842 &cksum_flags); 843 844 NXGE_DEBUG_MSG((NULL, TX_CTL, 845 "==> nxge_start(14): pull msg - " 846 "len %d pkt_len %d ngathers %d", 847 len, pkt_len, ngathers)); 848 /* Pull all message blocks from b_cont */ 849 if (is_lso) { 850 mp = nmp_lso_save; 851 goto nxge_start_fail_lso; 852 } 853 if ((msgpullup(mp, -1)) == NULL) { 854 goto nxge_start_fail2; 855 } 856 goto nxge_start_fail2; 857 } 858 } /* while (nmp) */ 859 860 tx_msg_p->tx_message = mp; 861 tx_desc_p = &tx_desc_ring_vp[sop_index]; 862 #if defined(__i386) 863 npi_desc_handle.regp = (uint32_t)tx_desc_p; 864 #else 865 npi_desc_handle.regp = (uint64_t)tx_desc_p; 866 #endif 867 868 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 869 pkthdrp->reserved = 0; 870 hdrp->value = 0; 871 bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 872 873 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 874 tdc_stats->tx_jumbo_pkts++; 875 } 876 877 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 878 if (pkt_len < min_len) { 879 /* Assume we use bcopy to premapped buffers */ 880 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 881 NXGE_DEBUG_MSG((NULL, TX_CTL, 882 "==> nxge_start(14-1): < (msg_min + 16)" 883 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 884 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 885 bzero((kaddr + pkt_len), (min_len - pkt_len)); 886 pkt_len = tx_msg_p->tx_msg_size = min_len; 887 888 sop_tx_desc_p->bits.hdw.tr_len = min_len; 889 890 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 891 tx_desc_p->value = sop_tx_desc_p->value; 892 893 NXGE_DEBUG_MSG((NULL, TX_CTL, 894 "==> nxge_start(14-2): < msg_min - " 895 "len %d pkt_len %d min_len %d ngathers %d", 896 len, pkt_len, min_len, ngathers)); 897 } 898 899 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 900 cksum_flags)); 901 { 902 uint64_t tmp_len; 903 904 /* pkt_len already includes 16 + paddings!! */ 905 /* Update the control header length */ 906 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 907 tmp_len = hdrp->value | 908 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 909 910 NXGE_DEBUG_MSG((nxgep, TX_CTL, 911 "==> nxge_start(15_x1): setting SOP " 912 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 913 "0x%llx hdrp->value 0x%llx", 914 tot_xfer_len, tot_xfer_len, pkt_len, 915 tmp_len, hdrp->value)); 916 #if defined(_BIG_ENDIAN) 917 hdrp->value = ddi_swap64(tmp_len); 918 #else 919 hdrp->value = tmp_len; 920 #endif 921 NXGE_DEBUG_MSG((nxgep, 922 TX_CTL, "==> nxge_start(15_x2): setting SOP " 923 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 924 "tmp_len 0x%llx hdrp->value 0x%llx", 925 tot_xfer_len, pkt_len, 926 tmp_len, hdrp->value)); 927 } 928 929 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 930 "wr_index %d " 931 "tot_xfer_len (%d) pkt_len %d npads %d", 932 sop_index, 933 tot_xfer_len, pkt_len, 934 npads)); 935 936 sop_tx_desc_p->bits.hdw.sop = 1; 937 sop_tx_desc_p->bits.hdw.mark = mark_mode; 938 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 939 940 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 941 942 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 943 944 #ifdef NXGE_DEBUG 945 npi_desc_handle.nxgep = nxgep; 946 npi_desc_handle.function.function = nxgep->function_num; 947 npi_desc_handle.function.instance = nxgep->instance; 948 949 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 950 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 951 save_desc_p->value, 952 sad, 953 save_desc_p->bits.hdw.tr_len, 954 xfer_len, 955 save_desc_p->bits.hdw.num_ptr, 956 save_desc_p->bits.hdw.mark, 957 save_desc_p->bits.hdw.sop)); 958 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 959 960 dump_len = (pkt_len > 128) ? 128: pkt_len; 961 NXGE_DEBUG_MSG((nxgep, TX_CTL, 962 "==> nxge_start: dump packets(17) (after sop set, len " 963 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 964 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 965 (char *)hdrp, 966 nxge_dump_packet((char *)hdrp, dump_len))); 967 NXGE_DEBUG_MSG((nxgep, TX_CTL, 968 "==> nxge_start(18): TX desc sync: sop_index %d", 969 sop_index)); 970 #endif 971 972 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 973 (void) ddi_dma_sync(tx_desc_dma_handle, 974 sop_index * sizeof (tx_desc_t), 975 ngathers * sizeof (tx_desc_t), 976 DDI_DMA_SYNC_FORDEV); 977 978 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 979 "cs_off = 0x%02X cs_s_off = 0x%02X " 980 "pkt_len %d ngathers %d sop_index %d\n", 981 stuff_offset, start_offset, 982 pkt_len, ngathers, sop_index)); 983 } else { /* more than one descriptor and wrap around */ 984 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 985 (void) ddi_dma_sync(tx_desc_dma_handle, 986 sop_index * sizeof (tx_desc_t), 987 nsdescs * sizeof (tx_desc_t), 988 DDI_DMA_SYNC_FORDEV); 989 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 990 "cs_off = 0x%02X cs_s_off = 0x%02X " 991 "pkt_len %d ngathers %d sop_index %d\n", 992 stuff_offset, start_offset, 993 pkt_len, ngathers, sop_index)); 994 995 (void) ddi_dma_sync(tx_desc_dma_handle, 996 0, 997 (ngathers - nsdescs) * sizeof (tx_desc_t), 998 DDI_DMA_SYNC_FORDEV); 999 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 1000 "cs_off = 0x%02X cs_s_off = 0x%02X " 1001 "pkt_len %d ngathers %d sop_index %d\n", 1002 stuff_offset, start_offset, 1003 pkt_len, ngathers, sop_index)); 1004 } 1005 1006 tail_index = tx_ring_p->wr_index; 1007 tail_wrap = tx_ring_p->wr_index_wrap; 1008 1009 tx_ring_p->wr_index = i; 1010 if (tx_ring_p->wr_index <= tail_index) { 1011 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 1012 B_FALSE : B_TRUE); 1013 } 1014 1015 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 1016 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 1017 tx_ring_p->tdc, 1018 tx_ring_p->wr_index, 1019 tx_ring_p->wr_index_wrap, 1020 ngathers, 1021 tx_ring_p->descs_pending)); 1022 1023 if (is_lso) { 1024 lso_ngathers += ngathers; 1025 if (mp_chain != NULL) { 1026 mp = mp_chain; 1027 mp_chain = mp_chain->b_next; 1028 mp->b_next = NULL; 1029 if (nxge_lso_kick_cnt == lso_ngathers) { 1030 tx_ring_p->descs_pending += lso_ngathers; 1031 { 1032 tx_ring_kick_t kick; 1033 1034 kick.value = 0; 1035 kick.bits.ldw.wrap = 1036 tx_ring_p->wr_index_wrap; 1037 kick.bits.ldw.tail = 1038 (uint16_t)tx_ring_p->wr_index; 1039 1040 /* Kick the Transmit kick register */ 1041 TXDMA_REG_WRITE64( 1042 NXGE_DEV_NPI_HANDLE(nxgep), 1043 TX_RING_KICK_REG, 1044 (uint8_t)tx_ring_p->tdc, 1045 kick.value); 1046 tdc_stats->tx_starts++; 1047 1048 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1049 "==> nxge_start: more LSO: " 1050 "LSO_CNT %d", 1051 lso_ngathers)); 1052 } 1053 lso_ngathers = 0; 1054 ngathers = 0; 1055 cur_index_lso = sop_index = tx_ring_p->wr_index; 1056 lso_tail_wrap = tx_ring_p->wr_index_wrap; 1057 } 1058 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1059 "==> nxge_start: lso again: " 1060 "lso_gathers %d ngathers %d cur_index_lso %d " 1061 "wr_index %d sop_index %d", 1062 lso_ngathers, ngathers, cur_index_lso, 1063 tx_ring_p->wr_index, sop_index)); 1064 1065 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1066 "==> nxge_start: next : count %d", 1067 lso_ngathers)); 1068 lso_again = B_TRUE; 1069 goto start_again; 1070 } 1071 ngathers = lso_ngathers; 1072 } 1073 1074 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 1075 1076 { 1077 tx_ring_kick_t kick; 1078 1079 kick.value = 0; 1080 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 1081 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 1082 1083 /* Kick start the Transmit kick register */ 1084 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 1085 TX_RING_KICK_REG, 1086 (uint8_t)tx_ring_p->tdc, 1087 kick.value); 1088 } 1089 1090 tx_ring_p->descs_pending += ngathers; 1091 tdc_stats->tx_starts++; 1092 1093 if (isLDOMservice(nxgep)) { 1094 tx_ring_p->tx_ring_busy = B_FALSE; 1095 if (tx_ring_p->tx_ring_offline) { 1096 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1097 NXGE_TX_RING_OFFLINED); 1098 } 1099 } 1100 1101 MUTEX_EXIT(&tx_ring_p->lock); 1102 1103 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1104 return (status); 1105 1106 nxge_start_fail_lso: 1107 status = 0; 1108 good_packet = B_FALSE; 1109 if (mp != NULL) { 1110 freemsg(mp); 1111 } 1112 if (mp_chain != NULL) { 1113 freemsg(mp_chain); 1114 } 1115 if (!lso_again && !ngathers) { 1116 if (isLDOMservice(nxgep)) { 1117 tx_ring_p->tx_ring_busy = B_FALSE; 1118 if (tx_ring_p->tx_ring_offline) { 1119 (void) atomic_swap_32( 1120 &tx_ring_p->tx_ring_offline, 1121 NXGE_TX_RING_OFFLINED); 1122 } 1123 } 1124 1125 MUTEX_EXIT(&tx_ring_p->lock); 1126 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1127 "==> nxge_start: lso exit (nothing changed)")); 1128 goto nxge_start_fail1; 1129 } 1130 1131 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1132 "==> nxge_start (channel %d): before lso " 1133 "lso_gathers %d ngathers %d cur_index_lso %d " 1134 "wr_index %d sop_index %d lso_again %d", 1135 tx_ring_p->tdc, 1136 lso_ngathers, ngathers, cur_index_lso, 1137 tx_ring_p->wr_index, sop_index, lso_again)); 1138 1139 if (lso_again) { 1140 lso_ngathers += ngathers; 1141 ngathers = lso_ngathers; 1142 sop_index = cur_index_lso; 1143 tx_ring_p->wr_index = sop_index; 1144 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1145 } 1146 1147 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1148 "==> nxge_start (channel %d): after lso " 1149 "lso_gathers %d ngathers %d cur_index_lso %d " 1150 "wr_index %d sop_index %d lso_again %d", 1151 tx_ring_p->tdc, 1152 lso_ngathers, ngathers, cur_index_lso, 1153 tx_ring_p->wr_index, sop_index, lso_again)); 1154 1155 nxge_start_fail2: 1156 if (good_packet == B_FALSE) { 1157 cur_index = sop_index; 1158 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1159 for (i = 0; i < ngathers; i++) { 1160 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1161 #if defined(__i386) 1162 npi_handle.regp = (uint32_t)tx_desc_p; 1163 #else 1164 npi_handle.regp = (uint64_t)tx_desc_p; 1165 #endif 1166 tx_msg_p = &tx_msg_ring[cur_index]; 1167 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1168 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1169 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1170 "tx_desc_p = %X index = %d", 1171 tx_desc_p, tx_ring_p->rd_index)); 1172 (void) dvma_unload(tx_msg_p->dvma_handle, 1173 0, -1); 1174 tx_msg_p->dvma_handle = NULL; 1175 if (tx_ring_p->dvma_wr_index == 1176 tx_ring_p->dvma_wrap_mask) 1177 tx_ring_p->dvma_wr_index = 0; 1178 else 1179 tx_ring_p->dvma_wr_index++; 1180 tx_ring_p->dvma_pending--; 1181 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1182 if (ddi_dma_unbind_handle( 1183 tx_msg_p->dma_handle)) { 1184 cmn_err(CE_WARN, "!nxge_start: " 1185 "ddi_dma_unbind_handle failed"); 1186 } 1187 } 1188 tx_msg_p->flags.dma_type = USE_NONE; 1189 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1190 tx_ring_p->tx_wrap_mask); 1191 1192 } 1193 } 1194 1195 if (isLDOMservice(nxgep)) { 1196 tx_ring_p->tx_ring_busy = B_FALSE; 1197 if (tx_ring_p->tx_ring_offline) { 1198 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1199 NXGE_TX_RING_OFFLINED); 1200 } 1201 } 1202 1203 MUTEX_EXIT(&tx_ring_p->lock); 1204 1205 nxge_start_fail1: 1206 /* Add FMA to check the access handle nxge_hregh */ 1207 1208 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1209 return (status); 1210 } 1211 1212 /* Software LSO starts here */ 1213 static void 1214 nxge_hcksum_retrieve(mblk_t *mp, 1215 uint32_t *start, uint32_t *stuff, uint32_t *end, 1216 uint32_t *value, uint32_t *flags) 1217 { 1218 if (mp->b_datap->db_type == M_DATA) { 1219 if (flags != NULL) { 1220 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1221 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1222 HCK_FULLCKSUM_OK); 1223 if ((*flags & (HCK_PARTIALCKSUM | 1224 HCK_FULLCKSUM)) != 0) { 1225 if (value != NULL) 1226 *value = (uint32_t)DB_CKSUM16(mp); 1227 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1228 if (start != NULL) 1229 *start = 1230 (uint32_t)DB_CKSUMSTART(mp); 1231 if (stuff != NULL) 1232 *stuff = 1233 (uint32_t)DB_CKSUMSTUFF(mp); 1234 if (end != NULL) 1235 *end = 1236 (uint32_t)DB_CKSUMEND(mp); 1237 } 1238 } 1239 } 1240 } 1241 } 1242 1243 static void 1244 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1245 { 1246 ASSERT(DB_TYPE(mp) == M_DATA); 1247 1248 *mss = 0; 1249 if (flags != NULL) { 1250 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1251 if ((*flags != 0) && (mss != NULL)) { 1252 *mss = (uint32_t)DB_LSOMSS(mp); 1253 } 1254 NXGE_DEBUG_MSG((NULL, TX_CTL, 1255 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1256 *mss, *flags)); 1257 } 1258 1259 NXGE_DEBUG_MSG((NULL, TX_CTL, 1260 "<== nxge_lso_info_get: mss %d", *mss)); 1261 } 1262 1263 /* 1264 * Do Soft LSO on the oversized packet. 1265 * 1266 * 1. Create a chain of message for headers. 1267 * 2. Fill up header messages with proper information. 1268 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1269 * each new message with necessary adjustments. 1270 * * Unchange the ethernet header for DIX frames. (by default) 1271 * * IP Total Length field is updated to MSS or less(only for the last one). 1272 * * IP Identification value is incremented by one for each packet. 1273 * * TCP sequence Number is recalculated according to the payload length. 1274 * * Set FIN and/or PSH flags for the *last* packet if applied. 1275 * * TCP partial Checksum 1276 * 4. Update LSO information in the first message header. 1277 * 5. Release the original message header. 1278 */ 1279 static mblk_t * 1280 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1281 { 1282 uint32_t hckflags; 1283 int pktlen; 1284 int hdrlen; 1285 int segnum; 1286 int i; 1287 struct ether_vlan_header *evh; 1288 int ehlen, iphlen, tcphlen; 1289 struct ip *oiph, *niph; 1290 struct tcphdr *otcph, *ntcph; 1291 int available, len, left; 1292 uint16_t ip_id; 1293 uint32_t tcp_seq; 1294 #ifdef __sparc 1295 uint32_t tcp_seq_tmp; 1296 #endif 1297 mblk_t *datamp; 1298 uchar_t *rptr; 1299 mblk_t *nmp; 1300 mblk_t *cmp; 1301 mblk_t *mp_chain; 1302 boolean_t do_cleanup = B_FALSE; 1303 t_uscalar_t start_offset = 0; 1304 t_uscalar_t stuff_offset = 0; 1305 t_uscalar_t value = 0; 1306 uint16_t l4_len; 1307 ipaddr_t src, dst; 1308 uint32_t cksum, sum, l4cksum; 1309 1310 NXGE_DEBUG_MSG((NULL, TX_CTL, 1311 "==> nxge_do_softlso")); 1312 /* 1313 * check the length of LSO packet payload and calculate the number of 1314 * segments to be generated. 1315 */ 1316 pktlen = msgsize(mp); 1317 evh = (struct ether_vlan_header *)mp->b_rptr; 1318 1319 /* VLAN? */ 1320 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1321 ehlen = sizeof (struct ether_vlan_header); 1322 else 1323 ehlen = sizeof (struct ether_header); 1324 oiph = (struct ip *)(mp->b_rptr + ehlen); 1325 iphlen = oiph->ip_hl * 4; 1326 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1327 tcphlen = otcph->th_off * 4; 1328 1329 l4_len = pktlen - ehlen - iphlen; 1330 1331 NXGE_DEBUG_MSG((NULL, TX_CTL, 1332 "==> nxge_do_softlso: mss %d oiph $%p " 1333 "original ip_sum oiph->ip_sum 0x%x " 1334 "original tcp_sum otcph->th_sum 0x%x " 1335 "oiph->ip_len %d pktlen %d ehlen %d " 1336 "l4_len %d (0x%x) ip_len - iphlen %d ", 1337 mss, 1338 oiph, 1339 oiph->ip_sum, 1340 otcph->th_sum, 1341 ntohs(oiph->ip_len), pktlen, 1342 ehlen, 1343 l4_len, 1344 l4_len, 1345 ntohs(oiph->ip_len) - iphlen)); 1346 1347 /* IPv4 + TCP */ 1348 if (!(oiph->ip_v == IPV4_VERSION)) { 1349 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1350 "<== nxge_do_softlso: not IPV4 " 1351 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1352 ntohs(oiph->ip_len), pktlen, ehlen, 1353 tcphlen)); 1354 freemsg(mp); 1355 return (NULL); 1356 } 1357 1358 if (!(oiph->ip_p == IPPROTO_TCP)) { 1359 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1360 "<== nxge_do_softlso: not TCP " 1361 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1362 ntohs(oiph->ip_len), pktlen, ehlen, 1363 tcphlen)); 1364 freemsg(mp); 1365 return (NULL); 1366 } 1367 1368 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1369 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1370 "<== nxge_do_softlso: len not matched " 1371 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1372 ntohs(oiph->ip_len), pktlen, ehlen, 1373 tcphlen)); 1374 freemsg(mp); 1375 return (NULL); 1376 } 1377 1378 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1379 tcphlen = otcph->th_off * 4; 1380 1381 /* TCP flags can not include URG, RST, or SYN */ 1382 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1383 1384 hdrlen = ehlen + iphlen + tcphlen; 1385 1386 VERIFY(MBLKL(mp) >= hdrlen); 1387 1388 if (MBLKL(mp) > hdrlen) { 1389 datamp = mp; 1390 rptr = mp->b_rptr + hdrlen; 1391 } else { /* = */ 1392 datamp = mp->b_cont; 1393 rptr = datamp->b_rptr; 1394 } 1395 1396 NXGE_DEBUG_MSG((NULL, TX_CTL, 1397 "nxge_do_softlso: otcph $%p pktlen: %d, " 1398 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1399 "mblkl(mp): %d, mblkl(datamp): %d", 1400 otcph, 1401 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1402 (int)MBLKL(mp), (int)MBLKL(datamp))); 1403 1404 hckflags = 0; 1405 nxge_hcksum_retrieve(mp, 1406 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1407 1408 dst = oiph->ip_dst.s_addr; 1409 src = oiph->ip_src.s_addr; 1410 1411 cksum = (dst >> 16) + (dst & 0xFFFF) + 1412 (src >> 16) + (src & 0xFFFF); 1413 l4cksum = cksum + IP_TCP_CSUM_COMP; 1414 1415 sum = l4_len + l4cksum; 1416 sum = (sum & 0xFFFF) + (sum >> 16); 1417 1418 NXGE_DEBUG_MSG((NULL, TX_CTL, 1419 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1420 "hckflags 0x%x start_offset %d stuff_offset %d " 1421 "value (original) 0x%x th_sum 0x%x " 1422 "pktlen %d l4_len %d (0x%x) " 1423 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1424 dst, src, 1425 (sum & 0xffff), (~sum & 0xffff), 1426 hckflags, start_offset, stuff_offset, 1427 value, otcph->th_sum, 1428 pktlen, 1429 l4_len, 1430 l4_len, 1431 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1432 (int)MBLKL(datamp), 1433 nxge_dump_packet((char *)evh, 12))); 1434 1435 /* 1436 * Start to process. 1437 */ 1438 available = pktlen - hdrlen; 1439 segnum = (available - 1) / mss + 1; 1440 1441 NXGE_DEBUG_MSG((NULL, TX_CTL, 1442 "==> nxge_do_softlso: pktlen %d " 1443 "MBLKL(mp): %d, MBLKL(datamp): %d " 1444 "available %d mss %d segnum %d", 1445 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1446 available, 1447 mss, 1448 segnum)); 1449 1450 VERIFY(segnum >= 2); 1451 1452 /* 1453 * Try to pre-allocate all header messages 1454 */ 1455 mp_chain = NULL; 1456 for (i = 0; i < segnum; i++) { 1457 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1458 /* Clean up the mp_chain */ 1459 while (mp_chain != NULL) { 1460 nmp = mp_chain; 1461 mp_chain = mp_chain->b_next; 1462 freemsg(nmp); 1463 } 1464 NXGE_DEBUG_MSG((NULL, TX_CTL, 1465 "<== nxge_do_softlso: " 1466 "Could not allocate enough messages for headers!")); 1467 freemsg(mp); 1468 return (NULL); 1469 } 1470 nmp->b_next = mp_chain; 1471 mp_chain = nmp; 1472 1473 NXGE_DEBUG_MSG((NULL, TX_CTL, 1474 "==> nxge_do_softlso: " 1475 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1476 mp, nmp, mp_chain, mp_chain->b_next)); 1477 } 1478 1479 NXGE_DEBUG_MSG((NULL, TX_CTL, 1480 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1481 mp, nmp, mp_chain)); 1482 1483 /* 1484 * Associate payload with new packets 1485 */ 1486 cmp = mp_chain; 1487 left = available; 1488 while (cmp != NULL) { 1489 nmp = dupb(datamp); 1490 if (nmp == NULL) { 1491 do_cleanup = B_TRUE; 1492 NXGE_DEBUG_MSG((NULL, TX_CTL, 1493 "==>nxge_do_softlso: " 1494 "Can not dupb(datamp), have to do clean up")); 1495 goto cleanup_allocated_msgs; 1496 } 1497 1498 NXGE_DEBUG_MSG((NULL, TX_CTL, 1499 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1500 "dupb nmp $%p len %d left %d msd %d ", 1501 mp, cmp, nmp, len, left, mss)); 1502 1503 cmp->b_cont = nmp; 1504 nmp->b_rptr = rptr; 1505 len = (left < mss) ? left : mss; 1506 left -= len; 1507 1508 NXGE_DEBUG_MSG((NULL, TX_CTL, 1509 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1510 "dupb nmp $%p len %d left %d mss %d ", 1511 mp, cmp, nmp, len, left, mss)); 1512 NXGE_DEBUG_MSG((NULL, TX_CTL, 1513 "nxge_do_softlso: before available: %d, " 1514 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1515 available, left, len, segnum, (int)MBLKL(nmp))); 1516 1517 len -= MBLKL(nmp); 1518 NXGE_DEBUG_MSG((NULL, TX_CTL, 1519 "nxge_do_softlso: after available: %d, " 1520 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1521 available, left, len, segnum, (int)MBLKL(nmp))); 1522 1523 while (len > 0) { 1524 mblk_t *mmp = NULL; 1525 1526 NXGE_DEBUG_MSG((NULL, TX_CTL, 1527 "nxge_do_softlso: (4) len > 0 available: %d, " 1528 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1529 available, left, len, segnum, (int)MBLKL(nmp))); 1530 1531 if (datamp->b_cont != NULL) { 1532 datamp = datamp->b_cont; 1533 rptr = datamp->b_rptr; 1534 mmp = dupb(datamp); 1535 if (mmp == NULL) { 1536 do_cleanup = B_TRUE; 1537 NXGE_DEBUG_MSG((NULL, TX_CTL, 1538 "==> nxge_do_softlso: " 1539 "Can not dupb(datamp) (1), :" 1540 "have to do clean up")); 1541 NXGE_DEBUG_MSG((NULL, TX_CTL, 1542 "==> nxge_do_softlso: " 1543 "available: %d, left: %d, " 1544 "len: %d, MBLKL(nmp): %d", 1545 available, left, len, 1546 (int)MBLKL(nmp))); 1547 goto cleanup_allocated_msgs; 1548 } 1549 } else { 1550 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1551 "==> nxge_do_softlso: " 1552 "(1)available: %d, left: %d, " 1553 "len: %d, MBLKL(nmp): %d", 1554 available, left, len, 1555 (int)MBLKL(nmp))); 1556 cmn_err(CE_PANIC, 1557 "==> nxge_do_softlso: " 1558 "Pointers must have been corrupted!\n" 1559 "datamp: $%p, nmp: $%p, rptr: $%p", 1560 (void *)datamp, 1561 (void *)nmp, 1562 (void *)rptr); 1563 } 1564 nmp->b_cont = mmp; 1565 nmp = mmp; 1566 len -= MBLKL(nmp); 1567 } 1568 if (len < 0) { 1569 nmp->b_wptr += len; 1570 rptr = nmp->b_wptr; 1571 NXGE_DEBUG_MSG((NULL, TX_CTL, 1572 "(5) len < 0 (less than 0)" 1573 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1574 available, left, len, (int)MBLKL(nmp))); 1575 1576 } else if (len == 0) { 1577 if (datamp->b_cont != NULL) { 1578 NXGE_DEBUG_MSG((NULL, TX_CTL, 1579 "(5) len == 0" 1580 "available: %d, left: %d, len: %d, " 1581 "MBLKL(nmp): %d", 1582 available, left, len, (int)MBLKL(nmp))); 1583 datamp = datamp->b_cont; 1584 rptr = datamp->b_rptr; 1585 } else { 1586 NXGE_DEBUG_MSG((NULL, TX_CTL, 1587 "(6)available b_cont == NULL : %d, " 1588 "left: %d, len: %d, MBLKL(nmp): %d", 1589 available, left, len, (int)MBLKL(nmp))); 1590 1591 VERIFY(cmp->b_next == NULL); 1592 VERIFY(left == 0); 1593 break; /* Done! */ 1594 } 1595 } 1596 cmp = cmp->b_next; 1597 1598 NXGE_DEBUG_MSG((NULL, TX_CTL, 1599 "(7) do_softlso: " 1600 "next mp in mp_chain available len != 0 : %d, " 1601 "left: %d, len: %d, MBLKL(nmp): %d", 1602 available, left, len, (int)MBLKL(nmp))); 1603 } 1604 1605 /* 1606 * From now, start to fill up all headers for the first message 1607 * Hardware checksum flags need to be updated separately for FULLCKSUM 1608 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1609 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1610 * required fields need to be updated properly. 1611 */ 1612 nmp = mp_chain; 1613 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1614 nmp->b_wptr = nmp->b_rptr + hdrlen; 1615 niph = (struct ip *)(nmp->b_rptr + ehlen); 1616 niph->ip_len = htons(mss + iphlen + tcphlen); 1617 ip_id = ntohs(niph->ip_id); 1618 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1619 #ifdef __sparc 1620 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1621 tcp_seq = ntohl(tcp_seq_tmp); 1622 #else 1623 tcp_seq = ntohl(ntcph->th_seq); 1624 #endif 1625 1626 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1627 1628 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1629 DB_CKSUMSTART(nmp) = start_offset; 1630 DB_CKSUMSTUFF(nmp) = stuff_offset; 1631 1632 /* calculate IP checksum and TCP pseudo header checksum */ 1633 niph->ip_sum = 0; 1634 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1635 1636 l4_len = mss + tcphlen; 1637 sum = htons(l4_len) + l4cksum; 1638 sum = (sum & 0xFFFF) + (sum >> 16); 1639 ntcph->th_sum = (sum & 0xffff); 1640 1641 NXGE_DEBUG_MSG((NULL, TX_CTL, 1642 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1643 "mss %d pktlen %d l4_len %d (0x%x) " 1644 "MBLKL(mp): %d, MBLKL(datamp): %d " 1645 "ip_sum 0x%x " 1646 "th_sum 0x%x sum 0x%x ) " 1647 "dump first ip->tcp %s", 1648 nmp, mp_chain, 1649 mss, 1650 pktlen, 1651 l4_len, 1652 l4_len, 1653 (int)MBLKL(mp), (int)MBLKL(datamp), 1654 niph->ip_sum, 1655 ntcph->th_sum, 1656 sum, 1657 nxge_dump_packet((char *)niph, 52))); 1658 1659 cmp = nmp; 1660 while ((nmp = nmp->b_next)->b_next != NULL) { 1661 NXGE_DEBUG_MSG((NULL, TX_CTL, 1662 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1663 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1664 nmp->b_wptr = nmp->b_rptr + hdrlen; 1665 niph = (struct ip *)(nmp->b_rptr + ehlen); 1666 niph->ip_id = htons(++ip_id); 1667 niph->ip_len = htons(mss + iphlen + tcphlen); 1668 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1669 tcp_seq += mss; 1670 1671 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1672 1673 #ifdef __sparc 1674 tcp_seq_tmp = htonl(tcp_seq); 1675 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1676 #else 1677 ntcph->th_seq = htonl(tcp_seq); 1678 #endif 1679 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1680 DB_CKSUMSTART(nmp) = start_offset; 1681 DB_CKSUMSTUFF(nmp) = stuff_offset; 1682 1683 /* calculate IP checksum and TCP pseudo header checksum */ 1684 niph->ip_sum = 0; 1685 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1686 ntcph->th_sum = (sum & 0xffff); 1687 1688 NXGE_DEBUG_MSG((NULL, TX_CTL, 1689 "==> nxge_do_softlso: middle ip_sum 0x%x " 1690 "th_sum 0x%x " 1691 " mp $%p (mp_chain $%p) pktlen %d " 1692 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1693 niph->ip_sum, 1694 ntcph->th_sum, 1695 nmp, mp_chain, 1696 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1697 } 1698 1699 /* Last segment */ 1700 /* 1701 * Set FIN and/or PSH flags if present only in the last packet. 1702 * The ip_len could be different from prior packets. 1703 */ 1704 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1705 nmp->b_wptr = nmp->b_rptr + hdrlen; 1706 niph = (struct ip *)(nmp->b_rptr + ehlen); 1707 niph->ip_id = htons(++ip_id); 1708 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1709 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1710 tcp_seq += mss; 1711 #ifdef __sparc 1712 tcp_seq_tmp = htonl(tcp_seq); 1713 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1714 #else 1715 ntcph->th_seq = htonl(tcp_seq); 1716 #endif 1717 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1718 1719 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1720 DB_CKSUMSTART(nmp) = start_offset; 1721 DB_CKSUMSTUFF(nmp) = stuff_offset; 1722 1723 /* calculate IP checksum and TCP pseudo header checksum */ 1724 niph->ip_sum = 0; 1725 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1726 1727 l4_len = ntohs(niph->ip_len) - iphlen; 1728 sum = htons(l4_len) + l4cksum; 1729 sum = (sum & 0xFFFF) + (sum >> 16); 1730 ntcph->th_sum = (sum & 0xffff); 1731 1732 NXGE_DEBUG_MSG((NULL, TX_CTL, 1733 "==> nxge_do_softlso: last next " 1734 "niph->ip_sum 0x%x " 1735 "ntcph->th_sum 0x%x sum 0x%x " 1736 "dump last ip->tcp %s " 1737 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1738 "l4_len %d (0x%x) " 1739 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1740 niph->ip_sum, 1741 ntcph->th_sum, sum, 1742 nxge_dump_packet((char *)niph, 52), 1743 cmp, nmp, mp_chain, 1744 pktlen, pktlen, 1745 l4_len, 1746 l4_len, 1747 (int)MBLKL(mp), (int)MBLKL(datamp))); 1748 1749 cleanup_allocated_msgs: 1750 if (do_cleanup) { 1751 NXGE_DEBUG_MSG((NULL, TX_CTL, 1752 "==> nxge_do_softlso: " 1753 "Failed allocating messages, " 1754 "have to clean up and fail!")); 1755 while (mp_chain != NULL) { 1756 nmp = mp_chain; 1757 mp_chain = mp_chain->b_next; 1758 freemsg(nmp); 1759 } 1760 } 1761 /* 1762 * We're done here, so just free the original message and return the 1763 * new message chain, that could be NULL if failed, back to the caller. 1764 */ 1765 freemsg(mp); 1766 1767 NXGE_DEBUG_MSG((NULL, TX_CTL, 1768 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1769 return (mp_chain); 1770 } 1771 1772 /* 1773 * Will be called before NIC driver do further operation on the message. 1774 * The input message may include LSO information, if so, go to softlso logic 1775 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1776 * The return could be the same non-LSO message or a message chain for LSO case. 1777 * 1778 * The driver needs to call this function per packet and process the whole chain 1779 * if applied. 1780 */ 1781 static mblk_t * 1782 nxge_lso_eliminate(mblk_t *mp) 1783 { 1784 uint32_t lsoflags; 1785 uint32_t mss; 1786 1787 NXGE_DEBUG_MSG((NULL, TX_CTL, 1788 "==>nxge_lso_eliminate:")); 1789 nxge_lso_info_get(mp, &mss, &lsoflags); 1790 1791 if (lsoflags & HW_LSO) { 1792 mblk_t *nmp; 1793 1794 NXGE_DEBUG_MSG((NULL, TX_CTL, 1795 "==>nxge_lso_eliminate:" 1796 "HW_LSO:mss %d mp $%p", 1797 mss, mp)); 1798 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1799 NXGE_DEBUG_MSG((NULL, TX_CTL, 1800 "<== nxge_lso_eliminate: " 1801 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1802 nmp, mss, mp)); 1803 return (nmp); 1804 } else { 1805 NXGE_DEBUG_MSG((NULL, TX_CTL, 1806 "<== nxge_lso_eliminate_ " 1807 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1808 nmp, mss, mp)); 1809 return (NULL); 1810 } 1811 } 1812 1813 NXGE_DEBUG_MSG((NULL, TX_CTL, 1814 "<== nxge_lso_eliminate")); 1815 return (mp); 1816 } 1817 1818 static uint32_t 1819 nxge_csgen(uint16_t *adr, int len) 1820 { 1821 int i, odd; 1822 uint32_t sum = 0; 1823 uint32_t c = 0; 1824 1825 odd = len % 2; 1826 for (i = 0; i < (len / 2); i++) { 1827 sum += (adr[i] & 0xffff); 1828 } 1829 if (odd) { 1830 sum += adr[len / 2] & 0xff00; 1831 } 1832 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 1833 sum &= 0xffff; 1834 sum += c; 1835 } 1836 return (~sum & 0xffff); 1837 } 1838