1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/mac_provider.h> 27 #include <sys/nxge/nxge_impl.h> 28 #include <sys/nxge/nxge_hio.h> 29 #include <npi_tx_wr64.h> 30 31 /* Software LSO required header files */ 32 #include <netinet/tcp.h> 33 #include <inet/ip_impl.h> 34 #include <inet/tcp.h> 35 36 extern uint64_t mac_pkt_hash(uint_t, mblk_t *mp, uint8_t policy, 37 boolean_t is_outbound); 38 39 static mblk_t *nxge_lso_eliminate(mblk_t *); 40 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 41 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 42 static void nxge_hcksum_retrieve(mblk_t *, 43 uint32_t *, uint32_t *, uint32_t *, 44 uint32_t *, uint32_t *); 45 static uint32_t nxge_csgen(uint16_t *, int); 46 47 extern uint32_t nxge_reclaim_pending; 48 extern uint32_t nxge_bcopy_thresh; 49 extern uint32_t nxge_dvma_thresh; 50 extern uint32_t nxge_dma_stream_thresh; 51 extern uint32_t nxge_tx_minfree; 52 extern uint32_t nxge_tx_intr_thres; 53 extern uint32_t nxge_tx_max_gathers; 54 extern uint32_t nxge_tx_tiny_pack; 55 extern uint32_t nxge_tx_use_bcopy; 56 extern nxge_tx_mode_t nxge_tx_scheme; 57 uint32_t nxge_lso_kick_cnt = 2; 58 59 60 void 61 nxge_tx_ring_task(void *arg) 62 { 63 p_tx_ring_t ring = (p_tx_ring_t)arg; 64 65 MUTEX_ENTER(&ring->lock); 66 (void) nxge_txdma_reclaim(ring->nxgep, ring, 0); 67 MUTEX_EXIT(&ring->lock); 68 69 if (!isLDOMguest(ring->nxgep) && !ring->tx_ring_offline) 70 mac_tx_ring_update(ring->nxgep->mach, ring->tx_ring_handle); 71 #if defined(sun4v) 72 else { 73 nxge_hio_data_t *nhd = 74 (nxge_hio_data_t *)ring->nxgep->nxge_hw_p->hio; 75 nx_vio_fp_t *vio = &nhd->hio.vio; 76 77 /* Call back vnet. */ 78 if (vio->cb.vio_net_tx_update) { 79 (*vio->cb.vio_net_tx_update)(ring->nxgep->hio_vr->vhp); 80 } 81 } 82 #endif 83 } 84 85 static void 86 nxge_tx_ring_dispatch(p_tx_ring_t ring) 87 { 88 /* 89 * Kick the ring task to reclaim some buffers. 90 */ 91 (void) ddi_taskq_dispatch(ring->taskq, 92 nxge_tx_ring_task, (void *)ring, DDI_SLEEP); 93 } 94 95 mblk_t * 96 nxge_tx_ring_send(void *arg, mblk_t *mp) 97 { 98 p_nxge_ring_handle_t nrhp = (p_nxge_ring_handle_t)arg; 99 p_nxge_t nxgep; 100 p_tx_ring_t tx_ring_p; 101 int status, channel; 102 103 ASSERT(nrhp != NULL); 104 nxgep = nrhp->nxgep; 105 channel = nxgep->pt_config.hw_config.tdc.start + nrhp->index; 106 tx_ring_p = nxgep->tx_rings->rings[channel]; 107 108 ASSERT(nxgep == tx_ring_p->nxgep); 109 110 #ifdef DEBUG 111 if (isLDOMservice(nxgep)) { 112 ASSERT(!tx_ring_p->tx_ring_offline); 113 } 114 #endif 115 116 status = nxge_start(nxgep, tx_ring_p, mp); 117 if (status) { 118 nxge_tx_ring_dispatch(tx_ring_p); 119 return (mp); 120 } 121 122 return ((mblk_t *)NULL); 123 } 124 125 #if defined(sun4v) 126 127 /* 128 * Hashing policy for load balancing over the set of TX rings 129 * available to the driver. 130 */ 131 static uint8_t nxge_tx_hash_policy = MAC_PKT_HASH_L4; 132 133 /* 134 * nxge_m_tx() is needed for Hybrid I/O operation of the vnet in 135 * the guest domain. See CR 6778758 for long term solution. 136 * 137 * The guest domain driver will for now hash the packet 138 * to pick a DMA channel from the only group it has group 0. 139 */ 140 141 mblk_t * 142 nxge_m_tx(void *arg, mblk_t *mp) 143 { 144 p_nxge_t nxgep = (p_nxge_t)arg; 145 mblk_t *next; 146 uint64_t rindex; 147 p_tx_ring_t tx_ring_p; 148 int status; 149 150 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_m_tx")); 151 152 /* 153 * Hash to pick a ring from Group 0, the only TX group 154 * for a guest domain driver. 155 */ 156 rindex = mac_pkt_hash(DL_ETHER, mp, nxge_tx_hash_policy, B_TRUE); 157 rindex = rindex % nxgep->pt_config.tdc_grps[0].max_tdcs; 158 159 /* 160 * Get the ring handle. 161 */ 162 tx_ring_p = nxgep->tx_rings->rings[rindex]; 163 164 while (mp != NULL) { 165 next = mp->b_next; 166 mp->b_next = NULL; 167 168 status = nxge_start(nxgep, tx_ring_p, mp); 169 if (status != 0) { 170 mp->b_next = next; 171 nxge_tx_ring_dispatch(tx_ring_p); 172 return (mp); 173 } 174 175 mp = next; 176 } 177 178 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_m_tx")); 179 return ((mblk_t *)NULL); 180 } 181 182 #endif 183 184 int 185 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 186 { 187 int status = 0; 188 p_tx_desc_t tx_desc_ring_vp; 189 npi_handle_t npi_desc_handle; 190 nxge_os_dma_handle_t tx_desc_dma_handle; 191 p_tx_desc_t tx_desc_p; 192 p_tx_msg_t tx_msg_ring; 193 p_tx_msg_t tx_msg_p; 194 tx_desc_t tx_desc, *tmp_desc_p; 195 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 196 p_tx_pkt_header_t hdrp; 197 tx_pkt_hdr_all_t tmp_hdrp; 198 p_tx_pkt_hdr_all_t pkthdrp; 199 uint8_t npads = 0; 200 uint64_t dma_ioaddr; 201 uint32_t dma_flags; 202 int last_bidx; 203 uint8_t *b_rptr; 204 caddr_t kaddr; 205 uint32_t nmblks; 206 uint32_t ngathers; 207 uint32_t clen; 208 int len; 209 uint32_t pkt_len, pack_len, min_len; 210 uint32_t bcopy_thresh; 211 int i, cur_index, sop_index; 212 uint16_t tail_index; 213 boolean_t tail_wrap = B_FALSE; 214 nxge_dma_common_t desc_area; 215 nxge_os_dma_handle_t dma_handle; 216 ddi_dma_cookie_t dma_cookie; 217 npi_handle_t npi_handle; 218 p_mblk_t nmp; 219 p_mblk_t t_mp; 220 uint32_t ncookies; 221 boolean_t good_packet; 222 boolean_t mark_mode = B_FALSE; 223 p_nxge_stats_t statsp; 224 p_nxge_tx_ring_stats_t tdc_stats; 225 t_uscalar_t start_offset = 0; 226 t_uscalar_t stuff_offset = 0; 227 t_uscalar_t end_offset = 0; 228 t_uscalar_t value = 0; 229 t_uscalar_t cksum_flags = 0; 230 boolean_t cksum_on = B_FALSE; 231 uint32_t boff = 0; 232 uint64_t tot_xfer_len = 0; 233 boolean_t header_set = B_FALSE; 234 #ifdef NXGE_DEBUG 235 p_tx_desc_t tx_desc_ring_pp; 236 p_tx_desc_t tx_desc_pp; 237 tx_desc_t *save_desc_p; 238 int dump_len; 239 int sad_len; 240 uint64_t sad; 241 int xfer_len; 242 uint32_t msgsize; 243 #endif 244 p_mblk_t mp_chain = NULL; 245 boolean_t is_lso = B_FALSE; 246 boolean_t lso_again; 247 int cur_index_lso; 248 p_mblk_t nmp_lso_save; 249 uint32_t lso_ngathers; 250 boolean_t lso_tail_wrap = B_FALSE; 251 252 NXGE_DEBUG_MSG((nxgep, TX_CTL, 253 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 254 NXGE_DEBUG_MSG((nxgep, TX_CTL, 255 "==> nxge_start: Starting tdc %d desc pending %d", 256 tx_ring_p->tdc, tx_ring_p->descs_pending)); 257 258 statsp = nxgep->statsp; 259 260 if (!isLDOMguest(nxgep)) { 261 switch (nxgep->mac.portmode) { 262 default: 263 if (nxgep->statsp->port_stats.lb_mode == 264 nxge_lb_normal) { 265 if (!statsp->mac_stats.link_up) { 266 freemsg(mp); 267 NXGE_DEBUG_MSG((nxgep, TX_CTL, 268 "==> nxge_start: " 269 "link not up")); 270 goto nxge_start_fail1; 271 } 272 } 273 break; 274 case PORT_10G_FIBER: 275 /* 276 * For the following modes, check the link status 277 * before sending the packet out: 278 * nxge_lb_normal, 279 * nxge_lb_ext10g, 280 * nxge_lb_ext1000, 281 * nxge_lb_ext100, 282 * nxge_lb_ext10. 283 */ 284 if (nxgep->statsp->port_stats.lb_mode < 285 nxge_lb_phy10g) { 286 if (!statsp->mac_stats.link_up) { 287 freemsg(mp); 288 NXGE_DEBUG_MSG((nxgep, TX_CTL, 289 "==> nxge_start: " 290 "link not up")); 291 goto nxge_start_fail1; 292 } 293 } 294 break; 295 } 296 } 297 298 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 299 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 300 NXGE_DEBUG_MSG((nxgep, TX_CTL, 301 "==> nxge_start: hardware not initialized or stopped")); 302 freemsg(mp); 303 goto nxge_start_fail1; 304 } 305 306 if (nxgep->soft_lso_enable) { 307 mp_chain = nxge_lso_eliminate(mp); 308 NXGE_DEBUG_MSG((nxgep, TX_CTL, 309 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 310 mp, mp_chain)); 311 if (mp_chain == NULL) { 312 NXGE_ERROR_MSG((nxgep, TX_CTL, 313 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 314 mp_chain, mp)); 315 goto nxge_start_fail1; 316 } 317 if (mp_chain != mp) { 318 NXGE_DEBUG_MSG((nxgep, TX_CTL, 319 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 320 mp_chain, mp)); 321 is_lso = B_TRUE; 322 mp = mp_chain; 323 mp_chain = mp_chain->b_next; 324 mp->b_next = NULL; 325 } 326 } 327 328 hcksum_retrieve(mp, NULL, NULL, &start_offset, 329 &stuff_offset, &end_offset, &value, &cksum_flags); 330 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 331 start_offset += sizeof (ether_header_t); 332 stuff_offset += sizeof (ether_header_t); 333 } else { 334 start_offset += sizeof (struct ether_vlan_header); 335 stuff_offset += sizeof (struct ether_vlan_header); 336 } 337 338 if (cksum_flags & HCK_PARTIALCKSUM) { 339 NXGE_DEBUG_MSG((nxgep, TX_CTL, 340 "==> nxge_start: mp $%p len %d " 341 "cksum_flags 0x%x (partial checksum) ", 342 mp, MBLKL(mp), cksum_flags)); 343 cksum_on = B_TRUE; 344 } 345 346 pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 347 pkthdrp->reserved = 0; 348 tmp_hdrp.pkthdr.value = 0; 349 nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 350 0, 0, pkthdrp, 351 start_offset, stuff_offset); 352 353 lso_again = B_FALSE; 354 lso_ngathers = 0; 355 356 MUTEX_ENTER(&tx_ring_p->lock); 357 358 if (isLDOMservice(nxgep)) { 359 tx_ring_p->tx_ring_busy = B_TRUE; 360 if (tx_ring_p->tx_ring_offline) { 361 freemsg(mp); 362 tx_ring_p->tx_ring_busy = B_FALSE; 363 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 364 NXGE_TX_RING_OFFLINED); 365 MUTEX_EXIT(&tx_ring_p->lock); 366 return (status); 367 } 368 } 369 370 cur_index_lso = tx_ring_p->wr_index; 371 lso_tail_wrap = tx_ring_p->wr_index_wrap; 372 start_again: 373 ngathers = 0; 374 sop_index = tx_ring_p->wr_index; 375 #ifdef NXGE_DEBUG 376 if (tx_ring_p->descs_pending) { 377 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 378 "desc pending %d ", tx_ring_p->descs_pending)); 379 } 380 381 dump_len = (int)(MBLKL(mp)); 382 dump_len = (dump_len > 128) ? 128: dump_len; 383 384 NXGE_DEBUG_MSG((nxgep, TX_CTL, 385 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 386 "(Before header reserve: ORIGINAL LEN %d)", 387 tx_ring_p->tdc, 388 mp->b_rptr, 389 dump_len)); 390 391 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 392 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 393 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 394 #endif 395 396 tdc_stats = tx_ring_p->tdc_stats; 397 mark_mode = (tx_ring_p->descs_pending && 398 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 399 < nxge_tx_minfree)); 400 401 NXGE_DEBUG_MSG((nxgep, TX_CTL, 402 "TX Descriptor ring is channel %d mark mode %d", 403 tx_ring_p->tdc, mark_mode)); 404 405 if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) { 406 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, 407 (nxge_tx_minfree + lso_ngathers))) { 408 NXGE_DEBUG_MSG((nxgep, TX_CTL, 409 "TX Descriptor ring is full: channel %d", 410 tx_ring_p->tdc)); 411 NXGE_DEBUG_MSG((nxgep, TX_CTL, 412 "TX Descriptor ring is full: channel %d", 413 tx_ring_p->tdc)); 414 if (is_lso) { 415 /* 416 * free the current mp and mp_chain if not FULL. 417 */ 418 tdc_stats->tx_no_desc++; 419 NXGE_DEBUG_MSG((nxgep, TX_CTL, 420 "LSO packet: TX Descriptor ring is full: " 421 "channel %d", 422 tx_ring_p->tdc)); 423 goto nxge_start_fail_lso; 424 } else { 425 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 426 tdc_stats->tx_no_desc++; 427 428 if (isLDOMservice(nxgep)) { 429 tx_ring_p->tx_ring_busy = B_FALSE; 430 if (tx_ring_p->tx_ring_offline) { 431 (void) atomic_swap_32( 432 &tx_ring_p->tx_ring_offline, 433 NXGE_TX_RING_OFFLINED); 434 } 435 } 436 437 MUTEX_EXIT(&tx_ring_p->lock); 438 status = 1; 439 goto nxge_start_fail1; 440 } 441 } 442 } 443 444 nmp = mp; 445 i = sop_index = tx_ring_p->wr_index; 446 nmblks = 0; 447 ngathers = 0; 448 pkt_len = 0; 449 pack_len = 0; 450 clen = 0; 451 last_bidx = -1; 452 good_packet = B_TRUE; 453 454 desc_area = tx_ring_p->tdc_desc; 455 npi_handle = desc_area.npi_handle; 456 npi_desc_handle.regh = (nxge_os_acc_handle_t) 457 DMA_COMMON_ACC_HANDLE(desc_area); 458 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 459 tx_desc_dma_handle = (nxge_os_dma_handle_t) 460 DMA_COMMON_HANDLE(desc_area); 461 tx_msg_ring = tx_ring_p->tx_msg_ring; 462 463 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 464 sop_index, i)); 465 466 #ifdef NXGE_DEBUG 467 msgsize = msgdsize(nmp); 468 NXGE_DEBUG_MSG((nxgep, TX_CTL, 469 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 470 sop_index, i, msgsize)); 471 #endif 472 /* 473 * The first 16 bytes of the premapped buffer are reserved 474 * for header. No padding will be used. 475 */ 476 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 477 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 478 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 479 } else { 480 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 481 } 482 while (nmp) { 483 good_packet = B_TRUE; 484 b_rptr = nmp->b_rptr; 485 len = MBLKL(nmp); 486 if (len <= 0) { 487 nmp = nmp->b_cont; 488 continue; 489 } 490 nmblks++; 491 492 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 493 "len %d pkt_len %d pack_len %d", 494 nmblks, len, pkt_len, pack_len)); 495 /* 496 * Hardware limits the transfer length to 4K for NIU and 497 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 498 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 499 * If len is longer than the limit, then we break nmp into 500 * two chunks: Make the first chunk equal to the limit and 501 * the second chunk for the remaining data. If the second 502 * chunk is still larger than the limit, then it will be 503 * broken into two in the next pass. 504 */ 505 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 506 if ((t_mp = dupb(nmp)) != NULL) { 507 nmp->b_wptr = nmp->b_rptr + 508 (TX_MAX_TRANSFER_LENGTH 509 - TX_PKT_HEADER_SIZE); 510 t_mp->b_rptr = nmp->b_wptr; 511 t_mp->b_cont = nmp->b_cont; 512 nmp->b_cont = t_mp; 513 len = MBLKL(nmp); 514 } else { 515 if (is_lso) { 516 NXGE_DEBUG_MSG((nxgep, TX_CTL, 517 "LSO packet: dupb failed: " 518 "channel %d", 519 tx_ring_p->tdc)); 520 mp = nmp; 521 goto nxge_start_fail_lso; 522 } else { 523 good_packet = B_FALSE; 524 goto nxge_start_fail2; 525 } 526 } 527 } 528 tx_desc.value = 0; 529 tx_desc_p = &tx_desc_ring_vp[i]; 530 #ifdef NXGE_DEBUG 531 tx_desc_pp = &tx_desc_ring_pp[i]; 532 #endif 533 tx_msg_p = &tx_msg_ring[i]; 534 #if defined(__i386) 535 npi_desc_handle.regp = (uint32_t)tx_desc_p; 536 #else 537 npi_desc_handle.regp = (uint64_t)tx_desc_p; 538 #endif 539 if (!header_set && 540 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 541 (len >= bcopy_thresh))) { 542 header_set = B_TRUE; 543 bcopy_thresh += TX_PKT_HEADER_SIZE; 544 boff = 0; 545 pack_len = 0; 546 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 547 hdrp = (p_tx_pkt_header_t)kaddr; 548 clen = pkt_len; 549 dma_handle = tx_msg_p->buf_dma_handle; 550 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 551 (void) ddi_dma_sync(dma_handle, 552 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 553 DDI_DMA_SYNC_FORDEV); 554 555 tx_msg_p->flags.dma_type = USE_BCOPY; 556 goto nxge_start_control_header_only; 557 } 558 559 pkt_len += len; 560 pack_len += len; 561 562 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 563 "desc entry %d " 564 "DESC IOADDR $%p " 565 "desc_vp $%p tx_desc_p $%p " 566 "desc_pp $%p tx_desc_pp $%p " 567 "len %d pkt_len %d pack_len %d", 568 i, 569 DMA_COMMON_IOADDR(desc_area), 570 tx_desc_ring_vp, tx_desc_p, 571 tx_desc_ring_pp, tx_desc_pp, 572 len, pkt_len, pack_len)); 573 574 if (len < bcopy_thresh) { 575 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 576 "USE BCOPY: ")); 577 if (nxge_tx_tiny_pack) { 578 uint32_t blst = 579 TXDMA_DESC_NEXT_INDEX(i, -1, 580 tx_ring_p->tx_wrap_mask); 581 NXGE_DEBUG_MSG((nxgep, TX_CTL, 582 "==> nxge_start(5): pack")); 583 if ((pack_len <= bcopy_thresh) && 584 (last_bidx == blst)) { 585 NXGE_DEBUG_MSG((nxgep, TX_CTL, 586 "==> nxge_start: pack(6) " 587 "(pkt_len %d pack_len %d)", 588 pkt_len, pack_len)); 589 i = blst; 590 tx_desc_p = &tx_desc_ring_vp[i]; 591 #ifdef NXGE_DEBUG 592 tx_desc_pp = &tx_desc_ring_pp[i]; 593 #endif 594 tx_msg_p = &tx_msg_ring[i]; 595 boff = pack_len - len; 596 ngathers--; 597 } else if (pack_len > bcopy_thresh && 598 header_set) { 599 pack_len = len; 600 boff = 0; 601 bcopy_thresh = nxge_bcopy_thresh; 602 NXGE_DEBUG_MSG((nxgep, TX_CTL, 603 "==> nxge_start(7): > max NEW " 604 "bcopy thresh %d " 605 "pkt_len %d pack_len %d(next)", 606 bcopy_thresh, 607 pkt_len, pack_len)); 608 } 609 last_bidx = i; 610 } 611 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 612 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 613 hdrp = (p_tx_pkt_header_t)kaddr; 614 header_set = B_TRUE; 615 NXGE_DEBUG_MSG((nxgep, TX_CTL, 616 "==> nxge_start(7_x2): " 617 "pkt_len %d pack_len %d (new hdrp $%p)", 618 pkt_len, pack_len, hdrp)); 619 } 620 tx_msg_p->flags.dma_type = USE_BCOPY; 621 kaddr += boff; 622 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 623 "USE BCOPY: before bcopy " 624 "DESC IOADDR $%p entry %d " 625 "bcopy packets %d " 626 "bcopy kaddr $%p " 627 "bcopy ioaddr (SAD) $%p " 628 "bcopy clen %d " 629 "bcopy boff %d", 630 DMA_COMMON_IOADDR(desc_area), i, 631 tdc_stats->tx_hdr_pkts, 632 kaddr, 633 dma_ioaddr, 634 clen, 635 boff)); 636 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 637 "1USE BCOPY: ")); 638 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 639 "2USE BCOPY: ")); 640 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 641 "last USE BCOPY: copy from b_rptr $%p " 642 "to KADDR $%p (len %d offset %d", 643 b_rptr, kaddr, len, boff)); 644 645 bcopy(b_rptr, kaddr, len); 646 647 #ifdef NXGE_DEBUG 648 dump_len = (len > 128) ? 128: len; 649 NXGE_DEBUG_MSG((nxgep, TX_CTL, 650 "==> nxge_start: dump packets " 651 "(After BCOPY len %d)" 652 "(b_rptr $%p): %s", len, nmp->b_rptr, 653 nxge_dump_packet((char *)nmp->b_rptr, 654 dump_len))); 655 #endif 656 657 dma_handle = tx_msg_p->buf_dma_handle; 658 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 659 (void) ddi_dma_sync(dma_handle, 660 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 661 DDI_DMA_SYNC_FORDEV); 662 clen = len + boff; 663 tdc_stats->tx_hdr_pkts++; 664 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 665 "USE BCOPY: " 666 "DESC IOADDR $%p entry %d " 667 "bcopy packets %d " 668 "bcopy kaddr $%p " 669 "bcopy ioaddr (SAD) $%p " 670 "bcopy clen %d " 671 "bcopy boff %d", 672 DMA_COMMON_IOADDR(desc_area), 673 i, 674 tdc_stats->tx_hdr_pkts, 675 kaddr, 676 dma_ioaddr, 677 clen, 678 boff)); 679 } else { 680 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 681 "USE DVMA: len %d", len)); 682 tx_msg_p->flags.dma_type = USE_DMA; 683 dma_flags = DDI_DMA_WRITE; 684 if (len < nxge_dma_stream_thresh) { 685 dma_flags |= DDI_DMA_CONSISTENT; 686 } else { 687 dma_flags |= DDI_DMA_STREAMING; 688 } 689 690 dma_handle = tx_msg_p->dma_handle; 691 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 692 (caddr_t)b_rptr, len, dma_flags, 693 DDI_DMA_DONTWAIT, NULL, 694 &dma_cookie, &ncookies); 695 if (status == DDI_DMA_MAPPED) { 696 dma_ioaddr = dma_cookie.dmac_laddress; 697 len = (int)dma_cookie.dmac_size; 698 clen = (uint32_t)dma_cookie.dmac_size; 699 NXGE_DEBUG_MSG((nxgep, TX_CTL, 700 "==> nxge_start(12_1): " 701 "USE DVMA: len %d clen %d " 702 "ngathers %d", 703 len, clen, 704 ngathers)); 705 #if defined(__i386) 706 npi_desc_handle.regp = (uint32_t)tx_desc_p; 707 #else 708 npi_desc_handle.regp = (uint64_t)tx_desc_p; 709 #endif 710 while (ncookies > 1) { 711 ngathers++; 712 /* 713 * this is the fix for multiple 714 * cookies, which are basically 715 * a descriptor entry, we don't set 716 * SOP bit as well as related fields 717 */ 718 719 (void) npi_txdma_desc_gather_set( 720 npi_desc_handle, 721 &tx_desc, 722 (ngathers -1), 723 mark_mode, 724 ngathers, 725 dma_ioaddr, 726 clen); 727 728 tx_msg_p->tx_msg_size = clen; 729 NXGE_DEBUG_MSG((nxgep, TX_CTL, 730 "==> nxge_start: DMA " 731 "ncookie %d " 732 "ngathers %d " 733 "dma_ioaddr $%p len %d" 734 "desc $%p descp $%p (%d)", 735 ncookies, 736 ngathers, 737 dma_ioaddr, clen, 738 *tx_desc_p, tx_desc_p, i)); 739 740 ddi_dma_nextcookie(dma_handle, 741 &dma_cookie); 742 dma_ioaddr = 743 dma_cookie.dmac_laddress; 744 745 len = (int)dma_cookie.dmac_size; 746 clen = (uint32_t)dma_cookie.dmac_size; 747 NXGE_DEBUG_MSG((nxgep, TX_CTL, 748 "==> nxge_start(12_2): " 749 "USE DVMA: len %d clen %d ", 750 len, clen)); 751 752 i = TXDMA_DESC_NEXT_INDEX(i, 1, 753 tx_ring_p->tx_wrap_mask); 754 tx_desc_p = &tx_desc_ring_vp[i]; 755 756 #if defined(__i386) 757 npi_desc_handle.regp = 758 (uint32_t)tx_desc_p; 759 #else 760 npi_desc_handle.regp = 761 (uint64_t)tx_desc_p; 762 #endif 763 tx_msg_p = &tx_msg_ring[i]; 764 tx_msg_p->flags.dma_type = USE_NONE; 765 tx_desc.value = 0; 766 767 ncookies--; 768 } 769 tdc_stats->tx_ddi_pkts++; 770 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 771 "DMA: ddi packets %d", 772 tdc_stats->tx_ddi_pkts)); 773 } else { 774 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 775 "dma mapping failed for %d " 776 "bytes addr $%p flags %x (%d)", 777 len, b_rptr, status, status)); 778 good_packet = B_FALSE; 779 tdc_stats->tx_dma_bind_fail++; 780 tx_msg_p->flags.dma_type = USE_NONE; 781 if (is_lso) { 782 mp = nmp; 783 goto nxge_start_fail_lso; 784 } else { 785 goto nxge_start_fail2; 786 } 787 } 788 } /* ddi dvma */ 789 790 if (is_lso) { 791 nmp_lso_save = nmp; 792 } 793 nmp = nmp->b_cont; 794 nxge_start_control_header_only: 795 #if defined(__i386) 796 npi_desc_handle.regp = (uint32_t)tx_desc_p; 797 #else 798 npi_desc_handle.regp = (uint64_t)tx_desc_p; 799 #endif 800 ngathers++; 801 802 if (ngathers == 1) { 803 #ifdef NXGE_DEBUG 804 save_desc_p = &sop_tx_desc; 805 #endif 806 sop_tx_desc_p = &sop_tx_desc; 807 sop_tx_desc_p->value = 0; 808 sop_tx_desc_p->bits.hdw.tr_len = clen; 809 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 810 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 811 } else { 812 #ifdef NXGE_DEBUG 813 save_desc_p = &tx_desc; 814 #endif 815 tmp_desc_p = &tx_desc; 816 tmp_desc_p->value = 0; 817 tmp_desc_p->bits.hdw.tr_len = clen; 818 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 819 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 820 821 tx_desc_p->value = tmp_desc_p->value; 822 } 823 824 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 825 "Desc_entry %d ngathers %d " 826 "desc_vp $%p tx_desc_p $%p " 827 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 828 "dma_ioaddr (SAD) $%p mark %d", 829 i, ngathers, 830 tx_desc_ring_vp, tx_desc_p, 831 len, clen, pkt_len, pack_len, nmblks, 832 dma_ioaddr, mark_mode)); 833 834 #ifdef NXGE_DEBUG 835 npi_desc_handle.nxgep = nxgep; 836 npi_desc_handle.function.function = nxgep->function_num; 837 npi_desc_handle.function.instance = nxgep->instance; 838 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 839 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 840 TX_PKT_DESC_TR_LEN_SHIFT); 841 842 843 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 844 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 845 "mark %d sop %d\n", 846 save_desc_p->value, 847 sad, 848 save_desc_p->bits.hdw.tr_len, 849 xfer_len, 850 save_desc_p->bits.hdw.num_ptr, 851 save_desc_p->bits.hdw.mark, 852 save_desc_p->bits.hdw.sop)); 853 854 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 855 #endif 856 857 tx_msg_p->tx_msg_size = clen; 858 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 859 if (ngathers > nxge_tx_max_gathers) { 860 good_packet = B_FALSE; 861 hcksum_retrieve(mp, NULL, NULL, &start_offset, 862 &stuff_offset, &end_offset, &value, 863 &cksum_flags); 864 865 NXGE_DEBUG_MSG((NULL, TX_CTL, 866 "==> nxge_start(14): pull msg - " 867 "len %d pkt_len %d ngathers %d", 868 len, pkt_len, ngathers)); 869 /* Pull all message blocks from b_cont */ 870 if (is_lso) { 871 mp = nmp_lso_save; 872 goto nxge_start_fail_lso; 873 } 874 if ((msgpullup(mp, -1)) == NULL) { 875 goto nxge_start_fail2; 876 } 877 goto nxge_start_fail2; 878 } 879 } /* while (nmp) */ 880 881 tx_msg_p->tx_message = mp; 882 tx_desc_p = &tx_desc_ring_vp[sop_index]; 883 #if defined(__i386) 884 npi_desc_handle.regp = (uint32_t)tx_desc_p; 885 #else 886 npi_desc_handle.regp = (uint64_t)tx_desc_p; 887 #endif 888 889 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 890 pkthdrp->reserved = 0; 891 hdrp->value = 0; 892 bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 893 894 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 895 tdc_stats->tx_jumbo_pkts++; 896 } 897 898 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 899 if (pkt_len < min_len) { 900 /* Assume we use bcopy to premapped buffers */ 901 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 902 NXGE_DEBUG_MSG((NULL, TX_CTL, 903 "==> nxge_start(14-1): < (msg_min + 16)" 904 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 905 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 906 bzero((kaddr + pkt_len), (min_len - pkt_len)); 907 pkt_len = tx_msg_p->tx_msg_size = min_len; 908 909 sop_tx_desc_p->bits.hdw.tr_len = min_len; 910 911 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 912 tx_desc_p->value = sop_tx_desc_p->value; 913 914 NXGE_DEBUG_MSG((NULL, TX_CTL, 915 "==> nxge_start(14-2): < msg_min - " 916 "len %d pkt_len %d min_len %d ngathers %d", 917 len, pkt_len, min_len, ngathers)); 918 } 919 920 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 921 cksum_flags)); 922 { 923 uint64_t tmp_len; 924 925 /* pkt_len already includes 16 + paddings!! */ 926 /* Update the control header length */ 927 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 928 tmp_len = hdrp->value | 929 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 930 931 NXGE_DEBUG_MSG((nxgep, TX_CTL, 932 "==> nxge_start(15_x1): setting SOP " 933 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 934 "0x%llx hdrp->value 0x%llx", 935 tot_xfer_len, tot_xfer_len, pkt_len, 936 tmp_len, hdrp->value)); 937 #if defined(_BIG_ENDIAN) 938 hdrp->value = ddi_swap64(tmp_len); 939 #else 940 hdrp->value = tmp_len; 941 #endif 942 NXGE_DEBUG_MSG((nxgep, 943 TX_CTL, "==> nxge_start(15_x2): setting SOP " 944 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 945 "tmp_len 0x%llx hdrp->value 0x%llx", 946 tot_xfer_len, pkt_len, 947 tmp_len, hdrp->value)); 948 } 949 950 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 951 "wr_index %d " 952 "tot_xfer_len (%d) pkt_len %d npads %d", 953 sop_index, 954 tot_xfer_len, pkt_len, 955 npads)); 956 957 sop_tx_desc_p->bits.hdw.sop = 1; 958 sop_tx_desc_p->bits.hdw.mark = mark_mode; 959 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 960 961 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 962 963 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 964 965 #ifdef NXGE_DEBUG 966 npi_desc_handle.nxgep = nxgep; 967 npi_desc_handle.function.function = nxgep->function_num; 968 npi_desc_handle.function.instance = nxgep->instance; 969 970 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 971 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 972 save_desc_p->value, 973 sad, 974 save_desc_p->bits.hdw.tr_len, 975 xfer_len, 976 save_desc_p->bits.hdw.num_ptr, 977 save_desc_p->bits.hdw.mark, 978 save_desc_p->bits.hdw.sop)); 979 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 980 981 dump_len = (pkt_len > 128) ? 128: pkt_len; 982 NXGE_DEBUG_MSG((nxgep, TX_CTL, 983 "==> nxge_start: dump packets(17) (after sop set, len " 984 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 985 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 986 (char *)hdrp, 987 nxge_dump_packet((char *)hdrp, dump_len))); 988 NXGE_DEBUG_MSG((nxgep, TX_CTL, 989 "==> nxge_start(18): TX desc sync: sop_index %d", 990 sop_index)); 991 #endif 992 993 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 994 (void) ddi_dma_sync(tx_desc_dma_handle, 995 sop_index * sizeof (tx_desc_t), 996 ngathers * sizeof (tx_desc_t), 997 DDI_DMA_SYNC_FORDEV); 998 999 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 1000 "cs_off = 0x%02X cs_s_off = 0x%02X " 1001 "pkt_len %d ngathers %d sop_index %d\n", 1002 stuff_offset, start_offset, 1003 pkt_len, ngathers, sop_index)); 1004 } else { /* more than one descriptor and wrap around */ 1005 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 1006 (void) ddi_dma_sync(tx_desc_dma_handle, 1007 sop_index * sizeof (tx_desc_t), 1008 nsdescs * sizeof (tx_desc_t), 1009 DDI_DMA_SYNC_FORDEV); 1010 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 1011 "cs_off = 0x%02X cs_s_off = 0x%02X " 1012 "pkt_len %d ngathers %d sop_index %d\n", 1013 stuff_offset, start_offset, 1014 pkt_len, ngathers, sop_index)); 1015 1016 (void) ddi_dma_sync(tx_desc_dma_handle, 1017 0, 1018 (ngathers - nsdescs) * sizeof (tx_desc_t), 1019 DDI_DMA_SYNC_FORDEV); 1020 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 1021 "cs_off = 0x%02X cs_s_off = 0x%02X " 1022 "pkt_len %d ngathers %d sop_index %d\n", 1023 stuff_offset, start_offset, 1024 pkt_len, ngathers, sop_index)); 1025 } 1026 1027 tail_index = tx_ring_p->wr_index; 1028 tail_wrap = tx_ring_p->wr_index_wrap; 1029 1030 tx_ring_p->wr_index = i; 1031 if (tx_ring_p->wr_index <= tail_index) { 1032 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 1033 B_FALSE : B_TRUE); 1034 } 1035 1036 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 1037 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 1038 tx_ring_p->tdc, 1039 tx_ring_p->wr_index, 1040 tx_ring_p->wr_index_wrap, 1041 ngathers, 1042 tx_ring_p->descs_pending)); 1043 1044 if (is_lso) { 1045 lso_ngathers += ngathers; 1046 if (mp_chain != NULL) { 1047 mp = mp_chain; 1048 mp_chain = mp_chain->b_next; 1049 mp->b_next = NULL; 1050 if (nxge_lso_kick_cnt == lso_ngathers) { 1051 tx_ring_p->descs_pending += lso_ngathers; 1052 { 1053 tx_ring_kick_t kick; 1054 1055 kick.value = 0; 1056 kick.bits.ldw.wrap = 1057 tx_ring_p->wr_index_wrap; 1058 kick.bits.ldw.tail = 1059 (uint16_t)tx_ring_p->wr_index; 1060 1061 /* Kick the Transmit kick register */ 1062 TXDMA_REG_WRITE64( 1063 NXGE_DEV_NPI_HANDLE(nxgep), 1064 TX_RING_KICK_REG, 1065 (uint8_t)tx_ring_p->tdc, 1066 kick.value); 1067 tdc_stats->tx_starts++; 1068 1069 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1070 "==> nxge_start: more LSO: " 1071 "LSO_CNT %d", 1072 lso_ngathers)); 1073 } 1074 lso_ngathers = 0; 1075 ngathers = 0; 1076 cur_index_lso = sop_index = tx_ring_p->wr_index; 1077 lso_tail_wrap = tx_ring_p->wr_index_wrap; 1078 } 1079 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1080 "==> nxge_start: lso again: " 1081 "lso_gathers %d ngathers %d cur_index_lso %d " 1082 "wr_index %d sop_index %d", 1083 lso_ngathers, ngathers, cur_index_lso, 1084 tx_ring_p->wr_index, sop_index)); 1085 1086 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1087 "==> nxge_start: next : count %d", 1088 lso_ngathers)); 1089 lso_again = B_TRUE; 1090 goto start_again; 1091 } 1092 ngathers = lso_ngathers; 1093 } 1094 1095 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 1096 1097 { 1098 tx_ring_kick_t kick; 1099 1100 kick.value = 0; 1101 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 1102 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 1103 1104 /* Kick start the Transmit kick register */ 1105 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 1106 TX_RING_KICK_REG, 1107 (uint8_t)tx_ring_p->tdc, 1108 kick.value); 1109 } 1110 1111 tx_ring_p->descs_pending += ngathers; 1112 tdc_stats->tx_starts++; 1113 1114 if (isLDOMservice(nxgep)) { 1115 tx_ring_p->tx_ring_busy = B_FALSE; 1116 if (tx_ring_p->tx_ring_offline) { 1117 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1118 NXGE_TX_RING_OFFLINED); 1119 } 1120 } 1121 1122 MUTEX_EXIT(&tx_ring_p->lock); 1123 1124 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1125 return (status); 1126 1127 nxge_start_fail_lso: 1128 status = 0; 1129 good_packet = B_FALSE; 1130 if (mp != NULL) { 1131 freemsg(mp); 1132 } 1133 if (mp_chain != NULL) { 1134 freemsg(mp_chain); 1135 } 1136 if (!lso_again && !ngathers) { 1137 if (isLDOMservice(nxgep)) { 1138 tx_ring_p->tx_ring_busy = B_FALSE; 1139 if (tx_ring_p->tx_ring_offline) { 1140 (void) atomic_swap_32( 1141 &tx_ring_p->tx_ring_offline, 1142 NXGE_TX_RING_OFFLINED); 1143 } 1144 } 1145 1146 MUTEX_EXIT(&tx_ring_p->lock); 1147 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1148 "==> nxge_start: lso exit (nothing changed)")); 1149 goto nxge_start_fail1; 1150 } 1151 1152 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1153 "==> nxge_start (channel %d): before lso " 1154 "lso_gathers %d ngathers %d cur_index_lso %d " 1155 "wr_index %d sop_index %d lso_again %d", 1156 tx_ring_p->tdc, 1157 lso_ngathers, ngathers, cur_index_lso, 1158 tx_ring_p->wr_index, sop_index, lso_again)); 1159 1160 if (lso_again) { 1161 lso_ngathers += ngathers; 1162 ngathers = lso_ngathers; 1163 sop_index = cur_index_lso; 1164 tx_ring_p->wr_index = sop_index; 1165 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1166 } 1167 1168 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1169 "==> nxge_start (channel %d): after lso " 1170 "lso_gathers %d ngathers %d cur_index_lso %d " 1171 "wr_index %d sop_index %d lso_again %d", 1172 tx_ring_p->tdc, 1173 lso_ngathers, ngathers, cur_index_lso, 1174 tx_ring_p->wr_index, sop_index, lso_again)); 1175 1176 nxge_start_fail2: 1177 if (good_packet == B_FALSE) { 1178 cur_index = sop_index; 1179 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1180 for (i = 0; i < ngathers; i++) { 1181 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1182 #if defined(__i386) 1183 npi_handle.regp = (uint32_t)tx_desc_p; 1184 #else 1185 npi_handle.regp = (uint64_t)tx_desc_p; 1186 #endif 1187 tx_msg_p = &tx_msg_ring[cur_index]; 1188 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1189 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1190 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1191 "tx_desc_p = %X index = %d", 1192 tx_desc_p, tx_ring_p->rd_index)); 1193 (void) dvma_unload(tx_msg_p->dvma_handle, 1194 0, -1); 1195 tx_msg_p->dvma_handle = NULL; 1196 if (tx_ring_p->dvma_wr_index == 1197 tx_ring_p->dvma_wrap_mask) 1198 tx_ring_p->dvma_wr_index = 0; 1199 else 1200 tx_ring_p->dvma_wr_index++; 1201 tx_ring_p->dvma_pending--; 1202 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1203 if (ddi_dma_unbind_handle( 1204 tx_msg_p->dma_handle)) { 1205 cmn_err(CE_WARN, "!nxge_start: " 1206 "ddi_dma_unbind_handle failed"); 1207 } 1208 } 1209 tx_msg_p->flags.dma_type = USE_NONE; 1210 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1211 tx_ring_p->tx_wrap_mask); 1212 1213 } 1214 } 1215 1216 if (isLDOMservice(nxgep)) { 1217 tx_ring_p->tx_ring_busy = B_FALSE; 1218 if (tx_ring_p->tx_ring_offline) { 1219 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1220 NXGE_TX_RING_OFFLINED); 1221 } 1222 } 1223 1224 MUTEX_EXIT(&tx_ring_p->lock); 1225 1226 nxge_start_fail1: 1227 /* Add FMA to check the access handle nxge_hregh */ 1228 1229 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1230 return (status); 1231 } 1232 1233 /* Software LSO starts here */ 1234 static void 1235 nxge_hcksum_retrieve(mblk_t *mp, 1236 uint32_t *start, uint32_t *stuff, uint32_t *end, 1237 uint32_t *value, uint32_t *flags) 1238 { 1239 if (mp->b_datap->db_type == M_DATA) { 1240 if (flags != NULL) { 1241 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1242 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1243 HCK_FULLCKSUM_OK); 1244 if ((*flags & (HCK_PARTIALCKSUM | 1245 HCK_FULLCKSUM)) != 0) { 1246 if (value != NULL) 1247 *value = (uint32_t)DB_CKSUM16(mp); 1248 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1249 if (start != NULL) 1250 *start = 1251 (uint32_t)DB_CKSUMSTART(mp); 1252 if (stuff != NULL) 1253 *stuff = 1254 (uint32_t)DB_CKSUMSTUFF(mp); 1255 if (end != NULL) 1256 *end = 1257 (uint32_t)DB_CKSUMEND(mp); 1258 } 1259 } 1260 } 1261 } 1262 } 1263 1264 static void 1265 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1266 { 1267 ASSERT(DB_TYPE(mp) == M_DATA); 1268 1269 *mss = 0; 1270 if (flags != NULL) { 1271 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1272 if ((*flags != 0) && (mss != NULL)) { 1273 *mss = (uint32_t)DB_LSOMSS(mp); 1274 } 1275 NXGE_DEBUG_MSG((NULL, TX_CTL, 1276 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1277 *mss, *flags)); 1278 } 1279 1280 NXGE_DEBUG_MSG((NULL, TX_CTL, 1281 "<== nxge_lso_info_get: mss %d", *mss)); 1282 } 1283 1284 /* 1285 * Do Soft LSO on the oversized packet. 1286 * 1287 * 1. Create a chain of message for headers. 1288 * 2. Fill up header messages with proper information. 1289 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1290 * each new message with necessary adjustments. 1291 * * Unchange the ethernet header for DIX frames. (by default) 1292 * * IP Total Length field is updated to MSS or less(only for the last one). 1293 * * IP Identification value is incremented by one for each packet. 1294 * * TCP sequence Number is recalculated according to the payload length. 1295 * * Set FIN and/or PSH flags for the *last* packet if applied. 1296 * * TCP partial Checksum 1297 * 4. Update LSO information in the first message header. 1298 * 5. Release the original message header. 1299 */ 1300 static mblk_t * 1301 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1302 { 1303 uint32_t hckflags; 1304 int pktlen; 1305 int hdrlen; 1306 int segnum; 1307 int i; 1308 struct ether_vlan_header *evh; 1309 int ehlen, iphlen, tcphlen; 1310 struct ip *oiph, *niph; 1311 struct tcphdr *otcph, *ntcph; 1312 int available, len, left; 1313 uint16_t ip_id; 1314 uint32_t tcp_seq; 1315 #ifdef __sparc 1316 uint32_t tcp_seq_tmp; 1317 #endif 1318 mblk_t *datamp; 1319 uchar_t *rptr; 1320 mblk_t *nmp; 1321 mblk_t *cmp; 1322 mblk_t *mp_chain; 1323 boolean_t do_cleanup = B_FALSE; 1324 t_uscalar_t start_offset = 0; 1325 t_uscalar_t stuff_offset = 0; 1326 t_uscalar_t value = 0; 1327 uint16_t l4_len; 1328 ipaddr_t src, dst; 1329 uint32_t cksum, sum, l4cksum; 1330 1331 NXGE_DEBUG_MSG((NULL, TX_CTL, 1332 "==> nxge_do_softlso")); 1333 /* 1334 * check the length of LSO packet payload and calculate the number of 1335 * segments to be generated. 1336 */ 1337 pktlen = msgsize(mp); 1338 evh = (struct ether_vlan_header *)mp->b_rptr; 1339 1340 /* VLAN? */ 1341 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1342 ehlen = sizeof (struct ether_vlan_header); 1343 else 1344 ehlen = sizeof (struct ether_header); 1345 oiph = (struct ip *)(mp->b_rptr + ehlen); 1346 iphlen = oiph->ip_hl * 4; 1347 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1348 tcphlen = otcph->th_off * 4; 1349 1350 l4_len = pktlen - ehlen - iphlen; 1351 1352 NXGE_DEBUG_MSG((NULL, TX_CTL, 1353 "==> nxge_do_softlso: mss %d oiph $%p " 1354 "original ip_sum oiph->ip_sum 0x%x " 1355 "original tcp_sum otcph->th_sum 0x%x " 1356 "oiph->ip_len %d pktlen %d ehlen %d " 1357 "l4_len %d (0x%x) ip_len - iphlen %d ", 1358 mss, 1359 oiph, 1360 oiph->ip_sum, 1361 otcph->th_sum, 1362 ntohs(oiph->ip_len), pktlen, 1363 ehlen, 1364 l4_len, 1365 l4_len, 1366 ntohs(oiph->ip_len) - iphlen)); 1367 1368 /* IPv4 + TCP */ 1369 if (!(oiph->ip_v == IPV4_VERSION)) { 1370 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1371 "<== nxge_do_softlso: not IPV4 " 1372 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1373 ntohs(oiph->ip_len), pktlen, ehlen, 1374 tcphlen)); 1375 freemsg(mp); 1376 return (NULL); 1377 } 1378 1379 if (!(oiph->ip_p == IPPROTO_TCP)) { 1380 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1381 "<== nxge_do_softlso: not TCP " 1382 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1383 ntohs(oiph->ip_len), pktlen, ehlen, 1384 tcphlen)); 1385 freemsg(mp); 1386 return (NULL); 1387 } 1388 1389 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1390 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1391 "<== nxge_do_softlso: len not matched " 1392 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1393 ntohs(oiph->ip_len), pktlen, ehlen, 1394 tcphlen)); 1395 freemsg(mp); 1396 return (NULL); 1397 } 1398 1399 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1400 tcphlen = otcph->th_off * 4; 1401 1402 /* TCP flags can not include URG, RST, or SYN */ 1403 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1404 1405 hdrlen = ehlen + iphlen + tcphlen; 1406 1407 VERIFY(MBLKL(mp) >= hdrlen); 1408 1409 if (MBLKL(mp) > hdrlen) { 1410 datamp = mp; 1411 rptr = mp->b_rptr + hdrlen; 1412 } else { /* = */ 1413 datamp = mp->b_cont; 1414 rptr = datamp->b_rptr; 1415 } 1416 1417 NXGE_DEBUG_MSG((NULL, TX_CTL, 1418 "nxge_do_softlso: otcph $%p pktlen: %d, " 1419 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1420 "mblkl(mp): %d, mblkl(datamp): %d", 1421 otcph, 1422 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1423 (int)MBLKL(mp), (int)MBLKL(datamp))); 1424 1425 hckflags = 0; 1426 nxge_hcksum_retrieve(mp, 1427 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1428 1429 dst = oiph->ip_dst.s_addr; 1430 src = oiph->ip_src.s_addr; 1431 1432 cksum = (dst >> 16) + (dst & 0xFFFF) + 1433 (src >> 16) + (src & 0xFFFF); 1434 l4cksum = cksum + IP_TCP_CSUM_COMP; 1435 1436 sum = l4_len + l4cksum; 1437 sum = (sum & 0xFFFF) + (sum >> 16); 1438 1439 NXGE_DEBUG_MSG((NULL, TX_CTL, 1440 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1441 "hckflags 0x%x start_offset %d stuff_offset %d " 1442 "value (original) 0x%x th_sum 0x%x " 1443 "pktlen %d l4_len %d (0x%x) " 1444 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1445 dst, src, 1446 (sum & 0xffff), (~sum & 0xffff), 1447 hckflags, start_offset, stuff_offset, 1448 value, otcph->th_sum, 1449 pktlen, 1450 l4_len, 1451 l4_len, 1452 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1453 (int)MBLKL(datamp), 1454 nxge_dump_packet((char *)evh, 12))); 1455 1456 /* 1457 * Start to process. 1458 */ 1459 available = pktlen - hdrlen; 1460 segnum = (available - 1) / mss + 1; 1461 1462 NXGE_DEBUG_MSG((NULL, TX_CTL, 1463 "==> nxge_do_softlso: pktlen %d " 1464 "MBLKL(mp): %d, MBLKL(datamp): %d " 1465 "available %d mss %d segnum %d", 1466 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1467 available, 1468 mss, 1469 segnum)); 1470 1471 VERIFY(segnum >= 2); 1472 1473 /* 1474 * Try to pre-allocate all header messages 1475 */ 1476 mp_chain = NULL; 1477 for (i = 0; i < segnum; i++) { 1478 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1479 /* Clean up the mp_chain */ 1480 while (mp_chain != NULL) { 1481 nmp = mp_chain; 1482 mp_chain = mp_chain->b_next; 1483 freemsg(nmp); 1484 } 1485 NXGE_DEBUG_MSG((NULL, TX_CTL, 1486 "<== nxge_do_softlso: " 1487 "Could not allocate enough messages for headers!")); 1488 freemsg(mp); 1489 return (NULL); 1490 } 1491 nmp->b_next = mp_chain; 1492 mp_chain = nmp; 1493 1494 NXGE_DEBUG_MSG((NULL, TX_CTL, 1495 "==> nxge_do_softlso: " 1496 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1497 mp, nmp, mp_chain, mp_chain->b_next)); 1498 } 1499 1500 NXGE_DEBUG_MSG((NULL, TX_CTL, 1501 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1502 mp, nmp, mp_chain)); 1503 1504 /* 1505 * Associate payload with new packets 1506 */ 1507 cmp = mp_chain; 1508 left = available; 1509 while (cmp != NULL) { 1510 nmp = dupb(datamp); 1511 if (nmp == NULL) { 1512 do_cleanup = B_TRUE; 1513 NXGE_DEBUG_MSG((NULL, TX_CTL, 1514 "==>nxge_do_softlso: " 1515 "Can not dupb(datamp), have to do clean up")); 1516 goto cleanup_allocated_msgs; 1517 } 1518 1519 NXGE_DEBUG_MSG((NULL, TX_CTL, 1520 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1521 "dupb nmp $%p len %d left %d msd %d ", 1522 mp, cmp, nmp, len, left, mss)); 1523 1524 cmp->b_cont = nmp; 1525 nmp->b_rptr = rptr; 1526 len = (left < mss) ? left : mss; 1527 left -= len; 1528 1529 NXGE_DEBUG_MSG((NULL, TX_CTL, 1530 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1531 "dupb nmp $%p len %d left %d mss %d ", 1532 mp, cmp, nmp, len, left, mss)); 1533 NXGE_DEBUG_MSG((NULL, TX_CTL, 1534 "nxge_do_softlso: before available: %d, " 1535 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1536 available, left, len, segnum, (int)MBLKL(nmp))); 1537 1538 len -= MBLKL(nmp); 1539 NXGE_DEBUG_MSG((NULL, TX_CTL, 1540 "nxge_do_softlso: after available: %d, " 1541 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1542 available, left, len, segnum, (int)MBLKL(nmp))); 1543 1544 while (len > 0) { 1545 mblk_t *mmp = NULL; 1546 1547 NXGE_DEBUG_MSG((NULL, TX_CTL, 1548 "nxge_do_softlso: (4) len > 0 available: %d, " 1549 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1550 available, left, len, segnum, (int)MBLKL(nmp))); 1551 1552 if (datamp->b_cont != NULL) { 1553 datamp = datamp->b_cont; 1554 rptr = datamp->b_rptr; 1555 mmp = dupb(datamp); 1556 if (mmp == NULL) { 1557 do_cleanup = B_TRUE; 1558 NXGE_DEBUG_MSG((NULL, TX_CTL, 1559 "==> nxge_do_softlso: " 1560 "Can not dupb(datamp) (1), :" 1561 "have to do clean up")); 1562 NXGE_DEBUG_MSG((NULL, TX_CTL, 1563 "==> nxge_do_softlso: " 1564 "available: %d, left: %d, " 1565 "len: %d, MBLKL(nmp): %d", 1566 available, left, len, 1567 (int)MBLKL(nmp))); 1568 goto cleanup_allocated_msgs; 1569 } 1570 } else { 1571 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1572 "==> nxge_do_softlso: " 1573 "(1)available: %d, left: %d, " 1574 "len: %d, MBLKL(nmp): %d", 1575 available, left, len, 1576 (int)MBLKL(nmp))); 1577 cmn_err(CE_PANIC, 1578 "==> nxge_do_softlso: " 1579 "Pointers must have been corrupted!\n" 1580 "datamp: $%p, nmp: $%p, rptr: $%p", 1581 (void *)datamp, 1582 (void *)nmp, 1583 (void *)rptr); 1584 } 1585 nmp->b_cont = mmp; 1586 nmp = mmp; 1587 len -= MBLKL(nmp); 1588 } 1589 if (len < 0) { 1590 nmp->b_wptr += len; 1591 rptr = nmp->b_wptr; 1592 NXGE_DEBUG_MSG((NULL, TX_CTL, 1593 "(5) len < 0 (less than 0)" 1594 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1595 available, left, len, (int)MBLKL(nmp))); 1596 1597 } else if (len == 0) { 1598 if (datamp->b_cont != NULL) { 1599 NXGE_DEBUG_MSG((NULL, TX_CTL, 1600 "(5) len == 0" 1601 "available: %d, left: %d, len: %d, " 1602 "MBLKL(nmp): %d", 1603 available, left, len, (int)MBLKL(nmp))); 1604 datamp = datamp->b_cont; 1605 rptr = datamp->b_rptr; 1606 } else { 1607 NXGE_DEBUG_MSG((NULL, TX_CTL, 1608 "(6)available b_cont == NULL : %d, " 1609 "left: %d, len: %d, MBLKL(nmp): %d", 1610 available, left, len, (int)MBLKL(nmp))); 1611 1612 VERIFY(cmp->b_next == NULL); 1613 VERIFY(left == 0); 1614 break; /* Done! */ 1615 } 1616 } 1617 cmp = cmp->b_next; 1618 1619 NXGE_DEBUG_MSG((NULL, TX_CTL, 1620 "(7) do_softlso: " 1621 "next mp in mp_chain available len != 0 : %d, " 1622 "left: %d, len: %d, MBLKL(nmp): %d", 1623 available, left, len, (int)MBLKL(nmp))); 1624 } 1625 1626 /* 1627 * From now, start to fill up all headers for the first message 1628 * Hardware checksum flags need to be updated separately for FULLCKSUM 1629 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1630 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1631 * required fields need to be updated properly. 1632 */ 1633 nmp = mp_chain; 1634 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1635 nmp->b_wptr = nmp->b_rptr + hdrlen; 1636 niph = (struct ip *)(nmp->b_rptr + ehlen); 1637 niph->ip_len = htons(mss + iphlen + tcphlen); 1638 ip_id = ntohs(niph->ip_id); 1639 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1640 #ifdef __sparc 1641 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1642 tcp_seq = ntohl(tcp_seq_tmp); 1643 #else 1644 tcp_seq = ntohl(ntcph->th_seq); 1645 #endif 1646 1647 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1648 1649 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1650 DB_CKSUMSTART(nmp) = start_offset; 1651 DB_CKSUMSTUFF(nmp) = stuff_offset; 1652 1653 /* calculate IP checksum and TCP pseudo header checksum */ 1654 niph->ip_sum = 0; 1655 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1656 1657 l4_len = mss + tcphlen; 1658 sum = htons(l4_len) + l4cksum; 1659 sum = (sum & 0xFFFF) + (sum >> 16); 1660 ntcph->th_sum = (sum & 0xffff); 1661 1662 NXGE_DEBUG_MSG((NULL, TX_CTL, 1663 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1664 "mss %d pktlen %d l4_len %d (0x%x) " 1665 "MBLKL(mp): %d, MBLKL(datamp): %d " 1666 "ip_sum 0x%x " 1667 "th_sum 0x%x sum 0x%x ) " 1668 "dump first ip->tcp %s", 1669 nmp, mp_chain, 1670 mss, 1671 pktlen, 1672 l4_len, 1673 l4_len, 1674 (int)MBLKL(mp), (int)MBLKL(datamp), 1675 niph->ip_sum, 1676 ntcph->th_sum, 1677 sum, 1678 nxge_dump_packet((char *)niph, 52))); 1679 1680 cmp = nmp; 1681 while ((nmp = nmp->b_next)->b_next != NULL) { 1682 NXGE_DEBUG_MSG((NULL, TX_CTL, 1683 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1684 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1685 nmp->b_wptr = nmp->b_rptr + hdrlen; 1686 niph = (struct ip *)(nmp->b_rptr + ehlen); 1687 niph->ip_id = htons(++ip_id); 1688 niph->ip_len = htons(mss + iphlen + tcphlen); 1689 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1690 tcp_seq += mss; 1691 1692 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1693 1694 #ifdef __sparc 1695 tcp_seq_tmp = htonl(tcp_seq); 1696 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1697 #else 1698 ntcph->th_seq = htonl(tcp_seq); 1699 #endif 1700 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1701 DB_CKSUMSTART(nmp) = start_offset; 1702 DB_CKSUMSTUFF(nmp) = stuff_offset; 1703 1704 /* calculate IP checksum and TCP pseudo header checksum */ 1705 niph->ip_sum = 0; 1706 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1707 ntcph->th_sum = (sum & 0xffff); 1708 1709 NXGE_DEBUG_MSG((NULL, TX_CTL, 1710 "==> nxge_do_softlso: middle ip_sum 0x%x " 1711 "th_sum 0x%x " 1712 " mp $%p (mp_chain $%p) pktlen %d " 1713 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1714 niph->ip_sum, 1715 ntcph->th_sum, 1716 nmp, mp_chain, 1717 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1718 } 1719 1720 /* Last segment */ 1721 /* 1722 * Set FIN and/or PSH flags if present only in the last packet. 1723 * The ip_len could be different from prior packets. 1724 */ 1725 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1726 nmp->b_wptr = nmp->b_rptr + hdrlen; 1727 niph = (struct ip *)(nmp->b_rptr + ehlen); 1728 niph->ip_id = htons(++ip_id); 1729 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1730 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1731 tcp_seq += mss; 1732 #ifdef __sparc 1733 tcp_seq_tmp = htonl(tcp_seq); 1734 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1735 #else 1736 ntcph->th_seq = htonl(tcp_seq); 1737 #endif 1738 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1739 1740 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1741 DB_CKSUMSTART(nmp) = start_offset; 1742 DB_CKSUMSTUFF(nmp) = stuff_offset; 1743 1744 /* calculate IP checksum and TCP pseudo header checksum */ 1745 niph->ip_sum = 0; 1746 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1747 1748 l4_len = ntohs(niph->ip_len) - iphlen; 1749 sum = htons(l4_len) + l4cksum; 1750 sum = (sum & 0xFFFF) + (sum >> 16); 1751 ntcph->th_sum = (sum & 0xffff); 1752 1753 NXGE_DEBUG_MSG((NULL, TX_CTL, 1754 "==> nxge_do_softlso: last next " 1755 "niph->ip_sum 0x%x " 1756 "ntcph->th_sum 0x%x sum 0x%x " 1757 "dump last ip->tcp %s " 1758 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1759 "l4_len %d (0x%x) " 1760 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1761 niph->ip_sum, 1762 ntcph->th_sum, sum, 1763 nxge_dump_packet((char *)niph, 52), 1764 cmp, nmp, mp_chain, 1765 pktlen, pktlen, 1766 l4_len, 1767 l4_len, 1768 (int)MBLKL(mp), (int)MBLKL(datamp))); 1769 1770 cleanup_allocated_msgs: 1771 if (do_cleanup) { 1772 NXGE_DEBUG_MSG((NULL, TX_CTL, 1773 "==> nxge_do_softlso: " 1774 "Failed allocating messages, " 1775 "have to clean up and fail!")); 1776 while (mp_chain != NULL) { 1777 nmp = mp_chain; 1778 mp_chain = mp_chain->b_next; 1779 freemsg(nmp); 1780 } 1781 } 1782 /* 1783 * We're done here, so just free the original message and return the 1784 * new message chain, that could be NULL if failed, back to the caller. 1785 */ 1786 freemsg(mp); 1787 1788 NXGE_DEBUG_MSG((NULL, TX_CTL, 1789 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1790 return (mp_chain); 1791 } 1792 1793 /* 1794 * Will be called before NIC driver do further operation on the message. 1795 * The input message may include LSO information, if so, go to softlso logic 1796 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1797 * The return could be the same non-LSO message or a message chain for LSO case. 1798 * 1799 * The driver needs to call this function per packet and process the whole chain 1800 * if applied. 1801 */ 1802 static mblk_t * 1803 nxge_lso_eliminate(mblk_t *mp) 1804 { 1805 uint32_t lsoflags; 1806 uint32_t mss; 1807 1808 NXGE_DEBUG_MSG((NULL, TX_CTL, 1809 "==>nxge_lso_eliminate:")); 1810 nxge_lso_info_get(mp, &mss, &lsoflags); 1811 1812 if (lsoflags & HW_LSO) { 1813 mblk_t *nmp; 1814 1815 NXGE_DEBUG_MSG((NULL, TX_CTL, 1816 "==>nxge_lso_eliminate:" 1817 "HW_LSO:mss %d mp $%p", 1818 mss, mp)); 1819 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1820 NXGE_DEBUG_MSG((NULL, TX_CTL, 1821 "<== nxge_lso_eliminate: " 1822 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1823 nmp, mss, mp)); 1824 return (nmp); 1825 } else { 1826 NXGE_DEBUG_MSG((NULL, TX_CTL, 1827 "<== nxge_lso_eliminate_ " 1828 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1829 nmp, mss, mp)); 1830 return (NULL); 1831 } 1832 } 1833 1834 NXGE_DEBUG_MSG((NULL, TX_CTL, 1835 "<== nxge_lso_eliminate")); 1836 return (mp); 1837 } 1838 1839 static uint32_t 1840 nxge_csgen(uint16_t *adr, int len) 1841 { 1842 int i, odd; 1843 uint32_t sum = 0; 1844 uint32_t c = 0; 1845 1846 odd = len % 2; 1847 for (i = 0; i < (len / 2); i++) { 1848 sum += (adr[i] & 0xffff); 1849 } 1850 if (odd) { 1851 sum += adr[len / 2] & 0xff00; 1852 } 1853 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 1854 sum &= 0xffff; 1855 sum += c; 1856 } 1857 return (~sum & 0xffff); 1858 } 1859