1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/mac_provider.h> 27 #include <sys/nxge/nxge_impl.h> 28 #include <sys/nxge/nxge_hio.h> 29 #include <npi_tx_wr64.h> 30 31 /* Software LSO required header files */ 32 #include <netinet/tcp.h> 33 #include <inet/ip_impl.h> 34 #include <inet/tcp.h> 35 36 extern uint64_t mac_pkt_hash(uint_t, mblk_t *mp, uint8_t policy, 37 boolean_t is_outbound); 38 39 static mblk_t *nxge_lso_eliminate(mblk_t *); 40 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 41 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 42 static void nxge_hcksum_retrieve(mblk_t *, 43 uint32_t *, uint32_t *, uint32_t *, 44 uint32_t *, uint32_t *); 45 static uint32_t nxge_csgen(uint16_t *, int); 46 47 extern uint32_t nxge_reclaim_pending; 48 extern uint32_t nxge_bcopy_thresh; 49 extern uint32_t nxge_dvma_thresh; 50 extern uint32_t nxge_dma_stream_thresh; 51 extern uint32_t nxge_tx_minfree; 52 extern uint32_t nxge_tx_intr_thres; 53 extern uint32_t nxge_tx_max_gathers; 54 extern uint32_t nxge_tx_tiny_pack; 55 extern uint32_t nxge_tx_use_bcopy; 56 extern nxge_tx_mode_t nxge_tx_scheme; 57 uint32_t nxge_lso_kick_cnt = 2; 58 59 60 void 61 nxge_tx_ring_task(void *arg) 62 { 63 p_tx_ring_t ring = (p_tx_ring_t)arg; 64 65 MUTEX_ENTER(&ring->lock); 66 (void) nxge_txdma_reclaim(ring->nxgep, ring, 0); 67 MUTEX_EXIT(&ring->lock); 68 69 if (!isLDOMguest(ring->nxgep) && !ring->tx_ring_offline) 70 mac_tx_ring_update(ring->nxgep->mach, ring->tx_ring_handle); 71 #if defined(sun4v) 72 else { 73 nxge_hio_data_t *nhd = 74 (nxge_hio_data_t *)ring->nxgep->nxge_hw_p->hio; 75 nx_vio_fp_t *vio = &nhd->hio.vio; 76 77 /* Call back vnet. */ 78 if (vio->cb.vio_net_tx_update) { 79 (*vio->cb.vio_net_tx_update)(ring->nxgep->hio_vr->vhp); 80 } 81 } 82 #endif 83 } 84 85 static void 86 nxge_tx_ring_dispatch(p_tx_ring_t ring) 87 { 88 /* 89 * Kick the ring task to reclaim some buffers. 90 */ 91 (void) ddi_taskq_dispatch(ring->taskq, 92 nxge_tx_ring_task, (void *)ring, DDI_SLEEP); 93 } 94 95 mblk_t * 96 nxge_tx_ring_send(void *arg, mblk_t *mp) 97 { 98 p_nxge_ring_handle_t nrhp = (p_nxge_ring_handle_t)arg; 99 p_nxge_t nxgep; 100 p_tx_ring_t tx_ring_p; 101 int status, channel; 102 103 ASSERT(nrhp != NULL); 104 nxgep = nrhp->nxgep; 105 channel = nxgep->pt_config.hw_config.tdc.start + nrhp->index; 106 tx_ring_p = nxgep->tx_rings->rings[channel]; 107 108 /* 109 * We may be in a transition from offlined DMA to onlined 110 * DMA. 111 */ 112 if (tx_ring_p == NULL) { 113 ASSERT(tx_ring_p != NULL); 114 freemsg(mp); 115 return ((mblk_t *)NULL); 116 } 117 118 /* 119 * Valid DMA? 120 */ 121 ASSERT(nxgep == tx_ring_p->nxgep); 122 123 /* 124 * Make sure DMA is not offlined. 125 */ 126 if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline) { 127 ASSERT(!tx_ring_p->tx_ring_offline); 128 freemsg(mp); 129 return ((mblk_t *)NULL); 130 } 131 132 /* 133 * Transmit the packet. 134 */ 135 status = nxge_start(nxgep, tx_ring_p, mp); 136 if (status) { 137 nxge_tx_ring_dispatch(tx_ring_p); 138 return (mp); 139 } 140 141 return ((mblk_t *)NULL); 142 } 143 144 #if defined(sun4v) 145 146 /* 147 * Hashing policy for load balancing over the set of TX rings 148 * available to the driver. 149 */ 150 static uint8_t nxge_tx_hash_policy = MAC_PKT_HASH_L4; 151 152 /* 153 * nxge_m_tx() is needed for Hybrid I/O operation of the vnet in 154 * the guest domain. See CR 6778758 for long term solution. 155 * 156 * The guest domain driver will for now hash the packet 157 * to pick a DMA channel from the only group it has group 0. 158 */ 159 160 mblk_t * 161 nxge_m_tx(void *arg, mblk_t *mp) 162 { 163 p_nxge_t nxgep = (p_nxge_t)arg; 164 mblk_t *next; 165 uint64_t rindex; 166 p_tx_ring_t tx_ring_p; 167 int status; 168 169 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_m_tx")); 170 171 /* 172 * Hash to pick a ring from Group 0, the only TX group 173 * for a guest domain driver. 174 */ 175 rindex = mac_pkt_hash(DL_ETHER, mp, nxge_tx_hash_policy, B_TRUE); 176 rindex = rindex % nxgep->pt_config.tdc_grps[0].max_tdcs; 177 178 /* 179 * Get the ring handle. 180 */ 181 tx_ring_p = nxgep->tx_rings->rings[rindex]; 182 183 while (mp != NULL) { 184 next = mp->b_next; 185 mp->b_next = NULL; 186 187 status = nxge_start(nxgep, tx_ring_p, mp); 188 if (status != 0) { 189 mp->b_next = next; 190 nxge_tx_ring_dispatch(tx_ring_p); 191 return (mp); 192 } 193 194 mp = next; 195 } 196 197 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_m_tx")); 198 return ((mblk_t *)NULL); 199 } 200 201 #endif 202 203 int 204 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 205 { 206 int status = 0; 207 p_tx_desc_t tx_desc_ring_vp; 208 npi_handle_t npi_desc_handle; 209 nxge_os_dma_handle_t tx_desc_dma_handle; 210 p_tx_desc_t tx_desc_p; 211 p_tx_msg_t tx_msg_ring; 212 p_tx_msg_t tx_msg_p; 213 tx_desc_t tx_desc, *tmp_desc_p; 214 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 215 p_tx_pkt_header_t hdrp; 216 tx_pkt_hdr_all_t tmp_hdrp; 217 p_tx_pkt_hdr_all_t pkthdrp; 218 uint8_t npads = 0; 219 uint64_t dma_ioaddr; 220 uint32_t dma_flags; 221 int last_bidx; 222 uint8_t *b_rptr; 223 caddr_t kaddr; 224 uint32_t nmblks; 225 uint32_t ngathers; 226 uint32_t clen; 227 int len; 228 uint32_t pkt_len, pack_len, min_len; 229 uint32_t bcopy_thresh; 230 int i, cur_index, sop_index; 231 uint16_t tail_index; 232 boolean_t tail_wrap = B_FALSE; 233 nxge_dma_common_t desc_area; 234 nxge_os_dma_handle_t dma_handle; 235 ddi_dma_cookie_t dma_cookie; 236 npi_handle_t npi_handle; 237 p_mblk_t nmp; 238 p_mblk_t t_mp; 239 uint32_t ncookies; 240 boolean_t good_packet; 241 boolean_t mark_mode = B_FALSE; 242 p_nxge_stats_t statsp; 243 p_nxge_tx_ring_stats_t tdc_stats; 244 t_uscalar_t start_offset = 0; 245 t_uscalar_t stuff_offset = 0; 246 t_uscalar_t end_offset = 0; 247 t_uscalar_t value = 0; 248 t_uscalar_t cksum_flags = 0; 249 boolean_t cksum_on = B_FALSE; 250 uint32_t boff = 0; 251 uint64_t tot_xfer_len = 0; 252 boolean_t header_set = B_FALSE; 253 #ifdef NXGE_DEBUG 254 p_tx_desc_t tx_desc_ring_pp; 255 p_tx_desc_t tx_desc_pp; 256 tx_desc_t *save_desc_p; 257 int dump_len; 258 int sad_len; 259 uint64_t sad; 260 int xfer_len; 261 uint32_t msgsize; 262 #endif 263 p_mblk_t mp_chain = NULL; 264 boolean_t is_lso = B_FALSE; 265 boolean_t lso_again; 266 int cur_index_lso; 267 p_mblk_t nmp_lso_save; 268 uint32_t lso_ngathers; 269 boolean_t lso_tail_wrap = B_FALSE; 270 271 NXGE_DEBUG_MSG((nxgep, TX_CTL, 272 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 273 NXGE_DEBUG_MSG((nxgep, TX_CTL, 274 "==> nxge_start: Starting tdc %d desc pending %d", 275 tx_ring_p->tdc, tx_ring_p->descs_pending)); 276 277 statsp = nxgep->statsp; 278 279 if (!isLDOMguest(nxgep)) { 280 switch (nxgep->mac.portmode) { 281 default: 282 if (nxgep->statsp->port_stats.lb_mode == 283 nxge_lb_normal) { 284 if (!statsp->mac_stats.link_up) { 285 freemsg(mp); 286 NXGE_DEBUG_MSG((nxgep, TX_CTL, 287 "==> nxge_start: " 288 "link not up")); 289 goto nxge_start_fail1; 290 } 291 } 292 break; 293 case PORT_10G_FIBER: 294 /* 295 * For the following modes, check the link status 296 * before sending the packet out: 297 * nxge_lb_normal, 298 * nxge_lb_ext10g, 299 * nxge_lb_ext1000, 300 * nxge_lb_ext100, 301 * nxge_lb_ext10. 302 */ 303 if (nxgep->statsp->port_stats.lb_mode < 304 nxge_lb_phy10g) { 305 if (!statsp->mac_stats.link_up) { 306 freemsg(mp); 307 NXGE_DEBUG_MSG((nxgep, TX_CTL, 308 "==> nxge_start: " 309 "link not up")); 310 goto nxge_start_fail1; 311 } 312 } 313 break; 314 } 315 } 316 317 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 318 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 319 NXGE_DEBUG_MSG((nxgep, TX_CTL, 320 "==> nxge_start: hardware not initialized or stopped")); 321 freemsg(mp); 322 goto nxge_start_fail1; 323 } 324 325 if (nxgep->soft_lso_enable) { 326 mp_chain = nxge_lso_eliminate(mp); 327 NXGE_DEBUG_MSG((nxgep, TX_CTL, 328 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 329 mp, mp_chain)); 330 if (mp_chain == NULL) { 331 NXGE_ERROR_MSG((nxgep, TX_CTL, 332 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 333 mp_chain, mp)); 334 goto nxge_start_fail1; 335 } 336 if (mp_chain != mp) { 337 NXGE_DEBUG_MSG((nxgep, TX_CTL, 338 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 339 mp_chain, mp)); 340 is_lso = B_TRUE; 341 mp = mp_chain; 342 mp_chain = mp_chain->b_next; 343 mp->b_next = NULL; 344 } 345 } 346 347 hcksum_retrieve(mp, NULL, NULL, &start_offset, 348 &stuff_offset, &end_offset, &value, &cksum_flags); 349 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 350 start_offset += sizeof (ether_header_t); 351 stuff_offset += sizeof (ether_header_t); 352 } else { 353 start_offset += sizeof (struct ether_vlan_header); 354 stuff_offset += sizeof (struct ether_vlan_header); 355 } 356 357 if (cksum_flags & HCK_PARTIALCKSUM) { 358 NXGE_DEBUG_MSG((nxgep, TX_CTL, 359 "==> nxge_start: mp $%p len %d " 360 "cksum_flags 0x%x (partial checksum) ", 361 mp, MBLKL(mp), cksum_flags)); 362 cksum_on = B_TRUE; 363 } 364 365 pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 366 pkthdrp->reserved = 0; 367 tmp_hdrp.pkthdr.value = 0; 368 nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 369 0, 0, pkthdrp, 370 start_offset, stuff_offset); 371 372 lso_again = B_FALSE; 373 lso_ngathers = 0; 374 375 MUTEX_ENTER(&tx_ring_p->lock); 376 377 if (isLDOMservice(nxgep)) { 378 tx_ring_p->tx_ring_busy = B_TRUE; 379 if (tx_ring_p->tx_ring_offline) { 380 freemsg(mp); 381 tx_ring_p->tx_ring_busy = B_FALSE; 382 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 383 NXGE_TX_RING_OFFLINED); 384 MUTEX_EXIT(&tx_ring_p->lock); 385 return (status); 386 } 387 } 388 389 cur_index_lso = tx_ring_p->wr_index; 390 lso_tail_wrap = tx_ring_p->wr_index_wrap; 391 start_again: 392 ngathers = 0; 393 sop_index = tx_ring_p->wr_index; 394 #ifdef NXGE_DEBUG 395 if (tx_ring_p->descs_pending) { 396 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 397 "desc pending %d ", tx_ring_p->descs_pending)); 398 } 399 400 dump_len = (int)(MBLKL(mp)); 401 dump_len = (dump_len > 128) ? 128: dump_len; 402 403 NXGE_DEBUG_MSG((nxgep, TX_CTL, 404 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 405 "(Before header reserve: ORIGINAL LEN %d)", 406 tx_ring_p->tdc, 407 mp->b_rptr, 408 dump_len)); 409 410 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 411 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 412 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 413 #endif 414 415 tdc_stats = tx_ring_p->tdc_stats; 416 mark_mode = (tx_ring_p->descs_pending && 417 (((int)tx_ring_p->tx_ring_size - (int)tx_ring_p->descs_pending) < 418 (int)nxge_tx_minfree)); 419 420 NXGE_DEBUG_MSG((nxgep, TX_CTL, 421 "TX Descriptor ring is channel %d mark mode %d", 422 tx_ring_p->tdc, mark_mode)); 423 424 if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) { 425 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, 426 (nxge_tx_minfree + lso_ngathers))) { 427 NXGE_DEBUG_MSG((nxgep, TX_CTL, 428 "TX Descriptor ring is full: channel %d", 429 tx_ring_p->tdc)); 430 NXGE_DEBUG_MSG((nxgep, TX_CTL, 431 "TX Descriptor ring is full: channel %d", 432 tx_ring_p->tdc)); 433 if (is_lso) { 434 /* 435 * free the current mp and mp_chain if not FULL. 436 */ 437 tdc_stats->tx_no_desc++; 438 NXGE_DEBUG_MSG((nxgep, TX_CTL, 439 "LSO packet: TX Descriptor ring is full: " 440 "channel %d", 441 tx_ring_p->tdc)); 442 goto nxge_start_fail_lso; 443 } else { 444 (void) cas32((uint32_t *)&tx_ring_p->queueing, 445 0, 1); 446 tdc_stats->tx_no_desc++; 447 448 if (isLDOMservice(nxgep)) { 449 tx_ring_p->tx_ring_busy = B_FALSE; 450 if (tx_ring_p->tx_ring_offline) { 451 (void) atomic_swap_32( 452 &tx_ring_p->tx_ring_offline, 453 NXGE_TX_RING_OFFLINED); 454 } 455 } 456 457 MUTEX_EXIT(&tx_ring_p->lock); 458 status = 1; 459 goto nxge_start_fail1; 460 } 461 } 462 } 463 464 nmp = mp; 465 i = sop_index = tx_ring_p->wr_index; 466 nmblks = 0; 467 ngathers = 0; 468 pkt_len = 0; 469 pack_len = 0; 470 clen = 0; 471 last_bidx = -1; 472 good_packet = B_TRUE; 473 474 desc_area = tx_ring_p->tdc_desc; 475 npi_handle = desc_area.npi_handle; 476 npi_desc_handle.regh = (nxge_os_acc_handle_t) 477 DMA_COMMON_ACC_HANDLE(desc_area); 478 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 479 tx_desc_dma_handle = (nxge_os_dma_handle_t) 480 DMA_COMMON_HANDLE(desc_area); 481 tx_msg_ring = tx_ring_p->tx_msg_ring; 482 483 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 484 sop_index, i)); 485 486 #ifdef NXGE_DEBUG 487 msgsize = msgdsize(nmp); 488 NXGE_DEBUG_MSG((nxgep, TX_CTL, 489 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 490 sop_index, i, msgsize)); 491 #endif 492 /* 493 * The first 16 bytes of the premapped buffer are reserved 494 * for header. No padding will be used. 495 */ 496 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 497 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 498 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 499 } else { 500 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 501 } 502 while (nmp) { 503 good_packet = B_TRUE; 504 b_rptr = nmp->b_rptr; 505 len = MBLKL(nmp); 506 if (len <= 0) { 507 nmp = nmp->b_cont; 508 continue; 509 } 510 nmblks++; 511 512 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 513 "len %d pkt_len %d pack_len %d", 514 nmblks, len, pkt_len, pack_len)); 515 /* 516 * Hardware limits the transfer length to 4K for NIU and 517 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 518 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 519 * If len is longer than the limit, then we break nmp into 520 * two chunks: Make the first chunk equal to the limit and 521 * the second chunk for the remaining data. If the second 522 * chunk is still larger than the limit, then it will be 523 * broken into two in the next pass. 524 */ 525 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 526 if ((t_mp = dupb(nmp)) != NULL) { 527 nmp->b_wptr = nmp->b_rptr + 528 (TX_MAX_TRANSFER_LENGTH 529 - TX_PKT_HEADER_SIZE); 530 t_mp->b_rptr = nmp->b_wptr; 531 t_mp->b_cont = nmp->b_cont; 532 nmp->b_cont = t_mp; 533 len = MBLKL(nmp); 534 } else { 535 if (is_lso) { 536 NXGE_DEBUG_MSG((nxgep, TX_CTL, 537 "LSO packet: dupb failed: " 538 "channel %d", 539 tx_ring_p->tdc)); 540 mp = nmp; 541 goto nxge_start_fail_lso; 542 } else { 543 good_packet = B_FALSE; 544 goto nxge_start_fail2; 545 } 546 } 547 } 548 tx_desc.value = 0; 549 tx_desc_p = &tx_desc_ring_vp[i]; 550 #ifdef NXGE_DEBUG 551 tx_desc_pp = &tx_desc_ring_pp[i]; 552 #endif 553 tx_msg_p = &tx_msg_ring[i]; 554 #if defined(__i386) 555 npi_desc_handle.regp = (uint32_t)tx_desc_p; 556 #else 557 npi_desc_handle.regp = (uint64_t)tx_desc_p; 558 #endif 559 if (!header_set && 560 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 561 (len >= bcopy_thresh))) { 562 header_set = B_TRUE; 563 bcopy_thresh += TX_PKT_HEADER_SIZE; 564 boff = 0; 565 pack_len = 0; 566 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 567 hdrp = (p_tx_pkt_header_t)kaddr; 568 clen = pkt_len; 569 dma_handle = tx_msg_p->buf_dma_handle; 570 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 571 (void) ddi_dma_sync(dma_handle, 572 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 573 DDI_DMA_SYNC_FORDEV); 574 575 tx_msg_p->flags.dma_type = USE_BCOPY; 576 goto nxge_start_control_header_only; 577 } 578 579 pkt_len += len; 580 pack_len += len; 581 582 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 583 "desc entry %d " 584 "DESC IOADDR $%p " 585 "desc_vp $%p tx_desc_p $%p " 586 "desc_pp $%p tx_desc_pp $%p " 587 "len %d pkt_len %d pack_len %d", 588 i, 589 DMA_COMMON_IOADDR(desc_area), 590 tx_desc_ring_vp, tx_desc_p, 591 tx_desc_ring_pp, tx_desc_pp, 592 len, pkt_len, pack_len)); 593 594 if (len < bcopy_thresh) { 595 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 596 "USE BCOPY: ")); 597 if (nxge_tx_tiny_pack) { 598 uint32_t blst = 599 TXDMA_DESC_NEXT_INDEX(i, -1, 600 tx_ring_p->tx_wrap_mask); 601 NXGE_DEBUG_MSG((nxgep, TX_CTL, 602 "==> nxge_start(5): pack")); 603 if ((pack_len <= bcopy_thresh) && 604 (last_bidx == blst)) { 605 NXGE_DEBUG_MSG((nxgep, TX_CTL, 606 "==> nxge_start: pack(6) " 607 "(pkt_len %d pack_len %d)", 608 pkt_len, pack_len)); 609 i = blst; 610 tx_desc_p = &tx_desc_ring_vp[i]; 611 #ifdef NXGE_DEBUG 612 tx_desc_pp = &tx_desc_ring_pp[i]; 613 #endif 614 tx_msg_p = &tx_msg_ring[i]; 615 boff = pack_len - len; 616 ngathers--; 617 } else if (pack_len > bcopy_thresh && 618 header_set) { 619 pack_len = len; 620 boff = 0; 621 bcopy_thresh = nxge_bcopy_thresh; 622 NXGE_DEBUG_MSG((nxgep, TX_CTL, 623 "==> nxge_start(7): > max NEW " 624 "bcopy thresh %d " 625 "pkt_len %d pack_len %d(next)", 626 bcopy_thresh, 627 pkt_len, pack_len)); 628 } 629 last_bidx = i; 630 } 631 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 632 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 633 hdrp = (p_tx_pkt_header_t)kaddr; 634 header_set = B_TRUE; 635 NXGE_DEBUG_MSG((nxgep, TX_CTL, 636 "==> nxge_start(7_x2): " 637 "pkt_len %d pack_len %d (new hdrp $%p)", 638 pkt_len, pack_len, hdrp)); 639 } 640 tx_msg_p->flags.dma_type = USE_BCOPY; 641 kaddr += boff; 642 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 643 "USE BCOPY: before bcopy " 644 "DESC IOADDR $%p entry %d " 645 "bcopy packets %d " 646 "bcopy kaddr $%p " 647 "bcopy ioaddr (SAD) $%p " 648 "bcopy clen %d " 649 "bcopy boff %d", 650 DMA_COMMON_IOADDR(desc_area), i, 651 tdc_stats->tx_hdr_pkts, 652 kaddr, 653 dma_ioaddr, 654 clen, 655 boff)); 656 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 657 "1USE BCOPY: ")); 658 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 659 "2USE BCOPY: ")); 660 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 661 "last USE BCOPY: copy from b_rptr $%p " 662 "to KADDR $%p (len %d offset %d", 663 b_rptr, kaddr, len, boff)); 664 665 bcopy(b_rptr, kaddr, len); 666 667 #ifdef NXGE_DEBUG 668 dump_len = (len > 128) ? 128: len; 669 NXGE_DEBUG_MSG((nxgep, TX_CTL, 670 "==> nxge_start: dump packets " 671 "(After BCOPY len %d)" 672 "(b_rptr $%p): %s", len, nmp->b_rptr, 673 nxge_dump_packet((char *)nmp->b_rptr, 674 dump_len))); 675 #endif 676 677 dma_handle = tx_msg_p->buf_dma_handle; 678 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 679 (void) ddi_dma_sync(dma_handle, 680 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 681 DDI_DMA_SYNC_FORDEV); 682 clen = len + boff; 683 tdc_stats->tx_hdr_pkts++; 684 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 685 "USE BCOPY: " 686 "DESC IOADDR $%p entry %d " 687 "bcopy packets %d " 688 "bcopy kaddr $%p " 689 "bcopy ioaddr (SAD) $%p " 690 "bcopy clen %d " 691 "bcopy boff %d", 692 DMA_COMMON_IOADDR(desc_area), 693 i, 694 tdc_stats->tx_hdr_pkts, 695 kaddr, 696 dma_ioaddr, 697 clen, 698 boff)); 699 } else { 700 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 701 "USE DVMA: len %d", len)); 702 tx_msg_p->flags.dma_type = USE_DMA; 703 dma_flags = DDI_DMA_WRITE; 704 if (len < nxge_dma_stream_thresh) { 705 dma_flags |= DDI_DMA_CONSISTENT; 706 } else { 707 dma_flags |= DDI_DMA_STREAMING; 708 } 709 710 dma_handle = tx_msg_p->dma_handle; 711 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 712 (caddr_t)b_rptr, len, dma_flags, 713 DDI_DMA_DONTWAIT, NULL, 714 &dma_cookie, &ncookies); 715 if (status == DDI_DMA_MAPPED) { 716 dma_ioaddr = dma_cookie.dmac_laddress; 717 len = (int)dma_cookie.dmac_size; 718 clen = (uint32_t)dma_cookie.dmac_size; 719 NXGE_DEBUG_MSG((nxgep, TX_CTL, 720 "==> nxge_start(12_1): " 721 "USE DVMA: len %d clen %d " 722 "ngathers %d", 723 len, clen, 724 ngathers)); 725 #if defined(__i386) 726 npi_desc_handle.regp = (uint32_t)tx_desc_p; 727 #else 728 npi_desc_handle.regp = (uint64_t)tx_desc_p; 729 #endif 730 while (ncookies > 1) { 731 ngathers++; 732 /* 733 * this is the fix for multiple 734 * cookies, which are basically 735 * a descriptor entry, we don't set 736 * SOP bit as well as related fields 737 */ 738 739 (void) npi_txdma_desc_gather_set( 740 npi_desc_handle, 741 &tx_desc, 742 (ngathers -1), 743 mark_mode, 744 ngathers, 745 dma_ioaddr, 746 clen); 747 748 tx_msg_p->tx_msg_size = clen; 749 NXGE_DEBUG_MSG((nxgep, TX_CTL, 750 "==> nxge_start: DMA " 751 "ncookie %d " 752 "ngathers %d " 753 "dma_ioaddr $%p len %d" 754 "desc $%p descp $%p (%d)", 755 ncookies, 756 ngathers, 757 dma_ioaddr, clen, 758 *tx_desc_p, tx_desc_p, i)); 759 760 ddi_dma_nextcookie(dma_handle, 761 &dma_cookie); 762 dma_ioaddr = 763 dma_cookie.dmac_laddress; 764 765 len = (int)dma_cookie.dmac_size; 766 clen = (uint32_t)dma_cookie.dmac_size; 767 NXGE_DEBUG_MSG((nxgep, TX_CTL, 768 "==> nxge_start(12_2): " 769 "USE DVMA: len %d clen %d ", 770 len, clen)); 771 772 i = TXDMA_DESC_NEXT_INDEX(i, 1, 773 tx_ring_p->tx_wrap_mask); 774 tx_desc_p = &tx_desc_ring_vp[i]; 775 776 #if defined(__i386) 777 npi_desc_handle.regp = 778 (uint32_t)tx_desc_p; 779 #else 780 npi_desc_handle.regp = 781 (uint64_t)tx_desc_p; 782 #endif 783 tx_msg_p = &tx_msg_ring[i]; 784 tx_msg_p->flags.dma_type = USE_NONE; 785 tx_desc.value = 0; 786 787 ncookies--; 788 } 789 tdc_stats->tx_ddi_pkts++; 790 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 791 "DMA: ddi packets %d", 792 tdc_stats->tx_ddi_pkts)); 793 } else { 794 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 795 "dma mapping failed for %d " 796 "bytes addr $%p flags %x (%d)", 797 len, b_rptr, status, status)); 798 good_packet = B_FALSE; 799 tdc_stats->tx_dma_bind_fail++; 800 tx_msg_p->flags.dma_type = USE_NONE; 801 if (is_lso) { 802 mp = nmp; 803 goto nxge_start_fail_lso; 804 } else { 805 goto nxge_start_fail2; 806 } 807 } 808 } /* ddi dvma */ 809 810 if (is_lso) { 811 nmp_lso_save = nmp; 812 } 813 nmp = nmp->b_cont; 814 nxge_start_control_header_only: 815 #if defined(__i386) 816 npi_desc_handle.regp = (uint32_t)tx_desc_p; 817 #else 818 npi_desc_handle.regp = (uint64_t)tx_desc_p; 819 #endif 820 ngathers++; 821 822 if (ngathers == 1) { 823 #ifdef NXGE_DEBUG 824 save_desc_p = &sop_tx_desc; 825 #endif 826 sop_tx_desc_p = &sop_tx_desc; 827 sop_tx_desc_p->value = 0; 828 sop_tx_desc_p->bits.hdw.tr_len = clen; 829 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 830 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 831 } else { 832 #ifdef NXGE_DEBUG 833 save_desc_p = &tx_desc; 834 #endif 835 tmp_desc_p = &tx_desc; 836 tmp_desc_p->value = 0; 837 tmp_desc_p->bits.hdw.tr_len = clen; 838 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 839 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 840 841 tx_desc_p->value = tmp_desc_p->value; 842 } 843 844 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 845 "Desc_entry %d ngathers %d " 846 "desc_vp $%p tx_desc_p $%p " 847 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 848 "dma_ioaddr (SAD) $%p mark %d", 849 i, ngathers, 850 tx_desc_ring_vp, tx_desc_p, 851 len, clen, pkt_len, pack_len, nmblks, 852 dma_ioaddr, mark_mode)); 853 854 #ifdef NXGE_DEBUG 855 npi_desc_handle.nxgep = nxgep; 856 npi_desc_handle.function.function = nxgep->function_num; 857 npi_desc_handle.function.instance = nxgep->instance; 858 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 859 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 860 TX_PKT_DESC_TR_LEN_SHIFT); 861 862 863 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 864 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 865 "mark %d sop %d\n", 866 save_desc_p->value, 867 sad, 868 save_desc_p->bits.hdw.tr_len, 869 xfer_len, 870 save_desc_p->bits.hdw.num_ptr, 871 save_desc_p->bits.hdw.mark, 872 save_desc_p->bits.hdw.sop)); 873 874 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 875 #endif 876 877 tx_msg_p->tx_msg_size = clen; 878 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 879 if (ngathers > nxge_tx_max_gathers) { 880 good_packet = B_FALSE; 881 hcksum_retrieve(mp, NULL, NULL, &start_offset, 882 &stuff_offset, &end_offset, &value, 883 &cksum_flags); 884 885 NXGE_DEBUG_MSG((NULL, TX_CTL, 886 "==> nxge_start(14): pull msg - " 887 "len %d pkt_len %d ngathers %d", 888 len, pkt_len, ngathers)); 889 /* Pull all message blocks from b_cont */ 890 if (is_lso) { 891 mp = nmp_lso_save; 892 goto nxge_start_fail_lso; 893 } 894 if ((msgpullup(mp, -1)) == NULL) { 895 goto nxge_start_fail2; 896 } 897 goto nxge_start_fail2; 898 } 899 } /* while (nmp) */ 900 901 tx_msg_p->tx_message = mp; 902 tx_desc_p = &tx_desc_ring_vp[sop_index]; 903 #if defined(__i386) 904 npi_desc_handle.regp = (uint32_t)tx_desc_p; 905 #else 906 npi_desc_handle.regp = (uint64_t)tx_desc_p; 907 #endif 908 909 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 910 pkthdrp->reserved = 0; 911 hdrp->value = 0; 912 bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 913 914 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 915 tdc_stats->tx_jumbo_pkts++; 916 } 917 918 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 919 if (pkt_len < min_len) { 920 /* Assume we use bcopy to premapped buffers */ 921 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 922 NXGE_DEBUG_MSG((NULL, TX_CTL, 923 "==> nxge_start(14-1): < (msg_min + 16)" 924 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 925 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 926 bzero((kaddr + pkt_len), (min_len - pkt_len)); 927 pkt_len = tx_msg_p->tx_msg_size = min_len; 928 929 sop_tx_desc_p->bits.hdw.tr_len = min_len; 930 931 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 932 tx_desc_p->value = sop_tx_desc_p->value; 933 934 NXGE_DEBUG_MSG((NULL, TX_CTL, 935 "==> nxge_start(14-2): < msg_min - " 936 "len %d pkt_len %d min_len %d ngathers %d", 937 len, pkt_len, min_len, ngathers)); 938 } 939 940 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 941 cksum_flags)); 942 { 943 uint64_t tmp_len; 944 945 /* pkt_len already includes 16 + paddings!! */ 946 /* Update the control header length */ 947 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 948 tmp_len = hdrp->value | 949 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 950 951 NXGE_DEBUG_MSG((nxgep, TX_CTL, 952 "==> nxge_start(15_x1): setting SOP " 953 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 954 "0x%llx hdrp->value 0x%llx", 955 tot_xfer_len, tot_xfer_len, pkt_len, 956 tmp_len, hdrp->value)); 957 #if defined(_BIG_ENDIAN) 958 hdrp->value = ddi_swap64(tmp_len); 959 #else 960 hdrp->value = tmp_len; 961 #endif 962 NXGE_DEBUG_MSG((nxgep, 963 TX_CTL, "==> nxge_start(15_x2): setting SOP " 964 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 965 "tmp_len 0x%llx hdrp->value 0x%llx", 966 tot_xfer_len, pkt_len, 967 tmp_len, hdrp->value)); 968 } 969 970 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 971 "wr_index %d " 972 "tot_xfer_len (%d) pkt_len %d npads %d", 973 sop_index, 974 tot_xfer_len, pkt_len, 975 npads)); 976 977 sop_tx_desc_p->bits.hdw.sop = 1; 978 sop_tx_desc_p->bits.hdw.mark = mark_mode; 979 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 980 981 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 982 983 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 984 985 #ifdef NXGE_DEBUG 986 npi_desc_handle.nxgep = nxgep; 987 npi_desc_handle.function.function = nxgep->function_num; 988 npi_desc_handle.function.instance = nxgep->instance; 989 990 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 991 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 992 save_desc_p->value, 993 sad, 994 save_desc_p->bits.hdw.tr_len, 995 xfer_len, 996 save_desc_p->bits.hdw.num_ptr, 997 save_desc_p->bits.hdw.mark, 998 save_desc_p->bits.hdw.sop)); 999 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 1000 1001 dump_len = (pkt_len > 128) ? 128: pkt_len; 1002 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1003 "==> nxge_start: dump packets(17) (after sop set, len " 1004 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 1005 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 1006 (char *)hdrp, 1007 nxge_dump_packet((char *)hdrp, dump_len))); 1008 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1009 "==> nxge_start(18): TX desc sync: sop_index %d", 1010 sop_index)); 1011 #endif 1012 1013 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 1014 (void) ddi_dma_sync(tx_desc_dma_handle, 1015 sop_index * sizeof (tx_desc_t), 1016 ngathers * sizeof (tx_desc_t), 1017 DDI_DMA_SYNC_FORDEV); 1018 1019 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 1020 "cs_off = 0x%02X cs_s_off = 0x%02X " 1021 "pkt_len %d ngathers %d sop_index %d\n", 1022 stuff_offset, start_offset, 1023 pkt_len, ngathers, sop_index)); 1024 } else { /* more than one descriptor and wrap around */ 1025 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 1026 (void) ddi_dma_sync(tx_desc_dma_handle, 1027 sop_index * sizeof (tx_desc_t), 1028 nsdescs * sizeof (tx_desc_t), 1029 DDI_DMA_SYNC_FORDEV); 1030 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 1031 "cs_off = 0x%02X cs_s_off = 0x%02X " 1032 "pkt_len %d ngathers %d sop_index %d\n", 1033 stuff_offset, start_offset, 1034 pkt_len, ngathers, sop_index)); 1035 1036 (void) ddi_dma_sync(tx_desc_dma_handle, 1037 0, 1038 (ngathers - nsdescs) * sizeof (tx_desc_t), 1039 DDI_DMA_SYNC_FORDEV); 1040 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 1041 "cs_off = 0x%02X cs_s_off = 0x%02X " 1042 "pkt_len %d ngathers %d sop_index %d\n", 1043 stuff_offset, start_offset, 1044 pkt_len, ngathers, sop_index)); 1045 } 1046 1047 tail_index = tx_ring_p->wr_index; 1048 tail_wrap = tx_ring_p->wr_index_wrap; 1049 1050 tx_ring_p->wr_index = i; 1051 if (tx_ring_p->wr_index <= tail_index) { 1052 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 1053 B_FALSE : B_TRUE); 1054 } 1055 1056 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 1057 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 1058 tx_ring_p->tdc, 1059 tx_ring_p->wr_index, 1060 tx_ring_p->wr_index_wrap, 1061 ngathers, 1062 tx_ring_p->descs_pending)); 1063 1064 if (is_lso) { 1065 lso_ngathers += ngathers; 1066 if (mp_chain != NULL) { 1067 mp = mp_chain; 1068 mp_chain = mp_chain->b_next; 1069 mp->b_next = NULL; 1070 if (nxge_lso_kick_cnt == lso_ngathers) { 1071 tx_ring_p->descs_pending += lso_ngathers; 1072 { 1073 tx_ring_kick_t kick; 1074 1075 kick.value = 0; 1076 kick.bits.ldw.wrap = 1077 tx_ring_p->wr_index_wrap; 1078 kick.bits.ldw.tail = 1079 (uint16_t)tx_ring_p->wr_index; 1080 1081 /* Kick the Transmit kick register */ 1082 TXDMA_REG_WRITE64( 1083 NXGE_DEV_NPI_HANDLE(nxgep), 1084 TX_RING_KICK_REG, 1085 (uint8_t)tx_ring_p->tdc, 1086 kick.value); 1087 tdc_stats->tx_starts++; 1088 1089 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1090 "==> nxge_start: more LSO: " 1091 "LSO_CNT %d", 1092 lso_ngathers)); 1093 } 1094 lso_ngathers = 0; 1095 ngathers = 0; 1096 cur_index_lso = sop_index = tx_ring_p->wr_index; 1097 lso_tail_wrap = tx_ring_p->wr_index_wrap; 1098 } 1099 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1100 "==> nxge_start: lso again: " 1101 "lso_gathers %d ngathers %d cur_index_lso %d " 1102 "wr_index %d sop_index %d", 1103 lso_ngathers, ngathers, cur_index_lso, 1104 tx_ring_p->wr_index, sop_index)); 1105 1106 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1107 "==> nxge_start: next : count %d", 1108 lso_ngathers)); 1109 lso_again = B_TRUE; 1110 goto start_again; 1111 } 1112 ngathers = lso_ngathers; 1113 } 1114 1115 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 1116 1117 { 1118 tx_ring_kick_t kick; 1119 1120 kick.value = 0; 1121 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 1122 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 1123 1124 /* Kick start the Transmit kick register */ 1125 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 1126 TX_RING_KICK_REG, 1127 (uint8_t)tx_ring_p->tdc, 1128 kick.value); 1129 } 1130 1131 tx_ring_p->descs_pending += ngathers; 1132 tdc_stats->tx_starts++; 1133 1134 if (isLDOMservice(nxgep)) { 1135 tx_ring_p->tx_ring_busy = B_FALSE; 1136 if (tx_ring_p->tx_ring_offline) { 1137 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1138 NXGE_TX_RING_OFFLINED); 1139 } 1140 } 1141 1142 MUTEX_EXIT(&tx_ring_p->lock); 1143 1144 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1145 return (status); 1146 1147 nxge_start_fail_lso: 1148 status = 0; 1149 good_packet = B_FALSE; 1150 if (mp != NULL) { 1151 freemsg(mp); 1152 } 1153 if (mp_chain != NULL) { 1154 freemsg(mp_chain); 1155 } 1156 if (!lso_again && !ngathers) { 1157 if (isLDOMservice(nxgep)) { 1158 tx_ring_p->tx_ring_busy = B_FALSE; 1159 if (tx_ring_p->tx_ring_offline) { 1160 (void) atomic_swap_32( 1161 &tx_ring_p->tx_ring_offline, 1162 NXGE_TX_RING_OFFLINED); 1163 } 1164 } 1165 1166 MUTEX_EXIT(&tx_ring_p->lock); 1167 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1168 "==> nxge_start: lso exit (nothing changed)")); 1169 goto nxge_start_fail1; 1170 } 1171 1172 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1173 "==> nxge_start (channel %d): before lso " 1174 "lso_gathers %d ngathers %d cur_index_lso %d " 1175 "wr_index %d sop_index %d lso_again %d", 1176 tx_ring_p->tdc, 1177 lso_ngathers, ngathers, cur_index_lso, 1178 tx_ring_p->wr_index, sop_index, lso_again)); 1179 1180 if (lso_again) { 1181 lso_ngathers += ngathers; 1182 ngathers = lso_ngathers; 1183 sop_index = cur_index_lso; 1184 tx_ring_p->wr_index = sop_index; 1185 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1186 } 1187 1188 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1189 "==> nxge_start (channel %d): after lso " 1190 "lso_gathers %d ngathers %d cur_index_lso %d " 1191 "wr_index %d sop_index %d lso_again %d", 1192 tx_ring_p->tdc, 1193 lso_ngathers, ngathers, cur_index_lso, 1194 tx_ring_p->wr_index, sop_index, lso_again)); 1195 1196 nxge_start_fail2: 1197 if (good_packet == B_FALSE) { 1198 cur_index = sop_index; 1199 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1200 for (i = 0; i < ngathers; i++) { 1201 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1202 #if defined(__i386) 1203 npi_handle.regp = (uint32_t)tx_desc_p; 1204 #else 1205 npi_handle.regp = (uint64_t)tx_desc_p; 1206 #endif 1207 tx_msg_p = &tx_msg_ring[cur_index]; 1208 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1209 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1210 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1211 "tx_desc_p = %X index = %d", 1212 tx_desc_p, tx_ring_p->rd_index)); 1213 (void) dvma_unload(tx_msg_p->dvma_handle, 1214 0, -1); 1215 tx_msg_p->dvma_handle = NULL; 1216 if (tx_ring_p->dvma_wr_index == 1217 tx_ring_p->dvma_wrap_mask) 1218 tx_ring_p->dvma_wr_index = 0; 1219 else 1220 tx_ring_p->dvma_wr_index++; 1221 tx_ring_p->dvma_pending--; 1222 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1223 if (ddi_dma_unbind_handle( 1224 tx_msg_p->dma_handle)) { 1225 cmn_err(CE_WARN, "!nxge_start: " 1226 "ddi_dma_unbind_handle failed"); 1227 } 1228 } 1229 tx_msg_p->flags.dma_type = USE_NONE; 1230 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1231 tx_ring_p->tx_wrap_mask); 1232 1233 } 1234 } 1235 1236 if (isLDOMservice(nxgep)) { 1237 tx_ring_p->tx_ring_busy = B_FALSE; 1238 if (tx_ring_p->tx_ring_offline) { 1239 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1240 NXGE_TX_RING_OFFLINED); 1241 } 1242 } 1243 1244 MUTEX_EXIT(&tx_ring_p->lock); 1245 1246 nxge_start_fail1: 1247 /* Add FMA to check the access handle nxge_hregh */ 1248 1249 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1250 return (status); 1251 } 1252 1253 /* Software LSO starts here */ 1254 static void 1255 nxge_hcksum_retrieve(mblk_t *mp, 1256 uint32_t *start, uint32_t *stuff, uint32_t *end, 1257 uint32_t *value, uint32_t *flags) 1258 { 1259 if (mp->b_datap->db_type == M_DATA) { 1260 if (flags != NULL) { 1261 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1262 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1263 HCK_FULLCKSUM_OK); 1264 if ((*flags & (HCK_PARTIALCKSUM | 1265 HCK_FULLCKSUM)) != 0) { 1266 if (value != NULL) 1267 *value = (uint32_t)DB_CKSUM16(mp); 1268 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1269 if (start != NULL) 1270 *start = 1271 (uint32_t)DB_CKSUMSTART(mp); 1272 if (stuff != NULL) 1273 *stuff = 1274 (uint32_t)DB_CKSUMSTUFF(mp); 1275 if (end != NULL) 1276 *end = 1277 (uint32_t)DB_CKSUMEND(mp); 1278 } 1279 } 1280 } 1281 } 1282 } 1283 1284 static void 1285 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1286 { 1287 ASSERT(DB_TYPE(mp) == M_DATA); 1288 1289 *mss = 0; 1290 if (flags != NULL) { 1291 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1292 if ((*flags != 0) && (mss != NULL)) { 1293 *mss = (uint32_t)DB_LSOMSS(mp); 1294 } 1295 NXGE_DEBUG_MSG((NULL, TX_CTL, 1296 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1297 *mss, *flags)); 1298 } 1299 1300 NXGE_DEBUG_MSG((NULL, TX_CTL, 1301 "<== nxge_lso_info_get: mss %d", *mss)); 1302 } 1303 1304 /* 1305 * Do Soft LSO on the oversized packet. 1306 * 1307 * 1. Create a chain of message for headers. 1308 * 2. Fill up header messages with proper information. 1309 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1310 * each new message with necessary adjustments. 1311 * * Unchange the ethernet header for DIX frames. (by default) 1312 * * IP Total Length field is updated to MSS or less(only for the last one). 1313 * * IP Identification value is incremented by one for each packet. 1314 * * TCP sequence Number is recalculated according to the payload length. 1315 * * Set FIN and/or PSH flags for the *last* packet if applied. 1316 * * TCP partial Checksum 1317 * 4. Update LSO information in the first message header. 1318 * 5. Release the original message header. 1319 */ 1320 static mblk_t * 1321 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1322 { 1323 uint32_t hckflags; 1324 int pktlen; 1325 int hdrlen; 1326 int segnum; 1327 int i; 1328 struct ether_vlan_header *evh; 1329 int ehlen, iphlen, tcphlen; 1330 struct ip *oiph, *niph; 1331 struct tcphdr *otcph, *ntcph; 1332 int available, len, left; 1333 uint16_t ip_id; 1334 uint32_t tcp_seq; 1335 #ifdef __sparc 1336 uint32_t tcp_seq_tmp; 1337 #endif 1338 mblk_t *datamp; 1339 uchar_t *rptr; 1340 mblk_t *nmp; 1341 mblk_t *cmp; 1342 mblk_t *mp_chain; 1343 boolean_t do_cleanup = B_FALSE; 1344 t_uscalar_t start_offset = 0; 1345 t_uscalar_t stuff_offset = 0; 1346 t_uscalar_t value = 0; 1347 uint16_t l4_len; 1348 ipaddr_t src, dst; 1349 uint32_t cksum, sum, l4cksum; 1350 1351 NXGE_DEBUG_MSG((NULL, TX_CTL, 1352 "==> nxge_do_softlso")); 1353 /* 1354 * check the length of LSO packet payload and calculate the number of 1355 * segments to be generated. 1356 */ 1357 pktlen = msgsize(mp); 1358 evh = (struct ether_vlan_header *)mp->b_rptr; 1359 1360 /* VLAN? */ 1361 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1362 ehlen = sizeof (struct ether_vlan_header); 1363 else 1364 ehlen = sizeof (struct ether_header); 1365 oiph = (struct ip *)(mp->b_rptr + ehlen); 1366 iphlen = oiph->ip_hl * 4; 1367 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1368 tcphlen = otcph->th_off * 4; 1369 1370 l4_len = pktlen - ehlen - iphlen; 1371 1372 NXGE_DEBUG_MSG((NULL, TX_CTL, 1373 "==> nxge_do_softlso: mss %d oiph $%p " 1374 "original ip_sum oiph->ip_sum 0x%x " 1375 "original tcp_sum otcph->th_sum 0x%x " 1376 "oiph->ip_len %d pktlen %d ehlen %d " 1377 "l4_len %d (0x%x) ip_len - iphlen %d ", 1378 mss, 1379 oiph, 1380 oiph->ip_sum, 1381 otcph->th_sum, 1382 ntohs(oiph->ip_len), pktlen, 1383 ehlen, 1384 l4_len, 1385 l4_len, 1386 ntohs(oiph->ip_len) - iphlen)); 1387 1388 /* IPv4 + TCP */ 1389 if (!(oiph->ip_v == IPV4_VERSION)) { 1390 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1391 "<== nxge_do_softlso: not IPV4 " 1392 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1393 ntohs(oiph->ip_len), pktlen, ehlen, 1394 tcphlen)); 1395 freemsg(mp); 1396 return (NULL); 1397 } 1398 1399 if (!(oiph->ip_p == IPPROTO_TCP)) { 1400 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1401 "<== nxge_do_softlso: not TCP " 1402 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1403 ntohs(oiph->ip_len), pktlen, ehlen, 1404 tcphlen)); 1405 freemsg(mp); 1406 return (NULL); 1407 } 1408 1409 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1410 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1411 "<== nxge_do_softlso: len not matched " 1412 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1413 ntohs(oiph->ip_len), pktlen, ehlen, 1414 tcphlen)); 1415 freemsg(mp); 1416 return (NULL); 1417 } 1418 1419 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1420 tcphlen = otcph->th_off * 4; 1421 1422 /* TCP flags can not include URG, RST, or SYN */ 1423 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1424 1425 hdrlen = ehlen + iphlen + tcphlen; 1426 1427 VERIFY(MBLKL(mp) >= hdrlen); 1428 1429 if (MBLKL(mp) > hdrlen) { 1430 datamp = mp; 1431 rptr = mp->b_rptr + hdrlen; 1432 } else { /* = */ 1433 datamp = mp->b_cont; 1434 rptr = datamp->b_rptr; 1435 } 1436 1437 NXGE_DEBUG_MSG((NULL, TX_CTL, 1438 "nxge_do_softlso: otcph $%p pktlen: %d, " 1439 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1440 "mblkl(mp): %d, mblkl(datamp): %d", 1441 otcph, 1442 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1443 (int)MBLKL(mp), (int)MBLKL(datamp))); 1444 1445 hckflags = 0; 1446 nxge_hcksum_retrieve(mp, 1447 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1448 1449 dst = oiph->ip_dst.s_addr; 1450 src = oiph->ip_src.s_addr; 1451 1452 cksum = (dst >> 16) + (dst & 0xFFFF) + 1453 (src >> 16) + (src & 0xFFFF); 1454 l4cksum = cksum + IP_TCP_CSUM_COMP; 1455 1456 sum = l4_len + l4cksum; 1457 sum = (sum & 0xFFFF) + (sum >> 16); 1458 1459 NXGE_DEBUG_MSG((NULL, TX_CTL, 1460 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1461 "hckflags 0x%x start_offset %d stuff_offset %d " 1462 "value (original) 0x%x th_sum 0x%x " 1463 "pktlen %d l4_len %d (0x%x) " 1464 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1465 dst, src, 1466 (sum & 0xffff), (~sum & 0xffff), 1467 hckflags, start_offset, stuff_offset, 1468 value, otcph->th_sum, 1469 pktlen, 1470 l4_len, 1471 l4_len, 1472 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1473 (int)MBLKL(datamp), 1474 nxge_dump_packet((char *)evh, 12))); 1475 1476 /* 1477 * Start to process. 1478 */ 1479 available = pktlen - hdrlen; 1480 segnum = (available - 1) / mss + 1; 1481 1482 NXGE_DEBUG_MSG((NULL, TX_CTL, 1483 "==> nxge_do_softlso: pktlen %d " 1484 "MBLKL(mp): %d, MBLKL(datamp): %d " 1485 "available %d mss %d segnum %d", 1486 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1487 available, 1488 mss, 1489 segnum)); 1490 1491 VERIFY(segnum >= 2); 1492 1493 /* 1494 * Try to pre-allocate all header messages 1495 */ 1496 mp_chain = NULL; 1497 for (i = 0; i < segnum; i++) { 1498 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1499 /* Clean up the mp_chain */ 1500 while (mp_chain != NULL) { 1501 nmp = mp_chain; 1502 mp_chain = mp_chain->b_next; 1503 freemsg(nmp); 1504 } 1505 NXGE_DEBUG_MSG((NULL, TX_CTL, 1506 "<== nxge_do_softlso: " 1507 "Could not allocate enough messages for headers!")); 1508 freemsg(mp); 1509 return (NULL); 1510 } 1511 nmp->b_next = mp_chain; 1512 mp_chain = nmp; 1513 1514 NXGE_DEBUG_MSG((NULL, TX_CTL, 1515 "==> nxge_do_softlso: " 1516 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1517 mp, nmp, mp_chain, mp_chain->b_next)); 1518 } 1519 1520 NXGE_DEBUG_MSG((NULL, TX_CTL, 1521 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1522 mp, nmp, mp_chain)); 1523 1524 /* 1525 * Associate payload with new packets 1526 */ 1527 cmp = mp_chain; 1528 left = available; 1529 while (cmp != NULL) { 1530 nmp = dupb(datamp); 1531 if (nmp == NULL) { 1532 do_cleanup = B_TRUE; 1533 NXGE_DEBUG_MSG((NULL, TX_CTL, 1534 "==>nxge_do_softlso: " 1535 "Can not dupb(datamp), have to do clean up")); 1536 goto cleanup_allocated_msgs; 1537 } 1538 1539 NXGE_DEBUG_MSG((NULL, TX_CTL, 1540 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1541 "dupb nmp $%p len %d left %d msd %d ", 1542 mp, cmp, nmp, len, left, mss)); 1543 1544 cmp->b_cont = nmp; 1545 nmp->b_rptr = rptr; 1546 len = (left < mss) ? left : mss; 1547 left -= len; 1548 1549 NXGE_DEBUG_MSG((NULL, TX_CTL, 1550 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1551 "dupb nmp $%p len %d left %d mss %d ", 1552 mp, cmp, nmp, len, left, mss)); 1553 NXGE_DEBUG_MSG((NULL, TX_CTL, 1554 "nxge_do_softlso: before available: %d, " 1555 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1556 available, left, len, segnum, (int)MBLKL(nmp))); 1557 1558 len -= MBLKL(nmp); 1559 NXGE_DEBUG_MSG((NULL, TX_CTL, 1560 "nxge_do_softlso: after available: %d, " 1561 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1562 available, left, len, segnum, (int)MBLKL(nmp))); 1563 1564 while (len > 0) { 1565 mblk_t *mmp = NULL; 1566 1567 NXGE_DEBUG_MSG((NULL, TX_CTL, 1568 "nxge_do_softlso: (4) len > 0 available: %d, " 1569 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1570 available, left, len, segnum, (int)MBLKL(nmp))); 1571 1572 if (datamp->b_cont != NULL) { 1573 datamp = datamp->b_cont; 1574 rptr = datamp->b_rptr; 1575 mmp = dupb(datamp); 1576 if (mmp == NULL) { 1577 do_cleanup = B_TRUE; 1578 NXGE_DEBUG_MSG((NULL, TX_CTL, 1579 "==> nxge_do_softlso: " 1580 "Can not dupb(datamp) (1), :" 1581 "have to do clean up")); 1582 NXGE_DEBUG_MSG((NULL, TX_CTL, 1583 "==> nxge_do_softlso: " 1584 "available: %d, left: %d, " 1585 "len: %d, MBLKL(nmp): %d", 1586 available, left, len, 1587 (int)MBLKL(nmp))); 1588 goto cleanup_allocated_msgs; 1589 } 1590 } else { 1591 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1592 "==> nxge_do_softlso: " 1593 "(1)available: %d, left: %d, " 1594 "len: %d, MBLKL(nmp): %d", 1595 available, left, len, 1596 (int)MBLKL(nmp))); 1597 cmn_err(CE_PANIC, 1598 "==> nxge_do_softlso: " 1599 "Pointers must have been corrupted!\n" 1600 "datamp: $%p, nmp: $%p, rptr: $%p", 1601 (void *)datamp, 1602 (void *)nmp, 1603 (void *)rptr); 1604 } 1605 nmp->b_cont = mmp; 1606 nmp = mmp; 1607 len -= MBLKL(nmp); 1608 } 1609 if (len < 0) { 1610 nmp->b_wptr += len; 1611 rptr = nmp->b_wptr; 1612 NXGE_DEBUG_MSG((NULL, TX_CTL, 1613 "(5) len < 0 (less than 0)" 1614 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1615 available, left, len, (int)MBLKL(nmp))); 1616 1617 } else if (len == 0) { 1618 if (datamp->b_cont != NULL) { 1619 NXGE_DEBUG_MSG((NULL, TX_CTL, 1620 "(5) len == 0" 1621 "available: %d, left: %d, len: %d, " 1622 "MBLKL(nmp): %d", 1623 available, left, len, (int)MBLKL(nmp))); 1624 datamp = datamp->b_cont; 1625 rptr = datamp->b_rptr; 1626 } else { 1627 NXGE_DEBUG_MSG((NULL, TX_CTL, 1628 "(6)available b_cont == NULL : %d, " 1629 "left: %d, len: %d, MBLKL(nmp): %d", 1630 available, left, len, (int)MBLKL(nmp))); 1631 1632 VERIFY(cmp->b_next == NULL); 1633 VERIFY(left == 0); 1634 break; /* Done! */ 1635 } 1636 } 1637 cmp = cmp->b_next; 1638 1639 NXGE_DEBUG_MSG((NULL, TX_CTL, 1640 "(7) do_softlso: " 1641 "next mp in mp_chain available len != 0 : %d, " 1642 "left: %d, len: %d, MBLKL(nmp): %d", 1643 available, left, len, (int)MBLKL(nmp))); 1644 } 1645 1646 /* 1647 * From now, start to fill up all headers for the first message 1648 * Hardware checksum flags need to be updated separately for FULLCKSUM 1649 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1650 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1651 * required fields need to be updated properly. 1652 */ 1653 nmp = mp_chain; 1654 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1655 nmp->b_wptr = nmp->b_rptr + hdrlen; 1656 niph = (struct ip *)(nmp->b_rptr + ehlen); 1657 niph->ip_len = htons(mss + iphlen + tcphlen); 1658 ip_id = ntohs(niph->ip_id); 1659 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1660 #ifdef __sparc 1661 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1662 tcp_seq = ntohl(tcp_seq_tmp); 1663 #else 1664 tcp_seq = ntohl(ntcph->th_seq); 1665 #endif 1666 1667 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1668 1669 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1670 DB_CKSUMSTART(nmp) = start_offset; 1671 DB_CKSUMSTUFF(nmp) = stuff_offset; 1672 1673 /* calculate IP checksum and TCP pseudo header checksum */ 1674 niph->ip_sum = 0; 1675 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1676 1677 l4_len = mss + tcphlen; 1678 sum = htons(l4_len) + l4cksum; 1679 sum = (sum & 0xFFFF) + (sum >> 16); 1680 ntcph->th_sum = (sum & 0xffff); 1681 1682 NXGE_DEBUG_MSG((NULL, TX_CTL, 1683 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1684 "mss %d pktlen %d l4_len %d (0x%x) " 1685 "MBLKL(mp): %d, MBLKL(datamp): %d " 1686 "ip_sum 0x%x " 1687 "th_sum 0x%x sum 0x%x ) " 1688 "dump first ip->tcp %s", 1689 nmp, mp_chain, 1690 mss, 1691 pktlen, 1692 l4_len, 1693 l4_len, 1694 (int)MBLKL(mp), (int)MBLKL(datamp), 1695 niph->ip_sum, 1696 ntcph->th_sum, 1697 sum, 1698 nxge_dump_packet((char *)niph, 52))); 1699 1700 cmp = nmp; 1701 while ((nmp = nmp->b_next)->b_next != NULL) { 1702 NXGE_DEBUG_MSG((NULL, TX_CTL, 1703 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1704 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1705 nmp->b_wptr = nmp->b_rptr + hdrlen; 1706 niph = (struct ip *)(nmp->b_rptr + ehlen); 1707 niph->ip_id = htons(++ip_id); 1708 niph->ip_len = htons(mss + iphlen + tcphlen); 1709 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1710 tcp_seq += mss; 1711 1712 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1713 1714 #ifdef __sparc 1715 tcp_seq_tmp = htonl(tcp_seq); 1716 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1717 #else 1718 ntcph->th_seq = htonl(tcp_seq); 1719 #endif 1720 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1721 DB_CKSUMSTART(nmp) = start_offset; 1722 DB_CKSUMSTUFF(nmp) = stuff_offset; 1723 1724 /* calculate IP checksum and TCP pseudo header checksum */ 1725 niph->ip_sum = 0; 1726 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1727 ntcph->th_sum = (sum & 0xffff); 1728 1729 NXGE_DEBUG_MSG((NULL, TX_CTL, 1730 "==> nxge_do_softlso: middle ip_sum 0x%x " 1731 "th_sum 0x%x " 1732 " mp $%p (mp_chain $%p) pktlen %d " 1733 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1734 niph->ip_sum, 1735 ntcph->th_sum, 1736 nmp, mp_chain, 1737 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1738 } 1739 1740 /* Last segment */ 1741 /* 1742 * Set FIN and/or PSH flags if present only in the last packet. 1743 * The ip_len could be different from prior packets. 1744 */ 1745 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1746 nmp->b_wptr = nmp->b_rptr + hdrlen; 1747 niph = (struct ip *)(nmp->b_rptr + ehlen); 1748 niph->ip_id = htons(++ip_id); 1749 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1750 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1751 tcp_seq += mss; 1752 #ifdef __sparc 1753 tcp_seq_tmp = htonl(tcp_seq); 1754 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1755 #else 1756 ntcph->th_seq = htonl(tcp_seq); 1757 #endif 1758 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1759 1760 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1761 DB_CKSUMSTART(nmp) = start_offset; 1762 DB_CKSUMSTUFF(nmp) = stuff_offset; 1763 1764 /* calculate IP checksum and TCP pseudo header checksum */ 1765 niph->ip_sum = 0; 1766 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1767 1768 l4_len = ntohs(niph->ip_len) - iphlen; 1769 sum = htons(l4_len) + l4cksum; 1770 sum = (sum & 0xFFFF) + (sum >> 16); 1771 ntcph->th_sum = (sum & 0xffff); 1772 1773 NXGE_DEBUG_MSG((NULL, TX_CTL, 1774 "==> nxge_do_softlso: last next " 1775 "niph->ip_sum 0x%x " 1776 "ntcph->th_sum 0x%x sum 0x%x " 1777 "dump last ip->tcp %s " 1778 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1779 "l4_len %d (0x%x) " 1780 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1781 niph->ip_sum, 1782 ntcph->th_sum, sum, 1783 nxge_dump_packet((char *)niph, 52), 1784 cmp, nmp, mp_chain, 1785 pktlen, pktlen, 1786 l4_len, 1787 l4_len, 1788 (int)MBLKL(mp), (int)MBLKL(datamp))); 1789 1790 cleanup_allocated_msgs: 1791 if (do_cleanup) { 1792 NXGE_DEBUG_MSG((NULL, TX_CTL, 1793 "==> nxge_do_softlso: " 1794 "Failed allocating messages, " 1795 "have to clean up and fail!")); 1796 while (mp_chain != NULL) { 1797 nmp = mp_chain; 1798 mp_chain = mp_chain->b_next; 1799 freemsg(nmp); 1800 } 1801 } 1802 /* 1803 * We're done here, so just free the original message and return the 1804 * new message chain, that could be NULL if failed, back to the caller. 1805 */ 1806 freemsg(mp); 1807 1808 NXGE_DEBUG_MSG((NULL, TX_CTL, 1809 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1810 return (mp_chain); 1811 } 1812 1813 /* 1814 * Will be called before NIC driver do further operation on the message. 1815 * The input message may include LSO information, if so, go to softlso logic 1816 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1817 * The return could be the same non-LSO message or a message chain for LSO case. 1818 * 1819 * The driver needs to call this function per packet and process the whole chain 1820 * if applied. 1821 */ 1822 static mblk_t * 1823 nxge_lso_eliminate(mblk_t *mp) 1824 { 1825 uint32_t lsoflags; 1826 uint32_t mss; 1827 1828 NXGE_DEBUG_MSG((NULL, TX_CTL, 1829 "==>nxge_lso_eliminate:")); 1830 nxge_lso_info_get(mp, &mss, &lsoflags); 1831 1832 if (lsoflags & HW_LSO) { 1833 mblk_t *nmp; 1834 1835 NXGE_DEBUG_MSG((NULL, TX_CTL, 1836 "==>nxge_lso_eliminate:" 1837 "HW_LSO:mss %d mp $%p", 1838 mss, mp)); 1839 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1840 NXGE_DEBUG_MSG((NULL, TX_CTL, 1841 "<== nxge_lso_eliminate: " 1842 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1843 nmp, mss, mp)); 1844 return (nmp); 1845 } else { 1846 NXGE_DEBUG_MSG((NULL, TX_CTL, 1847 "<== nxge_lso_eliminate_ " 1848 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1849 nmp, mss, mp)); 1850 return (NULL); 1851 } 1852 } 1853 1854 NXGE_DEBUG_MSG((NULL, TX_CTL, 1855 "<== nxge_lso_eliminate")); 1856 return (mp); 1857 } 1858 1859 static uint32_t 1860 nxge_csgen(uint16_t *adr, int len) 1861 { 1862 int i, odd; 1863 uint32_t sum = 0; 1864 uint32_t c = 0; 1865 1866 odd = len % 2; 1867 for (i = 0; i < (len / 2); i++) { 1868 sum += (adr[i] & 0xffff); 1869 } 1870 if (odd) { 1871 sum += adr[len / 2] & 0xff00; 1872 } 1873 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 1874 sum &= 0xffff; 1875 sum += c; 1876 } 1877 return (~sum & 0xffff); 1878 } 1879