1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/nxge/nxge_impl.h> 29 30 /* Software LSO required header files */ 31 #include <netinet/tcp.h> 32 #include <inet/ip_impl.h> 33 #include <inet/tcp.h> 34 35 static mblk_t *nxge_lso_eliminate(mblk_t *); 36 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 37 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 38 static void nxge_hcksum_retrieve(mblk_t *, 39 uint32_t *, uint32_t *, uint32_t *, 40 uint32_t *, uint32_t *); 41 static uint32_t nxge_csgen(uint16_t *, int); 42 43 extern uint32_t nxge_reclaim_pending; 44 extern uint32_t nxge_bcopy_thresh; 45 extern uint32_t nxge_dvma_thresh; 46 extern uint32_t nxge_dma_stream_thresh; 47 extern uint32_t nxge_tx_minfree; 48 extern uint32_t nxge_tx_intr_thres; 49 extern uint32_t nxge_tx_max_gathers; 50 extern uint32_t nxge_tx_tiny_pack; 51 extern uint32_t nxge_tx_use_bcopy; 52 extern uint32_t nxge_tx_lb_policy; 53 extern uint32_t nxge_no_tx_lb; 54 extern nxge_tx_mode_t nxge_tx_scheme; 55 uint32_t nxge_lso_kick_cnt = 2; 56 57 typedef struct _mac_tx_hint { 58 uint16_t sap; 59 uint16_t vid; 60 void *hash; 61 } mac_tx_hint_t, *p_mac_tx_hint_t; 62 63 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 64 65 int 66 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 67 { 68 int status = 0; 69 p_tx_desc_t tx_desc_ring_vp; 70 npi_handle_t npi_desc_handle; 71 nxge_os_dma_handle_t tx_desc_dma_handle; 72 p_tx_desc_t tx_desc_p; 73 p_tx_msg_t tx_msg_ring; 74 p_tx_msg_t tx_msg_p; 75 tx_desc_t tx_desc, *tmp_desc_p; 76 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 77 p_tx_pkt_header_t hdrp; 78 p_tx_pkt_hdr_all_t pkthdrp; 79 uint8_t npads = 0; 80 uint64_t dma_ioaddr; 81 uint32_t dma_flags; 82 int last_bidx; 83 uint8_t *b_rptr; 84 caddr_t kaddr; 85 uint32_t nmblks; 86 uint32_t ngathers; 87 uint32_t clen; 88 int len; 89 uint32_t pkt_len, pack_len, min_len; 90 uint32_t bcopy_thresh; 91 int i, cur_index, sop_index; 92 uint16_t tail_index; 93 boolean_t tail_wrap = B_FALSE; 94 nxge_dma_common_t desc_area; 95 nxge_os_dma_handle_t dma_handle; 96 ddi_dma_cookie_t dma_cookie; 97 npi_handle_t npi_handle; 98 p_mblk_t nmp; 99 p_mblk_t t_mp; 100 uint32_t ncookies; 101 boolean_t good_packet; 102 boolean_t mark_mode = B_FALSE; 103 p_nxge_stats_t statsp; 104 p_nxge_tx_ring_stats_t tdc_stats; 105 t_uscalar_t start_offset = 0; 106 t_uscalar_t stuff_offset = 0; 107 t_uscalar_t end_offset = 0; 108 t_uscalar_t value = 0; 109 t_uscalar_t cksum_flags = 0; 110 boolean_t cksum_on = B_FALSE; 111 uint32_t boff = 0; 112 uint64_t tot_xfer_len = 0, tmp_len = 0; 113 boolean_t header_set = B_FALSE; 114 #ifdef NXGE_DEBUG 115 p_tx_desc_t tx_desc_ring_pp; 116 p_tx_desc_t tx_desc_pp; 117 tx_desc_t *save_desc_p; 118 int dump_len; 119 int sad_len; 120 uint64_t sad; 121 int xfer_len; 122 uint32_t msgsize; 123 #endif 124 p_mblk_t mp_chain = NULL; 125 boolean_t is_lso = B_FALSE; 126 boolean_t lso_again; 127 int cur_index_lso; 128 p_mblk_t nmp_lso_save; 129 uint32_t lso_ngathers; 130 boolean_t lso_tail_wrap = B_FALSE; 131 132 NXGE_DEBUG_MSG((nxgep, TX_CTL, 133 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 134 NXGE_DEBUG_MSG((nxgep, TX_CTL, 135 "==> nxge_start: Starting tdc %d desc pending %d", 136 tx_ring_p->tdc, tx_ring_p->descs_pending)); 137 138 statsp = nxgep->statsp; 139 140 switch (nxgep->mac.portmode) { 141 default: 142 if (nxgep->statsp->port_stats.lb_mode == nxge_lb_normal) { 143 if (!statsp->mac_stats.link_up) { 144 freemsg(mp); 145 NXGE_DEBUG_MSG((nxgep, TX_CTL, 146 "==> nxge_start: " 147 "link not up")); 148 goto nxge_start_fail1; 149 } 150 } 151 break; 152 case PORT_10G_FIBER: 153 /* 154 * For the following modes, check the link status 155 * before sending the packet out: 156 * nxge_lb_normal, nxge_lb_ext10g, nxge_lb_phy10g 157 */ 158 if (nxgep->statsp->port_stats.lb_mode < nxge_lb_serdes10g) { 159 if (!statsp->mac_stats.link_up) { 160 freemsg(mp); 161 NXGE_DEBUG_MSG((nxgep, TX_CTL, 162 "==> nxge_start: " 163 "link not up")); 164 goto nxge_start_fail1; 165 } 166 } 167 break; 168 } 169 170 if (nxgep->soft_lso_enable) { 171 mp_chain = nxge_lso_eliminate(mp); 172 NXGE_DEBUG_MSG((nxgep, TX_CTL, 173 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 174 mp, mp_chain)); 175 if (mp_chain == NULL) { 176 NXGE_ERROR_MSG((nxgep, TX_CTL, 177 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 178 mp_chain, mp)); 179 goto nxge_start_fail1; 180 } 181 if (mp_chain != mp) { 182 NXGE_DEBUG_MSG((nxgep, TX_CTL, 183 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 184 mp_chain, mp)); 185 is_lso = B_TRUE; 186 mp = mp_chain; 187 mp_chain = mp_chain->b_next; 188 mp->b_next = NULL; 189 } 190 } 191 192 hcksum_retrieve(mp, NULL, NULL, &start_offset, 193 &stuff_offset, &end_offset, &value, &cksum_flags); 194 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 195 start_offset += sizeof (ether_header_t); 196 stuff_offset += sizeof (ether_header_t); 197 } else { 198 start_offset += sizeof (struct ether_vlan_header); 199 stuff_offset += sizeof (struct ether_vlan_header); 200 } 201 202 if (cksum_flags & HCK_PARTIALCKSUM) { 203 NXGE_DEBUG_MSG((nxgep, TX_CTL, 204 "==> nxge_start: mp $%p len %d " 205 "cksum_flags 0x%x (partial checksum) ", 206 mp, MBLKL(mp), cksum_flags)); 207 cksum_on = B_TRUE; 208 } 209 210 lso_again = B_FALSE; 211 lso_ngathers = 0; 212 213 MUTEX_ENTER(&tx_ring_p->lock); 214 cur_index_lso = tx_ring_p->wr_index; 215 lso_tail_wrap = tx_ring_p->wr_index_wrap; 216 start_again: 217 ngathers = 0; 218 sop_index = tx_ring_p->wr_index; 219 #ifdef NXGE_DEBUG 220 if (tx_ring_p->descs_pending) { 221 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 222 "desc pending %d ", tx_ring_p->descs_pending)); 223 } 224 225 dump_len = (int)(MBLKL(mp)); 226 dump_len = (dump_len > 128) ? 128: dump_len; 227 228 NXGE_DEBUG_MSG((nxgep, TX_CTL, 229 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 230 "(Before header reserve: ORIGINAL LEN %d)", 231 tx_ring_p->tdc, 232 mp->b_rptr, 233 dump_len)); 234 235 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 236 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 237 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 238 #endif 239 240 tdc_stats = tx_ring_p->tdc_stats; 241 mark_mode = (tx_ring_p->descs_pending && 242 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 243 < nxge_tx_minfree)); 244 245 NXGE_DEBUG_MSG((nxgep, TX_CTL, 246 "TX Descriptor ring is channel %d mark mode %d", 247 tx_ring_p->tdc, mark_mode)); 248 249 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { 250 NXGE_DEBUG_MSG((nxgep, TX_CTL, 251 "TX Descriptor ring is full: channel %d", 252 tx_ring_p->tdc)); 253 NXGE_DEBUG_MSG((nxgep, TX_CTL, 254 "TX Descriptor ring is full: channel %d", 255 tx_ring_p->tdc)); 256 if (is_lso) { 257 /* free the current mp and mp_chain if not FULL */ 258 tdc_stats->tx_no_desc++; 259 NXGE_DEBUG_MSG((nxgep, TX_CTL, 260 "LSO packet: TX Descriptor ring is full: " 261 "channel %d", 262 tx_ring_p->tdc)); 263 goto nxge_start_fail_lso; 264 } else { 265 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 266 tdc_stats->tx_no_desc++; 267 MUTEX_EXIT(&tx_ring_p->lock); 268 if (nxgep->resched_needed && !nxgep->resched_running) { 269 nxgep->resched_running = B_TRUE; 270 ddi_trigger_softintr(nxgep->resched_id); 271 } 272 status = 1; 273 goto nxge_start_fail1; 274 } 275 } 276 277 nmp = mp; 278 i = sop_index = tx_ring_p->wr_index; 279 nmblks = 0; 280 ngathers = 0; 281 pkt_len = 0; 282 pack_len = 0; 283 clen = 0; 284 last_bidx = -1; 285 good_packet = B_TRUE; 286 287 desc_area = tx_ring_p->tdc_desc; 288 npi_handle = desc_area.npi_handle; 289 npi_desc_handle.regh = (nxge_os_acc_handle_t) 290 DMA_COMMON_ACC_HANDLE(desc_area); 291 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 292 #ifdef NXGE_DEBUG 293 tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area); 294 #endif 295 tx_desc_dma_handle = (nxge_os_dma_handle_t) 296 DMA_COMMON_HANDLE(desc_area); 297 tx_msg_ring = tx_ring_p->tx_msg_ring; 298 299 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 300 sop_index, i)); 301 302 #ifdef NXGE_DEBUG 303 msgsize = msgdsize(nmp); 304 NXGE_DEBUG_MSG((nxgep, TX_CTL, 305 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 306 sop_index, i, msgsize)); 307 #endif 308 /* 309 * The first 16 bytes of the premapped buffer are reserved 310 * for header. No padding will be used. 311 */ 312 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 313 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 314 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 315 } else { 316 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 317 } 318 while (nmp) { 319 good_packet = B_TRUE; 320 b_rptr = nmp->b_rptr; 321 len = MBLKL(nmp); 322 if (len <= 0) { 323 nmp = nmp->b_cont; 324 continue; 325 } 326 nmblks++; 327 328 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 329 "len %d pkt_len %d pack_len %d", 330 nmblks, len, pkt_len, pack_len)); 331 /* 332 * Hardware limits the transfer length to 4K for NIU and 333 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 334 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 335 * If len is longer than the limit, then we break nmp into 336 * two chunks: Make the first chunk equal to the limit and 337 * the second chunk for the remaining data. If the second 338 * chunk is still larger than the limit, then it will be 339 * broken into two in the next pass. 340 */ 341 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 342 if ((t_mp = dupb(nmp)) != NULL) { 343 nmp->b_wptr = nmp->b_rptr + 344 (TX_MAX_TRANSFER_LENGTH 345 - TX_PKT_HEADER_SIZE); 346 t_mp->b_rptr = nmp->b_wptr; 347 t_mp->b_cont = nmp->b_cont; 348 nmp->b_cont = t_mp; 349 len = MBLKL(nmp); 350 } else { 351 if (is_lso) { 352 NXGE_DEBUG_MSG((nxgep, TX_CTL, 353 "LSO packet: dupb failed: " 354 "channel %d", 355 tx_ring_p->tdc)); 356 mp = nmp; 357 goto nxge_start_fail_lso; 358 } else { 359 good_packet = B_FALSE; 360 goto nxge_start_fail2; 361 } 362 } 363 } 364 tx_desc.value = 0; 365 tx_desc_p = &tx_desc_ring_vp[i]; 366 #ifdef NXGE_DEBUG 367 tx_desc_pp = &tx_desc_ring_pp[i]; 368 #endif 369 tx_msg_p = &tx_msg_ring[i]; 370 #if defined(__i386) 371 npi_desc_handle.regp = (uint32_t)tx_desc_p; 372 #else 373 npi_desc_handle.regp = (uint64_t)tx_desc_p; 374 #endif 375 if (!header_set && 376 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 377 (len >= bcopy_thresh))) { 378 header_set = B_TRUE; 379 bcopy_thresh += TX_PKT_HEADER_SIZE; 380 boff = 0; 381 pack_len = 0; 382 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 383 hdrp = (p_tx_pkt_header_t)kaddr; 384 clen = pkt_len; 385 dma_handle = tx_msg_p->buf_dma_handle; 386 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 387 (void) ddi_dma_sync(dma_handle, 388 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 389 DDI_DMA_SYNC_FORDEV); 390 391 tx_msg_p->flags.dma_type = USE_BCOPY; 392 goto nxge_start_control_header_only; 393 } 394 395 pkt_len += len; 396 pack_len += len; 397 398 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 399 "desc entry %d " 400 "DESC IOADDR $%p " 401 "desc_vp $%p tx_desc_p $%p " 402 "desc_pp $%p tx_desc_pp $%p " 403 "len %d pkt_len %d pack_len %d", 404 i, 405 DMA_COMMON_IOADDR(desc_area), 406 tx_desc_ring_vp, tx_desc_p, 407 tx_desc_ring_pp, tx_desc_pp, 408 len, pkt_len, pack_len)); 409 410 if (len < bcopy_thresh) { 411 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 412 "USE BCOPY: ")); 413 if (nxge_tx_tiny_pack) { 414 uint32_t blst = 415 TXDMA_DESC_NEXT_INDEX(i, -1, 416 tx_ring_p->tx_wrap_mask); 417 NXGE_DEBUG_MSG((nxgep, TX_CTL, 418 "==> nxge_start(5): pack")); 419 if ((pack_len <= bcopy_thresh) && 420 (last_bidx == blst)) { 421 NXGE_DEBUG_MSG((nxgep, TX_CTL, 422 "==> nxge_start: pack(6) " 423 "(pkt_len %d pack_len %d)", 424 pkt_len, pack_len)); 425 i = blst; 426 tx_desc_p = &tx_desc_ring_vp[i]; 427 #ifdef NXGE_DEBUG 428 tx_desc_pp = &tx_desc_ring_pp[i]; 429 #endif 430 tx_msg_p = &tx_msg_ring[i]; 431 boff = pack_len - len; 432 ngathers--; 433 } else if (pack_len > bcopy_thresh && 434 header_set) { 435 pack_len = len; 436 boff = 0; 437 bcopy_thresh = nxge_bcopy_thresh; 438 NXGE_DEBUG_MSG((nxgep, TX_CTL, 439 "==> nxge_start(7): > max NEW " 440 "bcopy thresh %d " 441 "pkt_len %d pack_len %d(next)", 442 bcopy_thresh, 443 pkt_len, pack_len)); 444 } 445 last_bidx = i; 446 } 447 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 448 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 449 hdrp = (p_tx_pkt_header_t)kaddr; 450 header_set = B_TRUE; 451 NXGE_DEBUG_MSG((nxgep, TX_CTL, 452 "==> nxge_start(7_x2): " 453 "pkt_len %d pack_len %d (new hdrp $%p)", 454 pkt_len, pack_len, hdrp)); 455 } 456 tx_msg_p->flags.dma_type = USE_BCOPY; 457 kaddr += boff; 458 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 459 "USE BCOPY: before bcopy " 460 "DESC IOADDR $%p entry %d " 461 "bcopy packets %d " 462 "bcopy kaddr $%p " 463 "bcopy ioaddr (SAD) $%p " 464 "bcopy clen %d " 465 "bcopy boff %d", 466 DMA_COMMON_IOADDR(desc_area), i, 467 tdc_stats->tx_hdr_pkts, 468 kaddr, 469 dma_ioaddr, 470 clen, 471 boff)); 472 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 473 "1USE BCOPY: ")); 474 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 475 "2USE BCOPY: ")); 476 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 477 "last USE BCOPY: copy from b_rptr $%p " 478 "to KADDR $%p (len %d offset %d", 479 b_rptr, kaddr, len, boff)); 480 481 bcopy(b_rptr, kaddr, len); 482 483 #ifdef NXGE_DEBUG 484 dump_len = (len > 128) ? 128: len; 485 NXGE_DEBUG_MSG((nxgep, TX_CTL, 486 "==> nxge_start: dump packets " 487 "(After BCOPY len %d)" 488 "(b_rptr $%p): %s", len, nmp->b_rptr, 489 nxge_dump_packet((char *)nmp->b_rptr, 490 dump_len))); 491 #endif 492 493 dma_handle = tx_msg_p->buf_dma_handle; 494 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 495 (void) ddi_dma_sync(dma_handle, 496 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 497 DDI_DMA_SYNC_FORDEV); 498 clen = len + boff; 499 tdc_stats->tx_hdr_pkts++; 500 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 501 "USE BCOPY: " 502 "DESC IOADDR $%p entry %d " 503 "bcopy packets %d " 504 "bcopy kaddr $%p " 505 "bcopy ioaddr (SAD) $%p " 506 "bcopy clen %d " 507 "bcopy boff %d", 508 DMA_COMMON_IOADDR(desc_area), 509 i, 510 tdc_stats->tx_hdr_pkts, 511 kaddr, 512 dma_ioaddr, 513 clen, 514 boff)); 515 } else { 516 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 517 "USE DVMA: len %d", len)); 518 tx_msg_p->flags.dma_type = USE_DMA; 519 dma_flags = DDI_DMA_WRITE; 520 if (len < nxge_dma_stream_thresh) { 521 dma_flags |= DDI_DMA_CONSISTENT; 522 } else { 523 dma_flags |= DDI_DMA_STREAMING; 524 } 525 526 dma_handle = tx_msg_p->dma_handle; 527 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 528 (caddr_t)b_rptr, len, dma_flags, 529 DDI_DMA_DONTWAIT, NULL, 530 &dma_cookie, &ncookies); 531 if (status == DDI_DMA_MAPPED) { 532 dma_ioaddr = dma_cookie.dmac_laddress; 533 len = (int)dma_cookie.dmac_size; 534 clen = (uint32_t)dma_cookie.dmac_size; 535 NXGE_DEBUG_MSG((nxgep, TX_CTL, 536 "==> nxge_start(12_1): " 537 "USE DVMA: len %d clen %d " 538 "ngathers %d", 539 len, clen, 540 ngathers)); 541 #if defined(__i386) 542 npi_desc_handle.regp = (uint32_t)tx_desc_p; 543 #else 544 npi_desc_handle.regp = (uint64_t)tx_desc_p; 545 #endif 546 while (ncookies > 1) { 547 ngathers++; 548 /* 549 * this is the fix for multiple 550 * cookies, which are basically 551 * a descriptor entry, we don't set 552 * SOP bit as well as related fields 553 */ 554 555 (void) npi_txdma_desc_gather_set( 556 npi_desc_handle, 557 &tx_desc, 558 (ngathers -1), 559 mark_mode, 560 ngathers, 561 dma_ioaddr, 562 clen); 563 564 tx_msg_p->tx_msg_size = clen; 565 NXGE_DEBUG_MSG((nxgep, TX_CTL, 566 "==> nxge_start: DMA " 567 "ncookie %d " 568 "ngathers %d " 569 "dma_ioaddr $%p len %d" 570 "desc $%p descp $%p (%d)", 571 ncookies, 572 ngathers, 573 dma_ioaddr, clen, 574 *tx_desc_p, tx_desc_p, i)); 575 576 ddi_dma_nextcookie(dma_handle, 577 &dma_cookie); 578 dma_ioaddr = 579 dma_cookie.dmac_laddress; 580 581 len = (int)dma_cookie.dmac_size; 582 clen = (uint32_t)dma_cookie.dmac_size; 583 NXGE_DEBUG_MSG((nxgep, TX_CTL, 584 "==> nxge_start(12_2): " 585 "USE DVMA: len %d clen %d ", 586 len, clen)); 587 588 i = TXDMA_DESC_NEXT_INDEX(i, 1, 589 tx_ring_p->tx_wrap_mask); 590 tx_desc_p = &tx_desc_ring_vp[i]; 591 592 npi_desc_handle.regp = 593 #if defined(__i386) 594 (uint32_t)tx_desc_p; 595 #else 596 (uint64_t)tx_desc_p; 597 #endif 598 tx_msg_p = &tx_msg_ring[i]; 599 tx_msg_p->flags.dma_type = USE_NONE; 600 tx_desc.value = 0; 601 602 ncookies--; 603 } 604 tdc_stats->tx_ddi_pkts++; 605 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 606 "DMA: ddi packets %d", 607 tdc_stats->tx_ddi_pkts)); 608 } else { 609 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 610 "dma mapping failed for %d " 611 "bytes addr $%p flags %x (%d)", 612 len, b_rptr, status, status)); 613 good_packet = B_FALSE; 614 tdc_stats->tx_dma_bind_fail++; 615 tx_msg_p->flags.dma_type = USE_NONE; 616 if (is_lso) { 617 mp = nmp; 618 goto nxge_start_fail_lso; 619 } else { 620 goto nxge_start_fail2; 621 } 622 } 623 } /* ddi dvma */ 624 625 if (is_lso) { 626 nmp_lso_save = nmp; 627 } 628 nmp = nmp->b_cont; 629 nxge_start_control_header_only: 630 #if defined(__i386) 631 npi_desc_handle.regp = (uint32_t)tx_desc_p; 632 #else 633 npi_desc_handle.regp = (uint64_t)tx_desc_p; 634 #endif 635 ngathers++; 636 637 if (ngathers == 1) { 638 #ifdef NXGE_DEBUG 639 save_desc_p = &sop_tx_desc; 640 #endif 641 sop_tx_desc_p = &sop_tx_desc; 642 sop_tx_desc_p->value = 0; 643 sop_tx_desc_p->bits.hdw.tr_len = clen; 644 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 645 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 646 } else { 647 #ifdef NXGE_DEBUG 648 save_desc_p = &tx_desc; 649 #endif 650 tmp_desc_p = &tx_desc; 651 tmp_desc_p->value = 0; 652 tmp_desc_p->bits.hdw.tr_len = clen; 653 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 654 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 655 656 tx_desc_p->value = tmp_desc_p->value; 657 } 658 659 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 660 "Desc_entry %d ngathers %d " 661 "desc_vp $%p tx_desc_p $%p " 662 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 663 "dma_ioaddr (SAD) $%p mark %d", 664 i, ngathers, 665 tx_desc_ring_vp, tx_desc_p, 666 len, clen, pkt_len, pack_len, nmblks, 667 dma_ioaddr, mark_mode)); 668 669 #ifdef NXGE_DEBUG 670 npi_desc_handle.nxgep = nxgep; 671 npi_desc_handle.function.function = nxgep->function_num; 672 npi_desc_handle.function.instance = nxgep->instance; 673 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 674 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 675 TX_PKT_DESC_TR_LEN_SHIFT); 676 677 678 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 679 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 680 "mark %d sop %d\n", 681 save_desc_p->value, 682 sad, 683 save_desc_p->bits.hdw.tr_len, 684 xfer_len, 685 save_desc_p->bits.hdw.num_ptr, 686 save_desc_p->bits.hdw.mark, 687 save_desc_p->bits.hdw.sop)); 688 689 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 690 #endif 691 692 tx_msg_p->tx_msg_size = clen; 693 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 694 if (ngathers > nxge_tx_max_gathers) { 695 good_packet = B_FALSE; 696 hcksum_retrieve(mp, NULL, NULL, &start_offset, 697 &stuff_offset, &end_offset, &value, 698 &cksum_flags); 699 700 NXGE_DEBUG_MSG((NULL, TX_CTL, 701 "==> nxge_start(14): pull msg - " 702 "len %d pkt_len %d ngathers %d", 703 len, pkt_len, ngathers)); 704 /* Pull all message blocks from b_cont */ 705 if (is_lso) { 706 mp = nmp_lso_save; 707 goto nxge_start_fail_lso; 708 } 709 if ((msgpullup(mp, -1)) == NULL) { 710 goto nxge_start_fail2; 711 } 712 goto nxge_start_fail2; 713 } 714 } /* while (nmp) */ 715 716 tx_msg_p->tx_message = mp; 717 tx_desc_p = &tx_desc_ring_vp[sop_index]; 718 #if defined(__i386) 719 npi_desc_handle.regp = (uint32_t)tx_desc_p; 720 #else 721 npi_desc_handle.regp = (uint64_t)tx_desc_p; 722 #endif 723 724 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 725 pkthdrp->reserved = 0; 726 hdrp->value = 0; 727 (void) nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 728 (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp); 729 730 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 731 tdc_stats->tx_jumbo_pkts++; 732 } 733 734 min_len = (nxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2)); 735 if (pkt_len < min_len) { 736 /* Assume we use bcopy to premapped buffers */ 737 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 738 NXGE_DEBUG_MSG((NULL, TX_CTL, 739 "==> nxge_start(14-1): < (msg_min + 16)" 740 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 741 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 742 bzero((kaddr + pkt_len), (min_len - pkt_len)); 743 pkt_len = tx_msg_p->tx_msg_size = min_len; 744 745 sop_tx_desc_p->bits.hdw.tr_len = min_len; 746 747 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 748 tx_desc_p->value = sop_tx_desc_p->value; 749 750 NXGE_DEBUG_MSG((NULL, TX_CTL, 751 "==> nxge_start(14-2): < msg_min - " 752 "len %d pkt_len %d min_len %d ngathers %d", 753 len, pkt_len, min_len, ngathers)); 754 } 755 756 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 757 cksum_flags)); 758 if (cksum_flags & HCK_PARTIALCKSUM) { 759 NXGE_DEBUG_MSG((nxgep, TX_CTL, 760 "==> nxge_start: cksum_flags 0x%x (partial checksum) ", 761 cksum_flags)); 762 cksum_on = B_TRUE; 763 NXGE_DEBUG_MSG((nxgep, TX_CTL, 764 "==> nxge_start: from IP cksum_flags 0x%x " 765 "(partial checksum) " 766 "start_offset %d stuff_offset %d", 767 cksum_flags, start_offset, stuff_offset)); 768 tmp_len = (uint64_t)(start_offset >> 1); 769 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT); 770 tmp_len = (uint64_t)(stuff_offset >> 1); 771 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT); 772 773 NXGE_DEBUG_MSG((nxgep, TX_CTL, 774 "==> nxge_start: from IP cksum_flags 0x%x " 775 "(partial checksum) " 776 "after SHIFT start_offset %d stuff_offset %d", 777 cksum_flags, start_offset, stuff_offset)); 778 } 779 { 780 uint64_t tmp_len; 781 782 /* pkt_len already includes 16 + paddings!! */ 783 /* Update the control header length */ 784 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 785 tmp_len = hdrp->value | 786 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 787 788 NXGE_DEBUG_MSG((nxgep, TX_CTL, 789 "==> nxge_start(15_x1): setting SOP " 790 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 791 "0x%llx hdrp->value 0x%llx", 792 tot_xfer_len, tot_xfer_len, pkt_len, 793 tmp_len, hdrp->value)); 794 #if defined(_BIG_ENDIAN) 795 hdrp->value = ddi_swap64(tmp_len); 796 #else 797 hdrp->value = tmp_len; 798 #endif 799 NXGE_DEBUG_MSG((nxgep, 800 TX_CTL, "==> nxge_start(15_x2): setting SOP " 801 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 802 "tmp_len 0x%llx hdrp->value 0x%llx", 803 tot_xfer_len, pkt_len, 804 tmp_len, hdrp->value)); 805 } 806 807 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 808 "wr_index %d " 809 "tot_xfer_len (%d) pkt_len %d npads %d", 810 sop_index, 811 tot_xfer_len, pkt_len, 812 npads)); 813 814 sop_tx_desc_p->bits.hdw.sop = 1; 815 sop_tx_desc_p->bits.hdw.mark = mark_mode; 816 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 817 818 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 819 820 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 821 822 #ifdef NXGE_DEBUG 823 npi_desc_handle.nxgep = nxgep; 824 npi_desc_handle.function.function = nxgep->function_num; 825 npi_desc_handle.function.instance = nxgep->instance; 826 827 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 828 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 829 save_desc_p->value, 830 sad, 831 save_desc_p->bits.hdw.tr_len, 832 xfer_len, 833 save_desc_p->bits.hdw.num_ptr, 834 save_desc_p->bits.hdw.mark, 835 save_desc_p->bits.hdw.sop)); 836 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 837 838 dump_len = (pkt_len > 128) ? 128: pkt_len; 839 NXGE_DEBUG_MSG((nxgep, TX_CTL, 840 "==> nxge_start: dump packets(17) (after sop set, len " 841 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 842 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 843 (char *)hdrp, 844 nxge_dump_packet((char *)hdrp, dump_len))); 845 NXGE_DEBUG_MSG((nxgep, TX_CTL, 846 "==> nxge_start(18): TX desc sync: sop_index %d", 847 sop_index)); 848 #endif 849 850 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 851 (void) ddi_dma_sync(tx_desc_dma_handle, 852 sop_index * sizeof (tx_desc_t), 853 ngathers * sizeof (tx_desc_t), 854 DDI_DMA_SYNC_FORDEV); 855 856 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 857 "cs_off = 0x%02X cs_s_off = 0x%02X " 858 "pkt_len %d ngathers %d sop_index %d\n", 859 stuff_offset, start_offset, 860 pkt_len, ngathers, sop_index)); 861 } else { /* more than one descriptor and wrap around */ 862 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 863 (void) ddi_dma_sync(tx_desc_dma_handle, 864 sop_index * sizeof (tx_desc_t), 865 nsdescs * sizeof (tx_desc_t), 866 DDI_DMA_SYNC_FORDEV); 867 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 868 "cs_off = 0x%02X cs_s_off = 0x%02X " 869 "pkt_len %d ngathers %d sop_index %d\n", 870 stuff_offset, start_offset, 871 pkt_len, ngathers, sop_index)); 872 873 (void) ddi_dma_sync(tx_desc_dma_handle, 874 0, 875 (ngathers - nsdescs) * sizeof (tx_desc_t), 876 DDI_DMA_SYNC_FORDEV); 877 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 878 "cs_off = 0x%02X cs_s_off = 0x%02X " 879 "pkt_len %d ngathers %d sop_index %d\n", 880 stuff_offset, start_offset, 881 pkt_len, ngathers, sop_index)); 882 } 883 884 tail_index = tx_ring_p->wr_index; 885 tail_wrap = tx_ring_p->wr_index_wrap; 886 887 tx_ring_p->wr_index = i; 888 if (tx_ring_p->wr_index <= tail_index) { 889 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 890 B_FALSE : B_TRUE); 891 } 892 893 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 894 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 895 tx_ring_p->tdc, 896 tx_ring_p->wr_index, 897 tx_ring_p->wr_index_wrap, 898 ngathers, 899 tx_ring_p->descs_pending)); 900 901 if (is_lso) { 902 lso_ngathers += ngathers; 903 if (mp_chain != NULL) { 904 mp = mp_chain; 905 mp_chain = mp_chain->b_next; 906 mp->b_next = NULL; 907 if (nxge_lso_kick_cnt == lso_ngathers) { 908 tx_ring_p->descs_pending += lso_ngathers; 909 { 910 tx_ring_kick_t kick; 911 912 kick.value = 0; 913 kick.bits.ldw.wrap = 914 tx_ring_p->wr_index_wrap; 915 kick.bits.ldw.tail = 916 (uint16_t)tx_ring_p->wr_index; 917 918 /* Kick the Transmit kick register */ 919 TXDMA_REG_WRITE64( 920 NXGE_DEV_NPI_HANDLE(nxgep), 921 TX_RING_KICK_REG, 922 (uint8_t)tx_ring_p->tdc, 923 kick.value); 924 tdc_stats->tx_starts++; 925 NXGE_DEBUG_MSG((nxgep, TX_CTL, 926 "==> nxge_start: more LSO: " 927 "LSO_CNT %d", 928 lso_gathers)); 929 } 930 lso_ngathers = 0; 931 ngathers = 0; 932 cur_index_lso = sop_index = tx_ring_p->wr_index; 933 lso_tail_wrap = tx_ring_p->wr_index_wrap; 934 } 935 NXGE_DEBUG_MSG((nxgep, TX_CTL, 936 "==> nxge_start: lso again: " 937 "lso_gathers %d ngathers %d cur_index_lso %d " 938 "wr_index %d sop_index %d", 939 lso_ngathers, ngathers, cur_index_lso, 940 tx_ring_p->wr_index, sop_index)); 941 942 NXGE_DEBUG_MSG((nxgep, TX_CTL, 943 "==> nxge_start: next : count %d", 944 lso_gathers)); 945 lso_again = B_TRUE; 946 goto start_again; 947 } 948 ngathers = lso_ngathers; 949 } 950 951 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 952 953 { 954 tx_ring_kick_t kick; 955 956 kick.value = 0; 957 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 958 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 959 960 /* Kick start the Transmit kick register */ 961 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 962 TX_RING_KICK_REG, 963 (uint8_t)tx_ring_p->tdc, 964 kick.value); 965 } 966 967 tx_ring_p->descs_pending += ngathers; 968 tdc_stats->tx_starts++; 969 970 MUTEX_EXIT(&tx_ring_p->lock); 971 972 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 973 974 return (status); 975 976 nxge_start_fail_lso: 977 status = 0; 978 good_packet = B_FALSE; 979 if (mp != NULL) { 980 freemsg(mp); 981 } 982 if (mp_chain != NULL) { 983 freemsg(mp_chain); 984 } 985 if (!lso_again && !ngathers) { 986 MUTEX_EXIT(&tx_ring_p->lock); 987 NXGE_DEBUG_MSG((nxgep, TX_CTL, 988 "==> nxge_start: lso exit (nothing changed)")); 989 goto nxge_start_fail1; 990 } 991 992 NXGE_DEBUG_MSG((nxgep, TX_CTL, 993 "==> nxge_start (channel %d): before lso " 994 "lso_gathers %d ngathers %d cur_index_lso %d " 995 "wr_index %d sop_index %d lso_again %d", 996 tx_ring_p->tdc, 997 lso_ngathers, ngathers, cur_index_lso, 998 tx_ring_p->wr_index, sop_index, lso_again)); 999 1000 if (lso_again) { 1001 lso_ngathers += ngathers; 1002 ngathers = lso_ngathers; 1003 sop_index = cur_index_lso; 1004 tx_ring_p->wr_index = sop_index; 1005 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1006 } 1007 1008 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1009 "==> nxge_start (channel %d): after lso " 1010 "lso_gathers %d ngathers %d cur_index_lso %d " 1011 "wr_index %d sop_index %d lso_again %d", 1012 tx_ring_p->tdc, 1013 lso_ngathers, ngathers, cur_index_lso, 1014 tx_ring_p->wr_index, sop_index, lso_again)); 1015 1016 nxge_start_fail2: 1017 if (good_packet == B_FALSE) { 1018 cur_index = sop_index; 1019 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1020 for (i = 0; i < ngathers; i++) { 1021 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1022 #if defined(__i386) 1023 npi_handle.regp = (uint32_t)tx_desc_p; 1024 #else 1025 npi_handle.regp = (uint64_t)tx_desc_p; 1026 #endif 1027 tx_msg_p = &tx_msg_ring[cur_index]; 1028 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1029 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1030 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1031 "tx_desc_p = %X index = %d", 1032 tx_desc_p, tx_ring_p->rd_index)); 1033 (void) dvma_unload(tx_msg_p->dvma_handle, 1034 0, -1); 1035 tx_msg_p->dvma_handle = NULL; 1036 if (tx_ring_p->dvma_wr_index == 1037 tx_ring_p->dvma_wrap_mask) 1038 tx_ring_p->dvma_wr_index = 0; 1039 else 1040 tx_ring_p->dvma_wr_index++; 1041 tx_ring_p->dvma_pending--; 1042 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1043 if (ddi_dma_unbind_handle( 1044 tx_msg_p->dma_handle)) { 1045 cmn_err(CE_WARN, "!nxge_start: " 1046 "ddi_dma_unbind_handle failed"); 1047 } 1048 } 1049 tx_msg_p->flags.dma_type = USE_NONE; 1050 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1051 tx_ring_p->tx_wrap_mask); 1052 1053 } 1054 1055 nxgep->resched_needed = B_TRUE; 1056 } 1057 1058 MUTEX_EXIT(&tx_ring_p->lock); 1059 1060 nxge_start_fail1: 1061 /* Add FMA to check the access handle nxge_hregh */ 1062 1063 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1064 1065 return (status); 1066 } 1067 1068 int 1069 nxge_serial_tx(mblk_t *mp, void *arg) 1070 { 1071 p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 1072 p_nxge_t nxgep = tx_ring_p->nxgep; 1073 1074 return (nxge_start(nxgep, tx_ring_p, mp)); 1075 } 1076 1077 boolean_t 1078 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 1079 { 1080 p_tx_ring_t *tx_rings; 1081 uint8_t ring_index; 1082 p_tx_ring_t tx_ring_p; 1083 1084 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 1085 1086 ASSERT(mp->b_next == NULL); 1087 1088 ring_index = nxge_tx_lb_ring_1(mp, nxgep->max_tdcs, hp); 1089 tx_rings = nxgep->tx_rings->rings; 1090 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: tx_rings $%p", 1091 tx_rings)); 1092 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: max_tdcs %d " 1093 "ring_index %d", nxgep->max_tdcs, ring_index)); 1094 1095 switch (nxge_tx_scheme) { 1096 case NXGE_USE_START: 1097 if (nxge_start(nxgep, tx_rings[ring_index], mp)) { 1098 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 1099 "ring index %d", ring_index)); 1100 return (B_FALSE); 1101 } 1102 break; 1103 1104 case NXGE_USE_SERIAL: 1105 default: 1106 tx_ring_p = tx_rings[ring_index]; 1107 nxge_serialize_enter(tx_ring_p->serial, mp); 1108 break; 1109 } 1110 1111 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 1112 ring_index)); 1113 1114 return (B_TRUE); 1115 } 1116 1117 /* 1118 * nxge_m_tx() - send a chain of packets 1119 */ 1120 mblk_t * 1121 nxge_m_tx(void *arg, mblk_t *mp) 1122 { 1123 p_nxge_t nxgep = (p_nxge_t)arg; 1124 mblk_t *next; 1125 mac_tx_hint_t hint; 1126 1127 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 1128 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1129 "==> nxge_m_tx: hardware not initialized")); 1130 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1131 "<== nxge_m_tx")); 1132 return (mp); 1133 } 1134 1135 hint.hash = NULL; 1136 hint.vid = 0; 1137 hint.sap = 0; 1138 1139 while (mp != NULL) { 1140 next = mp->b_next; 1141 mp->b_next = NULL; 1142 1143 /* 1144 * Until Nemo tx resource works, the mac driver 1145 * does the load balancing based on TCP port, 1146 * or CPU. For debugging, we use a system 1147 * configurable parameter. 1148 */ 1149 if (!nxge_send(nxgep, mp, &hint)) { 1150 mp->b_next = next; 1151 break; 1152 } 1153 1154 mp = next; 1155 1156 NXGE_DEBUG_MSG((NULL, TX_CTL, 1157 "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 1158 mp, next)); 1159 } 1160 1161 return (mp); 1162 } 1163 1164 int 1165 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 1166 { 1167 uint8_t ring_index = 0; 1168 uint8_t *tcp_port; 1169 p_mblk_t nmp; 1170 size_t mblk_len; 1171 size_t iph_len; 1172 size_t hdrs_size; 1173 uint8_t hdrs_buf[sizeof (struct ether_header) + 1174 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 1175 /* 1176 * allocate space big enough to cover 1177 * the max ip header length and the first 1178 * 4 bytes of the TCP/IP header. 1179 */ 1180 1181 boolean_t qos = B_FALSE; 1182 1183 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 1184 1185 if (hp->vid) { 1186 qos = B_TRUE; 1187 } 1188 switch (nxge_tx_lb_policy) { 1189 case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 1190 default: 1191 tcp_port = mp->b_rptr; 1192 if (!nxge_no_tx_lb && !qos && 1193 (ntohs(((p_ether_header_t)tcp_port)->ether_type) 1194 == ETHERTYPE_IP)) { 1195 nmp = mp; 1196 mblk_len = MBLKL(nmp); 1197 tcp_port = NULL; 1198 if (mblk_len > sizeof (struct ether_header) + 1199 sizeof (uint8_t)) { 1200 tcp_port = nmp->b_rptr + 1201 sizeof (struct ether_header); 1202 mblk_len -= sizeof (struct ether_header); 1203 iph_len = ((*tcp_port) & 0x0f) << 2; 1204 if (mblk_len > (iph_len + sizeof (uint32_t))) { 1205 tcp_port = nmp->b_rptr; 1206 } else { 1207 tcp_port = NULL; 1208 } 1209 } 1210 if (tcp_port == NULL) { 1211 hdrs_size = 0; 1212 ((p_ether_header_t)hdrs_buf)->ether_type = 0; 1213 while ((nmp) && (hdrs_size < 1214 sizeof (hdrs_buf))) { 1215 mblk_len = MBLKL(nmp); 1216 if (mblk_len >= 1217 (sizeof (hdrs_buf) - hdrs_size)) 1218 mblk_len = sizeof (hdrs_buf) - 1219 hdrs_size; 1220 bcopy(nmp->b_rptr, 1221 &hdrs_buf[hdrs_size], mblk_len); 1222 hdrs_size += mblk_len; 1223 nmp = nmp->b_cont; 1224 } 1225 tcp_port = hdrs_buf; 1226 } 1227 tcp_port += sizeof (ether_header_t); 1228 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1229 switch (tcp_port[9]) { 1230 case IPPROTO_TCP: 1231 case IPPROTO_UDP: 1232 case IPPROTO_ESP: 1233 tcp_port += ((*tcp_port) & 0x0f) << 2; 1234 ring_index = 1235 ((tcp_port[0] ^ 1236 tcp_port[1] ^ 1237 tcp_port[2] ^ 1238 tcp_port[3]) % maxtdcs); 1239 break; 1240 1241 case IPPROTO_AH: 1242 /* SPI starts at the 4th byte */ 1243 tcp_port += ((*tcp_port) & 0x0f) << 2; 1244 ring_index = 1245 ((tcp_port[4] ^ 1246 tcp_port[5] ^ 1247 tcp_port[6] ^ 1248 tcp_port[7]) % maxtdcs); 1249 break; 1250 1251 default: 1252 ring_index = tcp_port[19] % maxtdcs; 1253 break; 1254 } 1255 } else { /* fragmented packet */ 1256 ring_index = tcp_port[19] % maxtdcs; 1257 } 1258 } else { 1259 ring_index = mp->b_band % maxtdcs; 1260 } 1261 break; 1262 1263 case NXGE_TX_LB_HASH: 1264 if (hp->hash) { 1265 #if defined(__i386) 1266 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1267 #else 1268 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1269 #endif 1270 } else { 1271 ring_index = mp->b_band % maxtdcs; 1272 } 1273 break; 1274 1275 case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 1276 tcp_port = mp->b_rptr; 1277 ring_index = tcp_port[5] % maxtdcs; 1278 break; 1279 } 1280 1281 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 1282 1283 return (ring_index); 1284 } 1285 1286 uint_t 1287 nxge_reschedule(caddr_t arg) 1288 { 1289 p_nxge_t nxgep; 1290 1291 nxgep = (p_nxge_t)arg; 1292 1293 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 1294 1295 if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1296 nxgep->resched_needed) { 1297 mac_tx_update(nxgep->mach); 1298 nxgep->resched_needed = B_FALSE; 1299 nxgep->resched_running = B_FALSE; 1300 } 1301 1302 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 1303 return (DDI_INTR_CLAIMED); 1304 } 1305 1306 1307 /* Software LSO starts here */ 1308 static void 1309 nxge_hcksum_retrieve(mblk_t *mp, 1310 uint32_t *start, uint32_t *stuff, uint32_t *end, 1311 uint32_t *value, uint32_t *flags) 1312 { 1313 if (mp->b_datap->db_type == M_DATA) { 1314 if (flags != NULL) { 1315 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1316 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1317 HCK_FULLCKSUM_OK); 1318 if ((*flags & (HCK_PARTIALCKSUM | 1319 HCK_FULLCKSUM)) != 0) { 1320 if (value != NULL) 1321 *value = (uint32_t)DB_CKSUM16(mp); 1322 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1323 if (start != NULL) 1324 *start = 1325 (uint32_t)DB_CKSUMSTART(mp); 1326 if (stuff != NULL) 1327 *stuff = 1328 (uint32_t)DB_CKSUMSTUFF(mp); 1329 if (end != NULL) 1330 *end = 1331 (uint32_t)DB_CKSUMEND(mp); 1332 } 1333 } 1334 } 1335 } 1336 } 1337 1338 static void 1339 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1340 { 1341 ASSERT(DB_TYPE(mp) == M_DATA); 1342 1343 *mss = 0; 1344 if (flags != NULL) { 1345 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1346 if ((*flags != 0) && (mss != NULL)) { 1347 *mss = (uint32_t)DB_LSOMSS(mp); 1348 } 1349 NXGE_DEBUG_MSG((NULL, TX_CTL, 1350 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1351 *mss, *flags)); 1352 } 1353 1354 NXGE_DEBUG_MSG((NULL, TX_CTL, 1355 "<== nxge_lso_info_get: mss %d", *mss)); 1356 } 1357 1358 /* 1359 * Do Soft LSO on the oversized packet. 1360 * 1361 * 1. Create a chain of message for headers. 1362 * 2. Fill up header messages with proper information. 1363 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1364 * each new message with necessary adjustments. 1365 * * Unchange the ethernet header for DIX frames. (by default) 1366 * * IP Total Length field is updated to MSS or less(only for the last one). 1367 * * IP Identification value is incremented by one for each packet. 1368 * * TCP sequence Number is recalculated according to the payload length. 1369 * * Set FIN and/or PSH flags for the *last* packet if applied. 1370 * * TCP partial Checksum 1371 * 4. Update LSO information in the first message header. 1372 * 5. Release the original message header. 1373 */ 1374 static mblk_t * 1375 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1376 { 1377 uint32_t hckflags; 1378 int pktlen; 1379 int hdrlen; 1380 int segnum; 1381 int i; 1382 struct ether_vlan_header *evh; 1383 int ehlen, iphlen, tcphlen; 1384 struct ip *oiph, *niph; 1385 struct tcphdr *otcph, *ntcph; 1386 int available, len, left; 1387 uint16_t ip_id; 1388 uint32_t tcp_seq; 1389 #ifdef __sparc 1390 uint32_t tcp_seq_tmp; 1391 #endif 1392 mblk_t *datamp; 1393 uchar_t *rptr; 1394 mblk_t *nmp; 1395 mblk_t *cmp; 1396 mblk_t *mp_chain; 1397 boolean_t do_cleanup = B_FALSE; 1398 t_uscalar_t start_offset = 0; 1399 t_uscalar_t stuff_offset = 0; 1400 t_uscalar_t value = 0; 1401 uint16_t l4_len; 1402 ipaddr_t src, dst; 1403 uint32_t cksum, sum, l4cksum; 1404 1405 NXGE_DEBUG_MSG((NULL, TX_CTL, 1406 "==> nxge_do_softlso")); 1407 /* 1408 * check the length of LSO packet payload and calculate the number of 1409 * segments to be generated. 1410 */ 1411 pktlen = msgsize(mp); 1412 evh = (struct ether_vlan_header *)mp->b_rptr; 1413 1414 /* VLAN? */ 1415 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1416 ehlen = sizeof (struct ether_vlan_header); 1417 else 1418 ehlen = sizeof (struct ether_header); 1419 oiph = (struct ip *)(mp->b_rptr + ehlen); 1420 iphlen = oiph->ip_hl * 4; 1421 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1422 tcphlen = otcph->th_off * 4; 1423 1424 l4_len = pktlen - ehlen - iphlen; 1425 1426 NXGE_DEBUG_MSG((NULL, TX_CTL, 1427 "==> nxge_do_softlso: mss %d oiph $%p " 1428 "original ip_sum oiph->ip_sum 0x%x " 1429 "original tcp_sum otcph->th_sum 0x%x " 1430 "oiph->ip_len %d pktlen %d ehlen %d " 1431 "l4_len %d (0x%x) ip_len - iphlen %d ", 1432 mss, 1433 oiph, 1434 oiph->ip_sum, 1435 otcph->th_sum, 1436 ntohs(oiph->ip_len), pktlen, 1437 ehlen, 1438 l4_len, 1439 l4_len, 1440 ntohs(oiph->ip_len) - iphlen)); 1441 1442 /* IPv4 + TCP */ 1443 if (!(oiph->ip_v == IPV4_VERSION)) { 1444 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1445 "<== nxge_do_softlso: not IPV4 " 1446 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1447 ntohs(oiph->ip_len), pktlen, ehlen, 1448 tcphlen)); 1449 freemsg(mp); 1450 return (NULL); 1451 } 1452 1453 if (!(oiph->ip_p == IPPROTO_TCP)) { 1454 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1455 "<== nxge_do_softlso: not TCP " 1456 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1457 ntohs(oiph->ip_len), pktlen, ehlen, 1458 tcphlen)); 1459 freemsg(mp); 1460 return (NULL); 1461 } 1462 1463 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1464 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1465 "<== nxge_do_softlso: len not matched " 1466 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1467 ntohs(oiph->ip_len), pktlen, ehlen, 1468 tcphlen)); 1469 freemsg(mp); 1470 return (NULL); 1471 } 1472 1473 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1474 tcphlen = otcph->th_off * 4; 1475 1476 /* TCP flags can not include URG, RST, or SYN */ 1477 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1478 1479 hdrlen = ehlen + iphlen + tcphlen; 1480 1481 VERIFY(MBLKL(mp) >= hdrlen); 1482 1483 if (MBLKL(mp) > hdrlen) { 1484 datamp = mp; 1485 rptr = mp->b_rptr + hdrlen; 1486 } else { /* = */ 1487 datamp = mp->b_cont; 1488 rptr = datamp->b_rptr; 1489 } 1490 1491 NXGE_DEBUG_MSG((NULL, TX_CTL, 1492 "nxge_do_softlso: otcph $%p pktlen: %d, " 1493 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1494 "mblkl(mp): %d, mblkl(datamp): %d", 1495 otcph, 1496 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1497 (int)MBLKL(mp), (int)MBLKL(datamp))); 1498 1499 hckflags = 0; 1500 nxge_hcksum_retrieve(mp, 1501 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1502 1503 dst = oiph->ip_dst.s_addr; 1504 src = oiph->ip_src.s_addr; 1505 1506 cksum = (dst >> 16) + (dst & 0xFFFF) + 1507 (src >> 16) + (src & 0xFFFF); 1508 l4cksum = cksum + IP_TCP_CSUM_COMP; 1509 1510 sum = l4_len + l4cksum; 1511 sum = (sum & 0xFFFF) + (sum >> 16); 1512 1513 NXGE_DEBUG_MSG((NULL, TX_CTL, 1514 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1515 "hckflags 0x%x start_offset %d stuff_offset %d " 1516 "value (original) 0x%x th_sum 0x%x " 1517 "pktlen %d l4_len %d (0x%x) " 1518 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1519 dst, src, 1520 (sum & 0xffff), (~sum & 0xffff), 1521 hckflags, start_offset, stuff_offset, 1522 value, otcph->th_sum, 1523 pktlen, 1524 l4_len, 1525 l4_len, 1526 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1527 (int)MBLKL(datamp), 1528 nxge_dump_packet((char *)evh, 12))); 1529 1530 /* 1531 * Start to process. 1532 */ 1533 available = pktlen - hdrlen; 1534 segnum = (available - 1) / mss + 1; 1535 1536 NXGE_DEBUG_MSG((NULL, TX_CTL, 1537 "==> nxge_do_softlso: pktlen %d " 1538 "MBLKL(mp): %d, MBLKL(datamp): %d " 1539 "available %d mss %d segnum %d", 1540 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1541 available, 1542 mss, 1543 segnum)); 1544 1545 VERIFY(segnum >= 2); 1546 1547 /* 1548 * Try to pre-allocate all header messages 1549 */ 1550 mp_chain = NULL; 1551 for (i = 0; i < segnum; i++) { 1552 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1553 /* Clean up the mp_chain */ 1554 while (mp_chain != NULL) { 1555 nmp = mp_chain; 1556 mp_chain = mp_chain->b_next; 1557 freemsg(nmp); 1558 } 1559 NXGE_DEBUG_MSG((NULL, TX_CTL, 1560 "<== nxge_do_softlso: " 1561 "Could not allocate enough messages for headers!")); 1562 freemsg(mp); 1563 return (NULL); 1564 } 1565 nmp->b_next = mp_chain; 1566 mp_chain = nmp; 1567 1568 NXGE_DEBUG_MSG((NULL, TX_CTL, 1569 "==> nxge_do_softlso: " 1570 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1571 mp, nmp, mp_chain, mp_chain->b_next)); 1572 } 1573 1574 NXGE_DEBUG_MSG((NULL, TX_CTL, 1575 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1576 mp, nmp, mp_chain)); 1577 1578 /* 1579 * Associate payload with new packets 1580 */ 1581 cmp = mp_chain; 1582 left = available; 1583 while (cmp != NULL) { 1584 nmp = dupb(datamp); 1585 if (nmp == NULL) { 1586 do_cleanup = B_TRUE; 1587 NXGE_DEBUG_MSG((NULL, TX_CTL, 1588 "==>nxge_do_softlso: " 1589 "Can not dupb(datamp), have to do clean up")); 1590 goto cleanup_allocated_msgs; 1591 } 1592 1593 NXGE_DEBUG_MSG((NULL, TX_CTL, 1594 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1595 "dupb nmp $%p len %d left %d msd %d ", 1596 mp, cmp, nmp, len, left, mss)); 1597 1598 cmp->b_cont = nmp; 1599 nmp->b_rptr = rptr; 1600 len = (left < mss) ? left : mss; 1601 left -= len; 1602 1603 NXGE_DEBUG_MSG((NULL, TX_CTL, 1604 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1605 "dupb nmp $%p len %d left %d mss %d ", 1606 mp, cmp, nmp, len, left, mss)); 1607 NXGE_DEBUG_MSG((NULL, TX_CTL, 1608 "nxge_do_softlso: before available: %d, " 1609 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1610 available, left, len, segnum, (int)MBLKL(nmp))); 1611 1612 len -= MBLKL(nmp); 1613 NXGE_DEBUG_MSG((NULL, TX_CTL, 1614 "nxge_do_softlso: after available: %d, " 1615 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1616 available, left, len, segnum, (int)MBLKL(nmp))); 1617 1618 while (len > 0) { 1619 mblk_t *mmp = NULL; 1620 1621 NXGE_DEBUG_MSG((NULL, TX_CTL, 1622 "nxge_do_softlso: (4) len > 0 available: %d, " 1623 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1624 available, left, len, segnum, (int)MBLKL(nmp))); 1625 1626 if (datamp->b_cont != NULL) { 1627 datamp = datamp->b_cont; 1628 rptr = datamp->b_rptr; 1629 mmp = dupb(datamp); 1630 if (mmp == NULL) { 1631 do_cleanup = B_TRUE; 1632 NXGE_DEBUG_MSG((NULL, TX_CTL, 1633 "==> nxge_do_softlso: " 1634 "Can not dupb(datamp) (1), : 1635 "have to do clean up")); 1636 NXGE_DEBUG_MSG((NULL, TX_CTL, 1637 "==> nxge_do_softlso: " 1638 "available: %d, left: %d, " 1639 "len: %d, MBLKL(nmp): %d", 1640 available, left, len, 1641 (int)MBLKL(nmp))); 1642 goto cleanup_allocated_msgs; 1643 } 1644 } else { 1645 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1646 "==> nxge_do_softlso: " 1647 "(1)available: %d, left: %d, " 1648 "len: %d, MBLKL(nmp): %d", 1649 available, left, len, 1650 (int)MBLKL(nmp))); 1651 cmn_err(CE_PANIC, 1652 "==> nxge_do_softlso: " 1653 "Pointers must have been corrupted!\n" 1654 "datamp: $%p, nmp: $%p, rptr: $%p", 1655 (void *)datamp, 1656 (void *)nmp, 1657 (void *)rptr); 1658 } 1659 nmp->b_cont = mmp; 1660 nmp = mmp; 1661 len -= MBLKL(nmp); 1662 } 1663 if (len < 0) { 1664 nmp->b_wptr += len; 1665 rptr = nmp->b_wptr; 1666 NXGE_DEBUG_MSG((NULL, TX_CTL, 1667 "(5) len < 0 (less than 0)" 1668 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1669 available, left, len, (int)MBLKL(nmp))); 1670 1671 } else if (len == 0) { 1672 if (datamp->b_cont != NULL) { 1673 NXGE_DEBUG_MSG((NULL, TX_CTL, 1674 "(5) len == 0" 1675 "available: %d, left: %d, len: %d, " 1676 "MBLKL(nmp): %d", 1677 available, left, len, (int)MBLKL(nmp))); 1678 datamp = datamp->b_cont; 1679 rptr = datamp->b_rptr; 1680 } else { 1681 NXGE_DEBUG_MSG((NULL, TX_CTL, 1682 "(6)available b_cont == NULL : %d, " 1683 "left: %d, len: %d, MBLKL(nmp): %d", 1684 available, left, len, (int)MBLKL(nmp))); 1685 1686 VERIFY(cmp->b_next == NULL); 1687 VERIFY(left == 0); 1688 break; /* Done! */ 1689 } 1690 } 1691 cmp = cmp->b_next; 1692 1693 NXGE_DEBUG_MSG((NULL, TX_CTL, 1694 "(7) do_softlso: " 1695 "next mp in mp_chain available len != 0 : %d, " 1696 "left: %d, len: %d, MBLKL(nmp): %d", 1697 available, left, len, (int)MBLKL(nmp))); 1698 } 1699 1700 /* 1701 * From now, start to fill up all headers for the first message 1702 * Hardware checksum flags need to be updated separately for FULLCKSUM 1703 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1704 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1705 * required fields need to be updated properly. 1706 */ 1707 nmp = mp_chain; 1708 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1709 nmp->b_wptr = nmp->b_rptr + hdrlen; 1710 niph = (struct ip *)(nmp->b_rptr + ehlen); 1711 niph->ip_len = htons(mss + iphlen + tcphlen); 1712 ip_id = ntohs(niph->ip_id); 1713 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1714 #ifdef __sparc 1715 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1716 tcp_seq = ntohl(tcp_seq_tmp); 1717 #else 1718 tcp_seq = ntohl(ntcph->th_seq); 1719 #endif 1720 1721 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1722 1723 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1724 DB_CKSUMSTART(nmp) = start_offset; 1725 DB_CKSUMSTUFF(nmp) = stuff_offset; 1726 1727 /* calculate IP checksum and TCP pseudo header checksum */ 1728 niph->ip_sum = 0; 1729 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1730 1731 l4_len = mss + tcphlen; 1732 sum = htons(l4_len) + l4cksum; 1733 sum = (sum & 0xFFFF) + (sum >> 16); 1734 ntcph->th_sum = (sum & 0xffff); 1735 1736 NXGE_DEBUG_MSG((NULL, TX_CTL, 1737 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1738 "mss %d pktlen %d l4_len %d (0x%x) " 1739 "MBLKL(mp): %d, MBLKL(datamp): %d " 1740 "ip_sum 0x%x " 1741 "th_sum 0x%x sum 0x%x ) " 1742 "dump first ip->tcp %s", 1743 nmp, mp_chain, 1744 mss, 1745 pktlen, 1746 l4_len, 1747 l4_len, 1748 (int)MBLKL(mp), (int)MBLKL(datamp), 1749 niph->ip_sum, 1750 ntcph->th_sum, 1751 sum, 1752 nxge_dump_packet((char *)niph, 52))); 1753 1754 cmp = nmp; 1755 while ((nmp = nmp->b_next)->b_next != NULL) { 1756 NXGE_DEBUG_MSG((NULL, TX_CTL, 1757 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1758 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1759 nmp->b_wptr = nmp->b_rptr + hdrlen; 1760 niph = (struct ip *)(nmp->b_rptr + ehlen); 1761 niph->ip_id = htons(++ip_id); 1762 niph->ip_len = htons(mss + iphlen + tcphlen); 1763 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1764 tcp_seq += mss; 1765 1766 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1767 1768 #ifdef __sparc 1769 tcp_seq_tmp = htonl(tcp_seq); 1770 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1771 #else 1772 ntcph->th_seq = htonl(tcp_seq); 1773 #endif 1774 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1775 DB_CKSUMSTART(nmp) = start_offset; 1776 DB_CKSUMSTUFF(nmp) = stuff_offset; 1777 1778 /* calculate IP checksum and TCP pseudo header checksum */ 1779 niph->ip_sum = 0; 1780 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1781 ntcph->th_sum = (sum & 0xffff); 1782 1783 NXGE_DEBUG_MSG((NULL, TX_CTL, 1784 "==> nxge_do_softlso: middle ip_sum 0x%x " 1785 "th_sum 0x%x " 1786 " mp $%p (mp_chain $%p) pktlen %d " 1787 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1788 niph->ip_sum, 1789 ntcph->th_sum, 1790 nmp, mp_chain, 1791 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1792 } 1793 1794 /* Last segment */ 1795 /* 1796 * Set FIN and/or PSH flags if present only in the last packet. 1797 * The ip_len could be different from prior packets. 1798 */ 1799 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1800 nmp->b_wptr = nmp->b_rptr + hdrlen; 1801 niph = (struct ip *)(nmp->b_rptr + ehlen); 1802 niph->ip_id = htons(++ip_id); 1803 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1804 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1805 tcp_seq += mss; 1806 #ifdef __sparc 1807 tcp_seq_tmp = htonl(tcp_seq); 1808 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1809 #else 1810 ntcph->th_seq = htonl(tcp_seq); 1811 #endif 1812 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1813 1814 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1815 DB_CKSUMSTART(nmp) = start_offset; 1816 DB_CKSUMSTUFF(nmp) = stuff_offset; 1817 1818 /* calculate IP checksum and TCP pseudo header checksum */ 1819 niph->ip_sum = 0; 1820 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1821 1822 l4_len = ntohs(niph->ip_len) - iphlen; 1823 sum = htons(l4_len) + l4cksum; 1824 sum = (sum & 0xFFFF) + (sum >> 16); 1825 ntcph->th_sum = (sum & 0xffff); 1826 1827 NXGE_DEBUG_MSG((NULL, TX_CTL, 1828 "==> nxge_do_softlso: last next " 1829 "niph->ip_sum 0x%x " 1830 "ntcph->th_sum 0x%x sum 0x%x " 1831 "dump last ip->tcp %s " 1832 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1833 "l4_len %d (0x%x) " 1834 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1835 niph->ip_sum, 1836 ntcph->th_sum, sum, 1837 nxge_dump_packet((char *)niph, 52), 1838 cmp, nmp, mp_chain, 1839 pktlen, pktlen, 1840 l4_len, 1841 l4_len, 1842 (int)MBLKL(mp), (int)MBLKL(datamp))); 1843 1844 cleanup_allocated_msgs: 1845 if (do_cleanup) { 1846 NXGE_DEBUG_MSG((NULL, TX_CTL, 1847 "==> nxge_do_softlso: " 1848 "Failed allocating messages, " 1849 "have to clean up and fail!")); 1850 while (mp_chain != NULL) { 1851 nmp = mp_chain; 1852 mp_chain = mp_chain->b_next; 1853 freemsg(nmp); 1854 } 1855 } 1856 /* 1857 * We're done here, so just free the original message and return the 1858 * new message chain, that could be NULL if failed, back to the caller. 1859 */ 1860 freemsg(mp); 1861 1862 NXGE_DEBUG_MSG((NULL, TX_CTL, 1863 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1864 return (mp_chain); 1865 } 1866 1867 /* 1868 * Will be called before NIC driver do further operation on the message. 1869 * The input message may include LSO information, if so, go to softlso logic 1870 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1871 * The return could be the same non-LSO message or a message chain for LSO case. 1872 * 1873 * The driver needs to call this function per packet and process the whole chain 1874 * if applied. 1875 */ 1876 static mblk_t * 1877 nxge_lso_eliminate(mblk_t *mp) 1878 { 1879 uint32_t lsoflags; 1880 uint32_t mss; 1881 1882 NXGE_DEBUG_MSG((NULL, TX_CTL, 1883 "==>nxge_lso_eliminate:")); 1884 nxge_lso_info_get(mp, &mss, &lsoflags); 1885 1886 if (lsoflags & HW_LSO) { 1887 mblk_t *nmp; 1888 1889 NXGE_DEBUG_MSG((NULL, TX_CTL, 1890 "==>nxge_lso_eliminate:" 1891 "HW_LSO:mss %d mp $%p", 1892 mss, mp)); 1893 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1894 NXGE_DEBUG_MSG((NULL, TX_CTL, 1895 "<== nxge_lso_eliminate: " 1896 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1897 nmp, mss, mp)); 1898 return (nmp); 1899 } else { 1900 NXGE_DEBUG_MSG((NULL, TX_CTL, 1901 "<== nxge_lso_eliminate_ " 1902 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1903 nmp, mss, mp)); 1904 return (NULL); 1905 } 1906 } 1907 1908 NXGE_DEBUG_MSG((NULL, TX_CTL, 1909 "<== nxge_lso_eliminate")); 1910 return (mp); 1911 } 1912 1913 static uint32_t 1914 nxge_csgen(uint16_t *adr, int len) 1915 { 1916 int i, odd; 1917 uint32_t sum = 0; 1918 uint32_t c = 0; 1919 1920 odd = len % 2; 1921 for (i = 0; i < (len / 2); i++) { 1922 sum += (adr[i] & 0xffff); 1923 } 1924 if (odd) { 1925 sum += adr[len / 2] & 0xff00; 1926 } 1927 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 1928 sum &= 0xffff; 1929 sum += c; 1930 } 1931 return (~sum & 0xffff); 1932 } 1933