1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/nxge/nxge_impl.h> 29 30 /* Software LSO required header files */ 31 #include <netinet/tcp.h> 32 #include <inet/ip_impl.h> 33 #include <inet/tcp.h> 34 35 static mblk_t *nxge_lso_eliminate(mblk_t *); 36 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 37 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 38 static void nxge_hcksum_retrieve(mblk_t *, 39 uint32_t *, uint32_t *, uint32_t *, 40 uint32_t *, uint32_t *); 41 static uint32_t nxge_csgen(uint16_t *, int); 42 43 extern uint32_t nxge_reclaim_pending; 44 extern uint32_t nxge_bcopy_thresh; 45 extern uint32_t nxge_dvma_thresh; 46 extern uint32_t nxge_dma_stream_thresh; 47 extern uint32_t nxge_tx_minfree; 48 extern uint32_t nxge_tx_intr_thres; 49 extern uint32_t nxge_tx_max_gathers; 50 extern uint32_t nxge_tx_tiny_pack; 51 extern uint32_t nxge_tx_use_bcopy; 52 extern uint32_t nxge_tx_lb_policy; 53 extern uint32_t nxge_no_tx_lb; 54 extern nxge_tx_mode_t nxge_tx_scheme; 55 extern uint32_t nxge_lso_enable; 56 uint32_t nxge_lso_kick_cnt = 2; 57 58 typedef struct _mac_tx_hint { 59 uint16_t sap; 60 uint16_t vid; 61 void *hash; 62 } mac_tx_hint_t, *p_mac_tx_hint_t; 63 64 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 65 66 int 67 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 68 { 69 int status = 0; 70 p_tx_desc_t tx_desc_ring_vp; 71 npi_handle_t npi_desc_handle; 72 nxge_os_dma_handle_t tx_desc_dma_handle; 73 p_tx_desc_t tx_desc_p; 74 p_tx_msg_t tx_msg_ring; 75 p_tx_msg_t tx_msg_p; 76 tx_desc_t tx_desc, *tmp_desc_p; 77 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 78 p_tx_pkt_header_t hdrp; 79 p_tx_pkt_hdr_all_t pkthdrp; 80 uint8_t npads = 0; 81 uint64_t dma_ioaddr; 82 uint32_t dma_flags; 83 int last_bidx; 84 uint8_t *b_rptr; 85 caddr_t kaddr; 86 uint32_t nmblks; 87 uint32_t ngathers; 88 uint32_t clen; 89 int len; 90 uint32_t pkt_len, pack_len, min_len; 91 uint32_t bcopy_thresh; 92 int i, cur_index, sop_index; 93 uint16_t tail_index; 94 boolean_t tail_wrap = B_FALSE; 95 nxge_dma_common_t desc_area; 96 nxge_os_dma_handle_t dma_handle; 97 ddi_dma_cookie_t dma_cookie; 98 npi_handle_t npi_handle; 99 p_mblk_t nmp; 100 p_mblk_t t_mp; 101 uint32_t ncookies; 102 boolean_t good_packet; 103 boolean_t mark_mode = B_FALSE; 104 p_nxge_stats_t statsp; 105 p_nxge_tx_ring_stats_t tdc_stats; 106 t_uscalar_t start_offset = 0; 107 t_uscalar_t stuff_offset = 0; 108 t_uscalar_t end_offset = 0; 109 t_uscalar_t value = 0; 110 t_uscalar_t cksum_flags = 0; 111 boolean_t cksum_on = B_FALSE; 112 uint32_t boff = 0; 113 uint64_t tot_xfer_len = 0, tmp_len = 0; 114 boolean_t header_set = B_FALSE; 115 #ifdef NXGE_DEBUG 116 p_tx_desc_t tx_desc_ring_pp; 117 p_tx_desc_t tx_desc_pp; 118 tx_desc_t *save_desc_p; 119 int dump_len; 120 int sad_len; 121 uint64_t sad; 122 int xfer_len; 123 uint32_t msgsize; 124 #endif 125 p_mblk_t mp_chain = NULL; 126 boolean_t is_lso = B_FALSE; 127 boolean_t lso_again; 128 int cur_index_lso; 129 p_mblk_t nmp_lso_save; 130 uint32_t lso_ngathers; 131 boolean_t lso_tail_wrap = B_FALSE; 132 133 NXGE_DEBUG_MSG((nxgep, TX_CTL, 134 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 135 NXGE_DEBUG_MSG((nxgep, TX_CTL, 136 "==> nxge_start: Starting tdc %d desc pending %d", 137 tx_ring_p->tdc, tx_ring_p->descs_pending)); 138 139 statsp = nxgep->statsp; 140 141 if (nxgep->statsp->port_stats.lb_mode == nxge_lb_normal) { 142 if (!statsp->mac_stats.link_up) { 143 freemsg(mp); 144 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 145 "link not up or LB mode")); 146 goto nxge_start_fail1; 147 } 148 } 149 150 if (nxge_lso_enable) { 151 mp_chain = nxge_lso_eliminate(mp); 152 NXGE_DEBUG_MSG((nxgep, TX_CTL, 153 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 154 mp, mp_chain)); 155 if (mp_chain == NULL) { 156 NXGE_ERROR_MSG((nxgep, TX_CTL, 157 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 158 mp_chain, mp)); 159 goto nxge_start_fail1; 160 } 161 if (mp_chain != mp) { 162 NXGE_DEBUG_MSG((nxgep, TX_CTL, 163 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 164 mp_chain, mp)); 165 is_lso = B_TRUE; 166 mp = mp_chain; 167 mp_chain = mp_chain->b_next; 168 mp->b_next = NULL; 169 } 170 } 171 172 hcksum_retrieve(mp, NULL, NULL, &start_offset, 173 &stuff_offset, &end_offset, &value, &cksum_flags); 174 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 175 start_offset += sizeof (ether_header_t); 176 stuff_offset += sizeof (ether_header_t); 177 } else { 178 start_offset += sizeof (struct ether_vlan_header); 179 stuff_offset += sizeof (struct ether_vlan_header); 180 } 181 182 if (cksum_flags & HCK_PARTIALCKSUM) { 183 NXGE_DEBUG_MSG((nxgep, TX_CTL, 184 "==> nxge_start: mp $%p len %d " 185 "cksum_flags 0x%x (partial checksum) ", 186 mp, MBLKL(mp), cksum_flags)); 187 cksum_on = B_TRUE; 188 } 189 190 lso_again = B_FALSE; 191 lso_ngathers = 0; 192 193 MUTEX_ENTER(&tx_ring_p->lock); 194 cur_index_lso = tx_ring_p->wr_index; 195 lso_tail_wrap = tx_ring_p->wr_index_wrap; 196 start_again: 197 ngathers = 0; 198 sop_index = tx_ring_p->wr_index; 199 #ifdef NXGE_DEBUG 200 if (tx_ring_p->descs_pending) { 201 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 202 "desc pending %d ", tx_ring_p->descs_pending)); 203 } 204 205 dump_len = (int)(MBLKL(mp)); 206 dump_len = (dump_len > 128) ? 128: dump_len; 207 208 NXGE_DEBUG_MSG((nxgep, TX_CTL, 209 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 210 "(Before header reserve: ORIGINAL LEN %d)", 211 tx_ring_p->tdc, 212 mp->b_rptr, 213 dump_len)); 214 215 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 216 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 217 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 218 #endif 219 220 tdc_stats = tx_ring_p->tdc_stats; 221 mark_mode = (tx_ring_p->descs_pending && 222 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 223 < nxge_tx_minfree)); 224 225 NXGE_DEBUG_MSG((nxgep, TX_CTL, 226 "TX Descriptor ring is channel %d mark mode %d", 227 tx_ring_p->tdc, mark_mode)); 228 229 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { 230 NXGE_DEBUG_MSG((nxgep, TX_CTL, 231 "TX Descriptor ring is full: channel %d", 232 tx_ring_p->tdc)); 233 NXGE_DEBUG_MSG((nxgep, TX_CTL, 234 "TX Descriptor ring is full: channel %d", 235 tx_ring_p->tdc)); 236 if (is_lso) { 237 /* free the current mp and mp_chain if not FULL */ 238 tdc_stats->tx_no_desc++; 239 NXGE_DEBUG_MSG((nxgep, TX_CTL, 240 "LSO packet: TX Descriptor ring is full: " 241 "channel %d", 242 tx_ring_p->tdc)); 243 goto nxge_start_fail_lso; 244 } else { 245 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 246 tdc_stats->tx_no_desc++; 247 MUTEX_EXIT(&tx_ring_p->lock); 248 if (nxgep->resched_needed && !nxgep->resched_running) { 249 nxgep->resched_running = B_TRUE; 250 ddi_trigger_softintr(nxgep->resched_id); 251 } 252 status = 1; 253 goto nxge_start_fail1; 254 } 255 } 256 257 nmp = mp; 258 i = sop_index = tx_ring_p->wr_index; 259 nmblks = 0; 260 ngathers = 0; 261 pkt_len = 0; 262 pack_len = 0; 263 clen = 0; 264 last_bidx = -1; 265 good_packet = B_TRUE; 266 267 desc_area = tx_ring_p->tdc_desc; 268 npi_handle = desc_area.npi_handle; 269 npi_desc_handle.regh = (nxge_os_acc_handle_t) 270 DMA_COMMON_ACC_HANDLE(desc_area); 271 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 272 #ifdef NXGE_DEBUG 273 tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area); 274 #endif 275 tx_desc_dma_handle = (nxge_os_dma_handle_t) 276 DMA_COMMON_HANDLE(desc_area); 277 tx_msg_ring = tx_ring_p->tx_msg_ring; 278 279 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 280 sop_index, i)); 281 282 #ifdef NXGE_DEBUG 283 msgsize = msgdsize(nmp); 284 NXGE_DEBUG_MSG((nxgep, TX_CTL, 285 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 286 sop_index, i, msgsize)); 287 #endif 288 /* 289 * The first 16 bytes of the premapped buffer are reserved 290 * for header. No padding will be used. 291 */ 292 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 293 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 294 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 295 } else { 296 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 297 } 298 while (nmp) { 299 good_packet = B_TRUE; 300 b_rptr = nmp->b_rptr; 301 len = MBLKL(nmp); 302 if (len <= 0) { 303 nmp = nmp->b_cont; 304 continue; 305 } 306 nmblks++; 307 308 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 309 "len %d pkt_len %d pack_len %d", 310 nmblks, len, pkt_len, pack_len)); 311 /* 312 * Hardware limits the transfer length to 4K for NIU and 313 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 314 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 315 * If len is longer than the limit, then we break nmp into 316 * two chunks: Make the first chunk equal to the limit and 317 * the second chunk for the remaining data. If the second 318 * chunk is still larger than the limit, then it will be 319 * broken into two in the next pass. 320 */ 321 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 322 if ((t_mp = dupb(nmp)) != NULL) { 323 nmp->b_wptr = nmp->b_rptr + 324 (TX_MAX_TRANSFER_LENGTH 325 - TX_PKT_HEADER_SIZE); 326 t_mp->b_rptr = nmp->b_wptr; 327 t_mp->b_cont = nmp->b_cont; 328 nmp->b_cont = t_mp; 329 len = MBLKL(nmp); 330 } else { 331 if (is_lso) { 332 NXGE_DEBUG_MSG((nxgep, TX_CTL, 333 "LSO packet: dupb failed: " 334 "channel %d", 335 tx_ring_p->tdc)); 336 mp = nmp; 337 goto nxge_start_fail_lso; 338 } else { 339 good_packet = B_FALSE; 340 goto nxge_start_fail2; 341 } 342 } 343 } 344 tx_desc.value = 0; 345 tx_desc_p = &tx_desc_ring_vp[i]; 346 #ifdef NXGE_DEBUG 347 tx_desc_pp = &tx_desc_ring_pp[i]; 348 #endif 349 tx_msg_p = &tx_msg_ring[i]; 350 #if defined(__i386) 351 npi_desc_handle.regp = (uint32_t)tx_desc_p; 352 #else 353 npi_desc_handle.regp = (uint64_t)tx_desc_p; 354 #endif 355 if (!header_set && 356 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 357 (len >= bcopy_thresh))) { 358 header_set = B_TRUE; 359 bcopy_thresh += TX_PKT_HEADER_SIZE; 360 boff = 0; 361 pack_len = 0; 362 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 363 hdrp = (p_tx_pkt_header_t)kaddr; 364 clen = pkt_len; 365 dma_handle = tx_msg_p->buf_dma_handle; 366 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 367 (void) ddi_dma_sync(dma_handle, 368 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 369 DDI_DMA_SYNC_FORDEV); 370 371 tx_msg_p->flags.dma_type = USE_BCOPY; 372 goto nxge_start_control_header_only; 373 } 374 375 pkt_len += len; 376 pack_len += len; 377 378 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 379 "desc entry %d " 380 "DESC IOADDR $%p " 381 "desc_vp $%p tx_desc_p $%p " 382 "desc_pp $%p tx_desc_pp $%p " 383 "len %d pkt_len %d pack_len %d", 384 i, 385 DMA_COMMON_IOADDR(desc_area), 386 tx_desc_ring_vp, tx_desc_p, 387 tx_desc_ring_pp, tx_desc_pp, 388 len, pkt_len, pack_len)); 389 390 if (len < bcopy_thresh) { 391 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 392 "USE BCOPY: ")); 393 if (nxge_tx_tiny_pack) { 394 uint32_t blst = 395 TXDMA_DESC_NEXT_INDEX(i, -1, 396 tx_ring_p->tx_wrap_mask); 397 NXGE_DEBUG_MSG((nxgep, TX_CTL, 398 "==> nxge_start(5): pack")); 399 if ((pack_len <= bcopy_thresh) && 400 (last_bidx == blst)) { 401 NXGE_DEBUG_MSG((nxgep, TX_CTL, 402 "==> nxge_start: pack(6) " 403 "(pkt_len %d pack_len %d)", 404 pkt_len, pack_len)); 405 i = blst; 406 tx_desc_p = &tx_desc_ring_vp[i]; 407 #ifdef NXGE_DEBUG 408 tx_desc_pp = &tx_desc_ring_pp[i]; 409 #endif 410 tx_msg_p = &tx_msg_ring[i]; 411 boff = pack_len - len; 412 ngathers--; 413 } else if (pack_len > bcopy_thresh && 414 header_set) { 415 pack_len = len; 416 boff = 0; 417 bcopy_thresh = nxge_bcopy_thresh; 418 NXGE_DEBUG_MSG((nxgep, TX_CTL, 419 "==> nxge_start(7): > max NEW " 420 "bcopy thresh %d " 421 "pkt_len %d pack_len %d(next)", 422 bcopy_thresh, 423 pkt_len, pack_len)); 424 } 425 last_bidx = i; 426 } 427 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 428 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 429 hdrp = (p_tx_pkt_header_t)kaddr; 430 header_set = B_TRUE; 431 NXGE_DEBUG_MSG((nxgep, TX_CTL, 432 "==> nxge_start(7_x2): " 433 "pkt_len %d pack_len %d (new hdrp $%p)", 434 pkt_len, pack_len, hdrp)); 435 } 436 tx_msg_p->flags.dma_type = USE_BCOPY; 437 kaddr += boff; 438 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 439 "USE BCOPY: before bcopy " 440 "DESC IOADDR $%p entry %d " 441 "bcopy packets %d " 442 "bcopy kaddr $%p " 443 "bcopy ioaddr (SAD) $%p " 444 "bcopy clen %d " 445 "bcopy boff %d", 446 DMA_COMMON_IOADDR(desc_area), i, 447 tdc_stats->tx_hdr_pkts, 448 kaddr, 449 dma_ioaddr, 450 clen, 451 boff)); 452 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 453 "1USE BCOPY: ")); 454 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 455 "2USE BCOPY: ")); 456 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 457 "last USE BCOPY: copy from b_rptr $%p " 458 "to KADDR $%p (len %d offset %d", 459 b_rptr, kaddr, len, boff)); 460 461 bcopy(b_rptr, kaddr, len); 462 463 #ifdef NXGE_DEBUG 464 dump_len = (len > 128) ? 128: len; 465 NXGE_DEBUG_MSG((nxgep, TX_CTL, 466 "==> nxge_start: dump packets " 467 "(After BCOPY len %d)" 468 "(b_rptr $%p): %s", len, nmp->b_rptr, 469 nxge_dump_packet((char *)nmp->b_rptr, 470 dump_len))); 471 #endif 472 473 dma_handle = tx_msg_p->buf_dma_handle; 474 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 475 (void) ddi_dma_sync(dma_handle, 476 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 477 DDI_DMA_SYNC_FORDEV); 478 clen = len + boff; 479 tdc_stats->tx_hdr_pkts++; 480 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 481 "USE BCOPY: " 482 "DESC IOADDR $%p entry %d " 483 "bcopy packets %d " 484 "bcopy kaddr $%p " 485 "bcopy ioaddr (SAD) $%p " 486 "bcopy clen %d " 487 "bcopy boff %d", 488 DMA_COMMON_IOADDR(desc_area), 489 i, 490 tdc_stats->tx_hdr_pkts, 491 kaddr, 492 dma_ioaddr, 493 clen, 494 boff)); 495 } else { 496 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 497 "USE DVMA: len %d", len)); 498 tx_msg_p->flags.dma_type = USE_DMA; 499 dma_flags = DDI_DMA_WRITE; 500 if (len < nxge_dma_stream_thresh) { 501 dma_flags |= DDI_DMA_CONSISTENT; 502 } else { 503 dma_flags |= DDI_DMA_STREAMING; 504 } 505 506 dma_handle = tx_msg_p->dma_handle; 507 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 508 (caddr_t)b_rptr, len, dma_flags, 509 DDI_DMA_DONTWAIT, NULL, 510 &dma_cookie, &ncookies); 511 if (status == DDI_DMA_MAPPED) { 512 dma_ioaddr = dma_cookie.dmac_laddress; 513 len = (int)dma_cookie.dmac_size; 514 clen = (uint32_t)dma_cookie.dmac_size; 515 NXGE_DEBUG_MSG((nxgep, TX_CTL, 516 "==> nxge_start(12_1): " 517 "USE DVMA: len %d clen %d " 518 "ngathers %d", 519 len, clen, 520 ngathers)); 521 #if defined(__i386) 522 npi_desc_handle.regp = (uint32_t)tx_desc_p; 523 #else 524 npi_desc_handle.regp = (uint64_t)tx_desc_p; 525 #endif 526 while (ncookies > 1) { 527 ngathers++; 528 /* 529 * this is the fix for multiple 530 * cookies, which are basically 531 * a descriptor entry, we don't set 532 * SOP bit as well as related fields 533 */ 534 535 (void) npi_txdma_desc_gather_set( 536 npi_desc_handle, 537 &tx_desc, 538 (ngathers -1), 539 mark_mode, 540 ngathers, 541 dma_ioaddr, 542 clen); 543 544 tx_msg_p->tx_msg_size = clen; 545 NXGE_DEBUG_MSG((nxgep, TX_CTL, 546 "==> nxge_start: DMA " 547 "ncookie %d " 548 "ngathers %d " 549 "dma_ioaddr $%p len %d" 550 "desc $%p descp $%p (%d)", 551 ncookies, 552 ngathers, 553 dma_ioaddr, clen, 554 *tx_desc_p, tx_desc_p, i)); 555 556 ddi_dma_nextcookie(dma_handle, 557 &dma_cookie); 558 dma_ioaddr = 559 dma_cookie.dmac_laddress; 560 561 len = (int)dma_cookie.dmac_size; 562 clen = (uint32_t)dma_cookie.dmac_size; 563 NXGE_DEBUG_MSG((nxgep, TX_CTL, 564 "==> nxge_start(12_2): " 565 "USE DVMA: len %d clen %d ", 566 len, clen)); 567 568 i = TXDMA_DESC_NEXT_INDEX(i, 1, 569 tx_ring_p->tx_wrap_mask); 570 tx_desc_p = &tx_desc_ring_vp[i]; 571 572 npi_desc_handle.regp = 573 #if defined(__i386) 574 (uint32_t)tx_desc_p; 575 #else 576 (uint64_t)tx_desc_p; 577 #endif 578 tx_msg_p = &tx_msg_ring[i]; 579 tx_msg_p->flags.dma_type = USE_NONE; 580 tx_desc.value = 0; 581 582 ncookies--; 583 } 584 tdc_stats->tx_ddi_pkts++; 585 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 586 "DMA: ddi packets %d", 587 tdc_stats->tx_ddi_pkts)); 588 } else { 589 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 590 "dma mapping failed for %d " 591 "bytes addr $%p flags %x (%d)", 592 len, b_rptr, status, status)); 593 good_packet = B_FALSE; 594 tdc_stats->tx_dma_bind_fail++; 595 tx_msg_p->flags.dma_type = USE_NONE; 596 if (is_lso) { 597 mp = nmp; 598 goto nxge_start_fail_lso; 599 } else { 600 goto nxge_start_fail2; 601 } 602 } 603 } /* ddi dvma */ 604 605 if (is_lso) { 606 nmp_lso_save = nmp; 607 } 608 nmp = nmp->b_cont; 609 nxge_start_control_header_only: 610 #if defined(__i386) 611 npi_desc_handle.regp = (uint32_t)tx_desc_p; 612 #else 613 npi_desc_handle.regp = (uint64_t)tx_desc_p; 614 #endif 615 ngathers++; 616 617 if (ngathers == 1) { 618 #ifdef NXGE_DEBUG 619 save_desc_p = &sop_tx_desc; 620 #endif 621 sop_tx_desc_p = &sop_tx_desc; 622 sop_tx_desc_p->value = 0; 623 sop_tx_desc_p->bits.hdw.tr_len = clen; 624 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 625 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 626 } else { 627 #ifdef NXGE_DEBUG 628 save_desc_p = &tx_desc; 629 #endif 630 tmp_desc_p = &tx_desc; 631 tmp_desc_p->value = 0; 632 tmp_desc_p->bits.hdw.tr_len = clen; 633 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 634 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 635 636 tx_desc_p->value = tmp_desc_p->value; 637 } 638 639 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 640 "Desc_entry %d ngathers %d " 641 "desc_vp $%p tx_desc_p $%p " 642 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 643 "dma_ioaddr (SAD) $%p mark %d", 644 i, ngathers, 645 tx_desc_ring_vp, tx_desc_p, 646 len, clen, pkt_len, pack_len, nmblks, 647 dma_ioaddr, mark_mode)); 648 649 #ifdef NXGE_DEBUG 650 npi_desc_handle.nxgep = nxgep; 651 npi_desc_handle.function.function = nxgep->function_num; 652 npi_desc_handle.function.instance = nxgep->instance; 653 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 654 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 655 TX_PKT_DESC_TR_LEN_SHIFT); 656 657 658 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 659 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 660 "mark %d sop %d\n", 661 save_desc_p->value, 662 sad, 663 save_desc_p->bits.hdw.tr_len, 664 xfer_len, 665 save_desc_p->bits.hdw.num_ptr, 666 save_desc_p->bits.hdw.mark, 667 save_desc_p->bits.hdw.sop)); 668 669 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 670 #endif 671 672 tx_msg_p->tx_msg_size = clen; 673 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 674 if (ngathers > nxge_tx_max_gathers) { 675 good_packet = B_FALSE; 676 hcksum_retrieve(mp, NULL, NULL, &start_offset, 677 &stuff_offset, &end_offset, &value, 678 &cksum_flags); 679 680 NXGE_DEBUG_MSG((NULL, TX_CTL, 681 "==> nxge_start(14): pull msg - " 682 "len %d pkt_len %d ngathers %d", 683 len, pkt_len, ngathers)); 684 /* Pull all message blocks from b_cont */ 685 if (is_lso) { 686 mp = nmp_lso_save; 687 goto nxge_start_fail_lso; 688 } 689 if ((msgpullup(mp, -1)) == NULL) { 690 goto nxge_start_fail2; 691 } 692 goto nxge_start_fail2; 693 } 694 } /* while (nmp) */ 695 696 tx_msg_p->tx_message = mp; 697 tx_desc_p = &tx_desc_ring_vp[sop_index]; 698 #if defined(__i386) 699 npi_desc_handle.regp = (uint32_t)tx_desc_p; 700 #else 701 npi_desc_handle.regp = (uint64_t)tx_desc_p; 702 #endif 703 704 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 705 pkthdrp->reserved = 0; 706 hdrp->value = 0; 707 (void) nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 708 (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp); 709 710 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 711 tdc_stats->tx_jumbo_pkts++; 712 } 713 714 min_len = (nxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2)); 715 if (pkt_len < min_len) { 716 /* Assume we use bcopy to premapped buffers */ 717 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 718 NXGE_DEBUG_MSG((NULL, TX_CTL, 719 "==> nxge_start(14-1): < (msg_min + 16)" 720 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 721 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 722 bzero((kaddr + pkt_len), (min_len - pkt_len)); 723 pkt_len = tx_msg_p->tx_msg_size = min_len; 724 725 sop_tx_desc_p->bits.hdw.tr_len = min_len; 726 727 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 728 tx_desc_p->value = sop_tx_desc_p->value; 729 730 NXGE_DEBUG_MSG((NULL, TX_CTL, 731 "==> nxge_start(14-2): < msg_min - " 732 "len %d pkt_len %d min_len %d ngathers %d", 733 len, pkt_len, min_len, ngathers)); 734 } 735 736 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 737 cksum_flags)); 738 if (cksum_flags & HCK_PARTIALCKSUM) { 739 NXGE_DEBUG_MSG((nxgep, TX_CTL, 740 "==> nxge_start: cksum_flags 0x%x (partial checksum) ", 741 cksum_flags)); 742 cksum_on = B_TRUE; 743 NXGE_DEBUG_MSG((nxgep, TX_CTL, 744 "==> nxge_start: from IP cksum_flags 0x%x " 745 "(partial checksum) " 746 "start_offset %d stuff_offset %d", 747 cksum_flags, start_offset, stuff_offset)); 748 tmp_len = (uint64_t)(start_offset >> 1); 749 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT); 750 tmp_len = (uint64_t)(stuff_offset >> 1); 751 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT); 752 753 NXGE_DEBUG_MSG((nxgep, TX_CTL, 754 "==> nxge_start: from IP cksum_flags 0x%x " 755 "(partial checksum) " 756 "after SHIFT start_offset %d stuff_offset %d", 757 cksum_flags, start_offset, stuff_offset)); 758 } 759 { 760 uint64_t tmp_len; 761 762 /* pkt_len already includes 16 + paddings!! */ 763 /* Update the control header length */ 764 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 765 tmp_len = hdrp->value | 766 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 767 768 NXGE_DEBUG_MSG((nxgep, TX_CTL, 769 "==> nxge_start(15_x1): setting SOP " 770 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 771 "0x%llx hdrp->value 0x%llx", 772 tot_xfer_len, tot_xfer_len, pkt_len, 773 tmp_len, hdrp->value)); 774 #if defined(_BIG_ENDIAN) 775 hdrp->value = ddi_swap64(tmp_len); 776 #else 777 hdrp->value = tmp_len; 778 #endif 779 NXGE_DEBUG_MSG((nxgep, 780 TX_CTL, "==> nxge_start(15_x2): setting SOP " 781 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 782 "tmp_len 0x%llx hdrp->value 0x%llx", 783 tot_xfer_len, pkt_len, 784 tmp_len, hdrp->value)); 785 } 786 787 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 788 "wr_index %d " 789 "tot_xfer_len (%d) pkt_len %d npads %d", 790 sop_index, 791 tot_xfer_len, pkt_len, 792 npads)); 793 794 sop_tx_desc_p->bits.hdw.sop = 1; 795 sop_tx_desc_p->bits.hdw.mark = mark_mode; 796 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 797 798 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 799 800 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 801 802 #ifdef NXGE_DEBUG 803 npi_desc_handle.nxgep = nxgep; 804 npi_desc_handle.function.function = nxgep->function_num; 805 npi_desc_handle.function.instance = nxgep->instance; 806 807 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 808 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 809 save_desc_p->value, 810 sad, 811 save_desc_p->bits.hdw.tr_len, 812 xfer_len, 813 save_desc_p->bits.hdw.num_ptr, 814 save_desc_p->bits.hdw.mark, 815 save_desc_p->bits.hdw.sop)); 816 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 817 818 dump_len = (pkt_len > 128) ? 128: pkt_len; 819 NXGE_DEBUG_MSG((nxgep, TX_CTL, 820 "==> nxge_start: dump packets(17) (after sop set, len " 821 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 822 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 823 (char *)hdrp, 824 nxge_dump_packet((char *)hdrp, dump_len))); 825 NXGE_DEBUG_MSG((nxgep, TX_CTL, 826 "==> nxge_start(18): TX desc sync: sop_index %d", 827 sop_index)); 828 #endif 829 830 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 831 (void) ddi_dma_sync(tx_desc_dma_handle, 832 sop_index * sizeof (tx_desc_t), 833 ngathers * sizeof (tx_desc_t), 834 DDI_DMA_SYNC_FORDEV); 835 836 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 837 "cs_off = 0x%02X cs_s_off = 0x%02X " 838 "pkt_len %d ngathers %d sop_index %d\n", 839 stuff_offset, start_offset, 840 pkt_len, ngathers, sop_index)); 841 } else { /* more than one descriptor and wrap around */ 842 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 843 (void) ddi_dma_sync(tx_desc_dma_handle, 844 sop_index * sizeof (tx_desc_t), 845 nsdescs * sizeof (tx_desc_t), 846 DDI_DMA_SYNC_FORDEV); 847 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 848 "cs_off = 0x%02X cs_s_off = 0x%02X " 849 "pkt_len %d ngathers %d sop_index %d\n", 850 stuff_offset, start_offset, 851 pkt_len, ngathers, sop_index)); 852 853 (void) ddi_dma_sync(tx_desc_dma_handle, 854 0, 855 (ngathers - nsdescs) * sizeof (tx_desc_t), 856 DDI_DMA_SYNC_FORDEV); 857 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 858 "cs_off = 0x%02X cs_s_off = 0x%02X " 859 "pkt_len %d ngathers %d sop_index %d\n", 860 stuff_offset, start_offset, 861 pkt_len, ngathers, sop_index)); 862 } 863 864 tail_index = tx_ring_p->wr_index; 865 tail_wrap = tx_ring_p->wr_index_wrap; 866 867 tx_ring_p->wr_index = i; 868 if (tx_ring_p->wr_index <= tail_index) { 869 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 870 B_FALSE : B_TRUE); 871 } 872 873 tx_ring_p->descs_pending += ngathers; 874 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 875 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 876 tx_ring_p->tdc, 877 tx_ring_p->wr_index, 878 tx_ring_p->wr_index_wrap, 879 ngathers, 880 tx_ring_p->descs_pending)); 881 882 if (is_lso) { 883 lso_ngathers += ngathers; 884 if (mp_chain != NULL) { 885 mp = mp_chain; 886 mp_chain = mp_chain->b_next; 887 mp->b_next = NULL; 888 if (nxge_lso_kick_cnt == lso_ngathers) { 889 { 890 tx_ring_kick_t kick; 891 892 kick.value = 0; 893 kick.bits.ldw.wrap = 894 tx_ring_p->wr_index_wrap; 895 kick.bits.ldw.tail = 896 (uint16_t)tx_ring_p->wr_index; 897 898 /* Kick the Transmit kick register */ 899 TXDMA_REG_WRITE64( 900 NXGE_DEV_NPI_HANDLE(nxgep), 901 TX_RING_KICK_REG, 902 (uint8_t)tx_ring_p->tdc, 903 kick.value); 904 tdc_stats->tx_starts++; 905 NXGE_DEBUG_MSG((nxgep, TX_CTL, 906 "==> nxge_start: more LSO: " 907 "LSO_CNT %d", 908 lso_gathers)); 909 } 910 lso_ngathers = 0; 911 ngathers = 0; 912 cur_index_lso = sop_index = tx_ring_p->wr_index; 913 lso_tail_wrap = tx_ring_p->wr_index_wrap; 914 } 915 NXGE_DEBUG_MSG((nxgep, TX_CTL, 916 "==> nxge_start: lso again: " 917 "lso_gathers %d ngathers %d cur_index_lso %d " 918 "wr_index %d sop_index %d", 919 lso_ngathers, ngathers, cur_index_lso, 920 tx_ring_p->wr_index, sop_index)); 921 922 NXGE_DEBUG_MSG((nxgep, TX_CTL, 923 "==> nxge_start: next : count %d", 924 lso_gathers)); 925 lso_again = B_TRUE; 926 goto start_again; 927 } 928 } 929 930 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 931 932 { 933 tx_ring_kick_t kick; 934 935 kick.value = 0; 936 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 937 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 938 939 /* Kick start the Transmit kick register */ 940 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 941 TX_RING_KICK_REG, 942 (uint8_t)tx_ring_p->tdc, 943 kick.value); 944 } 945 946 tdc_stats->tx_starts++; 947 948 MUTEX_EXIT(&tx_ring_p->lock); 949 950 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 951 952 return (status); 953 954 nxge_start_fail_lso: 955 status = 0; 956 good_packet = B_FALSE; 957 if (mp != NULL) { 958 freemsg(mp); 959 } 960 if (mp_chain != NULL) { 961 freemsg(mp_chain); 962 } 963 if (!lso_again && !ngathers) { 964 MUTEX_EXIT(&tx_ring_p->lock); 965 NXGE_DEBUG_MSG((nxgep, TX_CTL, 966 "==> nxge_start: lso exit (nothing changed)")); 967 goto nxge_start_fail1; 968 } 969 970 NXGE_DEBUG_MSG((nxgep, TX_CTL, 971 "==> nxge_start (channel %d): before lso " 972 "lso_gathers %d ngathers %d cur_index_lso %d " 973 "wr_index %d sop_index %d lso_again %d", 974 tx_ring_p->tdc, 975 lso_ngathers, ngathers, cur_index_lso, 976 tx_ring_p->wr_index, sop_index, lso_again)); 977 978 if (lso_again) { 979 lso_ngathers += ngathers; 980 ngathers = lso_ngathers; 981 sop_index = cur_index_lso; 982 tx_ring_p->wr_index = sop_index; 983 tx_ring_p->wr_index_wrap = lso_tail_wrap; 984 } 985 986 NXGE_DEBUG_MSG((nxgep, TX_CTL, 987 "==> nxge_start (channel %d): after lso " 988 "lso_gathers %d ngathers %d cur_index_lso %d " 989 "wr_index %d sop_index %d lso_again %d", 990 tx_ring_p->tdc, 991 lso_ngathers, ngathers, cur_index_lso, 992 tx_ring_p->wr_index, sop_index, lso_again)); 993 994 nxge_start_fail2: 995 if (good_packet == B_FALSE) { 996 cur_index = sop_index; 997 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 998 for (i = 0; i < ngathers; i++) { 999 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1000 #if defined(__i386) 1001 npi_handle.regp = (uint32_t)tx_desc_p; 1002 #else 1003 npi_handle.regp = (uint64_t)tx_desc_p; 1004 #endif 1005 tx_msg_p = &tx_msg_ring[cur_index]; 1006 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1007 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1008 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1009 "tx_desc_p = %X index = %d", 1010 tx_desc_p, tx_ring_p->rd_index)); 1011 (void) dvma_unload(tx_msg_p->dvma_handle, 1012 0, -1); 1013 tx_msg_p->dvma_handle = NULL; 1014 if (tx_ring_p->dvma_wr_index == 1015 tx_ring_p->dvma_wrap_mask) 1016 tx_ring_p->dvma_wr_index = 0; 1017 else 1018 tx_ring_p->dvma_wr_index++; 1019 tx_ring_p->dvma_pending--; 1020 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1021 if (ddi_dma_unbind_handle( 1022 tx_msg_p->dma_handle)) { 1023 cmn_err(CE_WARN, "!nxge_start: " 1024 "ddi_dma_unbind_handle failed"); 1025 } 1026 } 1027 tx_msg_p->flags.dma_type = USE_NONE; 1028 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1029 tx_ring_p->tx_wrap_mask); 1030 1031 } 1032 1033 nxgep->resched_needed = B_TRUE; 1034 } 1035 1036 MUTEX_EXIT(&tx_ring_p->lock); 1037 1038 nxge_start_fail1: 1039 /* Add FMA to check the access handle nxge_hregh */ 1040 1041 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1042 1043 return (status); 1044 } 1045 1046 int 1047 nxge_serial_tx(mblk_t *mp, void *arg) 1048 { 1049 p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 1050 p_nxge_t nxgep = tx_ring_p->nxgep; 1051 1052 return (nxge_start(nxgep, tx_ring_p, mp)); 1053 } 1054 1055 boolean_t 1056 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 1057 { 1058 p_tx_ring_t *tx_rings; 1059 uint8_t ring_index; 1060 p_tx_ring_t tx_ring_p; 1061 1062 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 1063 1064 ASSERT(mp->b_next == NULL); 1065 1066 ring_index = nxge_tx_lb_ring_1(mp, nxgep->max_tdcs, hp); 1067 tx_rings = nxgep->tx_rings->rings; 1068 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: tx_rings $%p", 1069 tx_rings)); 1070 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: max_tdcs %d " 1071 "ring_index %d", nxgep->max_tdcs, ring_index)); 1072 1073 switch (nxge_tx_scheme) { 1074 case NXGE_USE_START: 1075 if (nxge_start(nxgep, tx_rings[ring_index], mp)) { 1076 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 1077 "ring index %d", ring_index)); 1078 return (B_FALSE); 1079 } 1080 break; 1081 1082 case NXGE_USE_SERIAL: 1083 default: 1084 tx_ring_p = tx_rings[ring_index]; 1085 nxge_serialize_enter(tx_ring_p->serial, mp); 1086 break; 1087 } 1088 1089 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 1090 ring_index)); 1091 1092 return (B_TRUE); 1093 } 1094 1095 /* 1096 * nxge_m_tx() - send a chain of packets 1097 */ 1098 mblk_t * 1099 nxge_m_tx(void *arg, mblk_t *mp) 1100 { 1101 p_nxge_t nxgep = (p_nxge_t)arg; 1102 mblk_t *next; 1103 mac_tx_hint_t hint; 1104 1105 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 1106 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1107 "==> nxge_m_tx: hardware not initialized")); 1108 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1109 "<== nxge_m_tx")); 1110 return (mp); 1111 } 1112 1113 hint.hash = NULL; 1114 hint.vid = 0; 1115 hint.sap = 0; 1116 1117 while (mp != NULL) { 1118 next = mp->b_next; 1119 mp->b_next = NULL; 1120 1121 /* 1122 * Until Nemo tx resource works, the mac driver 1123 * does the load balancing based on TCP port, 1124 * or CPU. For debugging, we use a system 1125 * configurable parameter. 1126 */ 1127 if (!nxge_send(nxgep, mp, &hint)) { 1128 mp->b_next = next; 1129 break; 1130 } 1131 1132 mp = next; 1133 1134 NXGE_DEBUG_MSG((NULL, TX_CTL, 1135 "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 1136 mp, next)); 1137 } 1138 1139 return (mp); 1140 } 1141 1142 int 1143 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 1144 { 1145 uint8_t ring_index = 0; 1146 uint8_t *tcp_port; 1147 p_mblk_t nmp; 1148 size_t mblk_len; 1149 size_t iph_len; 1150 size_t hdrs_size; 1151 uint8_t hdrs_buf[sizeof (struct ether_header) + 1152 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 1153 /* 1154 * allocate space big enough to cover 1155 * the max ip header length and the first 1156 * 4 bytes of the TCP/IP header. 1157 */ 1158 1159 boolean_t qos = B_FALSE; 1160 1161 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 1162 1163 if (hp->vid) { 1164 qos = B_TRUE; 1165 } 1166 switch (nxge_tx_lb_policy) { 1167 case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 1168 default: 1169 tcp_port = mp->b_rptr; 1170 if (!nxge_no_tx_lb && !qos && 1171 (ntohs(((p_ether_header_t)tcp_port)->ether_type) 1172 == ETHERTYPE_IP)) { 1173 nmp = mp; 1174 mblk_len = MBLKL(nmp); 1175 tcp_port = NULL; 1176 if (mblk_len > sizeof (struct ether_header) + 1177 sizeof (uint8_t)) { 1178 tcp_port = nmp->b_rptr + 1179 sizeof (struct ether_header); 1180 mblk_len -= sizeof (struct ether_header); 1181 iph_len = ((*tcp_port) & 0x0f) << 2; 1182 if (mblk_len > (iph_len + sizeof (uint32_t))) { 1183 tcp_port = nmp->b_rptr; 1184 } else { 1185 tcp_port = NULL; 1186 } 1187 } 1188 if (tcp_port == NULL) { 1189 hdrs_size = 0; 1190 ((p_ether_header_t)hdrs_buf)->ether_type = 0; 1191 while ((nmp) && (hdrs_size < 1192 sizeof (hdrs_buf))) { 1193 mblk_len = MBLKL(nmp); 1194 if (mblk_len >= 1195 (sizeof (hdrs_buf) - hdrs_size)) 1196 mblk_len = sizeof (hdrs_buf) - 1197 hdrs_size; 1198 bcopy(nmp->b_rptr, 1199 &hdrs_buf[hdrs_size], mblk_len); 1200 hdrs_size += mblk_len; 1201 nmp = nmp->b_cont; 1202 } 1203 tcp_port = hdrs_buf; 1204 } 1205 tcp_port += sizeof (ether_header_t); 1206 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1207 switch (tcp_port[9]) { 1208 case IPPROTO_TCP: 1209 case IPPROTO_UDP: 1210 case IPPROTO_ESP: 1211 tcp_port += ((*tcp_port) & 0x0f) << 2; 1212 ring_index = 1213 ((tcp_port[0] ^ 1214 tcp_port[1] ^ 1215 tcp_port[2] ^ 1216 tcp_port[3]) % maxtdcs); 1217 break; 1218 1219 case IPPROTO_AH: 1220 /* SPI starts at the 4th byte */ 1221 tcp_port += ((*tcp_port) & 0x0f) << 2; 1222 ring_index = 1223 ((tcp_port[4] ^ 1224 tcp_port[5] ^ 1225 tcp_port[6] ^ 1226 tcp_port[7]) % maxtdcs); 1227 break; 1228 1229 default: 1230 ring_index = tcp_port[19] % maxtdcs; 1231 break; 1232 } 1233 } else { /* fragmented packet */ 1234 ring_index = tcp_port[19] % maxtdcs; 1235 } 1236 } else { 1237 ring_index = mp->b_band % maxtdcs; 1238 } 1239 break; 1240 1241 case NXGE_TX_LB_HASH: 1242 if (hp->hash) { 1243 #if defined(__i386) 1244 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1245 #else 1246 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1247 #endif 1248 } else { 1249 ring_index = mp->b_band % maxtdcs; 1250 } 1251 break; 1252 1253 case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 1254 tcp_port = mp->b_rptr; 1255 ring_index = tcp_port[5] % maxtdcs; 1256 break; 1257 } 1258 1259 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 1260 1261 return (ring_index); 1262 } 1263 1264 uint_t 1265 nxge_reschedule(caddr_t arg) 1266 { 1267 p_nxge_t nxgep; 1268 1269 nxgep = (p_nxge_t)arg; 1270 1271 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 1272 1273 if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1274 nxgep->resched_needed) { 1275 mac_tx_update(nxgep->mach); 1276 nxgep->resched_needed = B_FALSE; 1277 nxgep->resched_running = B_FALSE; 1278 } 1279 1280 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 1281 return (DDI_INTR_CLAIMED); 1282 } 1283 1284 1285 /* Software LSO starts here */ 1286 static void 1287 nxge_hcksum_retrieve(mblk_t *mp, 1288 uint32_t *start, uint32_t *stuff, uint32_t *end, 1289 uint32_t *value, uint32_t *flags) 1290 { 1291 if (mp->b_datap->db_type == M_DATA) { 1292 if (flags != NULL) { 1293 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1294 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1295 HCK_FULLCKSUM_OK); 1296 if ((*flags & (HCK_PARTIALCKSUM | 1297 HCK_FULLCKSUM)) != 0) { 1298 if (value != NULL) 1299 *value = (uint32_t)DB_CKSUM16(mp); 1300 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1301 if (start != NULL) 1302 *start = 1303 (uint32_t)DB_CKSUMSTART(mp); 1304 if (stuff != NULL) 1305 *stuff = 1306 (uint32_t)DB_CKSUMSTUFF(mp); 1307 if (end != NULL) 1308 *end = 1309 (uint32_t)DB_CKSUMEND(mp); 1310 } 1311 } 1312 } 1313 } 1314 } 1315 1316 static void 1317 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1318 { 1319 ASSERT(DB_TYPE(mp) == M_DATA); 1320 1321 *mss = 0; 1322 if (flags != NULL) { 1323 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1324 if ((*flags != 0) && (mss != NULL)) { 1325 *mss = (uint32_t)DB_LSOMSS(mp); 1326 } 1327 NXGE_DEBUG_MSG((NULL, TX_CTL, 1328 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1329 *mss, *flags)); 1330 } 1331 1332 NXGE_DEBUG_MSG((NULL, TX_CTL, 1333 "<== nxge_lso_info_get: mss %d", *mss)); 1334 } 1335 1336 /* 1337 * Do Soft LSO on the oversized packet. 1338 * 1339 * 1. Create a chain of message for headers. 1340 * 2. Fill up header messages with proper information. 1341 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1342 * each new message with necessary adjustments. 1343 * * Unchange the ethernet header for DIX frames. (by default) 1344 * * IP Total Length field is updated to MSS or less(only for the last one). 1345 * * IP Identification value is incremented by one for each packet. 1346 * * TCP sequence Number is recalculated according to the payload length. 1347 * * Set FIN and/or PSH flags for the *last* packet if applied. 1348 * * TCP partial Checksum 1349 * 4. Update LSO information in the first message header. 1350 * 5. Release the original message header. 1351 */ 1352 static mblk_t * 1353 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1354 { 1355 uint32_t hckflags; 1356 int pktlen; 1357 int hdrlen; 1358 int segnum; 1359 int i; 1360 struct ether_vlan_header *evh; 1361 int ehlen, iphlen, tcphlen; 1362 struct ip *oiph, *niph; 1363 struct tcphdr *otcph, *ntcph; 1364 int available, len, left; 1365 uint16_t ip_id; 1366 uint32_t tcp_seq; 1367 #ifdef __sparc 1368 uint32_t tcp_seq_tmp; 1369 #endif 1370 mblk_t *datamp; 1371 uchar_t *rptr; 1372 mblk_t *nmp; 1373 mblk_t *cmp; 1374 mblk_t *mp_chain; 1375 boolean_t do_cleanup = B_FALSE; 1376 t_uscalar_t start_offset = 0; 1377 t_uscalar_t stuff_offset = 0; 1378 t_uscalar_t value = 0; 1379 uint16_t l4_len; 1380 ipaddr_t src, dst; 1381 uint32_t cksum, sum, l4cksum; 1382 1383 NXGE_DEBUG_MSG((NULL, TX_CTL, 1384 "==> nxge_do_softlso")); 1385 /* 1386 * check the length of LSO packet payload and calculate the number of 1387 * segments to be generated. 1388 */ 1389 pktlen = msgsize(mp); 1390 evh = (struct ether_vlan_header *)mp->b_rptr; 1391 1392 /* VLAN? */ 1393 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1394 ehlen = sizeof (struct ether_vlan_header); 1395 else 1396 ehlen = sizeof (struct ether_header); 1397 oiph = (struct ip *)(mp->b_rptr + ehlen); 1398 iphlen = oiph->ip_hl * 4; 1399 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1400 tcphlen = otcph->th_off * 4; 1401 1402 l4_len = pktlen - ehlen - iphlen; 1403 1404 NXGE_DEBUG_MSG((NULL, TX_CTL, 1405 "==> nxge_do_softlso: mss %d oiph $%p " 1406 "original ip_sum oiph->ip_sum 0x%x " 1407 "original tcp_sum otcph->th_sum 0x%x " 1408 "oiph->ip_len %d pktlen %d ehlen %d " 1409 "l4_len %d (0x%x) ip_len - iphlen %d ", 1410 mss, 1411 oiph, 1412 oiph->ip_sum, 1413 otcph->th_sum, 1414 ntohs(oiph->ip_len), pktlen, 1415 ehlen, 1416 l4_len, 1417 l4_len, 1418 ntohs(oiph->ip_len) - iphlen)); 1419 1420 /* IPv4 + TCP */ 1421 if (!(oiph->ip_v == IPV4_VERSION)) { 1422 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1423 "<== nxge_do_softlso: not IPV4 " 1424 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1425 ntohs(oiph->ip_len), pktlen, ehlen, 1426 tcphlen)); 1427 freemsg(mp); 1428 return (NULL); 1429 } 1430 1431 if (!(oiph->ip_p == IPPROTO_TCP)) { 1432 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1433 "<== nxge_do_softlso: not TCP " 1434 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1435 ntohs(oiph->ip_len), pktlen, ehlen, 1436 tcphlen)); 1437 freemsg(mp); 1438 return (NULL); 1439 } 1440 1441 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1442 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1443 "<== nxge_do_softlso: len not matched " 1444 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1445 ntohs(oiph->ip_len), pktlen, ehlen, 1446 tcphlen)); 1447 freemsg(mp); 1448 return (NULL); 1449 } 1450 1451 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1452 tcphlen = otcph->th_off * 4; 1453 1454 /* TCP flags can not include URG, RST, or SYN */ 1455 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1456 1457 hdrlen = ehlen + iphlen + tcphlen; 1458 1459 VERIFY(MBLKL(mp) >= hdrlen); 1460 1461 if (MBLKL(mp) > hdrlen) { 1462 datamp = mp; 1463 rptr = mp->b_rptr + hdrlen; 1464 } else { /* = */ 1465 datamp = mp->b_cont; 1466 rptr = datamp->b_rptr; 1467 } 1468 1469 NXGE_DEBUG_MSG((NULL, TX_CTL, 1470 "nxge_do_softlso: otcph $%p pktlen: %d, " 1471 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1472 "mblkl(mp): %d, mblkl(datamp): %d", 1473 otcph, 1474 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1475 (int)MBLKL(mp), (int)MBLKL(datamp))); 1476 1477 hckflags = 0; 1478 nxge_hcksum_retrieve(mp, 1479 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1480 1481 dst = oiph->ip_dst.s_addr; 1482 src = oiph->ip_src.s_addr; 1483 1484 cksum = (dst >> 16) + (dst & 0xFFFF) + 1485 (src >> 16) + (src & 0xFFFF); 1486 l4cksum = cksum + IP_TCP_CSUM_COMP; 1487 1488 sum = l4_len + l4cksum; 1489 sum = (sum & 0xFFFF) + (sum >> 16); 1490 1491 NXGE_DEBUG_MSG((NULL, TX_CTL, 1492 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1493 "hckflags 0x%x start_offset %d stuff_offset %d " 1494 "value (original) 0x%x th_sum 0x%x " 1495 "pktlen %d l4_len %d (0x%x) " 1496 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1497 dst, src, 1498 (sum & 0xffff), (~sum & 0xffff), 1499 hckflags, start_offset, stuff_offset, 1500 value, otcph->th_sum, 1501 pktlen, 1502 l4_len, 1503 l4_len, 1504 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1505 (int)MBLKL(datamp), 1506 nxge_dump_packet((char *)evh, 12))); 1507 1508 /* 1509 * Start to process. 1510 */ 1511 available = pktlen - hdrlen; 1512 segnum = (available - 1) / mss + 1; 1513 1514 NXGE_DEBUG_MSG((NULL, TX_CTL, 1515 "==> nxge_do_softlso: pktlen %d " 1516 "MBLKL(mp): %d, MBLKL(datamp): %d " 1517 "available %d mss %d segnum %d", 1518 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1519 available, 1520 mss, 1521 segnum)); 1522 1523 VERIFY(segnum >= 2); 1524 1525 /* 1526 * Try to pre-allocate all header messages 1527 */ 1528 mp_chain = NULL; 1529 for (i = 0; i < segnum; i++) { 1530 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1531 /* Clean up the mp_chain */ 1532 while (mp_chain != NULL) { 1533 nmp = mp_chain; 1534 mp_chain = mp_chain->b_next; 1535 freemsg(nmp); 1536 } 1537 NXGE_DEBUG_MSG((NULL, TX_CTL, 1538 "<== nxge_do_softlso: " 1539 "Could not allocate enough messages for headers!")); 1540 freemsg(mp); 1541 return (NULL); 1542 } 1543 nmp->b_next = mp_chain; 1544 mp_chain = nmp; 1545 1546 NXGE_DEBUG_MSG((NULL, TX_CTL, 1547 "==> nxge_do_softlso: " 1548 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1549 mp, nmp, mp_chain, mp_chain->b_next)); 1550 } 1551 1552 NXGE_DEBUG_MSG((NULL, TX_CTL, 1553 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1554 mp, nmp, mp_chain)); 1555 1556 /* 1557 * Associate payload with new packets 1558 */ 1559 cmp = mp_chain; 1560 left = available; 1561 while (cmp != NULL) { 1562 nmp = dupb(datamp); 1563 if (nmp == NULL) { 1564 do_cleanup = B_TRUE; 1565 NXGE_DEBUG_MSG((NULL, TX_CTL, 1566 "==>nxge_do_softlso: " 1567 "Can not dupb(datamp), have to do clean up")); 1568 goto cleanup_allocated_msgs; 1569 } 1570 1571 NXGE_DEBUG_MSG((NULL, TX_CTL, 1572 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1573 "dupb nmp $%p len %d left %d msd %d ", 1574 mp, cmp, nmp, len, left, mss)); 1575 1576 cmp->b_cont = nmp; 1577 nmp->b_rptr = rptr; 1578 len = (left < mss) ? left : mss; 1579 left -= len; 1580 1581 NXGE_DEBUG_MSG((NULL, TX_CTL, 1582 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1583 "dupb nmp $%p len %d left %d mss %d ", 1584 mp, cmp, nmp, len, left, mss)); 1585 NXGE_DEBUG_MSG((NULL, TX_CTL, 1586 "nxge_do_softlso: before available: %d, " 1587 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1588 available, left, len, segnum, (int)MBLKL(nmp))); 1589 1590 len -= MBLKL(nmp); 1591 NXGE_DEBUG_MSG((NULL, TX_CTL, 1592 "nxge_do_softlso: after available: %d, " 1593 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1594 available, left, len, segnum, (int)MBLKL(nmp))); 1595 1596 while (len > 0) { 1597 mblk_t *mmp = NULL; 1598 1599 NXGE_DEBUG_MSG((NULL, TX_CTL, 1600 "nxge_do_softlso: (4) len > 0 available: %d, " 1601 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1602 available, left, len, segnum, (int)MBLKL(nmp))); 1603 1604 if (datamp->b_cont != NULL) { 1605 datamp = datamp->b_cont; 1606 rptr = datamp->b_rptr; 1607 mmp = dupb(datamp); 1608 if (mmp == NULL) { 1609 do_cleanup = B_TRUE; 1610 NXGE_DEBUG_MSG((NULL, TX_CTL, 1611 "==> nxge_do_softlso: " 1612 "Can not dupb(datamp) (1), : 1613 "have to do clean up")); 1614 NXGE_DEBUG_MSG((NULL, TX_CTL, 1615 "==> nxge_do_softlso: " 1616 "available: %d, left: %d, " 1617 "len: %d, MBLKL(nmp): %d", 1618 available, left, len, 1619 (int)MBLKL(nmp))); 1620 goto cleanup_allocated_msgs; 1621 } 1622 } else { 1623 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1624 "==> nxge_do_softlso: " 1625 "(1)available: %d, left: %d, " 1626 "len: %d, MBLKL(nmp): %d", 1627 available, left, len, 1628 (int)MBLKL(nmp))); 1629 cmn_err(CE_PANIC, 1630 "==> nxge_do_softlso: " 1631 "Pointers must have been corrupted!\n" 1632 "datamp: $%p, nmp: $%p, rptr: $%p", 1633 (void *)datamp, 1634 (void *)nmp, 1635 (void *)rptr); 1636 } 1637 nmp->b_cont = mmp; 1638 nmp = mmp; 1639 len -= MBLKL(nmp); 1640 } 1641 if (len < 0) { 1642 nmp->b_wptr += len; 1643 rptr = nmp->b_wptr; 1644 NXGE_DEBUG_MSG((NULL, TX_CTL, 1645 "(5) len < 0 (less than 0)" 1646 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1647 available, left, len, (int)MBLKL(nmp))); 1648 1649 } else if (len == 0) { 1650 if (datamp->b_cont != NULL) { 1651 NXGE_DEBUG_MSG((NULL, TX_CTL, 1652 "(5) len == 0" 1653 "available: %d, left: %d, len: %d, " 1654 "MBLKL(nmp): %d", 1655 available, left, len, (int)MBLKL(nmp))); 1656 datamp = datamp->b_cont; 1657 rptr = datamp->b_rptr; 1658 } else { 1659 NXGE_DEBUG_MSG((NULL, TX_CTL, 1660 "(6)available b_cont == NULL : %d, " 1661 "left: %d, len: %d, MBLKL(nmp): %d", 1662 available, left, len, (int)MBLKL(nmp))); 1663 1664 VERIFY(cmp->b_next == NULL); 1665 VERIFY(left == 0); 1666 break; /* Done! */ 1667 } 1668 } 1669 cmp = cmp->b_next; 1670 1671 NXGE_DEBUG_MSG((NULL, TX_CTL, 1672 "(7) do_softlso: " 1673 "next mp in mp_chain available len != 0 : %d, " 1674 "left: %d, len: %d, MBLKL(nmp): %d", 1675 available, left, len, (int)MBLKL(nmp))); 1676 } 1677 1678 /* 1679 * From now, start to fill up all headers for the first message 1680 * Hardware checksum flags need to be updated separately for FULLCKSUM 1681 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1682 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1683 * required fields need to be updated properly. 1684 */ 1685 nmp = mp_chain; 1686 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1687 nmp->b_wptr = nmp->b_rptr + hdrlen; 1688 niph = (struct ip *)(nmp->b_rptr + ehlen); 1689 niph->ip_len = htons(mss + iphlen + tcphlen); 1690 ip_id = ntohs(niph->ip_id); 1691 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1692 #ifdef __sparc 1693 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1694 tcp_seq = ntohl(tcp_seq_tmp); 1695 #else 1696 tcp_seq = ntohl(ntcph->th_seq); 1697 #endif 1698 1699 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1700 1701 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1702 DB_CKSUMSTART(nmp) = start_offset; 1703 DB_CKSUMSTUFF(nmp) = stuff_offset; 1704 1705 /* calculate IP checksum and TCP pseudo header checksum */ 1706 niph->ip_sum = 0; 1707 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1708 1709 l4_len = mss + tcphlen; 1710 sum = htons(l4_len) + l4cksum; 1711 sum = (sum & 0xFFFF) + (sum >> 16); 1712 ntcph->th_sum = (sum & 0xffff); 1713 1714 NXGE_DEBUG_MSG((NULL, TX_CTL, 1715 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1716 "mss %d pktlen %d l4_len %d (0x%x) " 1717 "MBLKL(mp): %d, MBLKL(datamp): %d " 1718 "ip_sum 0x%x " 1719 "th_sum 0x%x sum 0x%x ) " 1720 "dump first ip->tcp %s", 1721 nmp, mp_chain, 1722 mss, 1723 pktlen, 1724 l4_len, 1725 l4_len, 1726 (int)MBLKL(mp), (int)MBLKL(datamp), 1727 niph->ip_sum, 1728 ntcph->th_sum, 1729 sum, 1730 nxge_dump_packet((char *)niph, 52))); 1731 1732 cmp = nmp; 1733 while ((nmp = nmp->b_next)->b_next != NULL) { 1734 NXGE_DEBUG_MSG((NULL, TX_CTL, 1735 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1736 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1737 nmp->b_wptr = nmp->b_rptr + hdrlen; 1738 niph = (struct ip *)(nmp->b_rptr + ehlen); 1739 niph->ip_id = htons(++ip_id); 1740 niph->ip_len = htons(mss + iphlen + tcphlen); 1741 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1742 tcp_seq += mss; 1743 1744 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1745 1746 #ifdef __sparc 1747 tcp_seq_tmp = htonl(tcp_seq); 1748 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1749 #else 1750 ntcph->th_seq = htonl(tcp_seq); 1751 #endif 1752 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1753 DB_CKSUMSTART(nmp) = start_offset; 1754 DB_CKSUMSTUFF(nmp) = stuff_offset; 1755 1756 /* calculate IP checksum and TCP pseudo header checksum */ 1757 niph->ip_sum = 0; 1758 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1759 ntcph->th_sum = (sum & 0xffff); 1760 1761 NXGE_DEBUG_MSG((NULL, TX_CTL, 1762 "==> nxge_do_softlso: middle ip_sum 0x%x " 1763 "th_sum 0x%x " 1764 " mp $%p (mp_chain $%p) pktlen %d " 1765 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1766 niph->ip_sum, 1767 ntcph->th_sum, 1768 nmp, mp_chain, 1769 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1770 } 1771 1772 /* Last segment */ 1773 /* 1774 * Set FIN and/or PSH flags if present only in the last packet. 1775 * The ip_len could be different from prior packets. 1776 */ 1777 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1778 nmp->b_wptr = nmp->b_rptr + hdrlen; 1779 niph = (struct ip *)(nmp->b_rptr + ehlen); 1780 niph->ip_id = htons(++ip_id); 1781 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1782 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1783 tcp_seq += mss; 1784 #ifdef __sparc 1785 tcp_seq_tmp = htonl(tcp_seq); 1786 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1787 #else 1788 ntcph->th_seq = htonl(tcp_seq); 1789 #endif 1790 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1791 1792 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1793 DB_CKSUMSTART(nmp) = start_offset; 1794 DB_CKSUMSTUFF(nmp) = stuff_offset; 1795 1796 /* calculate IP checksum and TCP pseudo header checksum */ 1797 niph->ip_sum = 0; 1798 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1799 1800 l4_len = ntohs(niph->ip_len) - iphlen; 1801 sum = htons(l4_len) + l4cksum; 1802 sum = (sum & 0xFFFF) + (sum >> 16); 1803 ntcph->th_sum = (sum & 0xffff); 1804 1805 NXGE_DEBUG_MSG((NULL, TX_CTL, 1806 "==> nxge_do_softlso: last next " 1807 "niph->ip_sum 0x%x " 1808 "ntcph->th_sum 0x%x sum 0x%x " 1809 "dump last ip->tcp %s " 1810 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1811 "l4_len %d (0x%x) " 1812 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1813 niph->ip_sum, 1814 ntcph->th_sum, sum, 1815 nxge_dump_packet((char *)niph, 52), 1816 cmp, nmp, mp_chain, 1817 pktlen, pktlen, 1818 l4_len, 1819 l4_len, 1820 (int)MBLKL(mp), (int)MBLKL(datamp))); 1821 1822 cleanup_allocated_msgs: 1823 if (do_cleanup) { 1824 NXGE_DEBUG_MSG((NULL, TX_CTL, 1825 "==> nxge_do_softlso: " 1826 "Failed allocating messages, " 1827 "have to clean up and fail!")); 1828 while (mp_chain != NULL) { 1829 nmp = mp_chain; 1830 mp_chain = mp_chain->b_next; 1831 freemsg(nmp); 1832 } 1833 } 1834 /* 1835 * We're done here, so just free the original message and return the 1836 * new message chain, that could be NULL if failed, back to the caller. 1837 */ 1838 freemsg(mp); 1839 1840 NXGE_DEBUG_MSG((NULL, TX_CTL, 1841 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1842 return (mp_chain); 1843 } 1844 1845 /* 1846 * Will be called before NIC driver do further operation on the message. 1847 * The input message may include LSO information, if so, go to softlso logic 1848 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1849 * The return could be the same non-LSO message or a message chain for LSO case. 1850 * 1851 * The driver needs to call this function per packet and process the whole chain 1852 * if applied. 1853 */ 1854 static mblk_t * 1855 nxge_lso_eliminate(mblk_t *mp) 1856 { 1857 uint32_t lsoflags; 1858 uint32_t mss; 1859 1860 NXGE_DEBUG_MSG((NULL, TX_CTL, 1861 "==>nxge_lso_eliminate:")); 1862 nxge_lso_info_get(mp, &mss, &lsoflags); 1863 1864 if (lsoflags & HW_LSO) { 1865 mblk_t *nmp; 1866 1867 NXGE_DEBUG_MSG((NULL, TX_CTL, 1868 "==>nxge_lso_eliminate:" 1869 "HW_LSO:mss %d mp $%p", 1870 mss, mp)); 1871 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1872 NXGE_DEBUG_MSG((NULL, TX_CTL, 1873 "<== nxge_lso_eliminate: " 1874 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1875 nmp, mss, mp)); 1876 return (nmp); 1877 } else { 1878 NXGE_DEBUG_MSG((NULL, TX_CTL, 1879 "<== nxge_lso_eliminate_ " 1880 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1881 nmp, mss, mp)); 1882 return (NULL); 1883 } 1884 } 1885 1886 NXGE_DEBUG_MSG((NULL, TX_CTL, 1887 "<== nxge_lso_eliminate")); 1888 return (mp); 1889 } 1890 1891 static uint32_t 1892 nxge_csgen(uint16_t *adr, int len) 1893 { 1894 int i, odd; 1895 uint32_t sum = 0; 1896 uint32_t c = 0; 1897 1898 odd = len % 2; 1899 for (i = 0; i < (len / 2); i++) { 1900 sum += (adr[i] & 0xffff); 1901 } 1902 if (odd) { 1903 sum += adr[len / 2] & 0xff00; 1904 } 1905 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 1906 sum &= 0xffff; 1907 sum += c; 1908 } 1909 return (~sum & 0xffff); 1910 } 1911