1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/nxge/nxge_impl.h> 29 #include <sys/nxge/nxge_hio.h> 30 #include <npi_tx_wr64.h> 31 32 /* Software LSO required header files */ 33 #include <netinet/tcp.h> 34 #include <inet/ip_impl.h> 35 #include <inet/tcp.h> 36 37 static mblk_t *nxge_lso_eliminate(mblk_t *); 38 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 39 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 40 static void nxge_hcksum_retrieve(mblk_t *, 41 uint32_t *, uint32_t *, uint32_t *, 42 uint32_t *, uint32_t *); 43 static uint32_t nxge_csgen(uint16_t *, int); 44 45 extern uint32_t nxge_reclaim_pending; 46 extern uint32_t nxge_bcopy_thresh; 47 extern uint32_t nxge_dvma_thresh; 48 extern uint32_t nxge_dma_stream_thresh; 49 extern uint32_t nxge_tx_minfree; 50 extern uint32_t nxge_tx_intr_thres; 51 extern uint32_t nxge_tx_max_gathers; 52 extern uint32_t nxge_tx_tiny_pack; 53 extern uint32_t nxge_tx_use_bcopy; 54 extern uint32_t nxge_tx_lb_policy; 55 extern uint32_t nxge_no_tx_lb; 56 extern nxge_tx_mode_t nxge_tx_scheme; 57 uint32_t nxge_lso_kick_cnt = 2; 58 59 typedef struct _mac_tx_hint { 60 uint16_t sap; 61 uint16_t vid; 62 void *hash; 63 } mac_tx_hint_t, *p_mac_tx_hint_t; 64 65 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 66 67 int 68 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 69 { 70 int status = 0; 71 p_tx_desc_t tx_desc_ring_vp; 72 npi_handle_t npi_desc_handle; 73 nxge_os_dma_handle_t tx_desc_dma_handle; 74 p_tx_desc_t tx_desc_p; 75 p_tx_msg_t tx_msg_ring; 76 p_tx_msg_t tx_msg_p; 77 tx_desc_t tx_desc, *tmp_desc_p; 78 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 79 p_tx_pkt_header_t hdrp; 80 tx_pkt_header_t tmp_hdrp; 81 p_tx_pkt_hdr_all_t pkthdrp; 82 uint8_t npads = 0; 83 uint64_t dma_ioaddr; 84 uint32_t dma_flags; 85 int last_bidx; 86 uint8_t *b_rptr; 87 caddr_t kaddr; 88 uint32_t nmblks; 89 uint32_t ngathers; 90 uint32_t clen; 91 int len; 92 uint32_t pkt_len, pack_len, min_len; 93 uint32_t bcopy_thresh; 94 int i, cur_index, sop_index; 95 uint16_t tail_index; 96 boolean_t tail_wrap = B_FALSE; 97 nxge_dma_common_t desc_area; 98 nxge_os_dma_handle_t dma_handle; 99 ddi_dma_cookie_t dma_cookie; 100 npi_handle_t npi_handle; 101 p_mblk_t nmp; 102 p_mblk_t t_mp; 103 uint32_t ncookies; 104 boolean_t good_packet; 105 boolean_t mark_mode = B_FALSE; 106 p_nxge_stats_t statsp; 107 p_nxge_tx_ring_stats_t tdc_stats; 108 t_uscalar_t start_offset = 0; 109 t_uscalar_t stuff_offset = 0; 110 t_uscalar_t end_offset = 0; 111 t_uscalar_t value = 0; 112 t_uscalar_t cksum_flags = 0; 113 boolean_t cksum_on = B_FALSE; 114 uint32_t boff = 0; 115 uint64_t tot_xfer_len = 0; 116 boolean_t header_set = B_FALSE; 117 #ifdef NXGE_DEBUG 118 p_tx_desc_t tx_desc_ring_pp; 119 p_tx_desc_t tx_desc_pp; 120 tx_desc_t *save_desc_p; 121 int dump_len; 122 int sad_len; 123 uint64_t sad; 124 int xfer_len; 125 uint32_t msgsize; 126 #endif 127 p_mblk_t mp_chain = NULL; 128 boolean_t is_lso = B_FALSE; 129 boolean_t lso_again; 130 int cur_index_lso; 131 p_mblk_t nmp_lso_save; 132 uint32_t lso_ngathers; 133 boolean_t lso_tail_wrap = B_FALSE; 134 135 NXGE_DEBUG_MSG((nxgep, TX_CTL, 136 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 137 NXGE_DEBUG_MSG((nxgep, TX_CTL, 138 "==> nxge_start: Starting tdc %d desc pending %d", 139 tx_ring_p->tdc, tx_ring_p->descs_pending)); 140 141 statsp = nxgep->statsp; 142 143 if (!isLDOMguest(nxgep)) { 144 switch (nxgep->mac.portmode) { 145 default: 146 if (nxgep->statsp->port_stats.lb_mode == 147 nxge_lb_normal) { 148 if (!statsp->mac_stats.link_up) { 149 freemsg(mp); 150 NXGE_DEBUG_MSG((nxgep, TX_CTL, 151 "==> nxge_start: " 152 "link not up")); 153 goto nxge_start_fail1; 154 } 155 } 156 break; 157 case PORT_10G_FIBER: 158 /* 159 * For the following modes, check the link status 160 * before sending the packet out: 161 * nxge_lb_normal, nxge_lb_ext10g, nxge_lb_phy10g 162 */ 163 if (nxgep->statsp->port_stats.lb_mode < 164 nxge_lb_serdes10g) { 165 if (!statsp->mac_stats.link_up) { 166 freemsg(mp); 167 NXGE_DEBUG_MSG((nxgep, TX_CTL, 168 "==> nxge_start: " 169 "link not up")); 170 goto nxge_start_fail1; 171 } 172 } 173 break; 174 } 175 } 176 177 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 178 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 179 NXGE_DEBUG_MSG((nxgep, TX_CTL, 180 "==> nxge_start: hardware not initialized or stopped")); 181 freemsg(mp); 182 goto nxge_start_fail1; 183 } 184 185 if (nxgep->soft_lso_enable) { 186 mp_chain = nxge_lso_eliminate(mp); 187 NXGE_DEBUG_MSG((nxgep, TX_CTL, 188 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 189 mp, mp_chain)); 190 if (mp_chain == NULL) { 191 NXGE_ERROR_MSG((nxgep, TX_CTL, 192 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 193 mp_chain, mp)); 194 goto nxge_start_fail1; 195 } 196 if (mp_chain != mp) { 197 NXGE_DEBUG_MSG((nxgep, TX_CTL, 198 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 199 mp_chain, mp)); 200 is_lso = B_TRUE; 201 mp = mp_chain; 202 mp_chain = mp_chain->b_next; 203 mp->b_next = NULL; 204 } 205 } 206 207 hcksum_retrieve(mp, NULL, NULL, &start_offset, 208 &stuff_offset, &end_offset, &value, &cksum_flags); 209 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 210 start_offset += sizeof (ether_header_t); 211 stuff_offset += sizeof (ether_header_t); 212 } else { 213 start_offset += sizeof (struct ether_vlan_header); 214 stuff_offset += sizeof (struct ether_vlan_header); 215 } 216 217 if (cksum_flags & HCK_PARTIALCKSUM) { 218 NXGE_DEBUG_MSG((nxgep, TX_CTL, 219 "==> nxge_start: mp $%p len %d " 220 "cksum_flags 0x%x (partial checksum) ", 221 mp, MBLKL(mp), cksum_flags)); 222 cksum_on = B_TRUE; 223 } 224 225 pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 226 pkthdrp->reserved = 0; 227 tmp_hdrp.value = 0; 228 nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 229 0, 0, pkthdrp, 230 start_offset, stuff_offset); 231 232 lso_again = B_FALSE; 233 lso_ngathers = 0; 234 235 MUTEX_ENTER(&tx_ring_p->lock); 236 cur_index_lso = tx_ring_p->wr_index; 237 lso_tail_wrap = tx_ring_p->wr_index_wrap; 238 start_again: 239 ngathers = 0; 240 sop_index = tx_ring_p->wr_index; 241 #ifdef NXGE_DEBUG 242 if (tx_ring_p->descs_pending) { 243 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 244 "desc pending %d ", tx_ring_p->descs_pending)); 245 } 246 247 dump_len = (int)(MBLKL(mp)); 248 dump_len = (dump_len > 128) ? 128: dump_len; 249 250 NXGE_DEBUG_MSG((nxgep, TX_CTL, 251 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 252 "(Before header reserve: ORIGINAL LEN %d)", 253 tx_ring_p->tdc, 254 mp->b_rptr, 255 dump_len)); 256 257 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 258 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 259 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 260 #endif 261 262 tdc_stats = tx_ring_p->tdc_stats; 263 mark_mode = (tx_ring_p->descs_pending && 264 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 265 < nxge_tx_minfree)); 266 267 NXGE_DEBUG_MSG((nxgep, TX_CTL, 268 "TX Descriptor ring is channel %d mark mode %d", 269 tx_ring_p->tdc, mark_mode)); 270 271 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { 272 NXGE_DEBUG_MSG((nxgep, TX_CTL, 273 "TX Descriptor ring is full: channel %d", 274 tx_ring_p->tdc)); 275 NXGE_DEBUG_MSG((nxgep, TX_CTL, 276 "TX Descriptor ring is full: channel %d", 277 tx_ring_p->tdc)); 278 if (is_lso) { 279 /* free the current mp and mp_chain if not FULL */ 280 tdc_stats->tx_no_desc++; 281 NXGE_DEBUG_MSG((nxgep, TX_CTL, 282 "LSO packet: TX Descriptor ring is full: " 283 "channel %d", 284 tx_ring_p->tdc)); 285 goto nxge_start_fail_lso; 286 } else { 287 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 288 tdc_stats->tx_no_desc++; 289 MUTEX_EXIT(&tx_ring_p->lock); 290 if (nxgep->resched_needed && !nxgep->resched_running) { 291 nxgep->resched_running = B_TRUE; 292 ddi_trigger_softintr(nxgep->resched_id); 293 } 294 status = 1; 295 goto nxge_start_fail1; 296 } 297 } 298 299 nmp = mp; 300 i = sop_index = tx_ring_p->wr_index; 301 nmblks = 0; 302 ngathers = 0; 303 pkt_len = 0; 304 pack_len = 0; 305 clen = 0; 306 last_bidx = -1; 307 good_packet = B_TRUE; 308 309 desc_area = tx_ring_p->tdc_desc; 310 npi_handle = desc_area.npi_handle; 311 npi_desc_handle.regh = (nxge_os_acc_handle_t) 312 DMA_COMMON_ACC_HANDLE(desc_area); 313 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 314 tx_desc_dma_handle = (nxge_os_dma_handle_t) 315 DMA_COMMON_HANDLE(desc_area); 316 tx_msg_ring = tx_ring_p->tx_msg_ring; 317 318 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 319 sop_index, i)); 320 321 #ifdef NXGE_DEBUG 322 msgsize = msgdsize(nmp); 323 NXGE_DEBUG_MSG((nxgep, TX_CTL, 324 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 325 sop_index, i, msgsize)); 326 #endif 327 /* 328 * The first 16 bytes of the premapped buffer are reserved 329 * for header. No padding will be used. 330 */ 331 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 332 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 333 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 334 } else { 335 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 336 } 337 while (nmp) { 338 good_packet = B_TRUE; 339 b_rptr = nmp->b_rptr; 340 len = MBLKL(nmp); 341 if (len <= 0) { 342 nmp = nmp->b_cont; 343 continue; 344 } 345 nmblks++; 346 347 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 348 "len %d pkt_len %d pack_len %d", 349 nmblks, len, pkt_len, pack_len)); 350 /* 351 * Hardware limits the transfer length to 4K for NIU and 352 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 353 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 354 * If len is longer than the limit, then we break nmp into 355 * two chunks: Make the first chunk equal to the limit and 356 * the second chunk for the remaining data. If the second 357 * chunk is still larger than the limit, then it will be 358 * broken into two in the next pass. 359 */ 360 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 361 if ((t_mp = dupb(nmp)) != NULL) { 362 nmp->b_wptr = nmp->b_rptr + 363 (TX_MAX_TRANSFER_LENGTH 364 - TX_PKT_HEADER_SIZE); 365 t_mp->b_rptr = nmp->b_wptr; 366 t_mp->b_cont = nmp->b_cont; 367 nmp->b_cont = t_mp; 368 len = MBLKL(nmp); 369 } else { 370 if (is_lso) { 371 NXGE_DEBUG_MSG((nxgep, TX_CTL, 372 "LSO packet: dupb failed: " 373 "channel %d", 374 tx_ring_p->tdc)); 375 mp = nmp; 376 goto nxge_start_fail_lso; 377 } else { 378 good_packet = B_FALSE; 379 goto nxge_start_fail2; 380 } 381 } 382 } 383 tx_desc.value = 0; 384 tx_desc_p = &tx_desc_ring_vp[i]; 385 #ifdef NXGE_DEBUG 386 tx_desc_pp = &tx_desc_ring_pp[i]; 387 #endif 388 tx_msg_p = &tx_msg_ring[i]; 389 #if defined(__i386) 390 npi_desc_handle.regp = (uint32_t)tx_desc_p; 391 #else 392 npi_desc_handle.regp = (uint64_t)tx_desc_p; 393 #endif 394 if (!header_set && 395 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 396 (len >= bcopy_thresh))) { 397 header_set = B_TRUE; 398 bcopy_thresh += TX_PKT_HEADER_SIZE; 399 boff = 0; 400 pack_len = 0; 401 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 402 hdrp = (p_tx_pkt_header_t)kaddr; 403 clen = pkt_len; 404 dma_handle = tx_msg_p->buf_dma_handle; 405 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 406 (void) ddi_dma_sync(dma_handle, 407 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 408 DDI_DMA_SYNC_FORDEV); 409 410 tx_msg_p->flags.dma_type = USE_BCOPY; 411 goto nxge_start_control_header_only; 412 } 413 414 pkt_len += len; 415 pack_len += len; 416 417 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 418 "desc entry %d " 419 "DESC IOADDR $%p " 420 "desc_vp $%p tx_desc_p $%p " 421 "desc_pp $%p tx_desc_pp $%p " 422 "len %d pkt_len %d pack_len %d", 423 i, 424 DMA_COMMON_IOADDR(desc_area), 425 tx_desc_ring_vp, tx_desc_p, 426 tx_desc_ring_pp, tx_desc_pp, 427 len, pkt_len, pack_len)); 428 429 if (len < bcopy_thresh) { 430 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 431 "USE BCOPY: ")); 432 if (nxge_tx_tiny_pack) { 433 uint32_t blst = 434 TXDMA_DESC_NEXT_INDEX(i, -1, 435 tx_ring_p->tx_wrap_mask); 436 NXGE_DEBUG_MSG((nxgep, TX_CTL, 437 "==> nxge_start(5): pack")); 438 if ((pack_len <= bcopy_thresh) && 439 (last_bidx == blst)) { 440 NXGE_DEBUG_MSG((nxgep, TX_CTL, 441 "==> nxge_start: pack(6) " 442 "(pkt_len %d pack_len %d)", 443 pkt_len, pack_len)); 444 i = blst; 445 tx_desc_p = &tx_desc_ring_vp[i]; 446 #ifdef NXGE_DEBUG 447 tx_desc_pp = &tx_desc_ring_pp[i]; 448 #endif 449 tx_msg_p = &tx_msg_ring[i]; 450 boff = pack_len - len; 451 ngathers--; 452 } else if (pack_len > bcopy_thresh && 453 header_set) { 454 pack_len = len; 455 boff = 0; 456 bcopy_thresh = nxge_bcopy_thresh; 457 NXGE_DEBUG_MSG((nxgep, TX_CTL, 458 "==> nxge_start(7): > max NEW " 459 "bcopy thresh %d " 460 "pkt_len %d pack_len %d(next)", 461 bcopy_thresh, 462 pkt_len, pack_len)); 463 } 464 last_bidx = i; 465 } 466 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 467 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 468 hdrp = (p_tx_pkt_header_t)kaddr; 469 header_set = B_TRUE; 470 NXGE_DEBUG_MSG((nxgep, TX_CTL, 471 "==> nxge_start(7_x2): " 472 "pkt_len %d pack_len %d (new hdrp $%p)", 473 pkt_len, pack_len, hdrp)); 474 } 475 tx_msg_p->flags.dma_type = USE_BCOPY; 476 kaddr += boff; 477 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 478 "USE BCOPY: before bcopy " 479 "DESC IOADDR $%p entry %d " 480 "bcopy packets %d " 481 "bcopy kaddr $%p " 482 "bcopy ioaddr (SAD) $%p " 483 "bcopy clen %d " 484 "bcopy boff %d", 485 DMA_COMMON_IOADDR(desc_area), i, 486 tdc_stats->tx_hdr_pkts, 487 kaddr, 488 dma_ioaddr, 489 clen, 490 boff)); 491 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 492 "1USE BCOPY: ")); 493 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 494 "2USE BCOPY: ")); 495 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 496 "last USE BCOPY: copy from b_rptr $%p " 497 "to KADDR $%p (len %d offset %d", 498 b_rptr, kaddr, len, boff)); 499 500 bcopy(b_rptr, kaddr, len); 501 502 #ifdef NXGE_DEBUG 503 dump_len = (len > 128) ? 128: len; 504 NXGE_DEBUG_MSG((nxgep, TX_CTL, 505 "==> nxge_start: dump packets " 506 "(After BCOPY len %d)" 507 "(b_rptr $%p): %s", len, nmp->b_rptr, 508 nxge_dump_packet((char *)nmp->b_rptr, 509 dump_len))); 510 #endif 511 512 dma_handle = tx_msg_p->buf_dma_handle; 513 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 514 (void) ddi_dma_sync(dma_handle, 515 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 516 DDI_DMA_SYNC_FORDEV); 517 clen = len + boff; 518 tdc_stats->tx_hdr_pkts++; 519 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 520 "USE BCOPY: " 521 "DESC IOADDR $%p entry %d " 522 "bcopy packets %d " 523 "bcopy kaddr $%p " 524 "bcopy ioaddr (SAD) $%p " 525 "bcopy clen %d " 526 "bcopy boff %d", 527 DMA_COMMON_IOADDR(desc_area), 528 i, 529 tdc_stats->tx_hdr_pkts, 530 kaddr, 531 dma_ioaddr, 532 clen, 533 boff)); 534 } else { 535 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 536 "USE DVMA: len %d", len)); 537 tx_msg_p->flags.dma_type = USE_DMA; 538 dma_flags = DDI_DMA_WRITE; 539 if (len < nxge_dma_stream_thresh) { 540 dma_flags |= DDI_DMA_CONSISTENT; 541 } else { 542 dma_flags |= DDI_DMA_STREAMING; 543 } 544 545 dma_handle = tx_msg_p->dma_handle; 546 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 547 (caddr_t)b_rptr, len, dma_flags, 548 DDI_DMA_DONTWAIT, NULL, 549 &dma_cookie, &ncookies); 550 if (status == DDI_DMA_MAPPED) { 551 dma_ioaddr = dma_cookie.dmac_laddress; 552 len = (int)dma_cookie.dmac_size; 553 clen = (uint32_t)dma_cookie.dmac_size; 554 NXGE_DEBUG_MSG((nxgep, TX_CTL, 555 "==> nxge_start(12_1): " 556 "USE DVMA: len %d clen %d " 557 "ngathers %d", 558 len, clen, 559 ngathers)); 560 #if defined(__i386) 561 npi_desc_handle.regp = (uint32_t)tx_desc_p; 562 #else 563 npi_desc_handle.regp = (uint64_t)tx_desc_p; 564 #endif 565 while (ncookies > 1) { 566 ngathers++; 567 /* 568 * this is the fix for multiple 569 * cookies, which are basically 570 * a descriptor entry, we don't set 571 * SOP bit as well as related fields 572 */ 573 574 (void) npi_txdma_desc_gather_set( 575 npi_desc_handle, 576 &tx_desc, 577 (ngathers -1), 578 mark_mode, 579 ngathers, 580 dma_ioaddr, 581 clen); 582 583 tx_msg_p->tx_msg_size = clen; 584 NXGE_DEBUG_MSG((nxgep, TX_CTL, 585 "==> nxge_start: DMA " 586 "ncookie %d " 587 "ngathers %d " 588 "dma_ioaddr $%p len %d" 589 "desc $%p descp $%p (%d)", 590 ncookies, 591 ngathers, 592 dma_ioaddr, clen, 593 *tx_desc_p, tx_desc_p, i)); 594 595 ddi_dma_nextcookie(dma_handle, 596 &dma_cookie); 597 dma_ioaddr = 598 dma_cookie.dmac_laddress; 599 600 len = (int)dma_cookie.dmac_size; 601 clen = (uint32_t)dma_cookie.dmac_size; 602 NXGE_DEBUG_MSG((nxgep, TX_CTL, 603 "==> nxge_start(12_2): " 604 "USE DVMA: len %d clen %d ", 605 len, clen)); 606 607 i = TXDMA_DESC_NEXT_INDEX(i, 1, 608 tx_ring_p->tx_wrap_mask); 609 tx_desc_p = &tx_desc_ring_vp[i]; 610 611 npi_desc_handle.regp = 612 #if defined(__i386) 613 (uint32_t)tx_desc_p; 614 #else 615 (uint64_t)tx_desc_p; 616 #endif 617 tx_msg_p = &tx_msg_ring[i]; 618 tx_msg_p->flags.dma_type = USE_NONE; 619 tx_desc.value = 0; 620 621 ncookies--; 622 } 623 tdc_stats->tx_ddi_pkts++; 624 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 625 "DMA: ddi packets %d", 626 tdc_stats->tx_ddi_pkts)); 627 } else { 628 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 629 "dma mapping failed for %d " 630 "bytes addr $%p flags %x (%d)", 631 len, b_rptr, status, status)); 632 good_packet = B_FALSE; 633 tdc_stats->tx_dma_bind_fail++; 634 tx_msg_p->flags.dma_type = USE_NONE; 635 if (is_lso) { 636 mp = nmp; 637 goto nxge_start_fail_lso; 638 } else { 639 goto nxge_start_fail2; 640 } 641 } 642 } /* ddi dvma */ 643 644 if (is_lso) { 645 nmp_lso_save = nmp; 646 } 647 nmp = nmp->b_cont; 648 nxge_start_control_header_only: 649 #if defined(__i386) 650 npi_desc_handle.regp = (uint32_t)tx_desc_p; 651 #else 652 npi_desc_handle.regp = (uint64_t)tx_desc_p; 653 #endif 654 ngathers++; 655 656 if (ngathers == 1) { 657 #ifdef NXGE_DEBUG 658 save_desc_p = &sop_tx_desc; 659 #endif 660 sop_tx_desc_p = &sop_tx_desc; 661 sop_tx_desc_p->value = 0; 662 sop_tx_desc_p->bits.hdw.tr_len = clen; 663 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 664 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 665 } else { 666 #ifdef NXGE_DEBUG 667 save_desc_p = &tx_desc; 668 #endif 669 tmp_desc_p = &tx_desc; 670 tmp_desc_p->value = 0; 671 tmp_desc_p->bits.hdw.tr_len = clen; 672 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 673 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 674 675 tx_desc_p->value = tmp_desc_p->value; 676 } 677 678 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 679 "Desc_entry %d ngathers %d " 680 "desc_vp $%p tx_desc_p $%p " 681 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 682 "dma_ioaddr (SAD) $%p mark %d", 683 i, ngathers, 684 tx_desc_ring_vp, tx_desc_p, 685 len, clen, pkt_len, pack_len, nmblks, 686 dma_ioaddr, mark_mode)); 687 688 #ifdef NXGE_DEBUG 689 npi_desc_handle.nxgep = nxgep; 690 npi_desc_handle.function.function = nxgep->function_num; 691 npi_desc_handle.function.instance = nxgep->instance; 692 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 693 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 694 TX_PKT_DESC_TR_LEN_SHIFT); 695 696 697 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 698 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 699 "mark %d sop %d\n", 700 save_desc_p->value, 701 sad, 702 save_desc_p->bits.hdw.tr_len, 703 xfer_len, 704 save_desc_p->bits.hdw.num_ptr, 705 save_desc_p->bits.hdw.mark, 706 save_desc_p->bits.hdw.sop)); 707 708 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 709 #endif 710 711 tx_msg_p->tx_msg_size = clen; 712 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 713 if (ngathers > nxge_tx_max_gathers) { 714 good_packet = B_FALSE; 715 hcksum_retrieve(mp, NULL, NULL, &start_offset, 716 &stuff_offset, &end_offset, &value, 717 &cksum_flags); 718 719 NXGE_DEBUG_MSG((NULL, TX_CTL, 720 "==> nxge_start(14): pull msg - " 721 "len %d pkt_len %d ngathers %d", 722 len, pkt_len, ngathers)); 723 /* Pull all message blocks from b_cont */ 724 if (is_lso) { 725 mp = nmp_lso_save; 726 goto nxge_start_fail_lso; 727 } 728 if ((msgpullup(mp, -1)) == NULL) { 729 goto nxge_start_fail2; 730 } 731 goto nxge_start_fail2; 732 } 733 } /* while (nmp) */ 734 735 tx_msg_p->tx_message = mp; 736 tx_desc_p = &tx_desc_ring_vp[sop_index]; 737 #if defined(__i386) 738 npi_desc_handle.regp = (uint32_t)tx_desc_p; 739 #else 740 npi_desc_handle.regp = (uint64_t)tx_desc_p; 741 #endif 742 743 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 744 pkthdrp->reserved = 0; 745 hdrp->value = 0; 746 bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 747 748 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 749 tdc_stats->tx_jumbo_pkts++; 750 } 751 752 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 753 if (pkt_len < min_len) { 754 /* Assume we use bcopy to premapped buffers */ 755 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 756 NXGE_DEBUG_MSG((NULL, TX_CTL, 757 "==> nxge_start(14-1): < (msg_min + 16)" 758 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 759 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 760 bzero((kaddr + pkt_len), (min_len - pkt_len)); 761 pkt_len = tx_msg_p->tx_msg_size = min_len; 762 763 sop_tx_desc_p->bits.hdw.tr_len = min_len; 764 765 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 766 tx_desc_p->value = sop_tx_desc_p->value; 767 768 NXGE_DEBUG_MSG((NULL, TX_CTL, 769 "==> nxge_start(14-2): < msg_min - " 770 "len %d pkt_len %d min_len %d ngathers %d", 771 len, pkt_len, min_len, ngathers)); 772 } 773 774 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 775 cksum_flags)); 776 { 777 uint64_t tmp_len; 778 779 /* pkt_len already includes 16 + paddings!! */ 780 /* Update the control header length */ 781 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 782 tmp_len = hdrp->value | 783 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 784 785 NXGE_DEBUG_MSG((nxgep, TX_CTL, 786 "==> nxge_start(15_x1): setting SOP " 787 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 788 "0x%llx hdrp->value 0x%llx", 789 tot_xfer_len, tot_xfer_len, pkt_len, 790 tmp_len, hdrp->value)); 791 #if defined(_BIG_ENDIAN) 792 hdrp->value = ddi_swap64(tmp_len); 793 #else 794 hdrp->value = tmp_len; 795 #endif 796 NXGE_DEBUG_MSG((nxgep, 797 TX_CTL, "==> nxge_start(15_x2): setting SOP " 798 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 799 "tmp_len 0x%llx hdrp->value 0x%llx", 800 tot_xfer_len, pkt_len, 801 tmp_len, hdrp->value)); 802 } 803 804 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 805 "wr_index %d " 806 "tot_xfer_len (%d) pkt_len %d npads %d", 807 sop_index, 808 tot_xfer_len, pkt_len, 809 npads)); 810 811 sop_tx_desc_p->bits.hdw.sop = 1; 812 sop_tx_desc_p->bits.hdw.mark = mark_mode; 813 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 814 815 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 816 817 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 818 819 #ifdef NXGE_DEBUG 820 npi_desc_handle.nxgep = nxgep; 821 npi_desc_handle.function.function = nxgep->function_num; 822 npi_desc_handle.function.instance = nxgep->instance; 823 824 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 825 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 826 save_desc_p->value, 827 sad, 828 save_desc_p->bits.hdw.tr_len, 829 xfer_len, 830 save_desc_p->bits.hdw.num_ptr, 831 save_desc_p->bits.hdw.mark, 832 save_desc_p->bits.hdw.sop)); 833 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 834 835 dump_len = (pkt_len > 128) ? 128: pkt_len; 836 NXGE_DEBUG_MSG((nxgep, TX_CTL, 837 "==> nxge_start: dump packets(17) (after sop set, len " 838 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 839 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 840 (char *)hdrp, 841 nxge_dump_packet((char *)hdrp, dump_len))); 842 NXGE_DEBUG_MSG((nxgep, TX_CTL, 843 "==> nxge_start(18): TX desc sync: sop_index %d", 844 sop_index)); 845 #endif 846 847 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 848 (void) ddi_dma_sync(tx_desc_dma_handle, 849 sop_index * sizeof (tx_desc_t), 850 ngathers * sizeof (tx_desc_t), 851 DDI_DMA_SYNC_FORDEV); 852 853 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 854 "cs_off = 0x%02X cs_s_off = 0x%02X " 855 "pkt_len %d ngathers %d sop_index %d\n", 856 stuff_offset, start_offset, 857 pkt_len, ngathers, sop_index)); 858 } else { /* more than one descriptor and wrap around */ 859 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 860 (void) ddi_dma_sync(tx_desc_dma_handle, 861 sop_index * sizeof (tx_desc_t), 862 nsdescs * sizeof (tx_desc_t), 863 DDI_DMA_SYNC_FORDEV); 864 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 865 "cs_off = 0x%02X cs_s_off = 0x%02X " 866 "pkt_len %d ngathers %d sop_index %d\n", 867 stuff_offset, start_offset, 868 pkt_len, ngathers, sop_index)); 869 870 (void) ddi_dma_sync(tx_desc_dma_handle, 871 0, 872 (ngathers - nsdescs) * sizeof (tx_desc_t), 873 DDI_DMA_SYNC_FORDEV); 874 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 875 "cs_off = 0x%02X cs_s_off = 0x%02X " 876 "pkt_len %d ngathers %d sop_index %d\n", 877 stuff_offset, start_offset, 878 pkt_len, ngathers, sop_index)); 879 } 880 881 tail_index = tx_ring_p->wr_index; 882 tail_wrap = tx_ring_p->wr_index_wrap; 883 884 tx_ring_p->wr_index = i; 885 if (tx_ring_p->wr_index <= tail_index) { 886 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 887 B_FALSE : B_TRUE); 888 } 889 890 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 891 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 892 tx_ring_p->tdc, 893 tx_ring_p->wr_index, 894 tx_ring_p->wr_index_wrap, 895 ngathers, 896 tx_ring_p->descs_pending)); 897 898 if (is_lso) { 899 lso_ngathers += ngathers; 900 if (mp_chain != NULL) { 901 mp = mp_chain; 902 mp_chain = mp_chain->b_next; 903 mp->b_next = NULL; 904 if (nxge_lso_kick_cnt == lso_ngathers) { 905 tx_ring_p->descs_pending += lso_ngathers; 906 { 907 tx_ring_kick_t kick; 908 909 kick.value = 0; 910 kick.bits.ldw.wrap = 911 tx_ring_p->wr_index_wrap; 912 kick.bits.ldw.tail = 913 (uint16_t)tx_ring_p->wr_index; 914 915 /* Kick the Transmit kick register */ 916 TXDMA_REG_WRITE64( 917 NXGE_DEV_NPI_HANDLE(nxgep), 918 TX_RING_KICK_REG, 919 (uint8_t)tx_ring_p->tdc, 920 kick.value); 921 tdc_stats->tx_starts++; 922 923 NXGE_DEBUG_MSG((nxgep, TX_CTL, 924 "==> nxge_start: more LSO: " 925 "LSO_CNT %d", 926 lso_ngathers)); 927 } 928 lso_ngathers = 0; 929 ngathers = 0; 930 cur_index_lso = sop_index = tx_ring_p->wr_index; 931 lso_tail_wrap = tx_ring_p->wr_index_wrap; 932 } 933 NXGE_DEBUG_MSG((nxgep, TX_CTL, 934 "==> nxge_start: lso again: " 935 "lso_gathers %d ngathers %d cur_index_lso %d " 936 "wr_index %d sop_index %d", 937 lso_ngathers, ngathers, cur_index_lso, 938 tx_ring_p->wr_index, sop_index)); 939 940 NXGE_DEBUG_MSG((nxgep, TX_CTL, 941 "==> nxge_start: next : count %d", 942 lso_ngathers)); 943 lso_again = B_TRUE; 944 goto start_again; 945 } 946 ngathers = lso_ngathers; 947 } 948 949 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 950 951 { 952 tx_ring_kick_t kick; 953 954 kick.value = 0; 955 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 956 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 957 958 /* Kick start the Transmit kick register */ 959 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 960 TX_RING_KICK_REG, 961 (uint8_t)tx_ring_p->tdc, 962 kick.value); 963 } 964 965 tx_ring_p->descs_pending += ngathers; 966 tdc_stats->tx_starts++; 967 968 tx_ring_p->tx_ring_state = TX_RING_STATE_IDLE; 969 970 MUTEX_EXIT(&tx_ring_p->lock); 971 972 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 973 974 return (status); 975 976 nxge_start_fail_lso: 977 status = 0; 978 good_packet = B_FALSE; 979 if (mp != NULL) { 980 freemsg(mp); 981 } 982 if (mp_chain != NULL) { 983 freemsg(mp_chain); 984 } 985 if (!lso_again && !ngathers) { 986 MUTEX_EXIT(&tx_ring_p->lock); 987 NXGE_DEBUG_MSG((nxgep, TX_CTL, 988 "==> nxge_start: lso exit (nothing changed)")); 989 goto nxge_start_fail1; 990 } 991 992 NXGE_DEBUG_MSG((nxgep, TX_CTL, 993 "==> nxge_start (channel %d): before lso " 994 "lso_gathers %d ngathers %d cur_index_lso %d " 995 "wr_index %d sop_index %d lso_again %d", 996 tx_ring_p->tdc, 997 lso_ngathers, ngathers, cur_index_lso, 998 tx_ring_p->wr_index, sop_index, lso_again)); 999 1000 if (lso_again) { 1001 lso_ngathers += ngathers; 1002 ngathers = lso_ngathers; 1003 sop_index = cur_index_lso; 1004 tx_ring_p->wr_index = sop_index; 1005 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1006 } 1007 1008 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1009 "==> nxge_start (channel %d): after lso " 1010 "lso_gathers %d ngathers %d cur_index_lso %d " 1011 "wr_index %d sop_index %d lso_again %d", 1012 tx_ring_p->tdc, 1013 lso_ngathers, ngathers, cur_index_lso, 1014 tx_ring_p->wr_index, sop_index, lso_again)); 1015 1016 nxge_start_fail2: 1017 if (good_packet == B_FALSE) { 1018 cur_index = sop_index; 1019 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1020 for (i = 0; i < ngathers; i++) { 1021 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1022 #if defined(__i386) 1023 npi_handle.regp = (uint32_t)tx_desc_p; 1024 #else 1025 npi_handle.regp = (uint64_t)tx_desc_p; 1026 #endif 1027 tx_msg_p = &tx_msg_ring[cur_index]; 1028 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1029 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1030 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1031 "tx_desc_p = %X index = %d", 1032 tx_desc_p, tx_ring_p->rd_index)); 1033 (void) dvma_unload(tx_msg_p->dvma_handle, 1034 0, -1); 1035 tx_msg_p->dvma_handle = NULL; 1036 if (tx_ring_p->dvma_wr_index == 1037 tx_ring_p->dvma_wrap_mask) 1038 tx_ring_p->dvma_wr_index = 0; 1039 else 1040 tx_ring_p->dvma_wr_index++; 1041 tx_ring_p->dvma_pending--; 1042 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1043 if (ddi_dma_unbind_handle( 1044 tx_msg_p->dma_handle)) { 1045 cmn_err(CE_WARN, "!nxge_start: " 1046 "ddi_dma_unbind_handle failed"); 1047 } 1048 } 1049 tx_msg_p->flags.dma_type = USE_NONE; 1050 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1051 tx_ring_p->tx_wrap_mask); 1052 1053 } 1054 1055 nxgep->resched_needed = B_TRUE; 1056 } 1057 1058 tx_ring_p->tx_ring_state = TX_RING_STATE_IDLE; 1059 1060 MUTEX_EXIT(&tx_ring_p->lock); 1061 1062 nxge_start_fail1: 1063 /* Add FMA to check the access handle nxge_hregh */ 1064 1065 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1066 1067 return (status); 1068 } 1069 1070 int 1071 nxge_serial_tx(mblk_t *mp, void *arg) 1072 { 1073 p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 1074 p_nxge_t nxgep = tx_ring_p->nxgep; 1075 1076 return (nxge_start(nxgep, tx_ring_p, mp)); 1077 } 1078 1079 boolean_t 1080 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 1081 { 1082 p_tx_ring_t *tx_rings; 1083 uint8_t ring_index; 1084 p_tx_ring_t tx_ring_p; 1085 nxge_grp_t *group; 1086 1087 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 1088 1089 ASSERT(mp->b_next == NULL); 1090 1091 group = nxgep->tx_set.group[0]; /* The default group */ 1092 ring_index = nxge_tx_lb_ring_1(mp, group->count, hp); 1093 1094 tx_rings = nxgep->tx_rings->rings; 1095 tx_ring_p = tx_rings[group->legend[ring_index]]; 1096 1097 MUTEX_ENTER(&tx_ring_p->lock); 1098 if (tx_ring_p->tx_ring_state == TX_RING_STATE_OFFLINE) { 1099 /* 1100 * OFFLINE means that it is in the process of being 1101 * shared - that is, it has been claimed by the HIO 1102 * code, but hasn't been unlinked from <group> yet. 1103 * So in this case use the first TDC, which always 1104 * belongs to the service domain and can't be shared. 1105 */ 1106 MUTEX_EXIT(&tx_ring_p->lock); 1107 1108 ring_index = 0; 1109 tx_ring_p = tx_rings[group->legend[ring_index]]; 1110 MUTEX_ENTER(&tx_ring_p->lock); 1111 tx_ring_p->tx_ring_state = TX_RING_STATE_BUSY; 1112 } else { 1113 /* 1114 * Otherwise, mark the TDC as BUSY: the HIO code 1115 * will wait until nxge_start() has completed. 1116 */ 1117 tx_ring_p->tx_ring_state = TX_RING_STATE_BUSY; 1118 } 1119 MUTEX_EXIT(&tx_ring_p->lock); 1120 1121 NXGE_DEBUG_MSG((nxgep, TX_CTL, "count %d, tx_rings[%d] = %p", 1122 (int)group->count, group->legend[ring_index], tx_ring_p)); 1123 1124 switch (nxge_tx_scheme) { 1125 case NXGE_USE_START: 1126 if (nxge_start(nxgep, tx_ring_p, mp)) { 1127 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 1128 "ring index %d", ring_index)); 1129 return (B_FALSE); 1130 } 1131 break; 1132 1133 case NXGE_USE_SERIAL: 1134 default: 1135 nxge_serialize_enter(tx_ring_p->serial, mp); 1136 break; 1137 } 1138 1139 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 1140 ring_index)); 1141 1142 return (B_TRUE); 1143 } 1144 1145 /* 1146 * nxge_m_tx() - send a chain of packets 1147 */ 1148 mblk_t * 1149 nxge_m_tx(void *arg, mblk_t *mp) 1150 { 1151 p_nxge_t nxgep = (p_nxge_t)arg; 1152 mblk_t *next; 1153 mac_tx_hint_t hint; 1154 1155 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_m_tx")); 1156 1157 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 1158 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 1159 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1160 "==> nxge_m_tx: hardware not initialized")); 1161 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1162 "<== nxge_m_tx")); 1163 freemsgchain(mp); 1164 mp = NULL; 1165 return (mp); 1166 } 1167 1168 hint.hash = NULL; 1169 hint.vid = 0; 1170 hint.sap = 0; 1171 1172 while (mp != NULL) { 1173 next = mp->b_next; 1174 mp->b_next = NULL; 1175 1176 /* 1177 * Until Nemo tx resource works, the mac driver 1178 * does the load balancing based on TCP port, 1179 * or CPU. For debugging, we use a system 1180 * configurable parameter. 1181 */ 1182 if (!nxge_send(nxgep, mp, &hint)) { 1183 mp->b_next = next; 1184 break; 1185 } 1186 1187 mp = next; 1188 1189 NXGE_DEBUG_MSG((NULL, TX_CTL, 1190 "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 1191 mp, next)); 1192 } 1193 1194 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_m_tx")); 1195 return (mp); 1196 } 1197 1198 int 1199 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 1200 { 1201 uint8_t ring_index = 0; 1202 uint8_t *tcp_port; 1203 p_mblk_t nmp; 1204 size_t mblk_len; 1205 size_t iph_len; 1206 size_t hdrs_size; 1207 uint8_t hdrs_buf[sizeof (struct ether_header) + 1208 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 1209 /* 1210 * allocate space big enough to cover 1211 * the max ip header length and the first 1212 * 4 bytes of the TCP/IP header. 1213 */ 1214 1215 boolean_t qos = B_FALSE; 1216 1217 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 1218 1219 if (hp->vid) { 1220 qos = B_TRUE; 1221 } 1222 switch (nxge_tx_lb_policy) { 1223 case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 1224 default: 1225 tcp_port = mp->b_rptr; 1226 if (!nxge_no_tx_lb && !qos && 1227 (ntohs(((p_ether_header_t)tcp_port)->ether_type) 1228 == ETHERTYPE_IP)) { 1229 nmp = mp; 1230 mblk_len = MBLKL(nmp); 1231 tcp_port = NULL; 1232 if (mblk_len > sizeof (struct ether_header) + 1233 sizeof (uint8_t)) { 1234 tcp_port = nmp->b_rptr + 1235 sizeof (struct ether_header); 1236 mblk_len -= sizeof (struct ether_header); 1237 iph_len = ((*tcp_port) & 0x0f) << 2; 1238 if (mblk_len > (iph_len + sizeof (uint32_t))) { 1239 tcp_port = nmp->b_rptr; 1240 } else { 1241 tcp_port = NULL; 1242 } 1243 } 1244 if (tcp_port == NULL) { 1245 hdrs_size = 0; 1246 ((p_ether_header_t)hdrs_buf)->ether_type = 0; 1247 while ((nmp) && (hdrs_size < 1248 sizeof (hdrs_buf))) { 1249 mblk_len = MBLKL(nmp); 1250 if (mblk_len >= 1251 (sizeof (hdrs_buf) - hdrs_size)) 1252 mblk_len = sizeof (hdrs_buf) - 1253 hdrs_size; 1254 bcopy(nmp->b_rptr, 1255 &hdrs_buf[hdrs_size], mblk_len); 1256 hdrs_size += mblk_len; 1257 nmp = nmp->b_cont; 1258 } 1259 tcp_port = hdrs_buf; 1260 } 1261 tcp_port += sizeof (ether_header_t); 1262 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1263 switch (tcp_port[9]) { 1264 case IPPROTO_TCP: 1265 case IPPROTO_UDP: 1266 case IPPROTO_ESP: 1267 tcp_port += ((*tcp_port) & 0x0f) << 2; 1268 ring_index = 1269 ((tcp_port[0] ^ 1270 tcp_port[1] ^ 1271 tcp_port[2] ^ 1272 tcp_port[3]) % maxtdcs); 1273 break; 1274 1275 case IPPROTO_AH: 1276 /* SPI starts at the 4th byte */ 1277 tcp_port += ((*tcp_port) & 0x0f) << 2; 1278 ring_index = 1279 ((tcp_port[4] ^ 1280 tcp_port[5] ^ 1281 tcp_port[6] ^ 1282 tcp_port[7]) % maxtdcs); 1283 break; 1284 1285 default: 1286 ring_index = tcp_port[19] % maxtdcs; 1287 break; 1288 } 1289 } else { /* fragmented packet */ 1290 ring_index = tcp_port[19] % maxtdcs; 1291 } 1292 } else { 1293 ring_index = mp->b_band % maxtdcs; 1294 } 1295 break; 1296 1297 case NXGE_TX_LB_HASH: 1298 if (hp->hash) { 1299 #if defined(__i386) 1300 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1301 #else 1302 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1303 #endif 1304 } else { 1305 ring_index = mp->b_band % maxtdcs; 1306 } 1307 break; 1308 1309 case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 1310 tcp_port = mp->b_rptr; 1311 ring_index = tcp_port[5] % maxtdcs; 1312 break; 1313 } 1314 1315 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 1316 1317 return (ring_index); 1318 } 1319 1320 uint_t 1321 nxge_reschedule(caddr_t arg) 1322 { 1323 p_nxge_t nxgep; 1324 1325 nxgep = (p_nxge_t)arg; 1326 1327 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 1328 1329 if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1330 nxgep->resched_needed) { 1331 if (!isLDOMguest(nxgep)) 1332 mac_tx_update(nxgep->mach); 1333 #if defined(sun4v) 1334 else { /* isLDOMguest(nxgep) */ 1335 nxge_hio_data_t *nhd = (nxge_hio_data_t *) 1336 nxgep->nxge_hw_p->hio; 1337 nx_vio_fp_t *vio = &nhd->hio.vio; 1338 1339 /* Call back vnet. */ 1340 if (vio->cb.vio_net_tx_update) { 1341 (*vio->cb.vio_net_tx_update) 1342 (nxgep->hio_vr->vhp); 1343 } 1344 } 1345 #endif 1346 nxgep->resched_needed = B_FALSE; 1347 nxgep->resched_running = B_FALSE; 1348 } 1349 1350 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 1351 return (DDI_INTR_CLAIMED); 1352 } 1353 1354 1355 /* Software LSO starts here */ 1356 static void 1357 nxge_hcksum_retrieve(mblk_t *mp, 1358 uint32_t *start, uint32_t *stuff, uint32_t *end, 1359 uint32_t *value, uint32_t *flags) 1360 { 1361 if (mp->b_datap->db_type == M_DATA) { 1362 if (flags != NULL) { 1363 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1364 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1365 HCK_FULLCKSUM_OK); 1366 if ((*flags & (HCK_PARTIALCKSUM | 1367 HCK_FULLCKSUM)) != 0) { 1368 if (value != NULL) 1369 *value = (uint32_t)DB_CKSUM16(mp); 1370 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1371 if (start != NULL) 1372 *start = 1373 (uint32_t)DB_CKSUMSTART(mp); 1374 if (stuff != NULL) 1375 *stuff = 1376 (uint32_t)DB_CKSUMSTUFF(mp); 1377 if (end != NULL) 1378 *end = 1379 (uint32_t)DB_CKSUMEND(mp); 1380 } 1381 } 1382 } 1383 } 1384 } 1385 1386 static void 1387 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1388 { 1389 ASSERT(DB_TYPE(mp) == M_DATA); 1390 1391 *mss = 0; 1392 if (flags != NULL) { 1393 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1394 if ((*flags != 0) && (mss != NULL)) { 1395 *mss = (uint32_t)DB_LSOMSS(mp); 1396 } 1397 NXGE_DEBUG_MSG((NULL, TX_CTL, 1398 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1399 *mss, *flags)); 1400 } 1401 1402 NXGE_DEBUG_MSG((NULL, TX_CTL, 1403 "<== nxge_lso_info_get: mss %d", *mss)); 1404 } 1405 1406 /* 1407 * Do Soft LSO on the oversized packet. 1408 * 1409 * 1. Create a chain of message for headers. 1410 * 2. Fill up header messages with proper information. 1411 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1412 * each new message with necessary adjustments. 1413 * * Unchange the ethernet header for DIX frames. (by default) 1414 * * IP Total Length field is updated to MSS or less(only for the last one). 1415 * * IP Identification value is incremented by one for each packet. 1416 * * TCP sequence Number is recalculated according to the payload length. 1417 * * Set FIN and/or PSH flags for the *last* packet if applied. 1418 * * TCP partial Checksum 1419 * 4. Update LSO information in the first message header. 1420 * 5. Release the original message header. 1421 */ 1422 static mblk_t * 1423 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1424 { 1425 uint32_t hckflags; 1426 int pktlen; 1427 int hdrlen; 1428 int segnum; 1429 int i; 1430 struct ether_vlan_header *evh; 1431 int ehlen, iphlen, tcphlen; 1432 struct ip *oiph, *niph; 1433 struct tcphdr *otcph, *ntcph; 1434 int available, len, left; 1435 uint16_t ip_id; 1436 uint32_t tcp_seq; 1437 #ifdef __sparc 1438 uint32_t tcp_seq_tmp; 1439 #endif 1440 mblk_t *datamp; 1441 uchar_t *rptr; 1442 mblk_t *nmp; 1443 mblk_t *cmp; 1444 mblk_t *mp_chain; 1445 boolean_t do_cleanup = B_FALSE; 1446 t_uscalar_t start_offset = 0; 1447 t_uscalar_t stuff_offset = 0; 1448 t_uscalar_t value = 0; 1449 uint16_t l4_len; 1450 ipaddr_t src, dst; 1451 uint32_t cksum, sum, l4cksum; 1452 1453 NXGE_DEBUG_MSG((NULL, TX_CTL, 1454 "==> nxge_do_softlso")); 1455 /* 1456 * check the length of LSO packet payload and calculate the number of 1457 * segments to be generated. 1458 */ 1459 pktlen = msgsize(mp); 1460 evh = (struct ether_vlan_header *)mp->b_rptr; 1461 1462 /* VLAN? */ 1463 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1464 ehlen = sizeof (struct ether_vlan_header); 1465 else 1466 ehlen = sizeof (struct ether_header); 1467 oiph = (struct ip *)(mp->b_rptr + ehlen); 1468 iphlen = oiph->ip_hl * 4; 1469 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1470 tcphlen = otcph->th_off * 4; 1471 1472 l4_len = pktlen - ehlen - iphlen; 1473 1474 NXGE_DEBUG_MSG((NULL, TX_CTL, 1475 "==> nxge_do_softlso: mss %d oiph $%p " 1476 "original ip_sum oiph->ip_sum 0x%x " 1477 "original tcp_sum otcph->th_sum 0x%x " 1478 "oiph->ip_len %d pktlen %d ehlen %d " 1479 "l4_len %d (0x%x) ip_len - iphlen %d ", 1480 mss, 1481 oiph, 1482 oiph->ip_sum, 1483 otcph->th_sum, 1484 ntohs(oiph->ip_len), pktlen, 1485 ehlen, 1486 l4_len, 1487 l4_len, 1488 ntohs(oiph->ip_len) - iphlen)); 1489 1490 /* IPv4 + TCP */ 1491 if (!(oiph->ip_v == IPV4_VERSION)) { 1492 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1493 "<== nxge_do_softlso: not IPV4 " 1494 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1495 ntohs(oiph->ip_len), pktlen, ehlen, 1496 tcphlen)); 1497 freemsg(mp); 1498 return (NULL); 1499 } 1500 1501 if (!(oiph->ip_p == IPPROTO_TCP)) { 1502 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1503 "<== nxge_do_softlso: not TCP " 1504 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1505 ntohs(oiph->ip_len), pktlen, ehlen, 1506 tcphlen)); 1507 freemsg(mp); 1508 return (NULL); 1509 } 1510 1511 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1512 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1513 "<== nxge_do_softlso: len not matched " 1514 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1515 ntohs(oiph->ip_len), pktlen, ehlen, 1516 tcphlen)); 1517 freemsg(mp); 1518 return (NULL); 1519 } 1520 1521 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1522 tcphlen = otcph->th_off * 4; 1523 1524 /* TCP flags can not include URG, RST, or SYN */ 1525 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1526 1527 hdrlen = ehlen + iphlen + tcphlen; 1528 1529 VERIFY(MBLKL(mp) >= hdrlen); 1530 1531 if (MBLKL(mp) > hdrlen) { 1532 datamp = mp; 1533 rptr = mp->b_rptr + hdrlen; 1534 } else { /* = */ 1535 datamp = mp->b_cont; 1536 rptr = datamp->b_rptr; 1537 } 1538 1539 NXGE_DEBUG_MSG((NULL, TX_CTL, 1540 "nxge_do_softlso: otcph $%p pktlen: %d, " 1541 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1542 "mblkl(mp): %d, mblkl(datamp): %d", 1543 otcph, 1544 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1545 (int)MBLKL(mp), (int)MBLKL(datamp))); 1546 1547 hckflags = 0; 1548 nxge_hcksum_retrieve(mp, 1549 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1550 1551 dst = oiph->ip_dst.s_addr; 1552 src = oiph->ip_src.s_addr; 1553 1554 cksum = (dst >> 16) + (dst & 0xFFFF) + 1555 (src >> 16) + (src & 0xFFFF); 1556 l4cksum = cksum + IP_TCP_CSUM_COMP; 1557 1558 sum = l4_len + l4cksum; 1559 sum = (sum & 0xFFFF) + (sum >> 16); 1560 1561 NXGE_DEBUG_MSG((NULL, TX_CTL, 1562 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1563 "hckflags 0x%x start_offset %d stuff_offset %d " 1564 "value (original) 0x%x th_sum 0x%x " 1565 "pktlen %d l4_len %d (0x%x) " 1566 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1567 dst, src, 1568 (sum & 0xffff), (~sum & 0xffff), 1569 hckflags, start_offset, stuff_offset, 1570 value, otcph->th_sum, 1571 pktlen, 1572 l4_len, 1573 l4_len, 1574 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1575 (int)MBLKL(datamp), 1576 nxge_dump_packet((char *)evh, 12))); 1577 1578 /* 1579 * Start to process. 1580 */ 1581 available = pktlen - hdrlen; 1582 segnum = (available - 1) / mss + 1; 1583 1584 NXGE_DEBUG_MSG((NULL, TX_CTL, 1585 "==> nxge_do_softlso: pktlen %d " 1586 "MBLKL(mp): %d, MBLKL(datamp): %d " 1587 "available %d mss %d segnum %d", 1588 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1589 available, 1590 mss, 1591 segnum)); 1592 1593 VERIFY(segnum >= 2); 1594 1595 /* 1596 * Try to pre-allocate all header messages 1597 */ 1598 mp_chain = NULL; 1599 for (i = 0; i < segnum; i++) { 1600 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1601 /* Clean up the mp_chain */ 1602 while (mp_chain != NULL) { 1603 nmp = mp_chain; 1604 mp_chain = mp_chain->b_next; 1605 freemsg(nmp); 1606 } 1607 NXGE_DEBUG_MSG((NULL, TX_CTL, 1608 "<== nxge_do_softlso: " 1609 "Could not allocate enough messages for headers!")); 1610 freemsg(mp); 1611 return (NULL); 1612 } 1613 nmp->b_next = mp_chain; 1614 mp_chain = nmp; 1615 1616 NXGE_DEBUG_MSG((NULL, TX_CTL, 1617 "==> nxge_do_softlso: " 1618 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1619 mp, nmp, mp_chain, mp_chain->b_next)); 1620 } 1621 1622 NXGE_DEBUG_MSG((NULL, TX_CTL, 1623 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1624 mp, nmp, mp_chain)); 1625 1626 /* 1627 * Associate payload with new packets 1628 */ 1629 cmp = mp_chain; 1630 left = available; 1631 while (cmp != NULL) { 1632 nmp = dupb(datamp); 1633 if (nmp == NULL) { 1634 do_cleanup = B_TRUE; 1635 NXGE_DEBUG_MSG((NULL, TX_CTL, 1636 "==>nxge_do_softlso: " 1637 "Can not dupb(datamp), have to do clean up")); 1638 goto cleanup_allocated_msgs; 1639 } 1640 1641 NXGE_DEBUG_MSG((NULL, TX_CTL, 1642 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1643 "dupb nmp $%p len %d left %d msd %d ", 1644 mp, cmp, nmp, len, left, mss)); 1645 1646 cmp->b_cont = nmp; 1647 nmp->b_rptr = rptr; 1648 len = (left < mss) ? left : mss; 1649 left -= len; 1650 1651 NXGE_DEBUG_MSG((NULL, TX_CTL, 1652 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1653 "dupb nmp $%p len %d left %d mss %d ", 1654 mp, cmp, nmp, len, left, mss)); 1655 NXGE_DEBUG_MSG((NULL, TX_CTL, 1656 "nxge_do_softlso: before available: %d, " 1657 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1658 available, left, len, segnum, (int)MBLKL(nmp))); 1659 1660 len -= MBLKL(nmp); 1661 NXGE_DEBUG_MSG((NULL, TX_CTL, 1662 "nxge_do_softlso: after available: %d, " 1663 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1664 available, left, len, segnum, (int)MBLKL(nmp))); 1665 1666 while (len > 0) { 1667 mblk_t *mmp = NULL; 1668 1669 NXGE_DEBUG_MSG((NULL, TX_CTL, 1670 "nxge_do_softlso: (4) len > 0 available: %d, " 1671 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1672 available, left, len, segnum, (int)MBLKL(nmp))); 1673 1674 if (datamp->b_cont != NULL) { 1675 datamp = datamp->b_cont; 1676 rptr = datamp->b_rptr; 1677 mmp = dupb(datamp); 1678 if (mmp == NULL) { 1679 do_cleanup = B_TRUE; 1680 NXGE_DEBUG_MSG((NULL, TX_CTL, 1681 "==> nxge_do_softlso: " 1682 "Can not dupb(datamp) (1), :" 1683 "have to do clean up")); 1684 NXGE_DEBUG_MSG((NULL, TX_CTL, 1685 "==> nxge_do_softlso: " 1686 "available: %d, left: %d, " 1687 "len: %d, MBLKL(nmp): %d", 1688 available, left, len, 1689 (int)MBLKL(nmp))); 1690 goto cleanup_allocated_msgs; 1691 } 1692 } else { 1693 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1694 "==> nxge_do_softlso: " 1695 "(1)available: %d, left: %d, " 1696 "len: %d, MBLKL(nmp): %d", 1697 available, left, len, 1698 (int)MBLKL(nmp))); 1699 cmn_err(CE_PANIC, 1700 "==> nxge_do_softlso: " 1701 "Pointers must have been corrupted!\n" 1702 "datamp: $%p, nmp: $%p, rptr: $%p", 1703 (void *)datamp, 1704 (void *)nmp, 1705 (void *)rptr); 1706 } 1707 nmp->b_cont = mmp; 1708 nmp = mmp; 1709 len -= MBLKL(nmp); 1710 } 1711 if (len < 0) { 1712 nmp->b_wptr += len; 1713 rptr = nmp->b_wptr; 1714 NXGE_DEBUG_MSG((NULL, TX_CTL, 1715 "(5) len < 0 (less than 0)" 1716 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1717 available, left, len, (int)MBLKL(nmp))); 1718 1719 } else if (len == 0) { 1720 if (datamp->b_cont != NULL) { 1721 NXGE_DEBUG_MSG((NULL, TX_CTL, 1722 "(5) len == 0" 1723 "available: %d, left: %d, len: %d, " 1724 "MBLKL(nmp): %d", 1725 available, left, len, (int)MBLKL(nmp))); 1726 datamp = datamp->b_cont; 1727 rptr = datamp->b_rptr; 1728 } else { 1729 NXGE_DEBUG_MSG((NULL, TX_CTL, 1730 "(6)available b_cont == NULL : %d, " 1731 "left: %d, len: %d, MBLKL(nmp): %d", 1732 available, left, len, (int)MBLKL(nmp))); 1733 1734 VERIFY(cmp->b_next == NULL); 1735 VERIFY(left == 0); 1736 break; /* Done! */ 1737 } 1738 } 1739 cmp = cmp->b_next; 1740 1741 NXGE_DEBUG_MSG((NULL, TX_CTL, 1742 "(7) do_softlso: " 1743 "next mp in mp_chain available len != 0 : %d, " 1744 "left: %d, len: %d, MBLKL(nmp): %d", 1745 available, left, len, (int)MBLKL(nmp))); 1746 } 1747 1748 /* 1749 * From now, start to fill up all headers for the first message 1750 * Hardware checksum flags need to be updated separately for FULLCKSUM 1751 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1752 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1753 * required fields need to be updated properly. 1754 */ 1755 nmp = mp_chain; 1756 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1757 nmp->b_wptr = nmp->b_rptr + hdrlen; 1758 niph = (struct ip *)(nmp->b_rptr + ehlen); 1759 niph->ip_len = htons(mss + iphlen + tcphlen); 1760 ip_id = ntohs(niph->ip_id); 1761 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1762 #ifdef __sparc 1763 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1764 tcp_seq = ntohl(tcp_seq_tmp); 1765 #else 1766 tcp_seq = ntohl(ntcph->th_seq); 1767 #endif 1768 1769 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1770 1771 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1772 DB_CKSUMSTART(nmp) = start_offset; 1773 DB_CKSUMSTUFF(nmp) = stuff_offset; 1774 1775 /* calculate IP checksum and TCP pseudo header checksum */ 1776 niph->ip_sum = 0; 1777 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1778 1779 l4_len = mss + tcphlen; 1780 sum = htons(l4_len) + l4cksum; 1781 sum = (sum & 0xFFFF) + (sum >> 16); 1782 ntcph->th_sum = (sum & 0xffff); 1783 1784 NXGE_DEBUG_MSG((NULL, TX_CTL, 1785 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1786 "mss %d pktlen %d l4_len %d (0x%x) " 1787 "MBLKL(mp): %d, MBLKL(datamp): %d " 1788 "ip_sum 0x%x " 1789 "th_sum 0x%x sum 0x%x ) " 1790 "dump first ip->tcp %s", 1791 nmp, mp_chain, 1792 mss, 1793 pktlen, 1794 l4_len, 1795 l4_len, 1796 (int)MBLKL(mp), (int)MBLKL(datamp), 1797 niph->ip_sum, 1798 ntcph->th_sum, 1799 sum, 1800 nxge_dump_packet((char *)niph, 52))); 1801 1802 cmp = nmp; 1803 while ((nmp = nmp->b_next)->b_next != NULL) { 1804 NXGE_DEBUG_MSG((NULL, TX_CTL, 1805 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1806 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1807 nmp->b_wptr = nmp->b_rptr + hdrlen; 1808 niph = (struct ip *)(nmp->b_rptr + ehlen); 1809 niph->ip_id = htons(++ip_id); 1810 niph->ip_len = htons(mss + iphlen + tcphlen); 1811 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1812 tcp_seq += mss; 1813 1814 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1815 1816 #ifdef __sparc 1817 tcp_seq_tmp = htonl(tcp_seq); 1818 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1819 #else 1820 ntcph->th_seq = htonl(tcp_seq); 1821 #endif 1822 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1823 DB_CKSUMSTART(nmp) = start_offset; 1824 DB_CKSUMSTUFF(nmp) = stuff_offset; 1825 1826 /* calculate IP checksum and TCP pseudo header checksum */ 1827 niph->ip_sum = 0; 1828 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1829 ntcph->th_sum = (sum & 0xffff); 1830 1831 NXGE_DEBUG_MSG((NULL, TX_CTL, 1832 "==> nxge_do_softlso: middle ip_sum 0x%x " 1833 "th_sum 0x%x " 1834 " mp $%p (mp_chain $%p) pktlen %d " 1835 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1836 niph->ip_sum, 1837 ntcph->th_sum, 1838 nmp, mp_chain, 1839 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1840 } 1841 1842 /* Last segment */ 1843 /* 1844 * Set FIN and/or PSH flags if present only in the last packet. 1845 * The ip_len could be different from prior packets. 1846 */ 1847 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1848 nmp->b_wptr = nmp->b_rptr + hdrlen; 1849 niph = (struct ip *)(nmp->b_rptr + ehlen); 1850 niph->ip_id = htons(++ip_id); 1851 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1852 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1853 tcp_seq += mss; 1854 #ifdef __sparc 1855 tcp_seq_tmp = htonl(tcp_seq); 1856 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1857 #else 1858 ntcph->th_seq = htonl(tcp_seq); 1859 #endif 1860 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1861 1862 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1863 DB_CKSUMSTART(nmp) = start_offset; 1864 DB_CKSUMSTUFF(nmp) = stuff_offset; 1865 1866 /* calculate IP checksum and TCP pseudo header checksum */ 1867 niph->ip_sum = 0; 1868 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1869 1870 l4_len = ntohs(niph->ip_len) - iphlen; 1871 sum = htons(l4_len) + l4cksum; 1872 sum = (sum & 0xFFFF) + (sum >> 16); 1873 ntcph->th_sum = (sum & 0xffff); 1874 1875 NXGE_DEBUG_MSG((NULL, TX_CTL, 1876 "==> nxge_do_softlso: last next " 1877 "niph->ip_sum 0x%x " 1878 "ntcph->th_sum 0x%x sum 0x%x " 1879 "dump last ip->tcp %s " 1880 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1881 "l4_len %d (0x%x) " 1882 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1883 niph->ip_sum, 1884 ntcph->th_sum, sum, 1885 nxge_dump_packet((char *)niph, 52), 1886 cmp, nmp, mp_chain, 1887 pktlen, pktlen, 1888 l4_len, 1889 l4_len, 1890 (int)MBLKL(mp), (int)MBLKL(datamp))); 1891 1892 cleanup_allocated_msgs: 1893 if (do_cleanup) { 1894 NXGE_DEBUG_MSG((NULL, TX_CTL, 1895 "==> nxge_do_softlso: " 1896 "Failed allocating messages, " 1897 "have to clean up and fail!")); 1898 while (mp_chain != NULL) { 1899 nmp = mp_chain; 1900 mp_chain = mp_chain->b_next; 1901 freemsg(nmp); 1902 } 1903 } 1904 /* 1905 * We're done here, so just free the original message and return the 1906 * new message chain, that could be NULL if failed, back to the caller. 1907 */ 1908 freemsg(mp); 1909 1910 NXGE_DEBUG_MSG((NULL, TX_CTL, 1911 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1912 return (mp_chain); 1913 } 1914 1915 /* 1916 * Will be called before NIC driver do further operation on the message. 1917 * The input message may include LSO information, if so, go to softlso logic 1918 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1919 * The return could be the same non-LSO message or a message chain for LSO case. 1920 * 1921 * The driver needs to call this function per packet and process the whole chain 1922 * if applied. 1923 */ 1924 static mblk_t * 1925 nxge_lso_eliminate(mblk_t *mp) 1926 { 1927 uint32_t lsoflags; 1928 uint32_t mss; 1929 1930 NXGE_DEBUG_MSG((NULL, TX_CTL, 1931 "==>nxge_lso_eliminate:")); 1932 nxge_lso_info_get(mp, &mss, &lsoflags); 1933 1934 if (lsoflags & HW_LSO) { 1935 mblk_t *nmp; 1936 1937 NXGE_DEBUG_MSG((NULL, TX_CTL, 1938 "==>nxge_lso_eliminate:" 1939 "HW_LSO:mss %d mp $%p", 1940 mss, mp)); 1941 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1942 NXGE_DEBUG_MSG((NULL, TX_CTL, 1943 "<== nxge_lso_eliminate: " 1944 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1945 nmp, mss, mp)); 1946 return (nmp); 1947 } else { 1948 NXGE_DEBUG_MSG((NULL, TX_CTL, 1949 "<== nxge_lso_eliminate_ " 1950 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1951 nmp, mss, mp)); 1952 return (NULL); 1953 } 1954 } 1955 1956 NXGE_DEBUG_MSG((NULL, TX_CTL, 1957 "<== nxge_lso_eliminate")); 1958 return (mp); 1959 } 1960 1961 static uint32_t 1962 nxge_csgen(uint16_t *adr, int len) 1963 { 1964 int i, odd; 1965 uint32_t sum = 0; 1966 uint32_t c = 0; 1967 1968 odd = len % 2; 1969 for (i = 0; i < (len / 2); i++) { 1970 sum += (adr[i] & 0xffff); 1971 } 1972 if (odd) { 1973 sum += adr[len / 2] & 0xff00; 1974 } 1975 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 1976 sum &= 0xffff; 1977 sum += c; 1978 } 1979 return (~sum & 0xffff); 1980 } 1981