1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/nxge/nxge_impl.h> 29 #include <sys/nxge/nxge_hio.h> 30 #include <npi_tx_wr64.h> 31 32 /* Software LSO required header files */ 33 #include <netinet/tcp.h> 34 #include <inet/ip_impl.h> 35 #include <inet/tcp.h> 36 37 static mblk_t *nxge_lso_eliminate(mblk_t *); 38 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 39 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 40 static void nxge_hcksum_retrieve(mblk_t *, 41 uint32_t *, uint32_t *, uint32_t *, 42 uint32_t *, uint32_t *); 43 static uint32_t nxge_csgen(uint16_t *, int); 44 45 extern uint32_t nxge_reclaim_pending; 46 extern uint32_t nxge_bcopy_thresh; 47 extern uint32_t nxge_dvma_thresh; 48 extern uint32_t nxge_dma_stream_thresh; 49 extern uint32_t nxge_tx_minfree; 50 extern uint32_t nxge_tx_intr_thres; 51 extern uint32_t nxge_tx_max_gathers; 52 extern uint32_t nxge_tx_tiny_pack; 53 extern uint32_t nxge_tx_use_bcopy; 54 extern uint32_t nxge_tx_lb_policy; 55 extern uint32_t nxge_no_tx_lb; 56 extern nxge_tx_mode_t nxge_tx_scheme; 57 uint32_t nxge_lso_kick_cnt = 2; 58 59 typedef struct _mac_tx_hint { 60 uint16_t sap; 61 uint16_t vid; 62 void *hash; 63 } mac_tx_hint_t, *p_mac_tx_hint_t; 64 65 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 66 67 int 68 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 69 { 70 int status = 0; 71 p_tx_desc_t tx_desc_ring_vp; 72 npi_handle_t npi_desc_handle; 73 nxge_os_dma_handle_t tx_desc_dma_handle; 74 p_tx_desc_t tx_desc_p; 75 p_tx_msg_t tx_msg_ring; 76 p_tx_msg_t tx_msg_p; 77 tx_desc_t tx_desc, *tmp_desc_p; 78 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 79 p_tx_pkt_header_t hdrp; 80 tx_pkt_header_t tmp_hdrp; 81 p_tx_pkt_hdr_all_t pkthdrp; 82 uint8_t npads = 0; 83 uint64_t dma_ioaddr; 84 uint32_t dma_flags; 85 int last_bidx; 86 uint8_t *b_rptr; 87 caddr_t kaddr; 88 uint32_t nmblks; 89 uint32_t ngathers; 90 uint32_t clen; 91 int len; 92 uint32_t pkt_len, pack_len, min_len; 93 uint32_t bcopy_thresh; 94 int i, cur_index, sop_index; 95 uint16_t tail_index; 96 boolean_t tail_wrap = B_FALSE; 97 nxge_dma_common_t desc_area; 98 nxge_os_dma_handle_t dma_handle; 99 ddi_dma_cookie_t dma_cookie; 100 npi_handle_t npi_handle; 101 p_mblk_t nmp; 102 p_mblk_t t_mp; 103 uint32_t ncookies; 104 boolean_t good_packet; 105 boolean_t mark_mode = B_FALSE; 106 p_nxge_stats_t statsp; 107 p_nxge_tx_ring_stats_t tdc_stats; 108 t_uscalar_t start_offset = 0; 109 t_uscalar_t stuff_offset = 0; 110 t_uscalar_t end_offset = 0; 111 t_uscalar_t value = 0; 112 t_uscalar_t cksum_flags = 0; 113 boolean_t cksum_on = B_FALSE; 114 uint32_t boff = 0; 115 uint64_t tot_xfer_len = 0; 116 boolean_t header_set = B_FALSE; 117 #ifdef NXGE_DEBUG 118 p_tx_desc_t tx_desc_ring_pp; 119 p_tx_desc_t tx_desc_pp; 120 tx_desc_t *save_desc_p; 121 int dump_len; 122 int sad_len; 123 uint64_t sad; 124 int xfer_len; 125 uint32_t msgsize; 126 #endif 127 p_mblk_t mp_chain = NULL; 128 boolean_t is_lso = B_FALSE; 129 boolean_t lso_again; 130 int cur_index_lso; 131 p_mblk_t nmp_lso_save; 132 uint32_t lso_ngathers; 133 boolean_t lso_tail_wrap = B_FALSE; 134 135 NXGE_DEBUG_MSG((nxgep, TX_CTL, 136 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 137 NXGE_DEBUG_MSG((nxgep, TX_CTL, 138 "==> nxge_start: Starting tdc %d desc pending %d", 139 tx_ring_p->tdc, tx_ring_p->descs_pending)); 140 141 statsp = nxgep->statsp; 142 143 if (!isLDOMguest(nxgep)) { 144 switch (nxgep->mac.portmode) { 145 default: 146 if (nxgep->statsp->port_stats.lb_mode == 147 nxge_lb_normal) { 148 if (!statsp->mac_stats.link_up) { 149 freemsg(mp); 150 NXGE_DEBUG_MSG((nxgep, TX_CTL, 151 "==> nxge_start: " 152 "link not up")); 153 goto nxge_start_fail1; 154 } 155 } 156 break; 157 case PORT_10G_FIBER: 158 /* 159 * For the following modes, check the link status 160 * before sending the packet out: 161 * nxge_lb_normal, nxge_lb_ext10g, nxge_lb_phy10g 162 */ 163 if (nxgep->statsp->port_stats.lb_mode < 164 nxge_lb_serdes10g) { 165 if (!statsp->mac_stats.link_up) { 166 freemsg(mp); 167 NXGE_DEBUG_MSG((nxgep, TX_CTL, 168 "==> nxge_start: " 169 "link not up")); 170 goto nxge_start_fail1; 171 } 172 } 173 break; 174 } 175 } 176 177 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 178 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 179 NXGE_DEBUG_MSG((nxgep, TX_CTL, 180 "==> nxge_start: hardware not initialized or stopped")); 181 freemsg(mp); 182 goto nxge_start_fail1; 183 } 184 185 if (nxgep->soft_lso_enable) { 186 mp_chain = nxge_lso_eliminate(mp); 187 NXGE_DEBUG_MSG((nxgep, TX_CTL, 188 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 189 mp, mp_chain)); 190 if (mp_chain == NULL) { 191 NXGE_ERROR_MSG((nxgep, TX_CTL, 192 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 193 mp_chain, mp)); 194 goto nxge_start_fail1; 195 } 196 if (mp_chain != mp) { 197 NXGE_DEBUG_MSG((nxgep, TX_CTL, 198 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 199 mp_chain, mp)); 200 is_lso = B_TRUE; 201 mp = mp_chain; 202 mp_chain = mp_chain->b_next; 203 mp->b_next = NULL; 204 } 205 } 206 207 hcksum_retrieve(mp, NULL, NULL, &start_offset, 208 &stuff_offset, &end_offset, &value, &cksum_flags); 209 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 210 start_offset += sizeof (ether_header_t); 211 stuff_offset += sizeof (ether_header_t); 212 } else { 213 start_offset += sizeof (struct ether_vlan_header); 214 stuff_offset += sizeof (struct ether_vlan_header); 215 } 216 217 if (cksum_flags & HCK_PARTIALCKSUM) { 218 NXGE_DEBUG_MSG((nxgep, TX_CTL, 219 "==> nxge_start: mp $%p len %d " 220 "cksum_flags 0x%x (partial checksum) ", 221 mp, MBLKL(mp), cksum_flags)); 222 cksum_on = B_TRUE; 223 } 224 225 pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 226 pkthdrp->reserved = 0; 227 tmp_hdrp.value = 0; 228 nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 229 0, 0, pkthdrp, 230 start_offset, stuff_offset); 231 232 lso_again = B_FALSE; 233 lso_ngathers = 0; 234 235 MUTEX_ENTER(&tx_ring_p->lock); 236 237 if (isLDOMservice(nxgep)) { 238 if (tx_ring_p->tx_ring_offline) { 239 freemsg(mp); 240 MUTEX_EXIT(&tx_ring_p->lock); 241 return (status); 242 } 243 } 244 245 cur_index_lso = tx_ring_p->wr_index; 246 lso_tail_wrap = tx_ring_p->wr_index_wrap; 247 start_again: 248 ngathers = 0; 249 sop_index = tx_ring_p->wr_index; 250 #ifdef NXGE_DEBUG 251 if (tx_ring_p->descs_pending) { 252 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 253 "desc pending %d ", tx_ring_p->descs_pending)); 254 } 255 256 dump_len = (int)(MBLKL(mp)); 257 dump_len = (dump_len > 128) ? 128: dump_len; 258 259 NXGE_DEBUG_MSG((nxgep, TX_CTL, 260 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 261 "(Before header reserve: ORIGINAL LEN %d)", 262 tx_ring_p->tdc, 263 mp->b_rptr, 264 dump_len)); 265 266 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 267 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 268 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 269 #endif 270 271 tdc_stats = tx_ring_p->tdc_stats; 272 mark_mode = (tx_ring_p->descs_pending && 273 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 274 < nxge_tx_minfree)); 275 276 NXGE_DEBUG_MSG((nxgep, TX_CTL, 277 "TX Descriptor ring is channel %d mark mode %d", 278 tx_ring_p->tdc, mark_mode)); 279 280 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { 281 NXGE_DEBUG_MSG((nxgep, TX_CTL, 282 "TX Descriptor ring is full: channel %d", 283 tx_ring_p->tdc)); 284 NXGE_DEBUG_MSG((nxgep, TX_CTL, 285 "TX Descriptor ring is full: channel %d", 286 tx_ring_p->tdc)); 287 if (is_lso) { 288 /* free the current mp and mp_chain if not FULL */ 289 tdc_stats->tx_no_desc++; 290 NXGE_DEBUG_MSG((nxgep, TX_CTL, 291 "LSO packet: TX Descriptor ring is full: " 292 "channel %d", 293 tx_ring_p->tdc)); 294 goto nxge_start_fail_lso; 295 } else { 296 boolean_t skip_sched = B_FALSE; 297 298 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 299 tdc_stats->tx_no_desc++; 300 if (isLDOMservice(nxgep) && 301 tx_ring_p->tx_ring_offline) { 302 (void) atomic_swap_32( 303 &tx_ring_p->tx_ring_offline, 304 NXGE_TX_RING_OFFLINED); 305 skip_sched = B_TRUE; 306 } 307 308 MUTEX_EXIT(&tx_ring_p->lock); 309 if (nxgep->resched_needed && 310 !nxgep->resched_running && !skip_sched) { 311 nxgep->resched_running = B_TRUE; 312 ddi_trigger_softintr(nxgep->resched_id); 313 } 314 status = 1; 315 goto nxge_start_fail1; 316 } 317 } 318 319 nmp = mp; 320 i = sop_index = tx_ring_p->wr_index; 321 nmblks = 0; 322 ngathers = 0; 323 pkt_len = 0; 324 pack_len = 0; 325 clen = 0; 326 last_bidx = -1; 327 good_packet = B_TRUE; 328 329 desc_area = tx_ring_p->tdc_desc; 330 npi_handle = desc_area.npi_handle; 331 npi_desc_handle.regh = (nxge_os_acc_handle_t) 332 DMA_COMMON_ACC_HANDLE(desc_area); 333 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 334 tx_desc_dma_handle = (nxge_os_dma_handle_t) 335 DMA_COMMON_HANDLE(desc_area); 336 tx_msg_ring = tx_ring_p->tx_msg_ring; 337 338 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 339 sop_index, i)); 340 341 #ifdef NXGE_DEBUG 342 msgsize = msgdsize(nmp); 343 NXGE_DEBUG_MSG((nxgep, TX_CTL, 344 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 345 sop_index, i, msgsize)); 346 #endif 347 /* 348 * The first 16 bytes of the premapped buffer are reserved 349 * for header. No padding will be used. 350 */ 351 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 352 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 353 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 354 } else { 355 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 356 } 357 while (nmp) { 358 good_packet = B_TRUE; 359 b_rptr = nmp->b_rptr; 360 len = MBLKL(nmp); 361 if (len <= 0) { 362 nmp = nmp->b_cont; 363 continue; 364 } 365 nmblks++; 366 367 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 368 "len %d pkt_len %d pack_len %d", 369 nmblks, len, pkt_len, pack_len)); 370 /* 371 * Hardware limits the transfer length to 4K for NIU and 372 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 373 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 374 * If len is longer than the limit, then we break nmp into 375 * two chunks: Make the first chunk equal to the limit and 376 * the second chunk for the remaining data. If the second 377 * chunk is still larger than the limit, then it will be 378 * broken into two in the next pass. 379 */ 380 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 381 if ((t_mp = dupb(nmp)) != NULL) { 382 nmp->b_wptr = nmp->b_rptr + 383 (TX_MAX_TRANSFER_LENGTH 384 - TX_PKT_HEADER_SIZE); 385 t_mp->b_rptr = nmp->b_wptr; 386 t_mp->b_cont = nmp->b_cont; 387 nmp->b_cont = t_mp; 388 len = MBLKL(nmp); 389 } else { 390 if (is_lso) { 391 NXGE_DEBUG_MSG((nxgep, TX_CTL, 392 "LSO packet: dupb failed: " 393 "channel %d", 394 tx_ring_p->tdc)); 395 mp = nmp; 396 goto nxge_start_fail_lso; 397 } else { 398 good_packet = B_FALSE; 399 goto nxge_start_fail2; 400 } 401 } 402 } 403 tx_desc.value = 0; 404 tx_desc_p = &tx_desc_ring_vp[i]; 405 #ifdef NXGE_DEBUG 406 tx_desc_pp = &tx_desc_ring_pp[i]; 407 #endif 408 tx_msg_p = &tx_msg_ring[i]; 409 #if defined(__i386) 410 npi_desc_handle.regp = (uint32_t)tx_desc_p; 411 #else 412 npi_desc_handle.regp = (uint64_t)tx_desc_p; 413 #endif 414 if (!header_set && 415 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 416 (len >= bcopy_thresh))) { 417 header_set = B_TRUE; 418 bcopy_thresh += TX_PKT_HEADER_SIZE; 419 boff = 0; 420 pack_len = 0; 421 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 422 hdrp = (p_tx_pkt_header_t)kaddr; 423 clen = pkt_len; 424 dma_handle = tx_msg_p->buf_dma_handle; 425 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 426 (void) ddi_dma_sync(dma_handle, 427 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 428 DDI_DMA_SYNC_FORDEV); 429 430 tx_msg_p->flags.dma_type = USE_BCOPY; 431 goto nxge_start_control_header_only; 432 } 433 434 pkt_len += len; 435 pack_len += len; 436 437 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 438 "desc entry %d " 439 "DESC IOADDR $%p " 440 "desc_vp $%p tx_desc_p $%p " 441 "desc_pp $%p tx_desc_pp $%p " 442 "len %d pkt_len %d pack_len %d", 443 i, 444 DMA_COMMON_IOADDR(desc_area), 445 tx_desc_ring_vp, tx_desc_p, 446 tx_desc_ring_pp, tx_desc_pp, 447 len, pkt_len, pack_len)); 448 449 if (len < bcopy_thresh) { 450 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 451 "USE BCOPY: ")); 452 if (nxge_tx_tiny_pack) { 453 uint32_t blst = 454 TXDMA_DESC_NEXT_INDEX(i, -1, 455 tx_ring_p->tx_wrap_mask); 456 NXGE_DEBUG_MSG((nxgep, TX_CTL, 457 "==> nxge_start(5): pack")); 458 if ((pack_len <= bcopy_thresh) && 459 (last_bidx == blst)) { 460 NXGE_DEBUG_MSG((nxgep, TX_CTL, 461 "==> nxge_start: pack(6) " 462 "(pkt_len %d pack_len %d)", 463 pkt_len, pack_len)); 464 i = blst; 465 tx_desc_p = &tx_desc_ring_vp[i]; 466 #ifdef NXGE_DEBUG 467 tx_desc_pp = &tx_desc_ring_pp[i]; 468 #endif 469 tx_msg_p = &tx_msg_ring[i]; 470 boff = pack_len - len; 471 ngathers--; 472 } else if (pack_len > bcopy_thresh && 473 header_set) { 474 pack_len = len; 475 boff = 0; 476 bcopy_thresh = nxge_bcopy_thresh; 477 NXGE_DEBUG_MSG((nxgep, TX_CTL, 478 "==> nxge_start(7): > max NEW " 479 "bcopy thresh %d " 480 "pkt_len %d pack_len %d(next)", 481 bcopy_thresh, 482 pkt_len, pack_len)); 483 } 484 last_bidx = i; 485 } 486 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 487 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 488 hdrp = (p_tx_pkt_header_t)kaddr; 489 header_set = B_TRUE; 490 NXGE_DEBUG_MSG((nxgep, TX_CTL, 491 "==> nxge_start(7_x2): " 492 "pkt_len %d pack_len %d (new hdrp $%p)", 493 pkt_len, pack_len, hdrp)); 494 } 495 tx_msg_p->flags.dma_type = USE_BCOPY; 496 kaddr += boff; 497 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 498 "USE BCOPY: before bcopy " 499 "DESC IOADDR $%p entry %d " 500 "bcopy packets %d " 501 "bcopy kaddr $%p " 502 "bcopy ioaddr (SAD) $%p " 503 "bcopy clen %d " 504 "bcopy boff %d", 505 DMA_COMMON_IOADDR(desc_area), i, 506 tdc_stats->tx_hdr_pkts, 507 kaddr, 508 dma_ioaddr, 509 clen, 510 boff)); 511 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 512 "1USE BCOPY: ")); 513 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 514 "2USE BCOPY: ")); 515 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 516 "last USE BCOPY: copy from b_rptr $%p " 517 "to KADDR $%p (len %d offset %d", 518 b_rptr, kaddr, len, boff)); 519 520 bcopy(b_rptr, kaddr, len); 521 522 #ifdef NXGE_DEBUG 523 dump_len = (len > 128) ? 128: len; 524 NXGE_DEBUG_MSG((nxgep, TX_CTL, 525 "==> nxge_start: dump packets " 526 "(After BCOPY len %d)" 527 "(b_rptr $%p): %s", len, nmp->b_rptr, 528 nxge_dump_packet((char *)nmp->b_rptr, 529 dump_len))); 530 #endif 531 532 dma_handle = tx_msg_p->buf_dma_handle; 533 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 534 (void) ddi_dma_sync(dma_handle, 535 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 536 DDI_DMA_SYNC_FORDEV); 537 clen = len + boff; 538 tdc_stats->tx_hdr_pkts++; 539 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 540 "USE BCOPY: " 541 "DESC IOADDR $%p entry %d " 542 "bcopy packets %d " 543 "bcopy kaddr $%p " 544 "bcopy ioaddr (SAD) $%p " 545 "bcopy clen %d " 546 "bcopy boff %d", 547 DMA_COMMON_IOADDR(desc_area), 548 i, 549 tdc_stats->tx_hdr_pkts, 550 kaddr, 551 dma_ioaddr, 552 clen, 553 boff)); 554 } else { 555 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 556 "USE DVMA: len %d", len)); 557 tx_msg_p->flags.dma_type = USE_DMA; 558 dma_flags = DDI_DMA_WRITE; 559 if (len < nxge_dma_stream_thresh) { 560 dma_flags |= DDI_DMA_CONSISTENT; 561 } else { 562 dma_flags |= DDI_DMA_STREAMING; 563 } 564 565 dma_handle = tx_msg_p->dma_handle; 566 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 567 (caddr_t)b_rptr, len, dma_flags, 568 DDI_DMA_DONTWAIT, NULL, 569 &dma_cookie, &ncookies); 570 if (status == DDI_DMA_MAPPED) { 571 dma_ioaddr = dma_cookie.dmac_laddress; 572 len = (int)dma_cookie.dmac_size; 573 clen = (uint32_t)dma_cookie.dmac_size; 574 NXGE_DEBUG_MSG((nxgep, TX_CTL, 575 "==> nxge_start(12_1): " 576 "USE DVMA: len %d clen %d " 577 "ngathers %d", 578 len, clen, 579 ngathers)); 580 #if defined(__i386) 581 npi_desc_handle.regp = (uint32_t)tx_desc_p; 582 #else 583 npi_desc_handle.regp = (uint64_t)tx_desc_p; 584 #endif 585 while (ncookies > 1) { 586 ngathers++; 587 /* 588 * this is the fix for multiple 589 * cookies, which are basically 590 * a descriptor entry, we don't set 591 * SOP bit as well as related fields 592 */ 593 594 (void) npi_txdma_desc_gather_set( 595 npi_desc_handle, 596 &tx_desc, 597 (ngathers -1), 598 mark_mode, 599 ngathers, 600 dma_ioaddr, 601 clen); 602 603 tx_msg_p->tx_msg_size = clen; 604 NXGE_DEBUG_MSG((nxgep, TX_CTL, 605 "==> nxge_start: DMA " 606 "ncookie %d " 607 "ngathers %d " 608 "dma_ioaddr $%p len %d" 609 "desc $%p descp $%p (%d)", 610 ncookies, 611 ngathers, 612 dma_ioaddr, clen, 613 *tx_desc_p, tx_desc_p, i)); 614 615 ddi_dma_nextcookie(dma_handle, 616 &dma_cookie); 617 dma_ioaddr = 618 dma_cookie.dmac_laddress; 619 620 len = (int)dma_cookie.dmac_size; 621 clen = (uint32_t)dma_cookie.dmac_size; 622 NXGE_DEBUG_MSG((nxgep, TX_CTL, 623 "==> nxge_start(12_2): " 624 "USE DVMA: len %d clen %d ", 625 len, clen)); 626 627 i = TXDMA_DESC_NEXT_INDEX(i, 1, 628 tx_ring_p->tx_wrap_mask); 629 tx_desc_p = &tx_desc_ring_vp[i]; 630 631 npi_desc_handle.regp = 632 #if defined(__i386) 633 (uint32_t)tx_desc_p; 634 #else 635 (uint64_t)tx_desc_p; 636 #endif 637 tx_msg_p = &tx_msg_ring[i]; 638 tx_msg_p->flags.dma_type = USE_NONE; 639 tx_desc.value = 0; 640 641 ncookies--; 642 } 643 tdc_stats->tx_ddi_pkts++; 644 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 645 "DMA: ddi packets %d", 646 tdc_stats->tx_ddi_pkts)); 647 } else { 648 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 649 "dma mapping failed for %d " 650 "bytes addr $%p flags %x (%d)", 651 len, b_rptr, status, status)); 652 good_packet = B_FALSE; 653 tdc_stats->tx_dma_bind_fail++; 654 tx_msg_p->flags.dma_type = USE_NONE; 655 if (is_lso) { 656 mp = nmp; 657 goto nxge_start_fail_lso; 658 } else { 659 goto nxge_start_fail2; 660 } 661 } 662 } /* ddi dvma */ 663 664 if (is_lso) { 665 nmp_lso_save = nmp; 666 } 667 nmp = nmp->b_cont; 668 nxge_start_control_header_only: 669 #if defined(__i386) 670 npi_desc_handle.regp = (uint32_t)tx_desc_p; 671 #else 672 npi_desc_handle.regp = (uint64_t)tx_desc_p; 673 #endif 674 ngathers++; 675 676 if (ngathers == 1) { 677 #ifdef NXGE_DEBUG 678 save_desc_p = &sop_tx_desc; 679 #endif 680 sop_tx_desc_p = &sop_tx_desc; 681 sop_tx_desc_p->value = 0; 682 sop_tx_desc_p->bits.hdw.tr_len = clen; 683 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 684 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 685 } else { 686 #ifdef NXGE_DEBUG 687 save_desc_p = &tx_desc; 688 #endif 689 tmp_desc_p = &tx_desc; 690 tmp_desc_p->value = 0; 691 tmp_desc_p->bits.hdw.tr_len = clen; 692 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 693 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 694 695 tx_desc_p->value = tmp_desc_p->value; 696 } 697 698 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 699 "Desc_entry %d ngathers %d " 700 "desc_vp $%p tx_desc_p $%p " 701 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 702 "dma_ioaddr (SAD) $%p mark %d", 703 i, ngathers, 704 tx_desc_ring_vp, tx_desc_p, 705 len, clen, pkt_len, pack_len, nmblks, 706 dma_ioaddr, mark_mode)); 707 708 #ifdef NXGE_DEBUG 709 npi_desc_handle.nxgep = nxgep; 710 npi_desc_handle.function.function = nxgep->function_num; 711 npi_desc_handle.function.instance = nxgep->instance; 712 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 713 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 714 TX_PKT_DESC_TR_LEN_SHIFT); 715 716 717 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 718 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 719 "mark %d sop %d\n", 720 save_desc_p->value, 721 sad, 722 save_desc_p->bits.hdw.tr_len, 723 xfer_len, 724 save_desc_p->bits.hdw.num_ptr, 725 save_desc_p->bits.hdw.mark, 726 save_desc_p->bits.hdw.sop)); 727 728 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 729 #endif 730 731 tx_msg_p->tx_msg_size = clen; 732 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 733 if (ngathers > nxge_tx_max_gathers) { 734 good_packet = B_FALSE; 735 hcksum_retrieve(mp, NULL, NULL, &start_offset, 736 &stuff_offset, &end_offset, &value, 737 &cksum_flags); 738 739 NXGE_DEBUG_MSG((NULL, TX_CTL, 740 "==> nxge_start(14): pull msg - " 741 "len %d pkt_len %d ngathers %d", 742 len, pkt_len, ngathers)); 743 /* Pull all message blocks from b_cont */ 744 if (is_lso) { 745 mp = nmp_lso_save; 746 goto nxge_start_fail_lso; 747 } 748 if ((msgpullup(mp, -1)) == NULL) { 749 goto nxge_start_fail2; 750 } 751 goto nxge_start_fail2; 752 } 753 } /* while (nmp) */ 754 755 tx_msg_p->tx_message = mp; 756 tx_desc_p = &tx_desc_ring_vp[sop_index]; 757 #if defined(__i386) 758 npi_desc_handle.regp = (uint32_t)tx_desc_p; 759 #else 760 npi_desc_handle.regp = (uint64_t)tx_desc_p; 761 #endif 762 763 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 764 pkthdrp->reserved = 0; 765 hdrp->value = 0; 766 bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 767 768 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 769 tdc_stats->tx_jumbo_pkts++; 770 } 771 772 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 773 if (pkt_len < min_len) { 774 /* Assume we use bcopy to premapped buffers */ 775 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 776 NXGE_DEBUG_MSG((NULL, TX_CTL, 777 "==> nxge_start(14-1): < (msg_min + 16)" 778 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 779 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 780 bzero((kaddr + pkt_len), (min_len - pkt_len)); 781 pkt_len = tx_msg_p->tx_msg_size = min_len; 782 783 sop_tx_desc_p->bits.hdw.tr_len = min_len; 784 785 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 786 tx_desc_p->value = sop_tx_desc_p->value; 787 788 NXGE_DEBUG_MSG((NULL, TX_CTL, 789 "==> nxge_start(14-2): < msg_min - " 790 "len %d pkt_len %d min_len %d ngathers %d", 791 len, pkt_len, min_len, ngathers)); 792 } 793 794 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 795 cksum_flags)); 796 { 797 uint64_t tmp_len; 798 799 /* pkt_len already includes 16 + paddings!! */ 800 /* Update the control header length */ 801 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 802 tmp_len = hdrp->value | 803 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 804 805 NXGE_DEBUG_MSG((nxgep, TX_CTL, 806 "==> nxge_start(15_x1): setting SOP " 807 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 808 "0x%llx hdrp->value 0x%llx", 809 tot_xfer_len, tot_xfer_len, pkt_len, 810 tmp_len, hdrp->value)); 811 #if defined(_BIG_ENDIAN) 812 hdrp->value = ddi_swap64(tmp_len); 813 #else 814 hdrp->value = tmp_len; 815 #endif 816 NXGE_DEBUG_MSG((nxgep, 817 TX_CTL, "==> nxge_start(15_x2): setting SOP " 818 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 819 "tmp_len 0x%llx hdrp->value 0x%llx", 820 tot_xfer_len, pkt_len, 821 tmp_len, hdrp->value)); 822 } 823 824 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 825 "wr_index %d " 826 "tot_xfer_len (%d) pkt_len %d npads %d", 827 sop_index, 828 tot_xfer_len, pkt_len, 829 npads)); 830 831 sop_tx_desc_p->bits.hdw.sop = 1; 832 sop_tx_desc_p->bits.hdw.mark = mark_mode; 833 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 834 835 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 836 837 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 838 839 #ifdef NXGE_DEBUG 840 npi_desc_handle.nxgep = nxgep; 841 npi_desc_handle.function.function = nxgep->function_num; 842 npi_desc_handle.function.instance = nxgep->instance; 843 844 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 845 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 846 save_desc_p->value, 847 sad, 848 save_desc_p->bits.hdw.tr_len, 849 xfer_len, 850 save_desc_p->bits.hdw.num_ptr, 851 save_desc_p->bits.hdw.mark, 852 save_desc_p->bits.hdw.sop)); 853 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 854 855 dump_len = (pkt_len > 128) ? 128: pkt_len; 856 NXGE_DEBUG_MSG((nxgep, TX_CTL, 857 "==> nxge_start: dump packets(17) (after sop set, len " 858 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 859 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 860 (char *)hdrp, 861 nxge_dump_packet((char *)hdrp, dump_len))); 862 NXGE_DEBUG_MSG((nxgep, TX_CTL, 863 "==> nxge_start(18): TX desc sync: sop_index %d", 864 sop_index)); 865 #endif 866 867 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 868 (void) ddi_dma_sync(tx_desc_dma_handle, 869 sop_index * sizeof (tx_desc_t), 870 ngathers * sizeof (tx_desc_t), 871 DDI_DMA_SYNC_FORDEV); 872 873 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 874 "cs_off = 0x%02X cs_s_off = 0x%02X " 875 "pkt_len %d ngathers %d sop_index %d\n", 876 stuff_offset, start_offset, 877 pkt_len, ngathers, sop_index)); 878 } else { /* more than one descriptor and wrap around */ 879 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 880 (void) ddi_dma_sync(tx_desc_dma_handle, 881 sop_index * sizeof (tx_desc_t), 882 nsdescs * sizeof (tx_desc_t), 883 DDI_DMA_SYNC_FORDEV); 884 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 885 "cs_off = 0x%02X cs_s_off = 0x%02X " 886 "pkt_len %d ngathers %d sop_index %d\n", 887 stuff_offset, start_offset, 888 pkt_len, ngathers, sop_index)); 889 890 (void) ddi_dma_sync(tx_desc_dma_handle, 891 0, 892 (ngathers - nsdescs) * sizeof (tx_desc_t), 893 DDI_DMA_SYNC_FORDEV); 894 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 895 "cs_off = 0x%02X cs_s_off = 0x%02X " 896 "pkt_len %d ngathers %d sop_index %d\n", 897 stuff_offset, start_offset, 898 pkt_len, ngathers, sop_index)); 899 } 900 901 tail_index = tx_ring_p->wr_index; 902 tail_wrap = tx_ring_p->wr_index_wrap; 903 904 tx_ring_p->wr_index = i; 905 if (tx_ring_p->wr_index <= tail_index) { 906 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 907 B_FALSE : B_TRUE); 908 } 909 910 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 911 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 912 tx_ring_p->tdc, 913 tx_ring_p->wr_index, 914 tx_ring_p->wr_index_wrap, 915 ngathers, 916 tx_ring_p->descs_pending)); 917 918 if (is_lso) { 919 lso_ngathers += ngathers; 920 if (mp_chain != NULL) { 921 mp = mp_chain; 922 mp_chain = mp_chain->b_next; 923 mp->b_next = NULL; 924 if (nxge_lso_kick_cnt == lso_ngathers) { 925 tx_ring_p->descs_pending += lso_ngathers; 926 { 927 tx_ring_kick_t kick; 928 929 kick.value = 0; 930 kick.bits.ldw.wrap = 931 tx_ring_p->wr_index_wrap; 932 kick.bits.ldw.tail = 933 (uint16_t)tx_ring_p->wr_index; 934 935 /* Kick the Transmit kick register */ 936 TXDMA_REG_WRITE64( 937 NXGE_DEV_NPI_HANDLE(nxgep), 938 TX_RING_KICK_REG, 939 (uint8_t)tx_ring_p->tdc, 940 kick.value); 941 tdc_stats->tx_starts++; 942 943 NXGE_DEBUG_MSG((nxgep, TX_CTL, 944 "==> nxge_start: more LSO: " 945 "LSO_CNT %d", 946 lso_ngathers)); 947 } 948 lso_ngathers = 0; 949 ngathers = 0; 950 cur_index_lso = sop_index = tx_ring_p->wr_index; 951 lso_tail_wrap = tx_ring_p->wr_index_wrap; 952 } 953 NXGE_DEBUG_MSG((nxgep, TX_CTL, 954 "==> nxge_start: lso again: " 955 "lso_gathers %d ngathers %d cur_index_lso %d " 956 "wr_index %d sop_index %d", 957 lso_ngathers, ngathers, cur_index_lso, 958 tx_ring_p->wr_index, sop_index)); 959 960 NXGE_DEBUG_MSG((nxgep, TX_CTL, 961 "==> nxge_start: next : count %d", 962 lso_ngathers)); 963 lso_again = B_TRUE; 964 goto start_again; 965 } 966 ngathers = lso_ngathers; 967 } 968 969 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 970 971 { 972 tx_ring_kick_t kick; 973 974 kick.value = 0; 975 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 976 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 977 978 /* Kick start the Transmit kick register */ 979 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 980 TX_RING_KICK_REG, 981 (uint8_t)tx_ring_p->tdc, 982 kick.value); 983 } 984 985 tx_ring_p->descs_pending += ngathers; 986 tdc_stats->tx_starts++; 987 988 if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline) 989 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 990 NXGE_TX_RING_OFFLINED); 991 992 MUTEX_EXIT(&tx_ring_p->lock); 993 994 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 995 996 return (status); 997 998 nxge_start_fail_lso: 999 status = 0; 1000 good_packet = B_FALSE; 1001 if (mp != NULL) { 1002 freemsg(mp); 1003 } 1004 if (mp_chain != NULL) { 1005 freemsg(mp_chain); 1006 } 1007 if (!lso_again && !ngathers) { 1008 if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline) 1009 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1010 NXGE_TX_RING_OFFLINED); 1011 MUTEX_EXIT(&tx_ring_p->lock); 1012 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1013 "==> nxge_start: lso exit (nothing changed)")); 1014 goto nxge_start_fail1; 1015 } 1016 1017 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1018 "==> nxge_start (channel %d): before lso " 1019 "lso_gathers %d ngathers %d cur_index_lso %d " 1020 "wr_index %d sop_index %d lso_again %d", 1021 tx_ring_p->tdc, 1022 lso_ngathers, ngathers, cur_index_lso, 1023 tx_ring_p->wr_index, sop_index, lso_again)); 1024 1025 if (lso_again) { 1026 lso_ngathers += ngathers; 1027 ngathers = lso_ngathers; 1028 sop_index = cur_index_lso; 1029 tx_ring_p->wr_index = sop_index; 1030 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1031 } 1032 1033 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1034 "==> nxge_start (channel %d): after lso " 1035 "lso_gathers %d ngathers %d cur_index_lso %d " 1036 "wr_index %d sop_index %d lso_again %d", 1037 tx_ring_p->tdc, 1038 lso_ngathers, ngathers, cur_index_lso, 1039 tx_ring_p->wr_index, sop_index, lso_again)); 1040 1041 nxge_start_fail2: 1042 if (good_packet == B_FALSE) { 1043 cur_index = sop_index; 1044 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1045 for (i = 0; i < ngathers; i++) { 1046 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1047 #if defined(__i386) 1048 npi_handle.regp = (uint32_t)tx_desc_p; 1049 #else 1050 npi_handle.regp = (uint64_t)tx_desc_p; 1051 #endif 1052 tx_msg_p = &tx_msg_ring[cur_index]; 1053 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1054 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1055 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1056 "tx_desc_p = %X index = %d", 1057 tx_desc_p, tx_ring_p->rd_index)); 1058 (void) dvma_unload(tx_msg_p->dvma_handle, 1059 0, -1); 1060 tx_msg_p->dvma_handle = NULL; 1061 if (tx_ring_p->dvma_wr_index == 1062 tx_ring_p->dvma_wrap_mask) 1063 tx_ring_p->dvma_wr_index = 0; 1064 else 1065 tx_ring_p->dvma_wr_index++; 1066 tx_ring_p->dvma_pending--; 1067 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1068 if (ddi_dma_unbind_handle( 1069 tx_msg_p->dma_handle)) { 1070 cmn_err(CE_WARN, "!nxge_start: " 1071 "ddi_dma_unbind_handle failed"); 1072 } 1073 } 1074 tx_msg_p->flags.dma_type = USE_NONE; 1075 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1076 tx_ring_p->tx_wrap_mask); 1077 1078 } 1079 1080 nxgep->resched_needed = B_TRUE; 1081 } 1082 1083 1084 if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline) 1085 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1086 NXGE_TX_RING_OFFLINED); 1087 1088 MUTEX_EXIT(&tx_ring_p->lock); 1089 1090 nxge_start_fail1: 1091 /* Add FMA to check the access handle nxge_hregh */ 1092 1093 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1094 1095 return (status); 1096 } 1097 1098 int 1099 nxge_serial_tx(mblk_t *mp, void *arg) 1100 { 1101 p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 1102 p_nxge_t nxgep = tx_ring_p->nxgep; 1103 int status = 0; 1104 1105 if (isLDOMservice(nxgep)) { 1106 if (tx_ring_p->tx_ring_offline) { 1107 freemsg(mp); 1108 return (status); 1109 } 1110 } 1111 1112 status = nxge_start(nxgep, tx_ring_p, mp); 1113 1114 return (status); 1115 } 1116 1117 boolean_t 1118 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 1119 { 1120 p_tx_ring_t *tx_rings; 1121 uint8_t ring_index; 1122 p_tx_ring_t tx_ring_p; 1123 nxge_grp_t *group; 1124 1125 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 1126 1127 ASSERT(mp->b_next == NULL); 1128 1129 group = nxgep->tx_set.group[0]; /* The default group */ 1130 ring_index = nxge_tx_lb_ring_1(mp, group->count, hp); 1131 1132 tx_rings = nxgep->tx_rings->rings; 1133 tx_ring_p = tx_rings[group->legend[ring_index]]; 1134 1135 if (isLDOMservice(nxgep)) { 1136 if (tx_ring_p->tx_ring_offline) { 1137 /* 1138 * OFFLINE means that it is in the process of being 1139 * shared - that is, it has been claimed by the HIO 1140 * code, but hasn't been unlinked from <group> yet. 1141 * So in this case use the first TDC, which always 1142 * belongs to the service domain and can't be shared. 1143 */ 1144 ring_index = 0; 1145 tx_ring_p = tx_rings[group->legend[ring_index]]; 1146 } 1147 } 1148 1149 NXGE_DEBUG_MSG((nxgep, TX_CTL, "count %d, tx_rings[%d] = %p", 1150 (int)group->count, group->legend[ring_index], tx_ring_p)); 1151 1152 switch (nxge_tx_scheme) { 1153 case NXGE_USE_START: 1154 if (nxge_start(nxgep, tx_ring_p, mp)) { 1155 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 1156 "ring index %d", ring_index)); 1157 return (B_FALSE); 1158 } 1159 break; 1160 1161 case NXGE_USE_SERIAL: 1162 default: 1163 nxge_serialize_enter(tx_ring_p->serial, mp); 1164 break; 1165 } 1166 1167 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 1168 ring_index)); 1169 1170 return (B_TRUE); 1171 } 1172 1173 /* 1174 * nxge_m_tx() - send a chain of packets 1175 */ 1176 mblk_t * 1177 nxge_m_tx(void *arg, mblk_t *mp) 1178 { 1179 p_nxge_t nxgep = (p_nxge_t)arg; 1180 mblk_t *next; 1181 mac_tx_hint_t hint; 1182 1183 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_m_tx")); 1184 1185 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 1186 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 1187 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1188 "==> nxge_m_tx: hardware not initialized")); 1189 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1190 "<== nxge_m_tx")); 1191 freemsgchain(mp); 1192 mp = NULL; 1193 return (mp); 1194 } 1195 1196 hint.hash = NULL; 1197 hint.vid = 0; 1198 hint.sap = 0; 1199 1200 while (mp != NULL) { 1201 next = mp->b_next; 1202 mp->b_next = NULL; 1203 1204 /* 1205 * Until Nemo tx resource works, the mac driver 1206 * does the load balancing based on TCP port, 1207 * or CPU. For debugging, we use a system 1208 * configurable parameter. 1209 */ 1210 if (!nxge_send(nxgep, mp, &hint)) { 1211 mp->b_next = next; 1212 break; 1213 } 1214 1215 mp = next; 1216 1217 NXGE_DEBUG_MSG((NULL, TX_CTL, 1218 "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 1219 mp, next)); 1220 } 1221 1222 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_m_tx")); 1223 return (mp); 1224 } 1225 1226 int 1227 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 1228 { 1229 uint8_t ring_index = 0; 1230 uint8_t *tcp_port; 1231 p_mblk_t nmp; 1232 size_t mblk_len; 1233 size_t iph_len; 1234 size_t hdrs_size; 1235 uint8_t hdrs_buf[sizeof (struct ether_header) + 1236 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 1237 /* 1238 * allocate space big enough to cover 1239 * the max ip header length and the first 1240 * 4 bytes of the TCP/IP header. 1241 */ 1242 1243 boolean_t qos = B_FALSE; 1244 1245 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 1246 1247 if (hp->vid) { 1248 qos = B_TRUE; 1249 } 1250 switch (nxge_tx_lb_policy) { 1251 case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 1252 default: 1253 tcp_port = mp->b_rptr; 1254 if (!nxge_no_tx_lb && !qos && 1255 (ntohs(((p_ether_header_t)tcp_port)->ether_type) 1256 == ETHERTYPE_IP)) { 1257 nmp = mp; 1258 mblk_len = MBLKL(nmp); 1259 tcp_port = NULL; 1260 if (mblk_len > sizeof (struct ether_header) + 1261 sizeof (uint8_t)) { 1262 tcp_port = nmp->b_rptr + 1263 sizeof (struct ether_header); 1264 mblk_len -= sizeof (struct ether_header); 1265 iph_len = ((*tcp_port) & 0x0f) << 2; 1266 if (mblk_len > (iph_len + sizeof (uint32_t))) { 1267 tcp_port = nmp->b_rptr; 1268 } else { 1269 tcp_port = NULL; 1270 } 1271 } 1272 if (tcp_port == NULL) { 1273 hdrs_size = 0; 1274 ((p_ether_header_t)hdrs_buf)->ether_type = 0; 1275 while ((nmp) && (hdrs_size < 1276 sizeof (hdrs_buf))) { 1277 mblk_len = MBLKL(nmp); 1278 if (mblk_len >= 1279 (sizeof (hdrs_buf) - hdrs_size)) 1280 mblk_len = sizeof (hdrs_buf) - 1281 hdrs_size; 1282 bcopy(nmp->b_rptr, 1283 &hdrs_buf[hdrs_size], mblk_len); 1284 hdrs_size += mblk_len; 1285 nmp = nmp->b_cont; 1286 } 1287 tcp_port = hdrs_buf; 1288 } 1289 tcp_port += sizeof (ether_header_t); 1290 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1291 switch (tcp_port[9]) { 1292 case IPPROTO_TCP: 1293 case IPPROTO_UDP: 1294 case IPPROTO_ESP: 1295 tcp_port += ((*tcp_port) & 0x0f) << 2; 1296 ring_index = 1297 ((tcp_port[0] ^ 1298 tcp_port[1] ^ 1299 tcp_port[2] ^ 1300 tcp_port[3]) % maxtdcs); 1301 break; 1302 1303 case IPPROTO_AH: 1304 /* SPI starts at the 4th byte */ 1305 tcp_port += ((*tcp_port) & 0x0f) << 2; 1306 ring_index = 1307 ((tcp_port[4] ^ 1308 tcp_port[5] ^ 1309 tcp_port[6] ^ 1310 tcp_port[7]) % maxtdcs); 1311 break; 1312 1313 default: 1314 ring_index = tcp_port[19] % maxtdcs; 1315 break; 1316 } 1317 } else { /* fragmented packet */ 1318 ring_index = tcp_port[19] % maxtdcs; 1319 } 1320 } else { 1321 ring_index = mp->b_band % maxtdcs; 1322 } 1323 break; 1324 1325 case NXGE_TX_LB_HASH: 1326 if (hp->hash) { 1327 #if defined(__i386) 1328 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1329 #else 1330 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1331 #endif 1332 } else { 1333 ring_index = mp->b_band % maxtdcs; 1334 } 1335 break; 1336 1337 case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 1338 tcp_port = mp->b_rptr; 1339 ring_index = tcp_port[5] % maxtdcs; 1340 break; 1341 } 1342 1343 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 1344 1345 return (ring_index); 1346 } 1347 1348 uint_t 1349 nxge_reschedule(caddr_t arg) 1350 { 1351 p_nxge_t nxgep; 1352 1353 nxgep = (p_nxge_t)arg; 1354 1355 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 1356 1357 if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1358 nxgep->resched_needed) { 1359 if (!isLDOMguest(nxgep)) 1360 mac_tx_update(nxgep->mach); 1361 #if defined(sun4v) 1362 else { /* isLDOMguest(nxgep) */ 1363 nxge_hio_data_t *nhd = (nxge_hio_data_t *) 1364 nxgep->nxge_hw_p->hio; 1365 nx_vio_fp_t *vio = &nhd->hio.vio; 1366 1367 /* Call back vnet. */ 1368 if (vio->cb.vio_net_tx_update) { 1369 (*vio->cb.vio_net_tx_update) 1370 (nxgep->hio_vr->vhp); 1371 } 1372 } 1373 #endif 1374 nxgep->resched_needed = B_FALSE; 1375 nxgep->resched_running = B_FALSE; 1376 } 1377 1378 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 1379 return (DDI_INTR_CLAIMED); 1380 } 1381 1382 1383 /* Software LSO starts here */ 1384 static void 1385 nxge_hcksum_retrieve(mblk_t *mp, 1386 uint32_t *start, uint32_t *stuff, uint32_t *end, 1387 uint32_t *value, uint32_t *flags) 1388 { 1389 if (mp->b_datap->db_type == M_DATA) { 1390 if (flags != NULL) { 1391 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1392 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1393 HCK_FULLCKSUM_OK); 1394 if ((*flags & (HCK_PARTIALCKSUM | 1395 HCK_FULLCKSUM)) != 0) { 1396 if (value != NULL) 1397 *value = (uint32_t)DB_CKSUM16(mp); 1398 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1399 if (start != NULL) 1400 *start = 1401 (uint32_t)DB_CKSUMSTART(mp); 1402 if (stuff != NULL) 1403 *stuff = 1404 (uint32_t)DB_CKSUMSTUFF(mp); 1405 if (end != NULL) 1406 *end = 1407 (uint32_t)DB_CKSUMEND(mp); 1408 } 1409 } 1410 } 1411 } 1412 } 1413 1414 static void 1415 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1416 { 1417 ASSERT(DB_TYPE(mp) == M_DATA); 1418 1419 *mss = 0; 1420 if (flags != NULL) { 1421 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1422 if ((*flags != 0) && (mss != NULL)) { 1423 *mss = (uint32_t)DB_LSOMSS(mp); 1424 } 1425 NXGE_DEBUG_MSG((NULL, TX_CTL, 1426 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1427 *mss, *flags)); 1428 } 1429 1430 NXGE_DEBUG_MSG((NULL, TX_CTL, 1431 "<== nxge_lso_info_get: mss %d", *mss)); 1432 } 1433 1434 /* 1435 * Do Soft LSO on the oversized packet. 1436 * 1437 * 1. Create a chain of message for headers. 1438 * 2. Fill up header messages with proper information. 1439 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1440 * each new message with necessary adjustments. 1441 * * Unchange the ethernet header for DIX frames. (by default) 1442 * * IP Total Length field is updated to MSS or less(only for the last one). 1443 * * IP Identification value is incremented by one for each packet. 1444 * * TCP sequence Number is recalculated according to the payload length. 1445 * * Set FIN and/or PSH flags for the *last* packet if applied. 1446 * * TCP partial Checksum 1447 * 4. Update LSO information in the first message header. 1448 * 5. Release the original message header. 1449 */ 1450 static mblk_t * 1451 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1452 { 1453 uint32_t hckflags; 1454 int pktlen; 1455 int hdrlen; 1456 int segnum; 1457 int i; 1458 struct ether_vlan_header *evh; 1459 int ehlen, iphlen, tcphlen; 1460 struct ip *oiph, *niph; 1461 struct tcphdr *otcph, *ntcph; 1462 int available, len, left; 1463 uint16_t ip_id; 1464 uint32_t tcp_seq; 1465 #ifdef __sparc 1466 uint32_t tcp_seq_tmp; 1467 #endif 1468 mblk_t *datamp; 1469 uchar_t *rptr; 1470 mblk_t *nmp; 1471 mblk_t *cmp; 1472 mblk_t *mp_chain; 1473 boolean_t do_cleanup = B_FALSE; 1474 t_uscalar_t start_offset = 0; 1475 t_uscalar_t stuff_offset = 0; 1476 t_uscalar_t value = 0; 1477 uint16_t l4_len; 1478 ipaddr_t src, dst; 1479 uint32_t cksum, sum, l4cksum; 1480 1481 NXGE_DEBUG_MSG((NULL, TX_CTL, 1482 "==> nxge_do_softlso")); 1483 /* 1484 * check the length of LSO packet payload and calculate the number of 1485 * segments to be generated. 1486 */ 1487 pktlen = msgsize(mp); 1488 evh = (struct ether_vlan_header *)mp->b_rptr; 1489 1490 /* VLAN? */ 1491 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1492 ehlen = sizeof (struct ether_vlan_header); 1493 else 1494 ehlen = sizeof (struct ether_header); 1495 oiph = (struct ip *)(mp->b_rptr + ehlen); 1496 iphlen = oiph->ip_hl * 4; 1497 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1498 tcphlen = otcph->th_off * 4; 1499 1500 l4_len = pktlen - ehlen - iphlen; 1501 1502 NXGE_DEBUG_MSG((NULL, TX_CTL, 1503 "==> nxge_do_softlso: mss %d oiph $%p " 1504 "original ip_sum oiph->ip_sum 0x%x " 1505 "original tcp_sum otcph->th_sum 0x%x " 1506 "oiph->ip_len %d pktlen %d ehlen %d " 1507 "l4_len %d (0x%x) ip_len - iphlen %d ", 1508 mss, 1509 oiph, 1510 oiph->ip_sum, 1511 otcph->th_sum, 1512 ntohs(oiph->ip_len), pktlen, 1513 ehlen, 1514 l4_len, 1515 l4_len, 1516 ntohs(oiph->ip_len) - iphlen)); 1517 1518 /* IPv4 + TCP */ 1519 if (!(oiph->ip_v == IPV4_VERSION)) { 1520 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1521 "<== nxge_do_softlso: not IPV4 " 1522 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1523 ntohs(oiph->ip_len), pktlen, ehlen, 1524 tcphlen)); 1525 freemsg(mp); 1526 return (NULL); 1527 } 1528 1529 if (!(oiph->ip_p == IPPROTO_TCP)) { 1530 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1531 "<== nxge_do_softlso: not TCP " 1532 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1533 ntohs(oiph->ip_len), pktlen, ehlen, 1534 tcphlen)); 1535 freemsg(mp); 1536 return (NULL); 1537 } 1538 1539 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1540 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1541 "<== nxge_do_softlso: len not matched " 1542 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1543 ntohs(oiph->ip_len), pktlen, ehlen, 1544 tcphlen)); 1545 freemsg(mp); 1546 return (NULL); 1547 } 1548 1549 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1550 tcphlen = otcph->th_off * 4; 1551 1552 /* TCP flags can not include URG, RST, or SYN */ 1553 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1554 1555 hdrlen = ehlen + iphlen + tcphlen; 1556 1557 VERIFY(MBLKL(mp) >= hdrlen); 1558 1559 if (MBLKL(mp) > hdrlen) { 1560 datamp = mp; 1561 rptr = mp->b_rptr + hdrlen; 1562 } else { /* = */ 1563 datamp = mp->b_cont; 1564 rptr = datamp->b_rptr; 1565 } 1566 1567 NXGE_DEBUG_MSG((NULL, TX_CTL, 1568 "nxge_do_softlso: otcph $%p pktlen: %d, " 1569 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1570 "mblkl(mp): %d, mblkl(datamp): %d", 1571 otcph, 1572 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1573 (int)MBLKL(mp), (int)MBLKL(datamp))); 1574 1575 hckflags = 0; 1576 nxge_hcksum_retrieve(mp, 1577 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1578 1579 dst = oiph->ip_dst.s_addr; 1580 src = oiph->ip_src.s_addr; 1581 1582 cksum = (dst >> 16) + (dst & 0xFFFF) + 1583 (src >> 16) + (src & 0xFFFF); 1584 l4cksum = cksum + IP_TCP_CSUM_COMP; 1585 1586 sum = l4_len + l4cksum; 1587 sum = (sum & 0xFFFF) + (sum >> 16); 1588 1589 NXGE_DEBUG_MSG((NULL, TX_CTL, 1590 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1591 "hckflags 0x%x start_offset %d stuff_offset %d " 1592 "value (original) 0x%x th_sum 0x%x " 1593 "pktlen %d l4_len %d (0x%x) " 1594 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1595 dst, src, 1596 (sum & 0xffff), (~sum & 0xffff), 1597 hckflags, start_offset, stuff_offset, 1598 value, otcph->th_sum, 1599 pktlen, 1600 l4_len, 1601 l4_len, 1602 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1603 (int)MBLKL(datamp), 1604 nxge_dump_packet((char *)evh, 12))); 1605 1606 /* 1607 * Start to process. 1608 */ 1609 available = pktlen - hdrlen; 1610 segnum = (available - 1) / mss + 1; 1611 1612 NXGE_DEBUG_MSG((NULL, TX_CTL, 1613 "==> nxge_do_softlso: pktlen %d " 1614 "MBLKL(mp): %d, MBLKL(datamp): %d " 1615 "available %d mss %d segnum %d", 1616 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1617 available, 1618 mss, 1619 segnum)); 1620 1621 VERIFY(segnum >= 2); 1622 1623 /* 1624 * Try to pre-allocate all header messages 1625 */ 1626 mp_chain = NULL; 1627 for (i = 0; i < segnum; i++) { 1628 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1629 /* Clean up the mp_chain */ 1630 while (mp_chain != NULL) { 1631 nmp = mp_chain; 1632 mp_chain = mp_chain->b_next; 1633 freemsg(nmp); 1634 } 1635 NXGE_DEBUG_MSG((NULL, TX_CTL, 1636 "<== nxge_do_softlso: " 1637 "Could not allocate enough messages for headers!")); 1638 freemsg(mp); 1639 return (NULL); 1640 } 1641 nmp->b_next = mp_chain; 1642 mp_chain = nmp; 1643 1644 NXGE_DEBUG_MSG((NULL, TX_CTL, 1645 "==> nxge_do_softlso: " 1646 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1647 mp, nmp, mp_chain, mp_chain->b_next)); 1648 } 1649 1650 NXGE_DEBUG_MSG((NULL, TX_CTL, 1651 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1652 mp, nmp, mp_chain)); 1653 1654 /* 1655 * Associate payload with new packets 1656 */ 1657 cmp = mp_chain; 1658 left = available; 1659 while (cmp != NULL) { 1660 nmp = dupb(datamp); 1661 if (nmp == NULL) { 1662 do_cleanup = B_TRUE; 1663 NXGE_DEBUG_MSG((NULL, TX_CTL, 1664 "==>nxge_do_softlso: " 1665 "Can not dupb(datamp), have to do clean up")); 1666 goto cleanup_allocated_msgs; 1667 } 1668 1669 NXGE_DEBUG_MSG((NULL, TX_CTL, 1670 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1671 "dupb nmp $%p len %d left %d msd %d ", 1672 mp, cmp, nmp, len, left, mss)); 1673 1674 cmp->b_cont = nmp; 1675 nmp->b_rptr = rptr; 1676 len = (left < mss) ? left : mss; 1677 left -= len; 1678 1679 NXGE_DEBUG_MSG((NULL, TX_CTL, 1680 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1681 "dupb nmp $%p len %d left %d mss %d ", 1682 mp, cmp, nmp, len, left, mss)); 1683 NXGE_DEBUG_MSG((NULL, TX_CTL, 1684 "nxge_do_softlso: before available: %d, " 1685 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1686 available, left, len, segnum, (int)MBLKL(nmp))); 1687 1688 len -= MBLKL(nmp); 1689 NXGE_DEBUG_MSG((NULL, TX_CTL, 1690 "nxge_do_softlso: after available: %d, " 1691 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1692 available, left, len, segnum, (int)MBLKL(nmp))); 1693 1694 while (len > 0) { 1695 mblk_t *mmp = NULL; 1696 1697 NXGE_DEBUG_MSG((NULL, TX_CTL, 1698 "nxge_do_softlso: (4) len > 0 available: %d, " 1699 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1700 available, left, len, segnum, (int)MBLKL(nmp))); 1701 1702 if (datamp->b_cont != NULL) { 1703 datamp = datamp->b_cont; 1704 rptr = datamp->b_rptr; 1705 mmp = dupb(datamp); 1706 if (mmp == NULL) { 1707 do_cleanup = B_TRUE; 1708 NXGE_DEBUG_MSG((NULL, TX_CTL, 1709 "==> nxge_do_softlso: " 1710 "Can not dupb(datamp) (1), :" 1711 "have to do clean up")); 1712 NXGE_DEBUG_MSG((NULL, TX_CTL, 1713 "==> nxge_do_softlso: " 1714 "available: %d, left: %d, " 1715 "len: %d, MBLKL(nmp): %d", 1716 available, left, len, 1717 (int)MBLKL(nmp))); 1718 goto cleanup_allocated_msgs; 1719 } 1720 } else { 1721 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1722 "==> nxge_do_softlso: " 1723 "(1)available: %d, left: %d, " 1724 "len: %d, MBLKL(nmp): %d", 1725 available, left, len, 1726 (int)MBLKL(nmp))); 1727 cmn_err(CE_PANIC, 1728 "==> nxge_do_softlso: " 1729 "Pointers must have been corrupted!\n" 1730 "datamp: $%p, nmp: $%p, rptr: $%p", 1731 (void *)datamp, 1732 (void *)nmp, 1733 (void *)rptr); 1734 } 1735 nmp->b_cont = mmp; 1736 nmp = mmp; 1737 len -= MBLKL(nmp); 1738 } 1739 if (len < 0) { 1740 nmp->b_wptr += len; 1741 rptr = nmp->b_wptr; 1742 NXGE_DEBUG_MSG((NULL, TX_CTL, 1743 "(5) len < 0 (less than 0)" 1744 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1745 available, left, len, (int)MBLKL(nmp))); 1746 1747 } else if (len == 0) { 1748 if (datamp->b_cont != NULL) { 1749 NXGE_DEBUG_MSG((NULL, TX_CTL, 1750 "(5) len == 0" 1751 "available: %d, left: %d, len: %d, " 1752 "MBLKL(nmp): %d", 1753 available, left, len, (int)MBLKL(nmp))); 1754 datamp = datamp->b_cont; 1755 rptr = datamp->b_rptr; 1756 } else { 1757 NXGE_DEBUG_MSG((NULL, TX_CTL, 1758 "(6)available b_cont == NULL : %d, " 1759 "left: %d, len: %d, MBLKL(nmp): %d", 1760 available, left, len, (int)MBLKL(nmp))); 1761 1762 VERIFY(cmp->b_next == NULL); 1763 VERIFY(left == 0); 1764 break; /* Done! */ 1765 } 1766 } 1767 cmp = cmp->b_next; 1768 1769 NXGE_DEBUG_MSG((NULL, TX_CTL, 1770 "(7) do_softlso: " 1771 "next mp in mp_chain available len != 0 : %d, " 1772 "left: %d, len: %d, MBLKL(nmp): %d", 1773 available, left, len, (int)MBLKL(nmp))); 1774 } 1775 1776 /* 1777 * From now, start to fill up all headers for the first message 1778 * Hardware checksum flags need to be updated separately for FULLCKSUM 1779 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1780 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1781 * required fields need to be updated properly. 1782 */ 1783 nmp = mp_chain; 1784 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1785 nmp->b_wptr = nmp->b_rptr + hdrlen; 1786 niph = (struct ip *)(nmp->b_rptr + ehlen); 1787 niph->ip_len = htons(mss + iphlen + tcphlen); 1788 ip_id = ntohs(niph->ip_id); 1789 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1790 #ifdef __sparc 1791 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1792 tcp_seq = ntohl(tcp_seq_tmp); 1793 #else 1794 tcp_seq = ntohl(ntcph->th_seq); 1795 #endif 1796 1797 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1798 1799 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1800 DB_CKSUMSTART(nmp) = start_offset; 1801 DB_CKSUMSTUFF(nmp) = stuff_offset; 1802 1803 /* calculate IP checksum and TCP pseudo header checksum */ 1804 niph->ip_sum = 0; 1805 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1806 1807 l4_len = mss + tcphlen; 1808 sum = htons(l4_len) + l4cksum; 1809 sum = (sum & 0xFFFF) + (sum >> 16); 1810 ntcph->th_sum = (sum & 0xffff); 1811 1812 NXGE_DEBUG_MSG((NULL, TX_CTL, 1813 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1814 "mss %d pktlen %d l4_len %d (0x%x) " 1815 "MBLKL(mp): %d, MBLKL(datamp): %d " 1816 "ip_sum 0x%x " 1817 "th_sum 0x%x sum 0x%x ) " 1818 "dump first ip->tcp %s", 1819 nmp, mp_chain, 1820 mss, 1821 pktlen, 1822 l4_len, 1823 l4_len, 1824 (int)MBLKL(mp), (int)MBLKL(datamp), 1825 niph->ip_sum, 1826 ntcph->th_sum, 1827 sum, 1828 nxge_dump_packet((char *)niph, 52))); 1829 1830 cmp = nmp; 1831 while ((nmp = nmp->b_next)->b_next != NULL) { 1832 NXGE_DEBUG_MSG((NULL, TX_CTL, 1833 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1834 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1835 nmp->b_wptr = nmp->b_rptr + hdrlen; 1836 niph = (struct ip *)(nmp->b_rptr + ehlen); 1837 niph->ip_id = htons(++ip_id); 1838 niph->ip_len = htons(mss + iphlen + tcphlen); 1839 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1840 tcp_seq += mss; 1841 1842 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1843 1844 #ifdef __sparc 1845 tcp_seq_tmp = htonl(tcp_seq); 1846 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1847 #else 1848 ntcph->th_seq = htonl(tcp_seq); 1849 #endif 1850 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1851 DB_CKSUMSTART(nmp) = start_offset; 1852 DB_CKSUMSTUFF(nmp) = stuff_offset; 1853 1854 /* calculate IP checksum and TCP pseudo header checksum */ 1855 niph->ip_sum = 0; 1856 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1857 ntcph->th_sum = (sum & 0xffff); 1858 1859 NXGE_DEBUG_MSG((NULL, TX_CTL, 1860 "==> nxge_do_softlso: middle ip_sum 0x%x " 1861 "th_sum 0x%x " 1862 " mp $%p (mp_chain $%p) pktlen %d " 1863 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1864 niph->ip_sum, 1865 ntcph->th_sum, 1866 nmp, mp_chain, 1867 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1868 } 1869 1870 /* Last segment */ 1871 /* 1872 * Set FIN and/or PSH flags if present only in the last packet. 1873 * The ip_len could be different from prior packets. 1874 */ 1875 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1876 nmp->b_wptr = nmp->b_rptr + hdrlen; 1877 niph = (struct ip *)(nmp->b_rptr + ehlen); 1878 niph->ip_id = htons(++ip_id); 1879 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1880 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1881 tcp_seq += mss; 1882 #ifdef __sparc 1883 tcp_seq_tmp = htonl(tcp_seq); 1884 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1885 #else 1886 ntcph->th_seq = htonl(tcp_seq); 1887 #endif 1888 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1889 1890 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1891 DB_CKSUMSTART(nmp) = start_offset; 1892 DB_CKSUMSTUFF(nmp) = stuff_offset; 1893 1894 /* calculate IP checksum and TCP pseudo header checksum */ 1895 niph->ip_sum = 0; 1896 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1897 1898 l4_len = ntohs(niph->ip_len) - iphlen; 1899 sum = htons(l4_len) + l4cksum; 1900 sum = (sum & 0xFFFF) + (sum >> 16); 1901 ntcph->th_sum = (sum & 0xffff); 1902 1903 NXGE_DEBUG_MSG((NULL, TX_CTL, 1904 "==> nxge_do_softlso: last next " 1905 "niph->ip_sum 0x%x " 1906 "ntcph->th_sum 0x%x sum 0x%x " 1907 "dump last ip->tcp %s " 1908 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1909 "l4_len %d (0x%x) " 1910 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1911 niph->ip_sum, 1912 ntcph->th_sum, sum, 1913 nxge_dump_packet((char *)niph, 52), 1914 cmp, nmp, mp_chain, 1915 pktlen, pktlen, 1916 l4_len, 1917 l4_len, 1918 (int)MBLKL(mp), (int)MBLKL(datamp))); 1919 1920 cleanup_allocated_msgs: 1921 if (do_cleanup) { 1922 NXGE_DEBUG_MSG((NULL, TX_CTL, 1923 "==> nxge_do_softlso: " 1924 "Failed allocating messages, " 1925 "have to clean up and fail!")); 1926 while (mp_chain != NULL) { 1927 nmp = mp_chain; 1928 mp_chain = mp_chain->b_next; 1929 freemsg(nmp); 1930 } 1931 } 1932 /* 1933 * We're done here, so just free the original message and return the 1934 * new message chain, that could be NULL if failed, back to the caller. 1935 */ 1936 freemsg(mp); 1937 1938 NXGE_DEBUG_MSG((NULL, TX_CTL, 1939 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1940 return (mp_chain); 1941 } 1942 1943 /* 1944 * Will be called before NIC driver do further operation on the message. 1945 * The input message may include LSO information, if so, go to softlso logic 1946 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1947 * The return could be the same non-LSO message or a message chain for LSO case. 1948 * 1949 * The driver needs to call this function per packet and process the whole chain 1950 * if applied. 1951 */ 1952 static mblk_t * 1953 nxge_lso_eliminate(mblk_t *mp) 1954 { 1955 uint32_t lsoflags; 1956 uint32_t mss; 1957 1958 NXGE_DEBUG_MSG((NULL, TX_CTL, 1959 "==>nxge_lso_eliminate:")); 1960 nxge_lso_info_get(mp, &mss, &lsoflags); 1961 1962 if (lsoflags & HW_LSO) { 1963 mblk_t *nmp; 1964 1965 NXGE_DEBUG_MSG((NULL, TX_CTL, 1966 "==>nxge_lso_eliminate:" 1967 "HW_LSO:mss %d mp $%p", 1968 mss, mp)); 1969 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1970 NXGE_DEBUG_MSG((NULL, TX_CTL, 1971 "<== nxge_lso_eliminate: " 1972 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1973 nmp, mss, mp)); 1974 return (nmp); 1975 } else { 1976 NXGE_DEBUG_MSG((NULL, TX_CTL, 1977 "<== nxge_lso_eliminate_ " 1978 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1979 nmp, mss, mp)); 1980 return (NULL); 1981 } 1982 } 1983 1984 NXGE_DEBUG_MSG((NULL, TX_CTL, 1985 "<== nxge_lso_eliminate")); 1986 return (mp); 1987 } 1988 1989 static uint32_t 1990 nxge_csgen(uint16_t *adr, int len) 1991 { 1992 int i, odd; 1993 uint32_t sum = 0; 1994 uint32_t c = 0; 1995 1996 odd = len % 2; 1997 for (i = 0; i < (len / 2); i++) { 1998 sum += (adr[i] & 0xffff); 1999 } 2000 if (odd) { 2001 sum += adr[len / 2] & 0xff00; 2002 } 2003 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 2004 sum &= 0xffff; 2005 sum += c; 2006 } 2007 return (~sum & 0xffff); 2008 } 2009