1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/nxge/nxge_impl.h> 29 #include <sys/nxge/nxge_hio.h> 30 #include <npi_tx_wr64.h> 31 32 /* Software LSO required header files */ 33 #include <netinet/tcp.h> 34 #include <inet/ip_impl.h> 35 #include <inet/tcp.h> 36 37 static mblk_t *nxge_lso_eliminate(mblk_t *); 38 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 39 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 40 static void nxge_hcksum_retrieve(mblk_t *, 41 uint32_t *, uint32_t *, uint32_t *, 42 uint32_t *, uint32_t *); 43 static uint32_t nxge_csgen(uint16_t *, int); 44 45 extern uint32_t nxge_reclaim_pending; 46 extern uint32_t nxge_bcopy_thresh; 47 extern uint32_t nxge_dvma_thresh; 48 extern uint32_t nxge_dma_stream_thresh; 49 extern uint32_t nxge_tx_minfree; 50 extern uint32_t nxge_tx_intr_thres; 51 extern uint32_t nxge_tx_max_gathers; 52 extern uint32_t nxge_tx_tiny_pack; 53 extern uint32_t nxge_tx_use_bcopy; 54 extern uint32_t nxge_tx_lb_policy; 55 extern uint32_t nxge_no_tx_lb; 56 extern nxge_tx_mode_t nxge_tx_scheme; 57 uint32_t nxge_lso_kick_cnt = 2; 58 59 typedef struct _mac_tx_hint { 60 uint16_t sap; 61 uint16_t vid; 62 void *hash; 63 } mac_tx_hint_t, *p_mac_tx_hint_t; 64 65 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 66 67 int 68 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 69 { 70 int status = 0; 71 p_tx_desc_t tx_desc_ring_vp; 72 npi_handle_t npi_desc_handle; 73 nxge_os_dma_handle_t tx_desc_dma_handle; 74 p_tx_desc_t tx_desc_p; 75 p_tx_msg_t tx_msg_ring; 76 p_tx_msg_t tx_msg_p; 77 tx_desc_t tx_desc, *tmp_desc_p; 78 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 79 p_tx_pkt_header_t hdrp; 80 tx_pkt_header_t tmp_hdrp; 81 p_tx_pkt_hdr_all_t pkthdrp; 82 uint8_t npads = 0; 83 uint64_t dma_ioaddr; 84 uint32_t dma_flags; 85 int last_bidx; 86 uint8_t *b_rptr; 87 caddr_t kaddr; 88 uint32_t nmblks; 89 uint32_t ngathers; 90 uint32_t clen; 91 int len; 92 uint32_t pkt_len, pack_len, min_len; 93 uint32_t bcopy_thresh; 94 int i, cur_index, sop_index; 95 uint16_t tail_index; 96 boolean_t tail_wrap = B_FALSE; 97 nxge_dma_common_t desc_area; 98 nxge_os_dma_handle_t dma_handle; 99 ddi_dma_cookie_t dma_cookie; 100 npi_handle_t npi_handle; 101 p_mblk_t nmp; 102 p_mblk_t t_mp; 103 uint32_t ncookies; 104 boolean_t good_packet; 105 boolean_t mark_mode = B_FALSE; 106 p_nxge_stats_t statsp; 107 p_nxge_tx_ring_stats_t tdc_stats; 108 t_uscalar_t start_offset = 0; 109 t_uscalar_t stuff_offset = 0; 110 t_uscalar_t end_offset = 0; 111 t_uscalar_t value = 0; 112 t_uscalar_t cksum_flags = 0; 113 boolean_t cksum_on = B_FALSE; 114 uint32_t boff = 0; 115 uint64_t tot_xfer_len = 0; 116 boolean_t header_set = B_FALSE; 117 #ifdef NXGE_DEBUG 118 p_tx_desc_t tx_desc_ring_pp; 119 p_tx_desc_t tx_desc_pp; 120 tx_desc_t *save_desc_p; 121 int dump_len; 122 int sad_len; 123 uint64_t sad; 124 int xfer_len; 125 uint32_t msgsize; 126 #endif 127 p_mblk_t mp_chain = NULL; 128 boolean_t is_lso = B_FALSE; 129 boolean_t lso_again; 130 int cur_index_lso; 131 p_mblk_t nmp_lso_save; 132 uint32_t lso_ngathers; 133 boolean_t lso_tail_wrap = B_FALSE; 134 135 NXGE_DEBUG_MSG((nxgep, TX_CTL, 136 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 137 NXGE_DEBUG_MSG((nxgep, TX_CTL, 138 "==> nxge_start: Starting tdc %d desc pending %d", 139 tx_ring_p->tdc, tx_ring_p->descs_pending)); 140 141 statsp = nxgep->statsp; 142 143 if (!isLDOMguest(nxgep)) { 144 switch (nxgep->mac.portmode) { 145 default: 146 if (nxgep->statsp->port_stats.lb_mode == 147 nxge_lb_normal) { 148 if (!statsp->mac_stats.link_up) { 149 freemsg(mp); 150 NXGE_DEBUG_MSG((nxgep, TX_CTL, 151 "==> nxge_start: " 152 "link not up")); 153 goto nxge_start_fail1; 154 } 155 } 156 break; 157 case PORT_10G_FIBER: 158 /* 159 * For the following modes, check the link status 160 * before sending the packet out: 161 * nxge_lb_normal, nxge_lb_ext10g, nxge_lb_phy10g 162 */ 163 if (nxgep->statsp->port_stats.lb_mode < 164 nxge_lb_serdes10g) { 165 if (!statsp->mac_stats.link_up) { 166 freemsg(mp); 167 NXGE_DEBUG_MSG((nxgep, TX_CTL, 168 "==> nxge_start: " 169 "link not up")); 170 goto nxge_start_fail1; 171 } 172 } 173 break; 174 } 175 } 176 177 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 178 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 179 NXGE_DEBUG_MSG((nxgep, TX_CTL, 180 "==> nxge_start: hardware not initialized or stopped")); 181 freemsg(mp); 182 goto nxge_start_fail1; 183 } 184 185 if (nxgep->soft_lso_enable) { 186 mp_chain = nxge_lso_eliminate(mp); 187 NXGE_DEBUG_MSG((nxgep, TX_CTL, 188 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 189 mp, mp_chain)); 190 if (mp_chain == NULL) { 191 NXGE_ERROR_MSG((nxgep, TX_CTL, 192 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 193 mp_chain, mp)); 194 goto nxge_start_fail1; 195 } 196 if (mp_chain != mp) { 197 NXGE_DEBUG_MSG((nxgep, TX_CTL, 198 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 199 mp_chain, mp)); 200 is_lso = B_TRUE; 201 mp = mp_chain; 202 mp_chain = mp_chain->b_next; 203 mp->b_next = NULL; 204 } 205 } 206 207 hcksum_retrieve(mp, NULL, NULL, &start_offset, 208 &stuff_offset, &end_offset, &value, &cksum_flags); 209 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 210 start_offset += sizeof (ether_header_t); 211 stuff_offset += sizeof (ether_header_t); 212 } else { 213 start_offset += sizeof (struct ether_vlan_header); 214 stuff_offset += sizeof (struct ether_vlan_header); 215 } 216 217 if (cksum_flags & HCK_PARTIALCKSUM) { 218 NXGE_DEBUG_MSG((nxgep, TX_CTL, 219 "==> nxge_start: mp $%p len %d " 220 "cksum_flags 0x%x (partial checksum) ", 221 mp, MBLKL(mp), cksum_flags)); 222 cksum_on = B_TRUE; 223 } 224 225 pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 226 pkthdrp->reserved = 0; 227 tmp_hdrp.value = 0; 228 nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 229 0, 0, pkthdrp, 230 start_offset, stuff_offset); 231 232 lso_again = B_FALSE; 233 lso_ngathers = 0; 234 235 MUTEX_ENTER(&tx_ring_p->lock); 236 237 if (isLDOMservice(nxgep)) { 238 tx_ring_p->tx_ring_busy = B_TRUE; 239 if (tx_ring_p->tx_ring_offline) { 240 freemsg(mp); 241 tx_ring_p->tx_ring_busy = B_FALSE; 242 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 243 NXGE_TX_RING_OFFLINED); 244 MUTEX_EXIT(&tx_ring_p->lock); 245 return (status); 246 } 247 } 248 249 cur_index_lso = tx_ring_p->wr_index; 250 lso_tail_wrap = tx_ring_p->wr_index_wrap; 251 start_again: 252 ngathers = 0; 253 sop_index = tx_ring_p->wr_index; 254 #ifdef NXGE_DEBUG 255 if (tx_ring_p->descs_pending) { 256 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 257 "desc pending %d ", tx_ring_p->descs_pending)); 258 } 259 260 dump_len = (int)(MBLKL(mp)); 261 dump_len = (dump_len > 128) ? 128: dump_len; 262 263 NXGE_DEBUG_MSG((nxgep, TX_CTL, 264 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 265 "(Before header reserve: ORIGINAL LEN %d)", 266 tx_ring_p->tdc, 267 mp->b_rptr, 268 dump_len)); 269 270 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 271 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 272 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 273 #endif 274 275 tdc_stats = tx_ring_p->tdc_stats; 276 mark_mode = (tx_ring_p->descs_pending && 277 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 278 < nxge_tx_minfree)); 279 280 NXGE_DEBUG_MSG((nxgep, TX_CTL, 281 "TX Descriptor ring is channel %d mark mode %d", 282 tx_ring_p->tdc, mark_mode)); 283 284 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { 285 NXGE_DEBUG_MSG((nxgep, TX_CTL, 286 "TX Descriptor ring is full: channel %d", 287 tx_ring_p->tdc)); 288 NXGE_DEBUG_MSG((nxgep, TX_CTL, 289 "TX Descriptor ring is full: channel %d", 290 tx_ring_p->tdc)); 291 if (is_lso) { 292 /* free the current mp and mp_chain if not FULL */ 293 tdc_stats->tx_no_desc++; 294 NXGE_DEBUG_MSG((nxgep, TX_CTL, 295 "LSO packet: TX Descriptor ring is full: " 296 "channel %d", 297 tx_ring_p->tdc)); 298 goto nxge_start_fail_lso; 299 } else { 300 boolean_t skip_sched = B_FALSE; 301 302 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 303 tdc_stats->tx_no_desc++; 304 305 if (isLDOMservice(nxgep)) { 306 tx_ring_p->tx_ring_busy = B_FALSE; 307 if (tx_ring_p->tx_ring_offline) { 308 (void) atomic_swap_32( 309 &tx_ring_p->tx_ring_offline, 310 NXGE_TX_RING_OFFLINED); 311 skip_sched = B_TRUE; 312 } 313 } 314 315 MUTEX_EXIT(&tx_ring_p->lock); 316 if (nxgep->resched_needed && 317 !nxgep->resched_running && !skip_sched) { 318 nxgep->resched_running = B_TRUE; 319 ddi_trigger_softintr(nxgep->resched_id); 320 } 321 status = 1; 322 goto nxge_start_fail1; 323 } 324 } 325 326 nmp = mp; 327 i = sop_index = tx_ring_p->wr_index; 328 nmblks = 0; 329 ngathers = 0; 330 pkt_len = 0; 331 pack_len = 0; 332 clen = 0; 333 last_bidx = -1; 334 good_packet = B_TRUE; 335 336 desc_area = tx_ring_p->tdc_desc; 337 npi_handle = desc_area.npi_handle; 338 npi_desc_handle.regh = (nxge_os_acc_handle_t) 339 DMA_COMMON_ACC_HANDLE(desc_area); 340 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 341 tx_desc_dma_handle = (nxge_os_dma_handle_t) 342 DMA_COMMON_HANDLE(desc_area); 343 tx_msg_ring = tx_ring_p->tx_msg_ring; 344 345 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 346 sop_index, i)); 347 348 #ifdef NXGE_DEBUG 349 msgsize = msgdsize(nmp); 350 NXGE_DEBUG_MSG((nxgep, TX_CTL, 351 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 352 sop_index, i, msgsize)); 353 #endif 354 /* 355 * The first 16 bytes of the premapped buffer are reserved 356 * for header. No padding will be used. 357 */ 358 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 359 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 360 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 361 } else { 362 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 363 } 364 while (nmp) { 365 good_packet = B_TRUE; 366 b_rptr = nmp->b_rptr; 367 len = MBLKL(nmp); 368 if (len <= 0) { 369 nmp = nmp->b_cont; 370 continue; 371 } 372 nmblks++; 373 374 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 375 "len %d pkt_len %d pack_len %d", 376 nmblks, len, pkt_len, pack_len)); 377 /* 378 * Hardware limits the transfer length to 4K for NIU and 379 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 380 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 381 * If len is longer than the limit, then we break nmp into 382 * two chunks: Make the first chunk equal to the limit and 383 * the second chunk for the remaining data. If the second 384 * chunk is still larger than the limit, then it will be 385 * broken into two in the next pass. 386 */ 387 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 388 if ((t_mp = dupb(nmp)) != NULL) { 389 nmp->b_wptr = nmp->b_rptr + 390 (TX_MAX_TRANSFER_LENGTH 391 - TX_PKT_HEADER_SIZE); 392 t_mp->b_rptr = nmp->b_wptr; 393 t_mp->b_cont = nmp->b_cont; 394 nmp->b_cont = t_mp; 395 len = MBLKL(nmp); 396 } else { 397 if (is_lso) { 398 NXGE_DEBUG_MSG((nxgep, TX_CTL, 399 "LSO packet: dupb failed: " 400 "channel %d", 401 tx_ring_p->tdc)); 402 mp = nmp; 403 goto nxge_start_fail_lso; 404 } else { 405 good_packet = B_FALSE; 406 goto nxge_start_fail2; 407 } 408 } 409 } 410 tx_desc.value = 0; 411 tx_desc_p = &tx_desc_ring_vp[i]; 412 #ifdef NXGE_DEBUG 413 tx_desc_pp = &tx_desc_ring_pp[i]; 414 #endif 415 tx_msg_p = &tx_msg_ring[i]; 416 #if defined(__i386) 417 npi_desc_handle.regp = (uint32_t)tx_desc_p; 418 #else 419 npi_desc_handle.regp = (uint64_t)tx_desc_p; 420 #endif 421 if (!header_set && 422 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 423 (len >= bcopy_thresh))) { 424 header_set = B_TRUE; 425 bcopy_thresh += TX_PKT_HEADER_SIZE; 426 boff = 0; 427 pack_len = 0; 428 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 429 hdrp = (p_tx_pkt_header_t)kaddr; 430 clen = pkt_len; 431 dma_handle = tx_msg_p->buf_dma_handle; 432 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 433 (void) ddi_dma_sync(dma_handle, 434 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 435 DDI_DMA_SYNC_FORDEV); 436 437 tx_msg_p->flags.dma_type = USE_BCOPY; 438 goto nxge_start_control_header_only; 439 } 440 441 pkt_len += len; 442 pack_len += len; 443 444 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 445 "desc entry %d " 446 "DESC IOADDR $%p " 447 "desc_vp $%p tx_desc_p $%p " 448 "desc_pp $%p tx_desc_pp $%p " 449 "len %d pkt_len %d pack_len %d", 450 i, 451 DMA_COMMON_IOADDR(desc_area), 452 tx_desc_ring_vp, tx_desc_p, 453 tx_desc_ring_pp, tx_desc_pp, 454 len, pkt_len, pack_len)); 455 456 if (len < bcopy_thresh) { 457 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 458 "USE BCOPY: ")); 459 if (nxge_tx_tiny_pack) { 460 uint32_t blst = 461 TXDMA_DESC_NEXT_INDEX(i, -1, 462 tx_ring_p->tx_wrap_mask); 463 NXGE_DEBUG_MSG((nxgep, TX_CTL, 464 "==> nxge_start(5): pack")); 465 if ((pack_len <= bcopy_thresh) && 466 (last_bidx == blst)) { 467 NXGE_DEBUG_MSG((nxgep, TX_CTL, 468 "==> nxge_start: pack(6) " 469 "(pkt_len %d pack_len %d)", 470 pkt_len, pack_len)); 471 i = blst; 472 tx_desc_p = &tx_desc_ring_vp[i]; 473 #ifdef NXGE_DEBUG 474 tx_desc_pp = &tx_desc_ring_pp[i]; 475 #endif 476 tx_msg_p = &tx_msg_ring[i]; 477 boff = pack_len - len; 478 ngathers--; 479 } else if (pack_len > bcopy_thresh && 480 header_set) { 481 pack_len = len; 482 boff = 0; 483 bcopy_thresh = nxge_bcopy_thresh; 484 NXGE_DEBUG_MSG((nxgep, TX_CTL, 485 "==> nxge_start(7): > max NEW " 486 "bcopy thresh %d " 487 "pkt_len %d pack_len %d(next)", 488 bcopy_thresh, 489 pkt_len, pack_len)); 490 } 491 last_bidx = i; 492 } 493 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 494 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 495 hdrp = (p_tx_pkt_header_t)kaddr; 496 header_set = B_TRUE; 497 NXGE_DEBUG_MSG((nxgep, TX_CTL, 498 "==> nxge_start(7_x2): " 499 "pkt_len %d pack_len %d (new hdrp $%p)", 500 pkt_len, pack_len, hdrp)); 501 } 502 tx_msg_p->flags.dma_type = USE_BCOPY; 503 kaddr += boff; 504 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 505 "USE BCOPY: before bcopy " 506 "DESC IOADDR $%p entry %d " 507 "bcopy packets %d " 508 "bcopy kaddr $%p " 509 "bcopy ioaddr (SAD) $%p " 510 "bcopy clen %d " 511 "bcopy boff %d", 512 DMA_COMMON_IOADDR(desc_area), i, 513 tdc_stats->tx_hdr_pkts, 514 kaddr, 515 dma_ioaddr, 516 clen, 517 boff)); 518 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 519 "1USE BCOPY: ")); 520 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 521 "2USE BCOPY: ")); 522 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 523 "last USE BCOPY: copy from b_rptr $%p " 524 "to KADDR $%p (len %d offset %d", 525 b_rptr, kaddr, len, boff)); 526 527 bcopy(b_rptr, kaddr, len); 528 529 #ifdef NXGE_DEBUG 530 dump_len = (len > 128) ? 128: len; 531 NXGE_DEBUG_MSG((nxgep, TX_CTL, 532 "==> nxge_start: dump packets " 533 "(After BCOPY len %d)" 534 "(b_rptr $%p): %s", len, nmp->b_rptr, 535 nxge_dump_packet((char *)nmp->b_rptr, 536 dump_len))); 537 #endif 538 539 dma_handle = tx_msg_p->buf_dma_handle; 540 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 541 (void) ddi_dma_sync(dma_handle, 542 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 543 DDI_DMA_SYNC_FORDEV); 544 clen = len + boff; 545 tdc_stats->tx_hdr_pkts++; 546 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 547 "USE BCOPY: " 548 "DESC IOADDR $%p entry %d " 549 "bcopy packets %d " 550 "bcopy kaddr $%p " 551 "bcopy ioaddr (SAD) $%p " 552 "bcopy clen %d " 553 "bcopy boff %d", 554 DMA_COMMON_IOADDR(desc_area), 555 i, 556 tdc_stats->tx_hdr_pkts, 557 kaddr, 558 dma_ioaddr, 559 clen, 560 boff)); 561 } else { 562 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 563 "USE DVMA: len %d", len)); 564 tx_msg_p->flags.dma_type = USE_DMA; 565 dma_flags = DDI_DMA_WRITE; 566 if (len < nxge_dma_stream_thresh) { 567 dma_flags |= DDI_DMA_CONSISTENT; 568 } else { 569 dma_flags |= DDI_DMA_STREAMING; 570 } 571 572 dma_handle = tx_msg_p->dma_handle; 573 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 574 (caddr_t)b_rptr, len, dma_flags, 575 DDI_DMA_DONTWAIT, NULL, 576 &dma_cookie, &ncookies); 577 if (status == DDI_DMA_MAPPED) { 578 dma_ioaddr = dma_cookie.dmac_laddress; 579 len = (int)dma_cookie.dmac_size; 580 clen = (uint32_t)dma_cookie.dmac_size; 581 NXGE_DEBUG_MSG((nxgep, TX_CTL, 582 "==> nxge_start(12_1): " 583 "USE DVMA: len %d clen %d " 584 "ngathers %d", 585 len, clen, 586 ngathers)); 587 #if defined(__i386) 588 npi_desc_handle.regp = (uint32_t)tx_desc_p; 589 #else 590 npi_desc_handle.regp = (uint64_t)tx_desc_p; 591 #endif 592 while (ncookies > 1) { 593 ngathers++; 594 /* 595 * this is the fix for multiple 596 * cookies, which are basically 597 * a descriptor entry, we don't set 598 * SOP bit as well as related fields 599 */ 600 601 (void) npi_txdma_desc_gather_set( 602 npi_desc_handle, 603 &tx_desc, 604 (ngathers -1), 605 mark_mode, 606 ngathers, 607 dma_ioaddr, 608 clen); 609 610 tx_msg_p->tx_msg_size = clen; 611 NXGE_DEBUG_MSG((nxgep, TX_CTL, 612 "==> nxge_start: DMA " 613 "ncookie %d " 614 "ngathers %d " 615 "dma_ioaddr $%p len %d" 616 "desc $%p descp $%p (%d)", 617 ncookies, 618 ngathers, 619 dma_ioaddr, clen, 620 *tx_desc_p, tx_desc_p, i)); 621 622 ddi_dma_nextcookie(dma_handle, 623 &dma_cookie); 624 dma_ioaddr = 625 dma_cookie.dmac_laddress; 626 627 len = (int)dma_cookie.dmac_size; 628 clen = (uint32_t)dma_cookie.dmac_size; 629 NXGE_DEBUG_MSG((nxgep, TX_CTL, 630 "==> nxge_start(12_2): " 631 "USE DVMA: len %d clen %d ", 632 len, clen)); 633 634 i = TXDMA_DESC_NEXT_INDEX(i, 1, 635 tx_ring_p->tx_wrap_mask); 636 tx_desc_p = &tx_desc_ring_vp[i]; 637 638 #if defined(__i386) 639 npi_desc_handle.regp = 640 (uint32_t)tx_desc_p; 641 #else 642 npi_desc_handle.regp = 643 (uint64_t)tx_desc_p; 644 #endif 645 tx_msg_p = &tx_msg_ring[i]; 646 tx_msg_p->flags.dma_type = USE_NONE; 647 tx_desc.value = 0; 648 649 ncookies--; 650 } 651 tdc_stats->tx_ddi_pkts++; 652 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 653 "DMA: ddi packets %d", 654 tdc_stats->tx_ddi_pkts)); 655 } else { 656 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 657 "dma mapping failed for %d " 658 "bytes addr $%p flags %x (%d)", 659 len, b_rptr, status, status)); 660 good_packet = B_FALSE; 661 tdc_stats->tx_dma_bind_fail++; 662 tx_msg_p->flags.dma_type = USE_NONE; 663 if (is_lso) { 664 mp = nmp; 665 goto nxge_start_fail_lso; 666 } else { 667 goto nxge_start_fail2; 668 } 669 } 670 } /* ddi dvma */ 671 672 if (is_lso) { 673 nmp_lso_save = nmp; 674 } 675 nmp = nmp->b_cont; 676 nxge_start_control_header_only: 677 #if defined(__i386) 678 npi_desc_handle.regp = (uint32_t)tx_desc_p; 679 #else 680 npi_desc_handle.regp = (uint64_t)tx_desc_p; 681 #endif 682 ngathers++; 683 684 if (ngathers == 1) { 685 #ifdef NXGE_DEBUG 686 save_desc_p = &sop_tx_desc; 687 #endif 688 sop_tx_desc_p = &sop_tx_desc; 689 sop_tx_desc_p->value = 0; 690 sop_tx_desc_p->bits.hdw.tr_len = clen; 691 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 692 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 693 } else { 694 #ifdef NXGE_DEBUG 695 save_desc_p = &tx_desc; 696 #endif 697 tmp_desc_p = &tx_desc; 698 tmp_desc_p->value = 0; 699 tmp_desc_p->bits.hdw.tr_len = clen; 700 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 701 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 702 703 tx_desc_p->value = tmp_desc_p->value; 704 } 705 706 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 707 "Desc_entry %d ngathers %d " 708 "desc_vp $%p tx_desc_p $%p " 709 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 710 "dma_ioaddr (SAD) $%p mark %d", 711 i, ngathers, 712 tx_desc_ring_vp, tx_desc_p, 713 len, clen, pkt_len, pack_len, nmblks, 714 dma_ioaddr, mark_mode)); 715 716 #ifdef NXGE_DEBUG 717 npi_desc_handle.nxgep = nxgep; 718 npi_desc_handle.function.function = nxgep->function_num; 719 npi_desc_handle.function.instance = nxgep->instance; 720 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 721 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 722 TX_PKT_DESC_TR_LEN_SHIFT); 723 724 725 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 726 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 727 "mark %d sop %d\n", 728 save_desc_p->value, 729 sad, 730 save_desc_p->bits.hdw.tr_len, 731 xfer_len, 732 save_desc_p->bits.hdw.num_ptr, 733 save_desc_p->bits.hdw.mark, 734 save_desc_p->bits.hdw.sop)); 735 736 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 737 #endif 738 739 tx_msg_p->tx_msg_size = clen; 740 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 741 if (ngathers > nxge_tx_max_gathers) { 742 good_packet = B_FALSE; 743 hcksum_retrieve(mp, NULL, NULL, &start_offset, 744 &stuff_offset, &end_offset, &value, 745 &cksum_flags); 746 747 NXGE_DEBUG_MSG((NULL, TX_CTL, 748 "==> nxge_start(14): pull msg - " 749 "len %d pkt_len %d ngathers %d", 750 len, pkt_len, ngathers)); 751 /* Pull all message blocks from b_cont */ 752 if (is_lso) { 753 mp = nmp_lso_save; 754 goto nxge_start_fail_lso; 755 } 756 if ((msgpullup(mp, -1)) == NULL) { 757 goto nxge_start_fail2; 758 } 759 goto nxge_start_fail2; 760 } 761 } /* while (nmp) */ 762 763 tx_msg_p->tx_message = mp; 764 tx_desc_p = &tx_desc_ring_vp[sop_index]; 765 #if defined(__i386) 766 npi_desc_handle.regp = (uint32_t)tx_desc_p; 767 #else 768 npi_desc_handle.regp = (uint64_t)tx_desc_p; 769 #endif 770 771 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 772 pkthdrp->reserved = 0; 773 hdrp->value = 0; 774 bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 775 776 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 777 tdc_stats->tx_jumbo_pkts++; 778 } 779 780 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 781 if (pkt_len < min_len) { 782 /* Assume we use bcopy to premapped buffers */ 783 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 784 NXGE_DEBUG_MSG((NULL, TX_CTL, 785 "==> nxge_start(14-1): < (msg_min + 16)" 786 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 787 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 788 bzero((kaddr + pkt_len), (min_len - pkt_len)); 789 pkt_len = tx_msg_p->tx_msg_size = min_len; 790 791 sop_tx_desc_p->bits.hdw.tr_len = min_len; 792 793 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 794 tx_desc_p->value = sop_tx_desc_p->value; 795 796 NXGE_DEBUG_MSG((NULL, TX_CTL, 797 "==> nxge_start(14-2): < msg_min - " 798 "len %d pkt_len %d min_len %d ngathers %d", 799 len, pkt_len, min_len, ngathers)); 800 } 801 802 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 803 cksum_flags)); 804 { 805 uint64_t tmp_len; 806 807 /* pkt_len already includes 16 + paddings!! */ 808 /* Update the control header length */ 809 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 810 tmp_len = hdrp->value | 811 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 812 813 NXGE_DEBUG_MSG((nxgep, TX_CTL, 814 "==> nxge_start(15_x1): setting SOP " 815 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 816 "0x%llx hdrp->value 0x%llx", 817 tot_xfer_len, tot_xfer_len, pkt_len, 818 tmp_len, hdrp->value)); 819 #if defined(_BIG_ENDIAN) 820 hdrp->value = ddi_swap64(tmp_len); 821 #else 822 hdrp->value = tmp_len; 823 #endif 824 NXGE_DEBUG_MSG((nxgep, 825 TX_CTL, "==> nxge_start(15_x2): setting SOP " 826 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 827 "tmp_len 0x%llx hdrp->value 0x%llx", 828 tot_xfer_len, pkt_len, 829 tmp_len, hdrp->value)); 830 } 831 832 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 833 "wr_index %d " 834 "tot_xfer_len (%d) pkt_len %d npads %d", 835 sop_index, 836 tot_xfer_len, pkt_len, 837 npads)); 838 839 sop_tx_desc_p->bits.hdw.sop = 1; 840 sop_tx_desc_p->bits.hdw.mark = mark_mode; 841 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 842 843 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 844 845 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 846 847 #ifdef NXGE_DEBUG 848 npi_desc_handle.nxgep = nxgep; 849 npi_desc_handle.function.function = nxgep->function_num; 850 npi_desc_handle.function.instance = nxgep->instance; 851 852 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 853 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 854 save_desc_p->value, 855 sad, 856 save_desc_p->bits.hdw.tr_len, 857 xfer_len, 858 save_desc_p->bits.hdw.num_ptr, 859 save_desc_p->bits.hdw.mark, 860 save_desc_p->bits.hdw.sop)); 861 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 862 863 dump_len = (pkt_len > 128) ? 128: pkt_len; 864 NXGE_DEBUG_MSG((nxgep, TX_CTL, 865 "==> nxge_start: dump packets(17) (after sop set, len " 866 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 867 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 868 (char *)hdrp, 869 nxge_dump_packet((char *)hdrp, dump_len))); 870 NXGE_DEBUG_MSG((nxgep, TX_CTL, 871 "==> nxge_start(18): TX desc sync: sop_index %d", 872 sop_index)); 873 #endif 874 875 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 876 (void) ddi_dma_sync(tx_desc_dma_handle, 877 sop_index * sizeof (tx_desc_t), 878 ngathers * sizeof (tx_desc_t), 879 DDI_DMA_SYNC_FORDEV); 880 881 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 882 "cs_off = 0x%02X cs_s_off = 0x%02X " 883 "pkt_len %d ngathers %d sop_index %d\n", 884 stuff_offset, start_offset, 885 pkt_len, ngathers, sop_index)); 886 } else { /* more than one descriptor and wrap around */ 887 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 888 (void) ddi_dma_sync(tx_desc_dma_handle, 889 sop_index * sizeof (tx_desc_t), 890 nsdescs * sizeof (tx_desc_t), 891 DDI_DMA_SYNC_FORDEV); 892 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 893 "cs_off = 0x%02X cs_s_off = 0x%02X " 894 "pkt_len %d ngathers %d sop_index %d\n", 895 stuff_offset, start_offset, 896 pkt_len, ngathers, sop_index)); 897 898 (void) ddi_dma_sync(tx_desc_dma_handle, 899 0, 900 (ngathers - nsdescs) * sizeof (tx_desc_t), 901 DDI_DMA_SYNC_FORDEV); 902 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 903 "cs_off = 0x%02X cs_s_off = 0x%02X " 904 "pkt_len %d ngathers %d sop_index %d\n", 905 stuff_offset, start_offset, 906 pkt_len, ngathers, sop_index)); 907 } 908 909 tail_index = tx_ring_p->wr_index; 910 tail_wrap = tx_ring_p->wr_index_wrap; 911 912 tx_ring_p->wr_index = i; 913 if (tx_ring_p->wr_index <= tail_index) { 914 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 915 B_FALSE : B_TRUE); 916 } 917 918 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 919 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 920 tx_ring_p->tdc, 921 tx_ring_p->wr_index, 922 tx_ring_p->wr_index_wrap, 923 ngathers, 924 tx_ring_p->descs_pending)); 925 926 if (is_lso) { 927 lso_ngathers += ngathers; 928 if (mp_chain != NULL) { 929 mp = mp_chain; 930 mp_chain = mp_chain->b_next; 931 mp->b_next = NULL; 932 if (nxge_lso_kick_cnt == lso_ngathers) { 933 tx_ring_p->descs_pending += lso_ngathers; 934 { 935 tx_ring_kick_t kick; 936 937 kick.value = 0; 938 kick.bits.ldw.wrap = 939 tx_ring_p->wr_index_wrap; 940 kick.bits.ldw.tail = 941 (uint16_t)tx_ring_p->wr_index; 942 943 /* Kick the Transmit kick register */ 944 TXDMA_REG_WRITE64( 945 NXGE_DEV_NPI_HANDLE(nxgep), 946 TX_RING_KICK_REG, 947 (uint8_t)tx_ring_p->tdc, 948 kick.value); 949 tdc_stats->tx_starts++; 950 951 NXGE_DEBUG_MSG((nxgep, TX_CTL, 952 "==> nxge_start: more LSO: " 953 "LSO_CNT %d", 954 lso_ngathers)); 955 } 956 lso_ngathers = 0; 957 ngathers = 0; 958 cur_index_lso = sop_index = tx_ring_p->wr_index; 959 lso_tail_wrap = tx_ring_p->wr_index_wrap; 960 } 961 NXGE_DEBUG_MSG((nxgep, TX_CTL, 962 "==> nxge_start: lso again: " 963 "lso_gathers %d ngathers %d cur_index_lso %d " 964 "wr_index %d sop_index %d", 965 lso_ngathers, ngathers, cur_index_lso, 966 tx_ring_p->wr_index, sop_index)); 967 968 NXGE_DEBUG_MSG((nxgep, TX_CTL, 969 "==> nxge_start: next : count %d", 970 lso_ngathers)); 971 lso_again = B_TRUE; 972 goto start_again; 973 } 974 ngathers = lso_ngathers; 975 } 976 977 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 978 979 { 980 tx_ring_kick_t kick; 981 982 kick.value = 0; 983 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 984 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 985 986 /* Kick start the Transmit kick register */ 987 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 988 TX_RING_KICK_REG, 989 (uint8_t)tx_ring_p->tdc, 990 kick.value); 991 } 992 993 tx_ring_p->descs_pending += ngathers; 994 tdc_stats->tx_starts++; 995 996 if (isLDOMservice(nxgep)) { 997 tx_ring_p->tx_ring_busy = B_FALSE; 998 if (tx_ring_p->tx_ring_offline) { 999 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1000 NXGE_TX_RING_OFFLINED); 1001 } 1002 } 1003 1004 MUTEX_EXIT(&tx_ring_p->lock); 1005 1006 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1007 1008 return (status); 1009 1010 nxge_start_fail_lso: 1011 status = 0; 1012 good_packet = B_FALSE; 1013 if (mp != NULL) { 1014 freemsg(mp); 1015 } 1016 if (mp_chain != NULL) { 1017 freemsg(mp_chain); 1018 } 1019 if (!lso_again && !ngathers) { 1020 if (isLDOMservice(nxgep)) { 1021 tx_ring_p->tx_ring_busy = B_FALSE; 1022 if (tx_ring_p->tx_ring_offline) { 1023 (void) atomic_swap_32( 1024 &tx_ring_p->tx_ring_offline, 1025 NXGE_TX_RING_OFFLINED); 1026 } 1027 } 1028 1029 MUTEX_EXIT(&tx_ring_p->lock); 1030 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1031 "==> nxge_start: lso exit (nothing changed)")); 1032 goto nxge_start_fail1; 1033 } 1034 1035 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1036 "==> nxge_start (channel %d): before lso " 1037 "lso_gathers %d ngathers %d cur_index_lso %d " 1038 "wr_index %d sop_index %d lso_again %d", 1039 tx_ring_p->tdc, 1040 lso_ngathers, ngathers, cur_index_lso, 1041 tx_ring_p->wr_index, sop_index, lso_again)); 1042 1043 if (lso_again) { 1044 lso_ngathers += ngathers; 1045 ngathers = lso_ngathers; 1046 sop_index = cur_index_lso; 1047 tx_ring_p->wr_index = sop_index; 1048 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1049 } 1050 1051 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1052 "==> nxge_start (channel %d): after lso " 1053 "lso_gathers %d ngathers %d cur_index_lso %d " 1054 "wr_index %d sop_index %d lso_again %d", 1055 tx_ring_p->tdc, 1056 lso_ngathers, ngathers, cur_index_lso, 1057 tx_ring_p->wr_index, sop_index, lso_again)); 1058 1059 nxge_start_fail2: 1060 if (good_packet == B_FALSE) { 1061 cur_index = sop_index; 1062 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1063 for (i = 0; i < ngathers; i++) { 1064 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1065 #if defined(__i386) 1066 npi_handle.regp = (uint32_t)tx_desc_p; 1067 #else 1068 npi_handle.regp = (uint64_t)tx_desc_p; 1069 #endif 1070 tx_msg_p = &tx_msg_ring[cur_index]; 1071 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1072 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1073 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1074 "tx_desc_p = %X index = %d", 1075 tx_desc_p, tx_ring_p->rd_index)); 1076 (void) dvma_unload(tx_msg_p->dvma_handle, 1077 0, -1); 1078 tx_msg_p->dvma_handle = NULL; 1079 if (tx_ring_p->dvma_wr_index == 1080 tx_ring_p->dvma_wrap_mask) 1081 tx_ring_p->dvma_wr_index = 0; 1082 else 1083 tx_ring_p->dvma_wr_index++; 1084 tx_ring_p->dvma_pending--; 1085 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1086 if (ddi_dma_unbind_handle( 1087 tx_msg_p->dma_handle)) { 1088 cmn_err(CE_WARN, "!nxge_start: " 1089 "ddi_dma_unbind_handle failed"); 1090 } 1091 } 1092 tx_msg_p->flags.dma_type = USE_NONE; 1093 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1094 tx_ring_p->tx_wrap_mask); 1095 1096 } 1097 1098 nxgep->resched_needed = B_TRUE; 1099 } 1100 1101 if (isLDOMservice(nxgep)) { 1102 tx_ring_p->tx_ring_busy = B_FALSE; 1103 if (tx_ring_p->tx_ring_offline) { 1104 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1105 NXGE_TX_RING_OFFLINED); 1106 } 1107 } 1108 1109 MUTEX_EXIT(&tx_ring_p->lock); 1110 1111 nxge_start_fail1: 1112 /* Add FMA to check the access handle nxge_hregh */ 1113 1114 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1115 1116 return (status); 1117 } 1118 1119 int 1120 nxge_serial_tx(mblk_t *mp, void *arg) 1121 { 1122 p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 1123 p_nxge_t nxgep = tx_ring_p->nxgep; 1124 int status = 0; 1125 1126 if (isLDOMservice(nxgep)) { 1127 if (tx_ring_p->tx_ring_offline) { 1128 freemsg(mp); 1129 return (status); 1130 } 1131 } 1132 1133 status = nxge_start(nxgep, tx_ring_p, mp); 1134 return (status); 1135 } 1136 1137 boolean_t 1138 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 1139 { 1140 p_tx_ring_t *tx_rings; 1141 uint8_t ring_index; 1142 p_tx_ring_t tx_ring_p; 1143 nxge_grp_t *group; 1144 1145 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 1146 1147 ASSERT(mp->b_next == NULL); 1148 1149 group = nxgep->tx_set.group[0]; /* The default group */ 1150 ring_index = nxge_tx_lb_ring_1(mp, group->count, hp); 1151 1152 tx_rings = nxgep->tx_rings->rings; 1153 tx_ring_p = tx_rings[group->legend[ring_index]]; 1154 1155 if (isLDOMservice(nxgep)) { 1156 if (tx_ring_p->tx_ring_offline) { 1157 /* 1158 * OFFLINE means that it is in the process of being 1159 * shared - that is, it has been claimed by the HIO 1160 * code, but hasn't been unlinked from <group> yet. 1161 * So in this case use the first TDC, which always 1162 * belongs to the service domain and can't be shared. 1163 */ 1164 ring_index = 0; 1165 tx_ring_p = tx_rings[group->legend[ring_index]]; 1166 } 1167 } 1168 1169 NXGE_DEBUG_MSG((nxgep, TX_CTL, "count %d, tx_rings[%d] = %p", 1170 (int)group->count, group->legend[ring_index], tx_ring_p)); 1171 1172 switch (nxge_tx_scheme) { 1173 case NXGE_USE_START: 1174 if (nxge_start(nxgep, tx_ring_p, mp)) { 1175 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 1176 "ring index %d", ring_index)); 1177 return (B_FALSE); 1178 } 1179 break; 1180 1181 case NXGE_USE_SERIAL: 1182 default: 1183 nxge_serialize_enter(tx_ring_p->serial, mp); 1184 break; 1185 } 1186 1187 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 1188 ring_index)); 1189 1190 return (B_TRUE); 1191 } 1192 1193 /* 1194 * nxge_m_tx() - send a chain of packets 1195 */ 1196 mblk_t * 1197 nxge_m_tx(void *arg, mblk_t *mp) 1198 { 1199 p_nxge_t nxgep = (p_nxge_t)arg; 1200 mblk_t *next; 1201 mac_tx_hint_t hint; 1202 1203 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_m_tx")); 1204 1205 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 1206 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 1207 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1208 "==> nxge_m_tx: hardware not initialized")); 1209 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1210 "<== nxge_m_tx")); 1211 freemsgchain(mp); 1212 mp = NULL; 1213 return (mp); 1214 } 1215 1216 hint.hash = NULL; 1217 hint.vid = 0; 1218 hint.sap = 0; 1219 1220 while (mp != NULL) { 1221 next = mp->b_next; 1222 mp->b_next = NULL; 1223 1224 /* 1225 * Until Nemo tx resource works, the mac driver 1226 * does the load balancing based on TCP port, 1227 * or CPU. For debugging, we use a system 1228 * configurable parameter. 1229 */ 1230 if (!nxge_send(nxgep, mp, &hint)) { 1231 mp->b_next = next; 1232 break; 1233 } 1234 1235 mp = next; 1236 1237 NXGE_DEBUG_MSG((NULL, TX_CTL, 1238 "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 1239 mp, next)); 1240 } 1241 1242 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_m_tx")); 1243 return (mp); 1244 } 1245 1246 int 1247 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 1248 { 1249 uint8_t ring_index = 0; 1250 uint8_t *tcp_port; 1251 p_mblk_t nmp; 1252 size_t mblk_len; 1253 size_t iph_len; 1254 size_t hdrs_size; 1255 uint8_t hdrs_buf[sizeof (struct ether_header) + 1256 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 1257 /* 1258 * allocate space big enough to cover 1259 * the max ip header length and the first 1260 * 4 bytes of the TCP/IP header. 1261 */ 1262 1263 boolean_t qos = B_FALSE; 1264 1265 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 1266 1267 if (hp->vid) { 1268 qos = B_TRUE; 1269 } 1270 switch (nxge_tx_lb_policy) { 1271 case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 1272 default: 1273 tcp_port = mp->b_rptr; 1274 if (!nxge_no_tx_lb && !qos && 1275 (ntohs(((p_ether_header_t)tcp_port)->ether_type) 1276 == ETHERTYPE_IP)) { 1277 nmp = mp; 1278 mblk_len = MBLKL(nmp); 1279 tcp_port = NULL; 1280 if (mblk_len > sizeof (struct ether_header) + 1281 sizeof (uint8_t)) { 1282 tcp_port = nmp->b_rptr + 1283 sizeof (struct ether_header); 1284 mblk_len -= sizeof (struct ether_header); 1285 iph_len = ((*tcp_port) & 0x0f) << 2; 1286 if (mblk_len > (iph_len + sizeof (uint32_t))) { 1287 tcp_port = nmp->b_rptr; 1288 } else { 1289 tcp_port = NULL; 1290 } 1291 } 1292 if (tcp_port == NULL) { 1293 hdrs_size = 0; 1294 ((p_ether_header_t)hdrs_buf)->ether_type = 0; 1295 while ((nmp) && (hdrs_size < 1296 sizeof (hdrs_buf))) { 1297 mblk_len = MBLKL(nmp); 1298 if (mblk_len >= 1299 (sizeof (hdrs_buf) - hdrs_size)) 1300 mblk_len = sizeof (hdrs_buf) - 1301 hdrs_size; 1302 bcopy(nmp->b_rptr, 1303 &hdrs_buf[hdrs_size], mblk_len); 1304 hdrs_size += mblk_len; 1305 nmp = nmp->b_cont; 1306 } 1307 tcp_port = hdrs_buf; 1308 } 1309 tcp_port += sizeof (ether_header_t); 1310 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1311 switch (tcp_port[9]) { 1312 case IPPROTO_TCP: 1313 case IPPROTO_UDP: 1314 case IPPROTO_ESP: 1315 tcp_port += ((*tcp_port) & 0x0f) << 2; 1316 ring_index = 1317 ((tcp_port[0] ^ 1318 tcp_port[1] ^ 1319 tcp_port[2] ^ 1320 tcp_port[3]) % maxtdcs); 1321 break; 1322 1323 case IPPROTO_AH: 1324 /* SPI starts at the 4th byte */ 1325 tcp_port += ((*tcp_port) & 0x0f) << 2; 1326 ring_index = 1327 ((tcp_port[4] ^ 1328 tcp_port[5] ^ 1329 tcp_port[6] ^ 1330 tcp_port[7]) % maxtdcs); 1331 break; 1332 1333 default: 1334 ring_index = tcp_port[19] % maxtdcs; 1335 break; 1336 } 1337 } else { /* fragmented packet */ 1338 ring_index = tcp_port[19] % maxtdcs; 1339 } 1340 } else { 1341 ring_index = mp->b_band % maxtdcs; 1342 } 1343 break; 1344 1345 case NXGE_TX_LB_HASH: 1346 if (hp->hash) { 1347 #if defined(__i386) 1348 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1349 #else 1350 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1351 #endif 1352 } else { 1353 ring_index = mp->b_band % maxtdcs; 1354 } 1355 break; 1356 1357 case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 1358 tcp_port = mp->b_rptr; 1359 ring_index = tcp_port[5] % maxtdcs; 1360 break; 1361 } 1362 1363 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 1364 1365 return (ring_index); 1366 } 1367 1368 uint_t 1369 nxge_reschedule(caddr_t arg) 1370 { 1371 p_nxge_t nxgep; 1372 1373 nxgep = (p_nxge_t)arg; 1374 1375 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 1376 1377 if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1378 nxgep->resched_needed) { 1379 if (!isLDOMguest(nxgep)) 1380 mac_tx_update(nxgep->mach); 1381 #if defined(sun4v) 1382 else { /* isLDOMguest(nxgep) */ 1383 nxge_hio_data_t *nhd = (nxge_hio_data_t *) 1384 nxgep->nxge_hw_p->hio; 1385 nx_vio_fp_t *vio = &nhd->hio.vio; 1386 1387 /* Call back vnet. */ 1388 if (vio->cb.vio_net_tx_update) { 1389 (*vio->cb.vio_net_tx_update) 1390 (nxgep->hio_vr->vhp); 1391 } 1392 } 1393 #endif 1394 nxgep->resched_needed = B_FALSE; 1395 nxgep->resched_running = B_FALSE; 1396 } 1397 1398 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 1399 return (DDI_INTR_CLAIMED); 1400 } 1401 1402 1403 /* Software LSO starts here */ 1404 static void 1405 nxge_hcksum_retrieve(mblk_t *mp, 1406 uint32_t *start, uint32_t *stuff, uint32_t *end, 1407 uint32_t *value, uint32_t *flags) 1408 { 1409 if (mp->b_datap->db_type == M_DATA) { 1410 if (flags != NULL) { 1411 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1412 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1413 HCK_FULLCKSUM_OK); 1414 if ((*flags & (HCK_PARTIALCKSUM | 1415 HCK_FULLCKSUM)) != 0) { 1416 if (value != NULL) 1417 *value = (uint32_t)DB_CKSUM16(mp); 1418 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1419 if (start != NULL) 1420 *start = 1421 (uint32_t)DB_CKSUMSTART(mp); 1422 if (stuff != NULL) 1423 *stuff = 1424 (uint32_t)DB_CKSUMSTUFF(mp); 1425 if (end != NULL) 1426 *end = 1427 (uint32_t)DB_CKSUMEND(mp); 1428 } 1429 } 1430 } 1431 } 1432 } 1433 1434 static void 1435 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1436 { 1437 ASSERT(DB_TYPE(mp) == M_DATA); 1438 1439 *mss = 0; 1440 if (flags != NULL) { 1441 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1442 if ((*flags != 0) && (mss != NULL)) { 1443 *mss = (uint32_t)DB_LSOMSS(mp); 1444 } 1445 NXGE_DEBUG_MSG((NULL, TX_CTL, 1446 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1447 *mss, *flags)); 1448 } 1449 1450 NXGE_DEBUG_MSG((NULL, TX_CTL, 1451 "<== nxge_lso_info_get: mss %d", *mss)); 1452 } 1453 1454 /* 1455 * Do Soft LSO on the oversized packet. 1456 * 1457 * 1. Create a chain of message for headers. 1458 * 2. Fill up header messages with proper information. 1459 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1460 * each new message with necessary adjustments. 1461 * * Unchange the ethernet header for DIX frames. (by default) 1462 * * IP Total Length field is updated to MSS or less(only for the last one). 1463 * * IP Identification value is incremented by one for each packet. 1464 * * TCP sequence Number is recalculated according to the payload length. 1465 * * Set FIN and/or PSH flags for the *last* packet if applied. 1466 * * TCP partial Checksum 1467 * 4. Update LSO information in the first message header. 1468 * 5. Release the original message header. 1469 */ 1470 static mblk_t * 1471 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1472 { 1473 uint32_t hckflags; 1474 int pktlen; 1475 int hdrlen; 1476 int segnum; 1477 int i; 1478 struct ether_vlan_header *evh; 1479 int ehlen, iphlen, tcphlen; 1480 struct ip *oiph, *niph; 1481 struct tcphdr *otcph, *ntcph; 1482 int available, len, left; 1483 uint16_t ip_id; 1484 uint32_t tcp_seq; 1485 #ifdef __sparc 1486 uint32_t tcp_seq_tmp; 1487 #endif 1488 mblk_t *datamp; 1489 uchar_t *rptr; 1490 mblk_t *nmp; 1491 mblk_t *cmp; 1492 mblk_t *mp_chain; 1493 boolean_t do_cleanup = B_FALSE; 1494 t_uscalar_t start_offset = 0; 1495 t_uscalar_t stuff_offset = 0; 1496 t_uscalar_t value = 0; 1497 uint16_t l4_len; 1498 ipaddr_t src, dst; 1499 uint32_t cksum, sum, l4cksum; 1500 1501 NXGE_DEBUG_MSG((NULL, TX_CTL, 1502 "==> nxge_do_softlso")); 1503 /* 1504 * check the length of LSO packet payload and calculate the number of 1505 * segments to be generated. 1506 */ 1507 pktlen = msgsize(mp); 1508 evh = (struct ether_vlan_header *)mp->b_rptr; 1509 1510 /* VLAN? */ 1511 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1512 ehlen = sizeof (struct ether_vlan_header); 1513 else 1514 ehlen = sizeof (struct ether_header); 1515 oiph = (struct ip *)(mp->b_rptr + ehlen); 1516 iphlen = oiph->ip_hl * 4; 1517 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1518 tcphlen = otcph->th_off * 4; 1519 1520 l4_len = pktlen - ehlen - iphlen; 1521 1522 NXGE_DEBUG_MSG((NULL, TX_CTL, 1523 "==> nxge_do_softlso: mss %d oiph $%p " 1524 "original ip_sum oiph->ip_sum 0x%x " 1525 "original tcp_sum otcph->th_sum 0x%x " 1526 "oiph->ip_len %d pktlen %d ehlen %d " 1527 "l4_len %d (0x%x) ip_len - iphlen %d ", 1528 mss, 1529 oiph, 1530 oiph->ip_sum, 1531 otcph->th_sum, 1532 ntohs(oiph->ip_len), pktlen, 1533 ehlen, 1534 l4_len, 1535 l4_len, 1536 ntohs(oiph->ip_len) - iphlen)); 1537 1538 /* IPv4 + TCP */ 1539 if (!(oiph->ip_v == IPV4_VERSION)) { 1540 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1541 "<== nxge_do_softlso: not IPV4 " 1542 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1543 ntohs(oiph->ip_len), pktlen, ehlen, 1544 tcphlen)); 1545 freemsg(mp); 1546 return (NULL); 1547 } 1548 1549 if (!(oiph->ip_p == IPPROTO_TCP)) { 1550 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1551 "<== nxge_do_softlso: not TCP " 1552 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1553 ntohs(oiph->ip_len), pktlen, ehlen, 1554 tcphlen)); 1555 freemsg(mp); 1556 return (NULL); 1557 } 1558 1559 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1560 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1561 "<== nxge_do_softlso: len not matched " 1562 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1563 ntohs(oiph->ip_len), pktlen, ehlen, 1564 tcphlen)); 1565 freemsg(mp); 1566 return (NULL); 1567 } 1568 1569 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1570 tcphlen = otcph->th_off * 4; 1571 1572 /* TCP flags can not include URG, RST, or SYN */ 1573 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1574 1575 hdrlen = ehlen + iphlen + tcphlen; 1576 1577 VERIFY(MBLKL(mp) >= hdrlen); 1578 1579 if (MBLKL(mp) > hdrlen) { 1580 datamp = mp; 1581 rptr = mp->b_rptr + hdrlen; 1582 } else { /* = */ 1583 datamp = mp->b_cont; 1584 rptr = datamp->b_rptr; 1585 } 1586 1587 NXGE_DEBUG_MSG((NULL, TX_CTL, 1588 "nxge_do_softlso: otcph $%p pktlen: %d, " 1589 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1590 "mblkl(mp): %d, mblkl(datamp): %d", 1591 otcph, 1592 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1593 (int)MBLKL(mp), (int)MBLKL(datamp))); 1594 1595 hckflags = 0; 1596 nxge_hcksum_retrieve(mp, 1597 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1598 1599 dst = oiph->ip_dst.s_addr; 1600 src = oiph->ip_src.s_addr; 1601 1602 cksum = (dst >> 16) + (dst & 0xFFFF) + 1603 (src >> 16) + (src & 0xFFFF); 1604 l4cksum = cksum + IP_TCP_CSUM_COMP; 1605 1606 sum = l4_len + l4cksum; 1607 sum = (sum & 0xFFFF) + (sum >> 16); 1608 1609 NXGE_DEBUG_MSG((NULL, TX_CTL, 1610 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1611 "hckflags 0x%x start_offset %d stuff_offset %d " 1612 "value (original) 0x%x th_sum 0x%x " 1613 "pktlen %d l4_len %d (0x%x) " 1614 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1615 dst, src, 1616 (sum & 0xffff), (~sum & 0xffff), 1617 hckflags, start_offset, stuff_offset, 1618 value, otcph->th_sum, 1619 pktlen, 1620 l4_len, 1621 l4_len, 1622 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1623 (int)MBLKL(datamp), 1624 nxge_dump_packet((char *)evh, 12))); 1625 1626 /* 1627 * Start to process. 1628 */ 1629 available = pktlen - hdrlen; 1630 segnum = (available - 1) / mss + 1; 1631 1632 NXGE_DEBUG_MSG((NULL, TX_CTL, 1633 "==> nxge_do_softlso: pktlen %d " 1634 "MBLKL(mp): %d, MBLKL(datamp): %d " 1635 "available %d mss %d segnum %d", 1636 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1637 available, 1638 mss, 1639 segnum)); 1640 1641 VERIFY(segnum >= 2); 1642 1643 /* 1644 * Try to pre-allocate all header messages 1645 */ 1646 mp_chain = NULL; 1647 for (i = 0; i < segnum; i++) { 1648 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1649 /* Clean up the mp_chain */ 1650 while (mp_chain != NULL) { 1651 nmp = mp_chain; 1652 mp_chain = mp_chain->b_next; 1653 freemsg(nmp); 1654 } 1655 NXGE_DEBUG_MSG((NULL, TX_CTL, 1656 "<== nxge_do_softlso: " 1657 "Could not allocate enough messages for headers!")); 1658 freemsg(mp); 1659 return (NULL); 1660 } 1661 nmp->b_next = mp_chain; 1662 mp_chain = nmp; 1663 1664 NXGE_DEBUG_MSG((NULL, TX_CTL, 1665 "==> nxge_do_softlso: " 1666 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1667 mp, nmp, mp_chain, mp_chain->b_next)); 1668 } 1669 1670 NXGE_DEBUG_MSG((NULL, TX_CTL, 1671 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1672 mp, nmp, mp_chain)); 1673 1674 /* 1675 * Associate payload with new packets 1676 */ 1677 cmp = mp_chain; 1678 left = available; 1679 while (cmp != NULL) { 1680 nmp = dupb(datamp); 1681 if (nmp == NULL) { 1682 do_cleanup = B_TRUE; 1683 NXGE_DEBUG_MSG((NULL, TX_CTL, 1684 "==>nxge_do_softlso: " 1685 "Can not dupb(datamp), have to do clean up")); 1686 goto cleanup_allocated_msgs; 1687 } 1688 1689 NXGE_DEBUG_MSG((NULL, TX_CTL, 1690 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1691 "dupb nmp $%p len %d left %d msd %d ", 1692 mp, cmp, nmp, len, left, mss)); 1693 1694 cmp->b_cont = nmp; 1695 nmp->b_rptr = rptr; 1696 len = (left < mss) ? left : mss; 1697 left -= len; 1698 1699 NXGE_DEBUG_MSG((NULL, TX_CTL, 1700 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1701 "dupb nmp $%p len %d left %d mss %d ", 1702 mp, cmp, nmp, len, left, mss)); 1703 NXGE_DEBUG_MSG((NULL, TX_CTL, 1704 "nxge_do_softlso: before available: %d, " 1705 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1706 available, left, len, segnum, (int)MBLKL(nmp))); 1707 1708 len -= MBLKL(nmp); 1709 NXGE_DEBUG_MSG((NULL, TX_CTL, 1710 "nxge_do_softlso: after available: %d, " 1711 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1712 available, left, len, segnum, (int)MBLKL(nmp))); 1713 1714 while (len > 0) { 1715 mblk_t *mmp = NULL; 1716 1717 NXGE_DEBUG_MSG((NULL, TX_CTL, 1718 "nxge_do_softlso: (4) len > 0 available: %d, " 1719 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1720 available, left, len, segnum, (int)MBLKL(nmp))); 1721 1722 if (datamp->b_cont != NULL) { 1723 datamp = datamp->b_cont; 1724 rptr = datamp->b_rptr; 1725 mmp = dupb(datamp); 1726 if (mmp == NULL) { 1727 do_cleanup = B_TRUE; 1728 NXGE_DEBUG_MSG((NULL, TX_CTL, 1729 "==> nxge_do_softlso: " 1730 "Can not dupb(datamp) (1), :" 1731 "have to do clean up")); 1732 NXGE_DEBUG_MSG((NULL, TX_CTL, 1733 "==> nxge_do_softlso: " 1734 "available: %d, left: %d, " 1735 "len: %d, MBLKL(nmp): %d", 1736 available, left, len, 1737 (int)MBLKL(nmp))); 1738 goto cleanup_allocated_msgs; 1739 } 1740 } else { 1741 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1742 "==> nxge_do_softlso: " 1743 "(1)available: %d, left: %d, " 1744 "len: %d, MBLKL(nmp): %d", 1745 available, left, len, 1746 (int)MBLKL(nmp))); 1747 cmn_err(CE_PANIC, 1748 "==> nxge_do_softlso: " 1749 "Pointers must have been corrupted!\n" 1750 "datamp: $%p, nmp: $%p, rptr: $%p", 1751 (void *)datamp, 1752 (void *)nmp, 1753 (void *)rptr); 1754 } 1755 nmp->b_cont = mmp; 1756 nmp = mmp; 1757 len -= MBLKL(nmp); 1758 } 1759 if (len < 0) { 1760 nmp->b_wptr += len; 1761 rptr = nmp->b_wptr; 1762 NXGE_DEBUG_MSG((NULL, TX_CTL, 1763 "(5) len < 0 (less than 0)" 1764 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1765 available, left, len, (int)MBLKL(nmp))); 1766 1767 } else if (len == 0) { 1768 if (datamp->b_cont != NULL) { 1769 NXGE_DEBUG_MSG((NULL, TX_CTL, 1770 "(5) len == 0" 1771 "available: %d, left: %d, len: %d, " 1772 "MBLKL(nmp): %d", 1773 available, left, len, (int)MBLKL(nmp))); 1774 datamp = datamp->b_cont; 1775 rptr = datamp->b_rptr; 1776 } else { 1777 NXGE_DEBUG_MSG((NULL, TX_CTL, 1778 "(6)available b_cont == NULL : %d, " 1779 "left: %d, len: %d, MBLKL(nmp): %d", 1780 available, left, len, (int)MBLKL(nmp))); 1781 1782 VERIFY(cmp->b_next == NULL); 1783 VERIFY(left == 0); 1784 break; /* Done! */ 1785 } 1786 } 1787 cmp = cmp->b_next; 1788 1789 NXGE_DEBUG_MSG((NULL, TX_CTL, 1790 "(7) do_softlso: " 1791 "next mp in mp_chain available len != 0 : %d, " 1792 "left: %d, len: %d, MBLKL(nmp): %d", 1793 available, left, len, (int)MBLKL(nmp))); 1794 } 1795 1796 /* 1797 * From now, start to fill up all headers for the first message 1798 * Hardware checksum flags need to be updated separately for FULLCKSUM 1799 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1800 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1801 * required fields need to be updated properly. 1802 */ 1803 nmp = mp_chain; 1804 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1805 nmp->b_wptr = nmp->b_rptr + hdrlen; 1806 niph = (struct ip *)(nmp->b_rptr + ehlen); 1807 niph->ip_len = htons(mss + iphlen + tcphlen); 1808 ip_id = ntohs(niph->ip_id); 1809 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1810 #ifdef __sparc 1811 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1812 tcp_seq = ntohl(tcp_seq_tmp); 1813 #else 1814 tcp_seq = ntohl(ntcph->th_seq); 1815 #endif 1816 1817 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1818 1819 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1820 DB_CKSUMSTART(nmp) = start_offset; 1821 DB_CKSUMSTUFF(nmp) = stuff_offset; 1822 1823 /* calculate IP checksum and TCP pseudo header checksum */ 1824 niph->ip_sum = 0; 1825 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1826 1827 l4_len = mss + tcphlen; 1828 sum = htons(l4_len) + l4cksum; 1829 sum = (sum & 0xFFFF) + (sum >> 16); 1830 ntcph->th_sum = (sum & 0xffff); 1831 1832 NXGE_DEBUG_MSG((NULL, TX_CTL, 1833 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1834 "mss %d pktlen %d l4_len %d (0x%x) " 1835 "MBLKL(mp): %d, MBLKL(datamp): %d " 1836 "ip_sum 0x%x " 1837 "th_sum 0x%x sum 0x%x ) " 1838 "dump first ip->tcp %s", 1839 nmp, mp_chain, 1840 mss, 1841 pktlen, 1842 l4_len, 1843 l4_len, 1844 (int)MBLKL(mp), (int)MBLKL(datamp), 1845 niph->ip_sum, 1846 ntcph->th_sum, 1847 sum, 1848 nxge_dump_packet((char *)niph, 52))); 1849 1850 cmp = nmp; 1851 while ((nmp = nmp->b_next)->b_next != NULL) { 1852 NXGE_DEBUG_MSG((NULL, TX_CTL, 1853 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1854 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1855 nmp->b_wptr = nmp->b_rptr + hdrlen; 1856 niph = (struct ip *)(nmp->b_rptr + ehlen); 1857 niph->ip_id = htons(++ip_id); 1858 niph->ip_len = htons(mss + iphlen + tcphlen); 1859 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1860 tcp_seq += mss; 1861 1862 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1863 1864 #ifdef __sparc 1865 tcp_seq_tmp = htonl(tcp_seq); 1866 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1867 #else 1868 ntcph->th_seq = htonl(tcp_seq); 1869 #endif 1870 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1871 DB_CKSUMSTART(nmp) = start_offset; 1872 DB_CKSUMSTUFF(nmp) = stuff_offset; 1873 1874 /* calculate IP checksum and TCP pseudo header checksum */ 1875 niph->ip_sum = 0; 1876 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1877 ntcph->th_sum = (sum & 0xffff); 1878 1879 NXGE_DEBUG_MSG((NULL, TX_CTL, 1880 "==> nxge_do_softlso: middle ip_sum 0x%x " 1881 "th_sum 0x%x " 1882 " mp $%p (mp_chain $%p) pktlen %d " 1883 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1884 niph->ip_sum, 1885 ntcph->th_sum, 1886 nmp, mp_chain, 1887 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1888 } 1889 1890 /* Last segment */ 1891 /* 1892 * Set FIN and/or PSH flags if present only in the last packet. 1893 * The ip_len could be different from prior packets. 1894 */ 1895 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1896 nmp->b_wptr = nmp->b_rptr + hdrlen; 1897 niph = (struct ip *)(nmp->b_rptr + ehlen); 1898 niph->ip_id = htons(++ip_id); 1899 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1900 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1901 tcp_seq += mss; 1902 #ifdef __sparc 1903 tcp_seq_tmp = htonl(tcp_seq); 1904 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1905 #else 1906 ntcph->th_seq = htonl(tcp_seq); 1907 #endif 1908 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1909 1910 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1911 DB_CKSUMSTART(nmp) = start_offset; 1912 DB_CKSUMSTUFF(nmp) = stuff_offset; 1913 1914 /* calculate IP checksum and TCP pseudo header checksum */ 1915 niph->ip_sum = 0; 1916 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1917 1918 l4_len = ntohs(niph->ip_len) - iphlen; 1919 sum = htons(l4_len) + l4cksum; 1920 sum = (sum & 0xFFFF) + (sum >> 16); 1921 ntcph->th_sum = (sum & 0xffff); 1922 1923 NXGE_DEBUG_MSG((NULL, TX_CTL, 1924 "==> nxge_do_softlso: last next " 1925 "niph->ip_sum 0x%x " 1926 "ntcph->th_sum 0x%x sum 0x%x " 1927 "dump last ip->tcp %s " 1928 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1929 "l4_len %d (0x%x) " 1930 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1931 niph->ip_sum, 1932 ntcph->th_sum, sum, 1933 nxge_dump_packet((char *)niph, 52), 1934 cmp, nmp, mp_chain, 1935 pktlen, pktlen, 1936 l4_len, 1937 l4_len, 1938 (int)MBLKL(mp), (int)MBLKL(datamp))); 1939 1940 cleanup_allocated_msgs: 1941 if (do_cleanup) { 1942 NXGE_DEBUG_MSG((NULL, TX_CTL, 1943 "==> nxge_do_softlso: " 1944 "Failed allocating messages, " 1945 "have to clean up and fail!")); 1946 while (mp_chain != NULL) { 1947 nmp = mp_chain; 1948 mp_chain = mp_chain->b_next; 1949 freemsg(nmp); 1950 } 1951 } 1952 /* 1953 * We're done here, so just free the original message and return the 1954 * new message chain, that could be NULL if failed, back to the caller. 1955 */ 1956 freemsg(mp); 1957 1958 NXGE_DEBUG_MSG((NULL, TX_CTL, 1959 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1960 return (mp_chain); 1961 } 1962 1963 /* 1964 * Will be called before NIC driver do further operation on the message. 1965 * The input message may include LSO information, if so, go to softlso logic 1966 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1967 * The return could be the same non-LSO message or a message chain for LSO case. 1968 * 1969 * The driver needs to call this function per packet and process the whole chain 1970 * if applied. 1971 */ 1972 static mblk_t * 1973 nxge_lso_eliminate(mblk_t *mp) 1974 { 1975 uint32_t lsoflags; 1976 uint32_t mss; 1977 1978 NXGE_DEBUG_MSG((NULL, TX_CTL, 1979 "==>nxge_lso_eliminate:")); 1980 nxge_lso_info_get(mp, &mss, &lsoflags); 1981 1982 if (lsoflags & HW_LSO) { 1983 mblk_t *nmp; 1984 1985 NXGE_DEBUG_MSG((NULL, TX_CTL, 1986 "==>nxge_lso_eliminate:" 1987 "HW_LSO:mss %d mp $%p", 1988 mss, mp)); 1989 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1990 NXGE_DEBUG_MSG((NULL, TX_CTL, 1991 "<== nxge_lso_eliminate: " 1992 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1993 nmp, mss, mp)); 1994 return (nmp); 1995 } else { 1996 NXGE_DEBUG_MSG((NULL, TX_CTL, 1997 "<== nxge_lso_eliminate_ " 1998 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1999 nmp, mss, mp)); 2000 return (NULL); 2001 } 2002 } 2003 2004 NXGE_DEBUG_MSG((NULL, TX_CTL, 2005 "<== nxge_lso_eliminate")); 2006 return (mp); 2007 } 2008 2009 static uint32_t 2010 nxge_csgen(uint16_t *adr, int len) 2011 { 2012 int i, odd; 2013 uint32_t sum = 0; 2014 uint32_t c = 0; 2015 2016 odd = len % 2; 2017 for (i = 0; i < (len / 2); i++) { 2018 sum += (adr[i] & 0xffff); 2019 } 2020 if (odd) { 2021 sum += adr[len / 2] & 0xff00; 2022 } 2023 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 2024 sum &= 0xffff; 2025 sum += c; 2026 } 2027 return (~sum & 0xffff); 2028 } 2029