1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/nxge/nxge_impl.h> 27 #include <sys/nxge/nxge_hio.h> 28 #include <npi_tx_wr64.h> 29 30 /* Software LSO required header files */ 31 #include <netinet/tcp.h> 32 #include <inet/ip_impl.h> 33 #include <inet/tcp.h> 34 35 static mblk_t *nxge_lso_eliminate(mblk_t *); 36 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 37 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 38 static void nxge_hcksum_retrieve(mblk_t *, 39 uint32_t *, uint32_t *, uint32_t *, 40 uint32_t *, uint32_t *); 41 static uint32_t nxge_csgen(uint16_t *, int); 42 43 extern uint32_t nxge_reclaim_pending; 44 extern uint32_t nxge_bcopy_thresh; 45 extern uint32_t nxge_dvma_thresh; 46 extern uint32_t nxge_dma_stream_thresh; 47 extern uint32_t nxge_tx_minfree; 48 extern uint32_t nxge_tx_intr_thres; 49 extern uint32_t nxge_tx_max_gathers; 50 extern uint32_t nxge_tx_tiny_pack; 51 extern uint32_t nxge_tx_use_bcopy; 52 extern uint32_t nxge_tx_lb_policy; 53 extern uint32_t nxge_no_tx_lb; 54 extern nxge_tx_mode_t nxge_tx_scheme; 55 uint32_t nxge_lso_kick_cnt = 2; 56 57 typedef struct _mac_tx_hint { 58 uint16_t sap; 59 uint16_t vid; 60 void *hash; 61 } mac_tx_hint_t, *p_mac_tx_hint_t; 62 63 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 64 65 int 66 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 67 { 68 int status = 0; 69 p_tx_desc_t tx_desc_ring_vp; 70 npi_handle_t npi_desc_handle; 71 nxge_os_dma_handle_t tx_desc_dma_handle; 72 p_tx_desc_t tx_desc_p; 73 p_tx_msg_t tx_msg_ring; 74 p_tx_msg_t tx_msg_p; 75 tx_desc_t tx_desc, *tmp_desc_p; 76 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 77 p_tx_pkt_header_t hdrp; 78 tx_pkt_header_t tmp_hdrp; 79 p_tx_pkt_hdr_all_t pkthdrp; 80 uint8_t npads = 0; 81 uint64_t dma_ioaddr; 82 uint32_t dma_flags; 83 int last_bidx; 84 uint8_t *b_rptr; 85 caddr_t kaddr; 86 uint32_t nmblks; 87 uint32_t ngathers; 88 uint32_t clen; 89 int len; 90 uint32_t pkt_len, pack_len, min_len; 91 uint32_t bcopy_thresh; 92 int i, cur_index, sop_index; 93 uint16_t tail_index; 94 boolean_t tail_wrap = B_FALSE; 95 nxge_dma_common_t desc_area; 96 nxge_os_dma_handle_t dma_handle; 97 ddi_dma_cookie_t dma_cookie; 98 npi_handle_t npi_handle; 99 p_mblk_t nmp; 100 p_mblk_t t_mp; 101 uint32_t ncookies; 102 boolean_t good_packet; 103 boolean_t mark_mode = B_FALSE; 104 p_nxge_stats_t statsp; 105 p_nxge_tx_ring_stats_t tdc_stats; 106 t_uscalar_t start_offset = 0; 107 t_uscalar_t stuff_offset = 0; 108 t_uscalar_t end_offset = 0; 109 t_uscalar_t value = 0; 110 t_uscalar_t cksum_flags = 0; 111 boolean_t cksum_on = B_FALSE; 112 uint32_t boff = 0; 113 uint64_t tot_xfer_len = 0; 114 boolean_t header_set = B_FALSE; 115 #ifdef NXGE_DEBUG 116 p_tx_desc_t tx_desc_ring_pp; 117 p_tx_desc_t tx_desc_pp; 118 tx_desc_t *save_desc_p; 119 int dump_len; 120 int sad_len; 121 uint64_t sad; 122 int xfer_len; 123 uint32_t msgsize; 124 #endif 125 p_mblk_t mp_chain = NULL; 126 boolean_t is_lso = B_FALSE; 127 boolean_t lso_again; 128 int cur_index_lso; 129 p_mblk_t nmp_lso_save; 130 uint32_t lso_ngathers; 131 boolean_t lso_tail_wrap = B_FALSE; 132 133 NXGE_DEBUG_MSG((nxgep, TX_CTL, 134 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 135 NXGE_DEBUG_MSG((nxgep, TX_CTL, 136 "==> nxge_start: Starting tdc %d desc pending %d", 137 tx_ring_p->tdc, tx_ring_p->descs_pending)); 138 139 statsp = nxgep->statsp; 140 141 if (!isLDOMguest(nxgep)) { 142 switch (nxgep->mac.portmode) { 143 default: 144 if (nxgep->statsp->port_stats.lb_mode == 145 nxge_lb_normal) { 146 if (!statsp->mac_stats.link_up) { 147 freemsg(mp); 148 NXGE_DEBUG_MSG((nxgep, TX_CTL, 149 "==> nxge_start: " 150 "link not up")); 151 goto nxge_start_fail1; 152 } 153 } 154 break; 155 case PORT_10G_FIBER: 156 /* 157 * For the following modes, check the link status 158 * before sending the packet out: 159 * nxge_lb_normal, nxge_lb_ext10g, nxge_lb_phy10g 160 */ 161 if (nxgep->statsp->port_stats.lb_mode < 162 nxge_lb_serdes10g) { 163 if (!statsp->mac_stats.link_up) { 164 freemsg(mp); 165 NXGE_DEBUG_MSG((nxgep, TX_CTL, 166 "==> nxge_start: " 167 "link not up")); 168 goto nxge_start_fail1; 169 } 170 } 171 break; 172 } 173 } 174 175 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 176 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 177 NXGE_DEBUG_MSG((nxgep, TX_CTL, 178 "==> nxge_start: hardware not initialized or stopped")); 179 freemsg(mp); 180 goto nxge_start_fail1; 181 } 182 183 if (nxgep->soft_lso_enable) { 184 mp_chain = nxge_lso_eliminate(mp); 185 NXGE_DEBUG_MSG((nxgep, TX_CTL, 186 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 187 mp, mp_chain)); 188 if (mp_chain == NULL) { 189 NXGE_ERROR_MSG((nxgep, TX_CTL, 190 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 191 mp_chain, mp)); 192 goto nxge_start_fail1; 193 } 194 if (mp_chain != mp) { 195 NXGE_DEBUG_MSG((nxgep, TX_CTL, 196 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 197 mp_chain, mp)); 198 is_lso = B_TRUE; 199 mp = mp_chain; 200 mp_chain = mp_chain->b_next; 201 mp->b_next = NULL; 202 } 203 } 204 205 hcksum_retrieve(mp, NULL, NULL, &start_offset, 206 &stuff_offset, &end_offset, &value, &cksum_flags); 207 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 208 start_offset += sizeof (ether_header_t); 209 stuff_offset += sizeof (ether_header_t); 210 } else { 211 start_offset += sizeof (struct ether_vlan_header); 212 stuff_offset += sizeof (struct ether_vlan_header); 213 } 214 215 if (cksum_flags & HCK_PARTIALCKSUM) { 216 NXGE_DEBUG_MSG((nxgep, TX_CTL, 217 "==> nxge_start: mp $%p len %d " 218 "cksum_flags 0x%x (partial checksum) ", 219 mp, MBLKL(mp), cksum_flags)); 220 cksum_on = B_TRUE; 221 } 222 223 pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 224 pkthdrp->reserved = 0; 225 tmp_hdrp.value = 0; 226 nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 227 0, 0, pkthdrp, 228 start_offset, stuff_offset); 229 230 lso_again = B_FALSE; 231 lso_ngathers = 0; 232 233 MUTEX_ENTER(&tx_ring_p->lock); 234 235 if (isLDOMservice(nxgep)) { 236 tx_ring_p->tx_ring_busy = B_TRUE; 237 if (tx_ring_p->tx_ring_offline) { 238 freemsg(mp); 239 tx_ring_p->tx_ring_busy = B_FALSE; 240 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 241 NXGE_TX_RING_OFFLINED); 242 MUTEX_EXIT(&tx_ring_p->lock); 243 return (status); 244 } 245 } 246 247 cur_index_lso = tx_ring_p->wr_index; 248 lso_tail_wrap = tx_ring_p->wr_index_wrap; 249 start_again: 250 ngathers = 0; 251 sop_index = tx_ring_p->wr_index; 252 #ifdef NXGE_DEBUG 253 if (tx_ring_p->descs_pending) { 254 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 255 "desc pending %d ", tx_ring_p->descs_pending)); 256 } 257 258 dump_len = (int)(MBLKL(mp)); 259 dump_len = (dump_len > 128) ? 128: dump_len; 260 261 NXGE_DEBUG_MSG((nxgep, TX_CTL, 262 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 263 "(Before header reserve: ORIGINAL LEN %d)", 264 tx_ring_p->tdc, 265 mp->b_rptr, 266 dump_len)); 267 268 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 269 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 270 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 271 #endif 272 273 tdc_stats = tx_ring_p->tdc_stats; 274 mark_mode = (tx_ring_p->descs_pending && 275 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 276 < nxge_tx_minfree)); 277 278 NXGE_DEBUG_MSG((nxgep, TX_CTL, 279 "TX Descriptor ring is channel %d mark mode %d", 280 tx_ring_p->tdc, mark_mode)); 281 282 if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) { 283 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, 284 (nxge_tx_minfree + lso_ngathers))) { 285 NXGE_DEBUG_MSG((nxgep, TX_CTL, 286 "TX Descriptor ring is full: channel %d", 287 tx_ring_p->tdc)); 288 NXGE_DEBUG_MSG((nxgep, TX_CTL, 289 "TX Descriptor ring is full: channel %d", 290 tx_ring_p->tdc)); 291 if (is_lso) { 292 /* 293 * free the current mp and mp_chain if not FULL. 294 */ 295 tdc_stats->tx_no_desc++; 296 NXGE_DEBUG_MSG((nxgep, TX_CTL, 297 "LSO packet: TX Descriptor ring is full: " 298 "channel %d", 299 tx_ring_p->tdc)); 300 goto nxge_start_fail_lso; 301 } else { 302 boolean_t skip_sched = B_FALSE; 303 304 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 305 tdc_stats->tx_no_desc++; 306 307 if (isLDOMservice(nxgep)) { 308 tx_ring_p->tx_ring_busy = B_FALSE; 309 if (tx_ring_p->tx_ring_offline) { 310 (void) atomic_swap_32( 311 &tx_ring_p->tx_ring_offline, 312 NXGE_TX_RING_OFFLINED); 313 skip_sched = B_TRUE; 314 } 315 } 316 317 MUTEX_EXIT(&tx_ring_p->lock); 318 if (nxgep->resched_needed && 319 !nxgep->resched_running && !skip_sched) { 320 nxgep->resched_running = B_TRUE; 321 ddi_trigger_softintr(nxgep->resched_id); 322 } 323 status = 1; 324 goto nxge_start_fail1; 325 } 326 } 327 } 328 329 nmp = mp; 330 i = sop_index = tx_ring_p->wr_index; 331 nmblks = 0; 332 ngathers = 0; 333 pkt_len = 0; 334 pack_len = 0; 335 clen = 0; 336 last_bidx = -1; 337 good_packet = B_TRUE; 338 339 desc_area = tx_ring_p->tdc_desc; 340 npi_handle = desc_area.npi_handle; 341 npi_desc_handle.regh = (nxge_os_acc_handle_t) 342 DMA_COMMON_ACC_HANDLE(desc_area); 343 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 344 tx_desc_dma_handle = (nxge_os_dma_handle_t) 345 DMA_COMMON_HANDLE(desc_area); 346 tx_msg_ring = tx_ring_p->tx_msg_ring; 347 348 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 349 sop_index, i)); 350 351 #ifdef NXGE_DEBUG 352 msgsize = msgdsize(nmp); 353 NXGE_DEBUG_MSG((nxgep, TX_CTL, 354 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 355 sop_index, i, msgsize)); 356 #endif 357 /* 358 * The first 16 bytes of the premapped buffer are reserved 359 * for header. No padding will be used. 360 */ 361 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 362 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 363 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 364 } else { 365 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 366 } 367 while (nmp) { 368 good_packet = B_TRUE; 369 b_rptr = nmp->b_rptr; 370 len = MBLKL(nmp); 371 if (len <= 0) { 372 nmp = nmp->b_cont; 373 continue; 374 } 375 nmblks++; 376 377 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 378 "len %d pkt_len %d pack_len %d", 379 nmblks, len, pkt_len, pack_len)); 380 /* 381 * Hardware limits the transfer length to 4K for NIU and 382 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 383 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 384 * If len is longer than the limit, then we break nmp into 385 * two chunks: Make the first chunk equal to the limit and 386 * the second chunk for the remaining data. If the second 387 * chunk is still larger than the limit, then it will be 388 * broken into two in the next pass. 389 */ 390 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 391 if ((t_mp = dupb(nmp)) != NULL) { 392 nmp->b_wptr = nmp->b_rptr + 393 (TX_MAX_TRANSFER_LENGTH 394 - TX_PKT_HEADER_SIZE); 395 t_mp->b_rptr = nmp->b_wptr; 396 t_mp->b_cont = nmp->b_cont; 397 nmp->b_cont = t_mp; 398 len = MBLKL(nmp); 399 } else { 400 if (is_lso) { 401 NXGE_DEBUG_MSG((nxgep, TX_CTL, 402 "LSO packet: dupb failed: " 403 "channel %d", 404 tx_ring_p->tdc)); 405 mp = nmp; 406 goto nxge_start_fail_lso; 407 } else { 408 good_packet = B_FALSE; 409 goto nxge_start_fail2; 410 } 411 } 412 } 413 tx_desc.value = 0; 414 tx_desc_p = &tx_desc_ring_vp[i]; 415 #ifdef NXGE_DEBUG 416 tx_desc_pp = &tx_desc_ring_pp[i]; 417 #endif 418 tx_msg_p = &tx_msg_ring[i]; 419 #if defined(__i386) 420 npi_desc_handle.regp = (uint32_t)tx_desc_p; 421 #else 422 npi_desc_handle.regp = (uint64_t)tx_desc_p; 423 #endif 424 if (!header_set && 425 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 426 (len >= bcopy_thresh))) { 427 header_set = B_TRUE; 428 bcopy_thresh += TX_PKT_HEADER_SIZE; 429 boff = 0; 430 pack_len = 0; 431 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 432 hdrp = (p_tx_pkt_header_t)kaddr; 433 clen = pkt_len; 434 dma_handle = tx_msg_p->buf_dma_handle; 435 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 436 (void) ddi_dma_sync(dma_handle, 437 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 438 DDI_DMA_SYNC_FORDEV); 439 440 tx_msg_p->flags.dma_type = USE_BCOPY; 441 goto nxge_start_control_header_only; 442 } 443 444 pkt_len += len; 445 pack_len += len; 446 447 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 448 "desc entry %d " 449 "DESC IOADDR $%p " 450 "desc_vp $%p tx_desc_p $%p " 451 "desc_pp $%p tx_desc_pp $%p " 452 "len %d pkt_len %d pack_len %d", 453 i, 454 DMA_COMMON_IOADDR(desc_area), 455 tx_desc_ring_vp, tx_desc_p, 456 tx_desc_ring_pp, tx_desc_pp, 457 len, pkt_len, pack_len)); 458 459 if (len < bcopy_thresh) { 460 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 461 "USE BCOPY: ")); 462 if (nxge_tx_tiny_pack) { 463 uint32_t blst = 464 TXDMA_DESC_NEXT_INDEX(i, -1, 465 tx_ring_p->tx_wrap_mask); 466 NXGE_DEBUG_MSG((nxgep, TX_CTL, 467 "==> nxge_start(5): pack")); 468 if ((pack_len <= bcopy_thresh) && 469 (last_bidx == blst)) { 470 NXGE_DEBUG_MSG((nxgep, TX_CTL, 471 "==> nxge_start: pack(6) " 472 "(pkt_len %d pack_len %d)", 473 pkt_len, pack_len)); 474 i = blst; 475 tx_desc_p = &tx_desc_ring_vp[i]; 476 #ifdef NXGE_DEBUG 477 tx_desc_pp = &tx_desc_ring_pp[i]; 478 #endif 479 tx_msg_p = &tx_msg_ring[i]; 480 boff = pack_len - len; 481 ngathers--; 482 } else if (pack_len > bcopy_thresh && 483 header_set) { 484 pack_len = len; 485 boff = 0; 486 bcopy_thresh = nxge_bcopy_thresh; 487 NXGE_DEBUG_MSG((nxgep, TX_CTL, 488 "==> nxge_start(7): > max NEW " 489 "bcopy thresh %d " 490 "pkt_len %d pack_len %d(next)", 491 bcopy_thresh, 492 pkt_len, pack_len)); 493 } 494 last_bidx = i; 495 } 496 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 497 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 498 hdrp = (p_tx_pkt_header_t)kaddr; 499 header_set = B_TRUE; 500 NXGE_DEBUG_MSG((nxgep, TX_CTL, 501 "==> nxge_start(7_x2): " 502 "pkt_len %d pack_len %d (new hdrp $%p)", 503 pkt_len, pack_len, hdrp)); 504 } 505 tx_msg_p->flags.dma_type = USE_BCOPY; 506 kaddr += boff; 507 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 508 "USE BCOPY: before bcopy " 509 "DESC IOADDR $%p entry %d " 510 "bcopy packets %d " 511 "bcopy kaddr $%p " 512 "bcopy ioaddr (SAD) $%p " 513 "bcopy clen %d " 514 "bcopy boff %d", 515 DMA_COMMON_IOADDR(desc_area), i, 516 tdc_stats->tx_hdr_pkts, 517 kaddr, 518 dma_ioaddr, 519 clen, 520 boff)); 521 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 522 "1USE BCOPY: ")); 523 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 524 "2USE BCOPY: ")); 525 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 526 "last USE BCOPY: copy from b_rptr $%p " 527 "to KADDR $%p (len %d offset %d", 528 b_rptr, kaddr, len, boff)); 529 530 bcopy(b_rptr, kaddr, len); 531 532 #ifdef NXGE_DEBUG 533 dump_len = (len > 128) ? 128: len; 534 NXGE_DEBUG_MSG((nxgep, TX_CTL, 535 "==> nxge_start: dump packets " 536 "(After BCOPY len %d)" 537 "(b_rptr $%p): %s", len, nmp->b_rptr, 538 nxge_dump_packet((char *)nmp->b_rptr, 539 dump_len))); 540 #endif 541 542 dma_handle = tx_msg_p->buf_dma_handle; 543 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 544 (void) ddi_dma_sync(dma_handle, 545 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 546 DDI_DMA_SYNC_FORDEV); 547 clen = len + boff; 548 tdc_stats->tx_hdr_pkts++; 549 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 550 "USE BCOPY: " 551 "DESC IOADDR $%p entry %d " 552 "bcopy packets %d " 553 "bcopy kaddr $%p " 554 "bcopy ioaddr (SAD) $%p " 555 "bcopy clen %d " 556 "bcopy boff %d", 557 DMA_COMMON_IOADDR(desc_area), 558 i, 559 tdc_stats->tx_hdr_pkts, 560 kaddr, 561 dma_ioaddr, 562 clen, 563 boff)); 564 } else { 565 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 566 "USE DVMA: len %d", len)); 567 tx_msg_p->flags.dma_type = USE_DMA; 568 dma_flags = DDI_DMA_WRITE; 569 if (len < nxge_dma_stream_thresh) { 570 dma_flags |= DDI_DMA_CONSISTENT; 571 } else { 572 dma_flags |= DDI_DMA_STREAMING; 573 } 574 575 dma_handle = tx_msg_p->dma_handle; 576 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 577 (caddr_t)b_rptr, len, dma_flags, 578 DDI_DMA_DONTWAIT, NULL, 579 &dma_cookie, &ncookies); 580 if (status == DDI_DMA_MAPPED) { 581 dma_ioaddr = dma_cookie.dmac_laddress; 582 len = (int)dma_cookie.dmac_size; 583 clen = (uint32_t)dma_cookie.dmac_size; 584 NXGE_DEBUG_MSG((nxgep, TX_CTL, 585 "==> nxge_start(12_1): " 586 "USE DVMA: len %d clen %d " 587 "ngathers %d", 588 len, clen, 589 ngathers)); 590 #if defined(__i386) 591 npi_desc_handle.regp = (uint32_t)tx_desc_p; 592 #else 593 npi_desc_handle.regp = (uint64_t)tx_desc_p; 594 #endif 595 while (ncookies > 1) { 596 ngathers++; 597 /* 598 * this is the fix for multiple 599 * cookies, which are basically 600 * a descriptor entry, we don't set 601 * SOP bit as well as related fields 602 */ 603 604 (void) npi_txdma_desc_gather_set( 605 npi_desc_handle, 606 &tx_desc, 607 (ngathers -1), 608 mark_mode, 609 ngathers, 610 dma_ioaddr, 611 clen); 612 613 tx_msg_p->tx_msg_size = clen; 614 NXGE_DEBUG_MSG((nxgep, TX_CTL, 615 "==> nxge_start: DMA " 616 "ncookie %d " 617 "ngathers %d " 618 "dma_ioaddr $%p len %d" 619 "desc $%p descp $%p (%d)", 620 ncookies, 621 ngathers, 622 dma_ioaddr, clen, 623 *tx_desc_p, tx_desc_p, i)); 624 625 ddi_dma_nextcookie(dma_handle, 626 &dma_cookie); 627 dma_ioaddr = 628 dma_cookie.dmac_laddress; 629 630 len = (int)dma_cookie.dmac_size; 631 clen = (uint32_t)dma_cookie.dmac_size; 632 NXGE_DEBUG_MSG((nxgep, TX_CTL, 633 "==> nxge_start(12_2): " 634 "USE DVMA: len %d clen %d ", 635 len, clen)); 636 637 i = TXDMA_DESC_NEXT_INDEX(i, 1, 638 tx_ring_p->tx_wrap_mask); 639 tx_desc_p = &tx_desc_ring_vp[i]; 640 641 #if defined(__i386) 642 npi_desc_handle.regp = 643 (uint32_t)tx_desc_p; 644 #else 645 npi_desc_handle.regp = 646 (uint64_t)tx_desc_p; 647 #endif 648 tx_msg_p = &tx_msg_ring[i]; 649 tx_msg_p->flags.dma_type = USE_NONE; 650 tx_desc.value = 0; 651 652 ncookies--; 653 } 654 tdc_stats->tx_ddi_pkts++; 655 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 656 "DMA: ddi packets %d", 657 tdc_stats->tx_ddi_pkts)); 658 } else { 659 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 660 "dma mapping failed for %d " 661 "bytes addr $%p flags %x (%d)", 662 len, b_rptr, status, status)); 663 good_packet = B_FALSE; 664 tdc_stats->tx_dma_bind_fail++; 665 tx_msg_p->flags.dma_type = USE_NONE; 666 if (is_lso) { 667 mp = nmp; 668 goto nxge_start_fail_lso; 669 } else { 670 goto nxge_start_fail2; 671 } 672 } 673 } /* ddi dvma */ 674 675 if (is_lso) { 676 nmp_lso_save = nmp; 677 } 678 nmp = nmp->b_cont; 679 nxge_start_control_header_only: 680 #if defined(__i386) 681 npi_desc_handle.regp = (uint32_t)tx_desc_p; 682 #else 683 npi_desc_handle.regp = (uint64_t)tx_desc_p; 684 #endif 685 ngathers++; 686 687 if (ngathers == 1) { 688 #ifdef NXGE_DEBUG 689 save_desc_p = &sop_tx_desc; 690 #endif 691 sop_tx_desc_p = &sop_tx_desc; 692 sop_tx_desc_p->value = 0; 693 sop_tx_desc_p->bits.hdw.tr_len = clen; 694 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 695 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 696 } else { 697 #ifdef NXGE_DEBUG 698 save_desc_p = &tx_desc; 699 #endif 700 tmp_desc_p = &tx_desc; 701 tmp_desc_p->value = 0; 702 tmp_desc_p->bits.hdw.tr_len = clen; 703 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 704 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 705 706 tx_desc_p->value = tmp_desc_p->value; 707 } 708 709 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 710 "Desc_entry %d ngathers %d " 711 "desc_vp $%p tx_desc_p $%p " 712 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 713 "dma_ioaddr (SAD) $%p mark %d", 714 i, ngathers, 715 tx_desc_ring_vp, tx_desc_p, 716 len, clen, pkt_len, pack_len, nmblks, 717 dma_ioaddr, mark_mode)); 718 719 #ifdef NXGE_DEBUG 720 npi_desc_handle.nxgep = nxgep; 721 npi_desc_handle.function.function = nxgep->function_num; 722 npi_desc_handle.function.instance = nxgep->instance; 723 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 724 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 725 TX_PKT_DESC_TR_LEN_SHIFT); 726 727 728 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 729 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 730 "mark %d sop %d\n", 731 save_desc_p->value, 732 sad, 733 save_desc_p->bits.hdw.tr_len, 734 xfer_len, 735 save_desc_p->bits.hdw.num_ptr, 736 save_desc_p->bits.hdw.mark, 737 save_desc_p->bits.hdw.sop)); 738 739 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 740 #endif 741 742 tx_msg_p->tx_msg_size = clen; 743 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 744 if (ngathers > nxge_tx_max_gathers) { 745 good_packet = B_FALSE; 746 hcksum_retrieve(mp, NULL, NULL, &start_offset, 747 &stuff_offset, &end_offset, &value, 748 &cksum_flags); 749 750 NXGE_DEBUG_MSG((NULL, TX_CTL, 751 "==> nxge_start(14): pull msg - " 752 "len %d pkt_len %d ngathers %d", 753 len, pkt_len, ngathers)); 754 /* Pull all message blocks from b_cont */ 755 if (is_lso) { 756 mp = nmp_lso_save; 757 goto nxge_start_fail_lso; 758 } 759 if ((msgpullup(mp, -1)) == NULL) { 760 goto nxge_start_fail2; 761 } 762 goto nxge_start_fail2; 763 } 764 } /* while (nmp) */ 765 766 tx_msg_p->tx_message = mp; 767 tx_desc_p = &tx_desc_ring_vp[sop_index]; 768 #if defined(__i386) 769 npi_desc_handle.regp = (uint32_t)tx_desc_p; 770 #else 771 npi_desc_handle.regp = (uint64_t)tx_desc_p; 772 #endif 773 774 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 775 pkthdrp->reserved = 0; 776 hdrp->value = 0; 777 bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 778 779 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 780 tdc_stats->tx_jumbo_pkts++; 781 } 782 783 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 784 if (pkt_len < min_len) { 785 /* Assume we use bcopy to premapped buffers */ 786 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 787 NXGE_DEBUG_MSG((NULL, TX_CTL, 788 "==> nxge_start(14-1): < (msg_min + 16)" 789 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 790 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 791 bzero((kaddr + pkt_len), (min_len - pkt_len)); 792 pkt_len = tx_msg_p->tx_msg_size = min_len; 793 794 sop_tx_desc_p->bits.hdw.tr_len = min_len; 795 796 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 797 tx_desc_p->value = sop_tx_desc_p->value; 798 799 NXGE_DEBUG_MSG((NULL, TX_CTL, 800 "==> nxge_start(14-2): < msg_min - " 801 "len %d pkt_len %d min_len %d ngathers %d", 802 len, pkt_len, min_len, ngathers)); 803 } 804 805 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 806 cksum_flags)); 807 { 808 uint64_t tmp_len; 809 810 /* pkt_len already includes 16 + paddings!! */ 811 /* Update the control header length */ 812 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 813 tmp_len = hdrp->value | 814 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 815 816 NXGE_DEBUG_MSG((nxgep, TX_CTL, 817 "==> nxge_start(15_x1): setting SOP " 818 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 819 "0x%llx hdrp->value 0x%llx", 820 tot_xfer_len, tot_xfer_len, pkt_len, 821 tmp_len, hdrp->value)); 822 #if defined(_BIG_ENDIAN) 823 hdrp->value = ddi_swap64(tmp_len); 824 #else 825 hdrp->value = tmp_len; 826 #endif 827 NXGE_DEBUG_MSG((nxgep, 828 TX_CTL, "==> nxge_start(15_x2): setting SOP " 829 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 830 "tmp_len 0x%llx hdrp->value 0x%llx", 831 tot_xfer_len, pkt_len, 832 tmp_len, hdrp->value)); 833 } 834 835 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 836 "wr_index %d " 837 "tot_xfer_len (%d) pkt_len %d npads %d", 838 sop_index, 839 tot_xfer_len, pkt_len, 840 npads)); 841 842 sop_tx_desc_p->bits.hdw.sop = 1; 843 sop_tx_desc_p->bits.hdw.mark = mark_mode; 844 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 845 846 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 847 848 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 849 850 #ifdef NXGE_DEBUG 851 npi_desc_handle.nxgep = nxgep; 852 npi_desc_handle.function.function = nxgep->function_num; 853 npi_desc_handle.function.instance = nxgep->instance; 854 855 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 856 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 857 save_desc_p->value, 858 sad, 859 save_desc_p->bits.hdw.tr_len, 860 xfer_len, 861 save_desc_p->bits.hdw.num_ptr, 862 save_desc_p->bits.hdw.mark, 863 save_desc_p->bits.hdw.sop)); 864 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 865 866 dump_len = (pkt_len > 128) ? 128: pkt_len; 867 NXGE_DEBUG_MSG((nxgep, TX_CTL, 868 "==> nxge_start: dump packets(17) (after sop set, len " 869 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 870 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 871 (char *)hdrp, 872 nxge_dump_packet((char *)hdrp, dump_len))); 873 NXGE_DEBUG_MSG((nxgep, TX_CTL, 874 "==> nxge_start(18): TX desc sync: sop_index %d", 875 sop_index)); 876 #endif 877 878 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 879 (void) ddi_dma_sync(tx_desc_dma_handle, 880 sop_index * sizeof (tx_desc_t), 881 ngathers * sizeof (tx_desc_t), 882 DDI_DMA_SYNC_FORDEV); 883 884 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 885 "cs_off = 0x%02X cs_s_off = 0x%02X " 886 "pkt_len %d ngathers %d sop_index %d\n", 887 stuff_offset, start_offset, 888 pkt_len, ngathers, sop_index)); 889 } else { /* more than one descriptor and wrap around */ 890 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 891 (void) ddi_dma_sync(tx_desc_dma_handle, 892 sop_index * sizeof (tx_desc_t), 893 nsdescs * sizeof (tx_desc_t), 894 DDI_DMA_SYNC_FORDEV); 895 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 896 "cs_off = 0x%02X cs_s_off = 0x%02X " 897 "pkt_len %d ngathers %d sop_index %d\n", 898 stuff_offset, start_offset, 899 pkt_len, ngathers, sop_index)); 900 901 (void) ddi_dma_sync(tx_desc_dma_handle, 902 0, 903 (ngathers - nsdescs) * sizeof (tx_desc_t), 904 DDI_DMA_SYNC_FORDEV); 905 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 906 "cs_off = 0x%02X cs_s_off = 0x%02X " 907 "pkt_len %d ngathers %d sop_index %d\n", 908 stuff_offset, start_offset, 909 pkt_len, ngathers, sop_index)); 910 } 911 912 tail_index = tx_ring_p->wr_index; 913 tail_wrap = tx_ring_p->wr_index_wrap; 914 915 tx_ring_p->wr_index = i; 916 if (tx_ring_p->wr_index <= tail_index) { 917 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 918 B_FALSE : B_TRUE); 919 } 920 921 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 922 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 923 tx_ring_p->tdc, 924 tx_ring_p->wr_index, 925 tx_ring_p->wr_index_wrap, 926 ngathers, 927 tx_ring_p->descs_pending)); 928 929 if (is_lso) { 930 lso_ngathers += ngathers; 931 if (mp_chain != NULL) { 932 mp = mp_chain; 933 mp_chain = mp_chain->b_next; 934 mp->b_next = NULL; 935 if (nxge_lso_kick_cnt == lso_ngathers) { 936 tx_ring_p->descs_pending += lso_ngathers; 937 { 938 tx_ring_kick_t kick; 939 940 kick.value = 0; 941 kick.bits.ldw.wrap = 942 tx_ring_p->wr_index_wrap; 943 kick.bits.ldw.tail = 944 (uint16_t)tx_ring_p->wr_index; 945 946 /* Kick the Transmit kick register */ 947 TXDMA_REG_WRITE64( 948 NXGE_DEV_NPI_HANDLE(nxgep), 949 TX_RING_KICK_REG, 950 (uint8_t)tx_ring_p->tdc, 951 kick.value); 952 tdc_stats->tx_starts++; 953 954 NXGE_DEBUG_MSG((nxgep, TX_CTL, 955 "==> nxge_start: more LSO: " 956 "LSO_CNT %d", 957 lso_ngathers)); 958 } 959 lso_ngathers = 0; 960 ngathers = 0; 961 cur_index_lso = sop_index = tx_ring_p->wr_index; 962 lso_tail_wrap = tx_ring_p->wr_index_wrap; 963 } 964 NXGE_DEBUG_MSG((nxgep, TX_CTL, 965 "==> nxge_start: lso again: " 966 "lso_gathers %d ngathers %d cur_index_lso %d " 967 "wr_index %d sop_index %d", 968 lso_ngathers, ngathers, cur_index_lso, 969 tx_ring_p->wr_index, sop_index)); 970 971 NXGE_DEBUG_MSG((nxgep, TX_CTL, 972 "==> nxge_start: next : count %d", 973 lso_ngathers)); 974 lso_again = B_TRUE; 975 goto start_again; 976 } 977 ngathers = lso_ngathers; 978 } 979 980 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 981 982 { 983 tx_ring_kick_t kick; 984 985 kick.value = 0; 986 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 987 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 988 989 /* Kick start the Transmit kick register */ 990 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 991 TX_RING_KICK_REG, 992 (uint8_t)tx_ring_p->tdc, 993 kick.value); 994 } 995 996 tx_ring_p->descs_pending += ngathers; 997 tdc_stats->tx_starts++; 998 999 if (isLDOMservice(nxgep)) { 1000 tx_ring_p->tx_ring_busy = B_FALSE; 1001 if (tx_ring_p->tx_ring_offline) { 1002 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1003 NXGE_TX_RING_OFFLINED); 1004 } 1005 } 1006 1007 MUTEX_EXIT(&tx_ring_p->lock); 1008 1009 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1010 1011 return (status); 1012 1013 nxge_start_fail_lso: 1014 status = 0; 1015 good_packet = B_FALSE; 1016 if (mp != NULL) { 1017 freemsg(mp); 1018 } 1019 if (mp_chain != NULL) { 1020 freemsg(mp_chain); 1021 } 1022 if (!lso_again && !ngathers) { 1023 if (isLDOMservice(nxgep)) { 1024 tx_ring_p->tx_ring_busy = B_FALSE; 1025 if (tx_ring_p->tx_ring_offline) { 1026 (void) atomic_swap_32( 1027 &tx_ring_p->tx_ring_offline, 1028 NXGE_TX_RING_OFFLINED); 1029 } 1030 } 1031 1032 MUTEX_EXIT(&tx_ring_p->lock); 1033 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1034 "==> nxge_start: lso exit (nothing changed)")); 1035 goto nxge_start_fail1; 1036 } 1037 1038 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1039 "==> nxge_start (channel %d): before lso " 1040 "lso_gathers %d ngathers %d cur_index_lso %d " 1041 "wr_index %d sop_index %d lso_again %d", 1042 tx_ring_p->tdc, 1043 lso_ngathers, ngathers, cur_index_lso, 1044 tx_ring_p->wr_index, sop_index, lso_again)); 1045 1046 if (lso_again) { 1047 lso_ngathers += ngathers; 1048 ngathers = lso_ngathers; 1049 sop_index = cur_index_lso; 1050 tx_ring_p->wr_index = sop_index; 1051 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1052 } 1053 1054 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1055 "==> nxge_start (channel %d): after lso " 1056 "lso_gathers %d ngathers %d cur_index_lso %d " 1057 "wr_index %d sop_index %d lso_again %d", 1058 tx_ring_p->tdc, 1059 lso_ngathers, ngathers, cur_index_lso, 1060 tx_ring_p->wr_index, sop_index, lso_again)); 1061 1062 nxge_start_fail2: 1063 if (good_packet == B_FALSE) { 1064 cur_index = sop_index; 1065 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1066 for (i = 0; i < ngathers; i++) { 1067 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1068 #if defined(__i386) 1069 npi_handle.regp = (uint32_t)tx_desc_p; 1070 #else 1071 npi_handle.regp = (uint64_t)tx_desc_p; 1072 #endif 1073 tx_msg_p = &tx_msg_ring[cur_index]; 1074 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1075 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1076 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1077 "tx_desc_p = %X index = %d", 1078 tx_desc_p, tx_ring_p->rd_index)); 1079 (void) dvma_unload(tx_msg_p->dvma_handle, 1080 0, -1); 1081 tx_msg_p->dvma_handle = NULL; 1082 if (tx_ring_p->dvma_wr_index == 1083 tx_ring_p->dvma_wrap_mask) 1084 tx_ring_p->dvma_wr_index = 0; 1085 else 1086 tx_ring_p->dvma_wr_index++; 1087 tx_ring_p->dvma_pending--; 1088 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1089 if (ddi_dma_unbind_handle( 1090 tx_msg_p->dma_handle)) { 1091 cmn_err(CE_WARN, "!nxge_start: " 1092 "ddi_dma_unbind_handle failed"); 1093 } 1094 } 1095 tx_msg_p->flags.dma_type = USE_NONE; 1096 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1097 tx_ring_p->tx_wrap_mask); 1098 1099 } 1100 1101 nxgep->resched_needed = B_TRUE; 1102 } 1103 1104 if (isLDOMservice(nxgep)) { 1105 tx_ring_p->tx_ring_busy = B_FALSE; 1106 if (tx_ring_p->tx_ring_offline) { 1107 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1108 NXGE_TX_RING_OFFLINED); 1109 } 1110 } 1111 1112 MUTEX_EXIT(&tx_ring_p->lock); 1113 1114 nxge_start_fail1: 1115 /* Add FMA to check the access handle nxge_hregh */ 1116 1117 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1118 1119 return (status); 1120 } 1121 1122 int 1123 nxge_serial_tx(mblk_t *mp, void *arg) 1124 { 1125 p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 1126 p_nxge_t nxgep = tx_ring_p->nxgep; 1127 int status = 0; 1128 1129 if (isLDOMservice(nxgep)) { 1130 if (tx_ring_p->tx_ring_offline) { 1131 freemsg(mp); 1132 return (status); 1133 } 1134 } 1135 1136 status = nxge_start(nxgep, tx_ring_p, mp); 1137 return (status); 1138 } 1139 1140 boolean_t 1141 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 1142 { 1143 p_tx_ring_t *tx_rings; 1144 uint8_t ring_index; 1145 p_tx_ring_t tx_ring_p; 1146 nxge_grp_t *group; 1147 1148 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 1149 1150 ASSERT(mp->b_next == NULL); 1151 1152 group = nxgep->tx_set.group[0]; /* The default group */ 1153 ring_index = nxge_tx_lb_ring_1(mp, group->count, hp); 1154 1155 tx_rings = nxgep->tx_rings->rings; 1156 tx_ring_p = tx_rings[group->legend[ring_index]]; 1157 1158 if (isLDOMservice(nxgep)) { 1159 if (tx_ring_p->tx_ring_offline) { 1160 /* 1161 * OFFLINE means that it is in the process of being 1162 * shared - that is, it has been claimed by the HIO 1163 * code, but hasn't been unlinked from <group> yet. 1164 * So in this case use the first TDC, which always 1165 * belongs to the service domain and can't be shared. 1166 */ 1167 ring_index = 0; 1168 tx_ring_p = tx_rings[group->legend[ring_index]]; 1169 } 1170 } 1171 1172 NXGE_DEBUG_MSG((nxgep, TX_CTL, "count %d, tx_rings[%d] = %p", 1173 (int)group->count, group->legend[ring_index], tx_ring_p)); 1174 1175 switch (nxge_tx_scheme) { 1176 case NXGE_USE_START: 1177 if (nxge_start(nxgep, tx_ring_p, mp)) { 1178 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 1179 "ring index %d", ring_index)); 1180 return (B_FALSE); 1181 } 1182 break; 1183 1184 case NXGE_USE_SERIAL: 1185 default: 1186 nxge_serialize_enter(tx_ring_p->serial, mp); 1187 break; 1188 } 1189 1190 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 1191 ring_index)); 1192 1193 return (B_TRUE); 1194 } 1195 1196 /* 1197 * nxge_m_tx() - send a chain of packets 1198 */ 1199 mblk_t * 1200 nxge_m_tx(void *arg, mblk_t *mp) 1201 { 1202 p_nxge_t nxgep = (p_nxge_t)arg; 1203 mblk_t *next; 1204 mac_tx_hint_t hint; 1205 1206 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_m_tx")); 1207 1208 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 1209 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 1210 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1211 "==> nxge_m_tx: hardware not initialized")); 1212 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1213 "<== nxge_m_tx")); 1214 freemsgchain(mp); 1215 mp = NULL; 1216 return (mp); 1217 } 1218 1219 hint.hash = NULL; 1220 hint.vid = 0; 1221 hint.sap = 0; 1222 1223 while (mp != NULL) { 1224 next = mp->b_next; 1225 mp->b_next = NULL; 1226 1227 /* 1228 * Until Nemo tx resource works, the mac driver 1229 * does the load balancing based on TCP port, 1230 * or CPU. For debugging, we use a system 1231 * configurable parameter. 1232 */ 1233 if (!nxge_send(nxgep, mp, &hint)) { 1234 mp->b_next = next; 1235 break; 1236 } 1237 1238 mp = next; 1239 1240 NXGE_DEBUG_MSG((NULL, TX_CTL, 1241 "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 1242 mp, next)); 1243 } 1244 1245 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_m_tx")); 1246 return (mp); 1247 } 1248 1249 int 1250 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 1251 { 1252 uint8_t ring_index = 0; 1253 uint8_t *tcp_port; 1254 p_mblk_t nmp; 1255 size_t mblk_len; 1256 size_t iph_len; 1257 size_t hdrs_size; 1258 uint8_t hdrs_buf[sizeof (struct ether_vlan_header) + 1259 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 1260 /* 1261 * allocate space big enough to cover 1262 * the max ip header length and the first 1263 * 4 bytes of the TCP/IP header. 1264 */ 1265 1266 boolean_t qos = B_FALSE; 1267 ushort_t eth_type; 1268 size_t eth_hdr_size; 1269 1270 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 1271 1272 if (hp->vid) { 1273 qos = B_TRUE; 1274 } 1275 switch (nxge_tx_lb_policy) { 1276 case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 1277 default: 1278 tcp_port = mp->b_rptr; 1279 eth_type = ntohs(((struct ether_header *)tcp_port)->ether_type); 1280 if (eth_type == VLAN_ETHERTYPE) { 1281 eth_type = ntohs(((struct ether_vlan_header *) 1282 tcp_port)->ether_type); 1283 eth_hdr_size = sizeof (struct ether_vlan_header); 1284 } else { 1285 eth_hdr_size = sizeof (struct ether_header); 1286 } 1287 1288 if (!nxge_no_tx_lb && !qos && eth_type == ETHERTYPE_IP) { 1289 nmp = mp; 1290 mblk_len = MBLKL(nmp); 1291 tcp_port = NULL; 1292 if (mblk_len > eth_hdr_size + sizeof (uint8_t)) { 1293 tcp_port = nmp->b_rptr + eth_hdr_size; 1294 mblk_len -= eth_hdr_size; 1295 iph_len = ((*tcp_port) & 0x0f) << 2; 1296 if (mblk_len > (iph_len + sizeof (uint32_t))) { 1297 tcp_port = nmp->b_rptr; 1298 } else { 1299 tcp_port = NULL; 1300 } 1301 } 1302 if (tcp_port == NULL) { 1303 hdrs_size = 0; 1304 while ((nmp) && (hdrs_size < 1305 sizeof (hdrs_buf))) { 1306 mblk_len = MBLKL(nmp); 1307 if (mblk_len >= 1308 (sizeof (hdrs_buf) - hdrs_size)) 1309 mblk_len = sizeof (hdrs_buf) - 1310 hdrs_size; 1311 bcopy(nmp->b_rptr, 1312 &hdrs_buf[hdrs_size], mblk_len); 1313 hdrs_size += mblk_len; 1314 nmp = nmp->b_cont; 1315 } 1316 tcp_port = hdrs_buf; 1317 } 1318 tcp_port += eth_hdr_size; 1319 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1320 switch (tcp_port[9]) { 1321 case IPPROTO_TCP: 1322 case IPPROTO_UDP: 1323 case IPPROTO_ESP: 1324 tcp_port += ((*tcp_port) & 0x0f) << 2; 1325 ring_index = 1326 ((tcp_port[0] ^ 1327 tcp_port[1] ^ 1328 tcp_port[2] ^ 1329 tcp_port[3]) % maxtdcs); 1330 break; 1331 1332 case IPPROTO_AH: 1333 /* SPI starts at the 4th byte */ 1334 tcp_port += ((*tcp_port) & 0x0f) << 2; 1335 ring_index = 1336 ((tcp_port[4] ^ 1337 tcp_port[5] ^ 1338 tcp_port[6] ^ 1339 tcp_port[7]) % maxtdcs); 1340 break; 1341 1342 default: 1343 ring_index = tcp_port[19] % maxtdcs; 1344 break; 1345 } 1346 } else { /* fragmented packet */ 1347 ring_index = tcp_port[19] % maxtdcs; 1348 } 1349 } else { 1350 ring_index = mp->b_band % maxtdcs; 1351 } 1352 break; 1353 1354 case NXGE_TX_LB_HASH: 1355 if (hp->hash) { 1356 #if defined(__i386) 1357 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1358 #else 1359 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1360 #endif 1361 } else { 1362 ring_index = mp->b_band % maxtdcs; 1363 } 1364 break; 1365 1366 case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 1367 tcp_port = mp->b_rptr; 1368 ring_index = tcp_port[5] % maxtdcs; 1369 break; 1370 } 1371 1372 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 1373 1374 return (ring_index); 1375 } 1376 1377 uint_t 1378 nxge_reschedule(caddr_t arg) 1379 { 1380 p_nxge_t nxgep; 1381 1382 nxgep = (p_nxge_t)arg; 1383 1384 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 1385 1386 if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1387 nxgep->resched_needed) { 1388 if (!isLDOMguest(nxgep)) 1389 mac_tx_update(nxgep->mach); 1390 #if defined(sun4v) 1391 else { /* isLDOMguest(nxgep) */ 1392 nxge_hio_data_t *nhd = (nxge_hio_data_t *) 1393 nxgep->nxge_hw_p->hio; 1394 nx_vio_fp_t *vio = &nhd->hio.vio; 1395 1396 /* Call back vnet. */ 1397 if (vio->cb.vio_net_tx_update) { 1398 (*vio->cb.vio_net_tx_update) 1399 (nxgep->hio_vr->vhp); 1400 } 1401 } 1402 #endif 1403 nxgep->resched_needed = B_FALSE; 1404 nxgep->resched_running = B_FALSE; 1405 } 1406 1407 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 1408 return (DDI_INTR_CLAIMED); 1409 } 1410 1411 1412 /* Software LSO starts here */ 1413 static void 1414 nxge_hcksum_retrieve(mblk_t *mp, 1415 uint32_t *start, uint32_t *stuff, uint32_t *end, 1416 uint32_t *value, uint32_t *flags) 1417 { 1418 if (mp->b_datap->db_type == M_DATA) { 1419 if (flags != NULL) { 1420 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1421 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1422 HCK_FULLCKSUM_OK); 1423 if ((*flags & (HCK_PARTIALCKSUM | 1424 HCK_FULLCKSUM)) != 0) { 1425 if (value != NULL) 1426 *value = (uint32_t)DB_CKSUM16(mp); 1427 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1428 if (start != NULL) 1429 *start = 1430 (uint32_t)DB_CKSUMSTART(mp); 1431 if (stuff != NULL) 1432 *stuff = 1433 (uint32_t)DB_CKSUMSTUFF(mp); 1434 if (end != NULL) 1435 *end = 1436 (uint32_t)DB_CKSUMEND(mp); 1437 } 1438 } 1439 } 1440 } 1441 } 1442 1443 static void 1444 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1445 { 1446 ASSERT(DB_TYPE(mp) == M_DATA); 1447 1448 *mss = 0; 1449 if (flags != NULL) { 1450 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1451 if ((*flags != 0) && (mss != NULL)) { 1452 *mss = (uint32_t)DB_LSOMSS(mp); 1453 } 1454 NXGE_DEBUG_MSG((NULL, TX_CTL, 1455 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1456 *mss, *flags)); 1457 } 1458 1459 NXGE_DEBUG_MSG((NULL, TX_CTL, 1460 "<== nxge_lso_info_get: mss %d", *mss)); 1461 } 1462 1463 /* 1464 * Do Soft LSO on the oversized packet. 1465 * 1466 * 1. Create a chain of message for headers. 1467 * 2. Fill up header messages with proper information. 1468 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1469 * each new message with necessary adjustments. 1470 * * Unchange the ethernet header for DIX frames. (by default) 1471 * * IP Total Length field is updated to MSS or less(only for the last one). 1472 * * IP Identification value is incremented by one for each packet. 1473 * * TCP sequence Number is recalculated according to the payload length. 1474 * * Set FIN and/or PSH flags for the *last* packet if applied. 1475 * * TCP partial Checksum 1476 * 4. Update LSO information in the first message header. 1477 * 5. Release the original message header. 1478 */ 1479 static mblk_t * 1480 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1481 { 1482 uint32_t hckflags; 1483 int pktlen; 1484 int hdrlen; 1485 int segnum; 1486 int i; 1487 struct ether_vlan_header *evh; 1488 int ehlen, iphlen, tcphlen; 1489 struct ip *oiph, *niph; 1490 struct tcphdr *otcph, *ntcph; 1491 int available, len, left; 1492 uint16_t ip_id; 1493 uint32_t tcp_seq; 1494 #ifdef __sparc 1495 uint32_t tcp_seq_tmp; 1496 #endif 1497 mblk_t *datamp; 1498 uchar_t *rptr; 1499 mblk_t *nmp; 1500 mblk_t *cmp; 1501 mblk_t *mp_chain; 1502 boolean_t do_cleanup = B_FALSE; 1503 t_uscalar_t start_offset = 0; 1504 t_uscalar_t stuff_offset = 0; 1505 t_uscalar_t value = 0; 1506 uint16_t l4_len; 1507 ipaddr_t src, dst; 1508 uint32_t cksum, sum, l4cksum; 1509 1510 NXGE_DEBUG_MSG((NULL, TX_CTL, 1511 "==> nxge_do_softlso")); 1512 /* 1513 * check the length of LSO packet payload and calculate the number of 1514 * segments to be generated. 1515 */ 1516 pktlen = msgsize(mp); 1517 evh = (struct ether_vlan_header *)mp->b_rptr; 1518 1519 /* VLAN? */ 1520 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1521 ehlen = sizeof (struct ether_vlan_header); 1522 else 1523 ehlen = sizeof (struct ether_header); 1524 oiph = (struct ip *)(mp->b_rptr + ehlen); 1525 iphlen = oiph->ip_hl * 4; 1526 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1527 tcphlen = otcph->th_off * 4; 1528 1529 l4_len = pktlen - ehlen - iphlen; 1530 1531 NXGE_DEBUG_MSG((NULL, TX_CTL, 1532 "==> nxge_do_softlso: mss %d oiph $%p " 1533 "original ip_sum oiph->ip_sum 0x%x " 1534 "original tcp_sum otcph->th_sum 0x%x " 1535 "oiph->ip_len %d pktlen %d ehlen %d " 1536 "l4_len %d (0x%x) ip_len - iphlen %d ", 1537 mss, 1538 oiph, 1539 oiph->ip_sum, 1540 otcph->th_sum, 1541 ntohs(oiph->ip_len), pktlen, 1542 ehlen, 1543 l4_len, 1544 l4_len, 1545 ntohs(oiph->ip_len) - iphlen)); 1546 1547 /* IPv4 + TCP */ 1548 if (!(oiph->ip_v == IPV4_VERSION)) { 1549 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1550 "<== nxge_do_softlso: not IPV4 " 1551 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1552 ntohs(oiph->ip_len), pktlen, ehlen, 1553 tcphlen)); 1554 freemsg(mp); 1555 return (NULL); 1556 } 1557 1558 if (!(oiph->ip_p == IPPROTO_TCP)) { 1559 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1560 "<== nxge_do_softlso: not TCP " 1561 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1562 ntohs(oiph->ip_len), pktlen, ehlen, 1563 tcphlen)); 1564 freemsg(mp); 1565 return (NULL); 1566 } 1567 1568 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1569 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1570 "<== nxge_do_softlso: len not matched " 1571 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1572 ntohs(oiph->ip_len), pktlen, ehlen, 1573 tcphlen)); 1574 freemsg(mp); 1575 return (NULL); 1576 } 1577 1578 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1579 tcphlen = otcph->th_off * 4; 1580 1581 /* TCP flags can not include URG, RST, or SYN */ 1582 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1583 1584 hdrlen = ehlen + iphlen + tcphlen; 1585 1586 VERIFY(MBLKL(mp) >= hdrlen); 1587 1588 if (MBLKL(mp) > hdrlen) { 1589 datamp = mp; 1590 rptr = mp->b_rptr + hdrlen; 1591 } else { /* = */ 1592 datamp = mp->b_cont; 1593 rptr = datamp->b_rptr; 1594 } 1595 1596 NXGE_DEBUG_MSG((NULL, TX_CTL, 1597 "nxge_do_softlso: otcph $%p pktlen: %d, " 1598 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1599 "mblkl(mp): %d, mblkl(datamp): %d", 1600 otcph, 1601 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1602 (int)MBLKL(mp), (int)MBLKL(datamp))); 1603 1604 hckflags = 0; 1605 nxge_hcksum_retrieve(mp, 1606 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1607 1608 dst = oiph->ip_dst.s_addr; 1609 src = oiph->ip_src.s_addr; 1610 1611 cksum = (dst >> 16) + (dst & 0xFFFF) + 1612 (src >> 16) + (src & 0xFFFF); 1613 l4cksum = cksum + IP_TCP_CSUM_COMP; 1614 1615 sum = l4_len + l4cksum; 1616 sum = (sum & 0xFFFF) + (sum >> 16); 1617 1618 NXGE_DEBUG_MSG((NULL, TX_CTL, 1619 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1620 "hckflags 0x%x start_offset %d stuff_offset %d " 1621 "value (original) 0x%x th_sum 0x%x " 1622 "pktlen %d l4_len %d (0x%x) " 1623 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1624 dst, src, 1625 (sum & 0xffff), (~sum & 0xffff), 1626 hckflags, start_offset, stuff_offset, 1627 value, otcph->th_sum, 1628 pktlen, 1629 l4_len, 1630 l4_len, 1631 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1632 (int)MBLKL(datamp), 1633 nxge_dump_packet((char *)evh, 12))); 1634 1635 /* 1636 * Start to process. 1637 */ 1638 available = pktlen - hdrlen; 1639 segnum = (available - 1) / mss + 1; 1640 1641 NXGE_DEBUG_MSG((NULL, TX_CTL, 1642 "==> nxge_do_softlso: pktlen %d " 1643 "MBLKL(mp): %d, MBLKL(datamp): %d " 1644 "available %d mss %d segnum %d", 1645 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1646 available, 1647 mss, 1648 segnum)); 1649 1650 VERIFY(segnum >= 2); 1651 1652 /* 1653 * Try to pre-allocate all header messages 1654 */ 1655 mp_chain = NULL; 1656 for (i = 0; i < segnum; i++) { 1657 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1658 /* Clean up the mp_chain */ 1659 while (mp_chain != NULL) { 1660 nmp = mp_chain; 1661 mp_chain = mp_chain->b_next; 1662 freemsg(nmp); 1663 } 1664 NXGE_DEBUG_MSG((NULL, TX_CTL, 1665 "<== nxge_do_softlso: " 1666 "Could not allocate enough messages for headers!")); 1667 freemsg(mp); 1668 return (NULL); 1669 } 1670 nmp->b_next = mp_chain; 1671 mp_chain = nmp; 1672 1673 NXGE_DEBUG_MSG((NULL, TX_CTL, 1674 "==> nxge_do_softlso: " 1675 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1676 mp, nmp, mp_chain, mp_chain->b_next)); 1677 } 1678 1679 NXGE_DEBUG_MSG((NULL, TX_CTL, 1680 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1681 mp, nmp, mp_chain)); 1682 1683 /* 1684 * Associate payload with new packets 1685 */ 1686 cmp = mp_chain; 1687 left = available; 1688 while (cmp != NULL) { 1689 nmp = dupb(datamp); 1690 if (nmp == NULL) { 1691 do_cleanup = B_TRUE; 1692 NXGE_DEBUG_MSG((NULL, TX_CTL, 1693 "==>nxge_do_softlso: " 1694 "Can not dupb(datamp), have to do clean up")); 1695 goto cleanup_allocated_msgs; 1696 } 1697 1698 NXGE_DEBUG_MSG((NULL, TX_CTL, 1699 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1700 "dupb nmp $%p len %d left %d msd %d ", 1701 mp, cmp, nmp, len, left, mss)); 1702 1703 cmp->b_cont = nmp; 1704 nmp->b_rptr = rptr; 1705 len = (left < mss) ? left : mss; 1706 left -= len; 1707 1708 NXGE_DEBUG_MSG((NULL, TX_CTL, 1709 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1710 "dupb nmp $%p len %d left %d mss %d ", 1711 mp, cmp, nmp, len, left, mss)); 1712 NXGE_DEBUG_MSG((NULL, TX_CTL, 1713 "nxge_do_softlso: before available: %d, " 1714 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1715 available, left, len, segnum, (int)MBLKL(nmp))); 1716 1717 len -= MBLKL(nmp); 1718 NXGE_DEBUG_MSG((NULL, TX_CTL, 1719 "nxge_do_softlso: after available: %d, " 1720 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1721 available, left, len, segnum, (int)MBLKL(nmp))); 1722 1723 while (len > 0) { 1724 mblk_t *mmp = NULL; 1725 1726 NXGE_DEBUG_MSG((NULL, TX_CTL, 1727 "nxge_do_softlso: (4) len > 0 available: %d, " 1728 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1729 available, left, len, segnum, (int)MBLKL(nmp))); 1730 1731 if (datamp->b_cont != NULL) { 1732 datamp = datamp->b_cont; 1733 rptr = datamp->b_rptr; 1734 mmp = dupb(datamp); 1735 if (mmp == NULL) { 1736 do_cleanup = B_TRUE; 1737 NXGE_DEBUG_MSG((NULL, TX_CTL, 1738 "==> nxge_do_softlso: " 1739 "Can not dupb(datamp) (1), :" 1740 "have to do clean up")); 1741 NXGE_DEBUG_MSG((NULL, TX_CTL, 1742 "==> nxge_do_softlso: " 1743 "available: %d, left: %d, " 1744 "len: %d, MBLKL(nmp): %d", 1745 available, left, len, 1746 (int)MBLKL(nmp))); 1747 goto cleanup_allocated_msgs; 1748 } 1749 } else { 1750 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1751 "==> nxge_do_softlso: " 1752 "(1)available: %d, left: %d, " 1753 "len: %d, MBLKL(nmp): %d", 1754 available, left, len, 1755 (int)MBLKL(nmp))); 1756 cmn_err(CE_PANIC, 1757 "==> nxge_do_softlso: " 1758 "Pointers must have been corrupted!\n" 1759 "datamp: $%p, nmp: $%p, rptr: $%p", 1760 (void *)datamp, 1761 (void *)nmp, 1762 (void *)rptr); 1763 } 1764 nmp->b_cont = mmp; 1765 nmp = mmp; 1766 len -= MBLKL(nmp); 1767 } 1768 if (len < 0) { 1769 nmp->b_wptr += len; 1770 rptr = nmp->b_wptr; 1771 NXGE_DEBUG_MSG((NULL, TX_CTL, 1772 "(5) len < 0 (less than 0)" 1773 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1774 available, left, len, (int)MBLKL(nmp))); 1775 1776 } else if (len == 0) { 1777 if (datamp->b_cont != NULL) { 1778 NXGE_DEBUG_MSG((NULL, TX_CTL, 1779 "(5) len == 0" 1780 "available: %d, left: %d, len: %d, " 1781 "MBLKL(nmp): %d", 1782 available, left, len, (int)MBLKL(nmp))); 1783 datamp = datamp->b_cont; 1784 rptr = datamp->b_rptr; 1785 } else { 1786 NXGE_DEBUG_MSG((NULL, TX_CTL, 1787 "(6)available b_cont == NULL : %d, " 1788 "left: %d, len: %d, MBLKL(nmp): %d", 1789 available, left, len, (int)MBLKL(nmp))); 1790 1791 VERIFY(cmp->b_next == NULL); 1792 VERIFY(left == 0); 1793 break; /* Done! */ 1794 } 1795 } 1796 cmp = cmp->b_next; 1797 1798 NXGE_DEBUG_MSG((NULL, TX_CTL, 1799 "(7) do_softlso: " 1800 "next mp in mp_chain available len != 0 : %d, " 1801 "left: %d, len: %d, MBLKL(nmp): %d", 1802 available, left, len, (int)MBLKL(nmp))); 1803 } 1804 1805 /* 1806 * From now, start to fill up all headers for the first message 1807 * Hardware checksum flags need to be updated separately for FULLCKSUM 1808 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1809 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1810 * required fields need to be updated properly. 1811 */ 1812 nmp = mp_chain; 1813 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1814 nmp->b_wptr = nmp->b_rptr + hdrlen; 1815 niph = (struct ip *)(nmp->b_rptr + ehlen); 1816 niph->ip_len = htons(mss + iphlen + tcphlen); 1817 ip_id = ntohs(niph->ip_id); 1818 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1819 #ifdef __sparc 1820 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1821 tcp_seq = ntohl(tcp_seq_tmp); 1822 #else 1823 tcp_seq = ntohl(ntcph->th_seq); 1824 #endif 1825 1826 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1827 1828 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1829 DB_CKSUMSTART(nmp) = start_offset; 1830 DB_CKSUMSTUFF(nmp) = stuff_offset; 1831 1832 /* calculate IP checksum and TCP pseudo header checksum */ 1833 niph->ip_sum = 0; 1834 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1835 1836 l4_len = mss + tcphlen; 1837 sum = htons(l4_len) + l4cksum; 1838 sum = (sum & 0xFFFF) + (sum >> 16); 1839 ntcph->th_sum = (sum & 0xffff); 1840 1841 NXGE_DEBUG_MSG((NULL, TX_CTL, 1842 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1843 "mss %d pktlen %d l4_len %d (0x%x) " 1844 "MBLKL(mp): %d, MBLKL(datamp): %d " 1845 "ip_sum 0x%x " 1846 "th_sum 0x%x sum 0x%x ) " 1847 "dump first ip->tcp %s", 1848 nmp, mp_chain, 1849 mss, 1850 pktlen, 1851 l4_len, 1852 l4_len, 1853 (int)MBLKL(mp), (int)MBLKL(datamp), 1854 niph->ip_sum, 1855 ntcph->th_sum, 1856 sum, 1857 nxge_dump_packet((char *)niph, 52))); 1858 1859 cmp = nmp; 1860 while ((nmp = nmp->b_next)->b_next != NULL) { 1861 NXGE_DEBUG_MSG((NULL, TX_CTL, 1862 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1863 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1864 nmp->b_wptr = nmp->b_rptr + hdrlen; 1865 niph = (struct ip *)(nmp->b_rptr + ehlen); 1866 niph->ip_id = htons(++ip_id); 1867 niph->ip_len = htons(mss + iphlen + tcphlen); 1868 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1869 tcp_seq += mss; 1870 1871 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1872 1873 #ifdef __sparc 1874 tcp_seq_tmp = htonl(tcp_seq); 1875 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1876 #else 1877 ntcph->th_seq = htonl(tcp_seq); 1878 #endif 1879 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1880 DB_CKSUMSTART(nmp) = start_offset; 1881 DB_CKSUMSTUFF(nmp) = stuff_offset; 1882 1883 /* calculate IP checksum and TCP pseudo header checksum */ 1884 niph->ip_sum = 0; 1885 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1886 ntcph->th_sum = (sum & 0xffff); 1887 1888 NXGE_DEBUG_MSG((NULL, TX_CTL, 1889 "==> nxge_do_softlso: middle ip_sum 0x%x " 1890 "th_sum 0x%x " 1891 " mp $%p (mp_chain $%p) pktlen %d " 1892 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1893 niph->ip_sum, 1894 ntcph->th_sum, 1895 nmp, mp_chain, 1896 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1897 } 1898 1899 /* Last segment */ 1900 /* 1901 * Set FIN and/or PSH flags if present only in the last packet. 1902 * The ip_len could be different from prior packets. 1903 */ 1904 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1905 nmp->b_wptr = nmp->b_rptr + hdrlen; 1906 niph = (struct ip *)(nmp->b_rptr + ehlen); 1907 niph->ip_id = htons(++ip_id); 1908 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1909 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1910 tcp_seq += mss; 1911 #ifdef __sparc 1912 tcp_seq_tmp = htonl(tcp_seq); 1913 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1914 #else 1915 ntcph->th_seq = htonl(tcp_seq); 1916 #endif 1917 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1918 1919 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1920 DB_CKSUMSTART(nmp) = start_offset; 1921 DB_CKSUMSTUFF(nmp) = stuff_offset; 1922 1923 /* calculate IP checksum and TCP pseudo header checksum */ 1924 niph->ip_sum = 0; 1925 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1926 1927 l4_len = ntohs(niph->ip_len) - iphlen; 1928 sum = htons(l4_len) + l4cksum; 1929 sum = (sum & 0xFFFF) + (sum >> 16); 1930 ntcph->th_sum = (sum & 0xffff); 1931 1932 NXGE_DEBUG_MSG((NULL, TX_CTL, 1933 "==> nxge_do_softlso: last next " 1934 "niph->ip_sum 0x%x " 1935 "ntcph->th_sum 0x%x sum 0x%x " 1936 "dump last ip->tcp %s " 1937 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1938 "l4_len %d (0x%x) " 1939 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1940 niph->ip_sum, 1941 ntcph->th_sum, sum, 1942 nxge_dump_packet((char *)niph, 52), 1943 cmp, nmp, mp_chain, 1944 pktlen, pktlen, 1945 l4_len, 1946 l4_len, 1947 (int)MBLKL(mp), (int)MBLKL(datamp))); 1948 1949 cleanup_allocated_msgs: 1950 if (do_cleanup) { 1951 NXGE_DEBUG_MSG((NULL, TX_CTL, 1952 "==> nxge_do_softlso: " 1953 "Failed allocating messages, " 1954 "have to clean up and fail!")); 1955 while (mp_chain != NULL) { 1956 nmp = mp_chain; 1957 mp_chain = mp_chain->b_next; 1958 freemsg(nmp); 1959 } 1960 } 1961 /* 1962 * We're done here, so just free the original message and return the 1963 * new message chain, that could be NULL if failed, back to the caller. 1964 */ 1965 freemsg(mp); 1966 1967 NXGE_DEBUG_MSG((NULL, TX_CTL, 1968 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1969 return (mp_chain); 1970 } 1971 1972 /* 1973 * Will be called before NIC driver do further operation on the message. 1974 * The input message may include LSO information, if so, go to softlso logic 1975 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1976 * The return could be the same non-LSO message or a message chain for LSO case. 1977 * 1978 * The driver needs to call this function per packet and process the whole chain 1979 * if applied. 1980 */ 1981 static mblk_t * 1982 nxge_lso_eliminate(mblk_t *mp) 1983 { 1984 uint32_t lsoflags; 1985 uint32_t mss; 1986 1987 NXGE_DEBUG_MSG((NULL, TX_CTL, 1988 "==>nxge_lso_eliminate:")); 1989 nxge_lso_info_get(mp, &mss, &lsoflags); 1990 1991 if (lsoflags & HW_LSO) { 1992 mblk_t *nmp; 1993 1994 NXGE_DEBUG_MSG((NULL, TX_CTL, 1995 "==>nxge_lso_eliminate:" 1996 "HW_LSO:mss %d mp $%p", 1997 mss, mp)); 1998 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1999 NXGE_DEBUG_MSG((NULL, TX_CTL, 2000 "<== nxge_lso_eliminate: " 2001 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 2002 nmp, mss, mp)); 2003 return (nmp); 2004 } else { 2005 NXGE_DEBUG_MSG((NULL, TX_CTL, 2006 "<== nxge_lso_eliminate_ " 2007 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 2008 nmp, mss, mp)); 2009 return (NULL); 2010 } 2011 } 2012 2013 NXGE_DEBUG_MSG((NULL, TX_CTL, 2014 "<== nxge_lso_eliminate")); 2015 return (mp); 2016 } 2017 2018 static uint32_t 2019 nxge_csgen(uint16_t *adr, int len) 2020 { 2021 int i, odd; 2022 uint32_t sum = 0; 2023 uint32_t c = 0; 2024 2025 odd = len % 2; 2026 for (i = 0; i < (len / 2); i++) { 2027 sum += (adr[i] & 0xffff); 2028 } 2029 if (odd) { 2030 sum += adr[len / 2] & 0xff00; 2031 } 2032 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 2033 sum &= 0xffff; 2034 sum += c; 2035 } 2036 return (~sum & 0xffff); 2037 } 2038