1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/nxge/nxge_impl.h> 27 #include <sys/nxge/nxge_hio.h> 28 #include <npi_tx_wr64.h> 29 30 /* Software LSO required header files */ 31 #include <netinet/tcp.h> 32 #include <inet/ip_impl.h> 33 #include <inet/tcp.h> 34 35 static mblk_t *nxge_lso_eliminate(mblk_t *); 36 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 37 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 38 static void nxge_hcksum_retrieve(mblk_t *, 39 uint32_t *, uint32_t *, uint32_t *, 40 uint32_t *, uint32_t *); 41 static uint32_t nxge_csgen(uint16_t *, int); 42 43 extern void nxge_txdma_freemsg_task(p_tx_ring_t ringp); 44 45 extern uint32_t nxge_reclaim_pending; 46 extern uint32_t nxge_bcopy_thresh; 47 extern uint32_t nxge_dvma_thresh; 48 extern uint32_t nxge_dma_stream_thresh; 49 extern uint32_t nxge_tx_minfree; 50 extern uint32_t nxge_tx_intr_thres; 51 extern uint32_t nxge_tx_max_gathers; 52 extern uint32_t nxge_tx_tiny_pack; 53 extern uint32_t nxge_tx_use_bcopy; 54 extern uint32_t nxge_tx_lb_policy; 55 extern uint32_t nxge_no_tx_lb; 56 extern nxge_tx_mode_t nxge_tx_scheme; 57 uint32_t nxge_lso_kick_cnt = 2; 58 59 typedef struct _mac_tx_hint { 60 uint16_t sap; 61 uint16_t vid; 62 void *hash; 63 } mac_tx_hint_t, *p_mac_tx_hint_t; 64 65 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 66 67 int 68 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 69 { 70 int status = 0; 71 p_tx_desc_t tx_desc_ring_vp; 72 npi_handle_t npi_desc_handle; 73 nxge_os_dma_handle_t tx_desc_dma_handle; 74 p_tx_desc_t tx_desc_p; 75 p_tx_msg_t tx_msg_ring; 76 p_tx_msg_t tx_msg_p; 77 tx_desc_t tx_desc, *tmp_desc_p; 78 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 79 p_tx_pkt_header_t hdrp; 80 tx_pkt_header_t tmp_hdrp; 81 p_tx_pkt_hdr_all_t pkthdrp; 82 uint8_t npads = 0; 83 uint64_t dma_ioaddr; 84 uint32_t dma_flags; 85 int last_bidx; 86 uint8_t *b_rptr; 87 caddr_t kaddr; 88 uint32_t nmblks; 89 uint32_t ngathers; 90 uint32_t clen; 91 int len; 92 uint32_t pkt_len, pack_len, min_len; 93 uint32_t bcopy_thresh; 94 int i, cur_index, sop_index; 95 uint16_t tail_index; 96 boolean_t tail_wrap = B_FALSE; 97 nxge_dma_common_t desc_area; 98 nxge_os_dma_handle_t dma_handle; 99 ddi_dma_cookie_t dma_cookie; 100 npi_handle_t npi_handle; 101 p_mblk_t nmp; 102 p_mblk_t t_mp; 103 uint32_t ncookies; 104 boolean_t good_packet; 105 boolean_t mark_mode = B_FALSE; 106 p_nxge_stats_t statsp; 107 p_nxge_tx_ring_stats_t tdc_stats; 108 t_uscalar_t start_offset = 0; 109 t_uscalar_t stuff_offset = 0; 110 t_uscalar_t end_offset = 0; 111 t_uscalar_t value = 0; 112 t_uscalar_t cksum_flags = 0; 113 boolean_t cksum_on = B_FALSE; 114 uint32_t boff = 0; 115 uint64_t tot_xfer_len = 0; 116 boolean_t header_set = B_FALSE; 117 #ifdef NXGE_DEBUG 118 p_tx_desc_t tx_desc_ring_pp; 119 p_tx_desc_t tx_desc_pp; 120 tx_desc_t *save_desc_p; 121 int dump_len; 122 int sad_len; 123 uint64_t sad; 124 int xfer_len; 125 uint32_t msgsize; 126 #endif 127 p_mblk_t mp_chain = NULL; 128 boolean_t is_lso = B_FALSE; 129 boolean_t lso_again; 130 int cur_index_lso; 131 p_mblk_t nmp_lso_save; 132 uint32_t lso_ngathers; 133 boolean_t lso_tail_wrap = B_FALSE; 134 135 NXGE_DEBUG_MSG((nxgep, TX_CTL, 136 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 137 NXGE_DEBUG_MSG((nxgep, TX_CTL, 138 "==> nxge_start: Starting tdc %d desc pending %d", 139 tx_ring_p->tdc, tx_ring_p->descs_pending)); 140 141 statsp = nxgep->statsp; 142 143 if (!isLDOMguest(nxgep)) { 144 switch (nxgep->mac.portmode) { 145 default: 146 if (nxgep->statsp->port_stats.lb_mode == 147 nxge_lb_normal) { 148 if (!statsp->mac_stats.link_up) { 149 freemsg(mp); 150 NXGE_DEBUG_MSG((nxgep, TX_CTL, 151 "==> nxge_start: " 152 "link not up")); 153 goto nxge_start_fail1; 154 } 155 } 156 break; 157 case PORT_10G_FIBER: 158 /* 159 * For the following modes, check the link status 160 * before sending the packet out: 161 * nxge_lb_normal, 162 * nxge_lb_ext10g, 163 * nxge_lb_ext1000, 164 * nxge_lb_ext100, 165 * nxge_lb_ext10. 166 */ 167 if (nxgep->statsp->port_stats.lb_mode < 168 nxge_lb_phy10g) { 169 if (!statsp->mac_stats.link_up) { 170 freemsg(mp); 171 NXGE_DEBUG_MSG((nxgep, TX_CTL, 172 "==> nxge_start: " 173 "link not up")); 174 goto nxge_start_fail1; 175 } 176 } 177 break; 178 } 179 } 180 181 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 182 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 183 NXGE_DEBUG_MSG((nxgep, TX_CTL, 184 "==> nxge_start: hardware not initialized or stopped")); 185 freemsg(mp); 186 goto nxge_start_fail1; 187 } 188 189 if (nxgep->soft_lso_enable) { 190 mp_chain = nxge_lso_eliminate(mp); 191 NXGE_DEBUG_MSG((nxgep, TX_CTL, 192 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 193 mp, mp_chain)); 194 if (mp_chain == NULL) { 195 NXGE_ERROR_MSG((nxgep, TX_CTL, 196 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 197 mp_chain, mp)); 198 goto nxge_start_fail1; 199 } 200 if (mp_chain != mp) { 201 NXGE_DEBUG_MSG((nxgep, TX_CTL, 202 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 203 mp_chain, mp)); 204 is_lso = B_TRUE; 205 mp = mp_chain; 206 mp_chain = mp_chain->b_next; 207 mp->b_next = NULL; 208 } 209 } 210 211 hcksum_retrieve(mp, NULL, NULL, &start_offset, 212 &stuff_offset, &end_offset, &value, &cksum_flags); 213 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 214 start_offset += sizeof (ether_header_t); 215 stuff_offset += sizeof (ether_header_t); 216 } else { 217 start_offset += sizeof (struct ether_vlan_header); 218 stuff_offset += sizeof (struct ether_vlan_header); 219 } 220 221 if (cksum_flags & HCK_PARTIALCKSUM) { 222 NXGE_DEBUG_MSG((nxgep, TX_CTL, 223 "==> nxge_start: mp $%p len %d " 224 "cksum_flags 0x%x (partial checksum) ", 225 mp, MBLKL(mp), cksum_flags)); 226 cksum_on = B_TRUE; 227 } 228 229 pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 230 pkthdrp->reserved = 0; 231 tmp_hdrp.value = 0; 232 nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 233 0, 0, pkthdrp, 234 start_offset, stuff_offset); 235 236 lso_again = B_FALSE; 237 lso_ngathers = 0; 238 239 MUTEX_ENTER(&tx_ring_p->lock); 240 241 if (isLDOMservice(nxgep)) { 242 tx_ring_p->tx_ring_busy = B_TRUE; 243 if (tx_ring_p->tx_ring_offline) { 244 freemsg(mp); 245 tx_ring_p->tx_ring_busy = B_FALSE; 246 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 247 NXGE_TX_RING_OFFLINED); 248 MUTEX_EXIT(&tx_ring_p->lock); 249 return (status); 250 } 251 } 252 253 cur_index_lso = tx_ring_p->wr_index; 254 lso_tail_wrap = tx_ring_p->wr_index_wrap; 255 start_again: 256 ngathers = 0; 257 sop_index = tx_ring_p->wr_index; 258 #ifdef NXGE_DEBUG 259 if (tx_ring_p->descs_pending) { 260 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 261 "desc pending %d ", tx_ring_p->descs_pending)); 262 } 263 264 dump_len = (int)(MBLKL(mp)); 265 dump_len = (dump_len > 128) ? 128: dump_len; 266 267 NXGE_DEBUG_MSG((nxgep, TX_CTL, 268 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 269 "(Before header reserve: ORIGINAL LEN %d)", 270 tx_ring_p->tdc, 271 mp->b_rptr, 272 dump_len)); 273 274 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 275 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 276 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 277 #endif 278 279 tdc_stats = tx_ring_p->tdc_stats; 280 mark_mode = (tx_ring_p->descs_pending && 281 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 282 < nxge_tx_minfree)); 283 284 NXGE_DEBUG_MSG((nxgep, TX_CTL, 285 "TX Descriptor ring is channel %d mark mode %d", 286 tx_ring_p->tdc, mark_mode)); 287 288 if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) { 289 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, 290 (nxge_tx_minfree + lso_ngathers))) { 291 NXGE_DEBUG_MSG((nxgep, TX_CTL, 292 "TX Descriptor ring is full: channel %d", 293 tx_ring_p->tdc)); 294 NXGE_DEBUG_MSG((nxgep, TX_CTL, 295 "TX Descriptor ring is full: channel %d", 296 tx_ring_p->tdc)); 297 if (is_lso) { 298 /* 299 * free the current mp and mp_chain if not FULL. 300 */ 301 tdc_stats->tx_no_desc++; 302 NXGE_DEBUG_MSG((nxgep, TX_CTL, 303 "LSO packet: TX Descriptor ring is full: " 304 "channel %d", 305 tx_ring_p->tdc)); 306 goto nxge_start_fail_lso; 307 } else { 308 boolean_t skip_sched = B_FALSE; 309 310 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 311 tdc_stats->tx_no_desc++; 312 313 if (isLDOMservice(nxgep)) { 314 tx_ring_p->tx_ring_busy = B_FALSE; 315 if (tx_ring_p->tx_ring_offline) { 316 (void) atomic_swap_32( 317 &tx_ring_p->tx_ring_offline, 318 NXGE_TX_RING_OFFLINED); 319 skip_sched = B_TRUE; 320 } 321 } 322 323 MUTEX_EXIT(&tx_ring_p->lock); 324 if (nxgep->resched_needed && 325 !nxgep->resched_running && !skip_sched) { 326 nxgep->resched_running = B_TRUE; 327 ddi_trigger_softintr(nxgep->resched_id); 328 } 329 status = 1; 330 goto nxge_start_fail1; 331 } 332 } 333 } 334 335 nmp = mp; 336 i = sop_index = tx_ring_p->wr_index; 337 nmblks = 0; 338 ngathers = 0; 339 pkt_len = 0; 340 pack_len = 0; 341 clen = 0; 342 last_bidx = -1; 343 good_packet = B_TRUE; 344 345 desc_area = tx_ring_p->tdc_desc; 346 npi_handle = desc_area.npi_handle; 347 npi_desc_handle.regh = (nxge_os_acc_handle_t) 348 DMA_COMMON_ACC_HANDLE(desc_area); 349 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 350 tx_desc_dma_handle = (nxge_os_dma_handle_t) 351 DMA_COMMON_HANDLE(desc_area); 352 tx_msg_ring = tx_ring_p->tx_msg_ring; 353 354 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 355 sop_index, i)); 356 357 #ifdef NXGE_DEBUG 358 msgsize = msgdsize(nmp); 359 NXGE_DEBUG_MSG((nxgep, TX_CTL, 360 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 361 sop_index, i, msgsize)); 362 #endif 363 /* 364 * The first 16 bytes of the premapped buffer are reserved 365 * for header. No padding will be used. 366 */ 367 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 368 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 369 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 370 } else { 371 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 372 } 373 while (nmp) { 374 good_packet = B_TRUE; 375 b_rptr = nmp->b_rptr; 376 len = MBLKL(nmp); 377 if (len <= 0) { 378 nmp = nmp->b_cont; 379 continue; 380 } 381 nmblks++; 382 383 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 384 "len %d pkt_len %d pack_len %d", 385 nmblks, len, pkt_len, pack_len)); 386 /* 387 * Hardware limits the transfer length to 4K for NIU and 388 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 389 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 390 * If len is longer than the limit, then we break nmp into 391 * two chunks: Make the first chunk equal to the limit and 392 * the second chunk for the remaining data. If the second 393 * chunk is still larger than the limit, then it will be 394 * broken into two in the next pass. 395 */ 396 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 397 if ((t_mp = dupb(nmp)) != NULL) { 398 nmp->b_wptr = nmp->b_rptr + 399 (TX_MAX_TRANSFER_LENGTH 400 - TX_PKT_HEADER_SIZE); 401 t_mp->b_rptr = nmp->b_wptr; 402 t_mp->b_cont = nmp->b_cont; 403 nmp->b_cont = t_mp; 404 len = MBLKL(nmp); 405 } else { 406 if (is_lso) { 407 NXGE_DEBUG_MSG((nxgep, TX_CTL, 408 "LSO packet: dupb failed: " 409 "channel %d", 410 tx_ring_p->tdc)); 411 mp = nmp; 412 goto nxge_start_fail_lso; 413 } else { 414 good_packet = B_FALSE; 415 goto nxge_start_fail2; 416 } 417 } 418 } 419 tx_desc.value = 0; 420 tx_desc_p = &tx_desc_ring_vp[i]; 421 #ifdef NXGE_DEBUG 422 tx_desc_pp = &tx_desc_ring_pp[i]; 423 #endif 424 tx_msg_p = &tx_msg_ring[i]; 425 #if defined(__i386) 426 npi_desc_handle.regp = (uint32_t)tx_desc_p; 427 #else 428 npi_desc_handle.regp = (uint64_t)tx_desc_p; 429 #endif 430 if (!header_set && 431 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 432 (len >= bcopy_thresh))) { 433 header_set = B_TRUE; 434 bcopy_thresh += TX_PKT_HEADER_SIZE; 435 boff = 0; 436 pack_len = 0; 437 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 438 hdrp = (p_tx_pkt_header_t)kaddr; 439 clen = pkt_len; 440 dma_handle = tx_msg_p->buf_dma_handle; 441 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 442 (void) ddi_dma_sync(dma_handle, 443 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 444 DDI_DMA_SYNC_FORDEV); 445 446 tx_msg_p->flags.dma_type = USE_BCOPY; 447 goto nxge_start_control_header_only; 448 } 449 450 pkt_len += len; 451 pack_len += len; 452 453 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 454 "desc entry %d " 455 "DESC IOADDR $%p " 456 "desc_vp $%p tx_desc_p $%p " 457 "desc_pp $%p tx_desc_pp $%p " 458 "len %d pkt_len %d pack_len %d", 459 i, 460 DMA_COMMON_IOADDR(desc_area), 461 tx_desc_ring_vp, tx_desc_p, 462 tx_desc_ring_pp, tx_desc_pp, 463 len, pkt_len, pack_len)); 464 465 if (len < bcopy_thresh) { 466 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 467 "USE BCOPY: ")); 468 if (nxge_tx_tiny_pack) { 469 uint32_t blst = 470 TXDMA_DESC_NEXT_INDEX(i, -1, 471 tx_ring_p->tx_wrap_mask); 472 NXGE_DEBUG_MSG((nxgep, TX_CTL, 473 "==> nxge_start(5): pack")); 474 if ((pack_len <= bcopy_thresh) && 475 (last_bidx == blst)) { 476 NXGE_DEBUG_MSG((nxgep, TX_CTL, 477 "==> nxge_start: pack(6) " 478 "(pkt_len %d pack_len %d)", 479 pkt_len, pack_len)); 480 i = blst; 481 tx_desc_p = &tx_desc_ring_vp[i]; 482 #ifdef NXGE_DEBUG 483 tx_desc_pp = &tx_desc_ring_pp[i]; 484 #endif 485 tx_msg_p = &tx_msg_ring[i]; 486 boff = pack_len - len; 487 ngathers--; 488 } else if (pack_len > bcopy_thresh && 489 header_set) { 490 pack_len = len; 491 boff = 0; 492 bcopy_thresh = nxge_bcopy_thresh; 493 NXGE_DEBUG_MSG((nxgep, TX_CTL, 494 "==> nxge_start(7): > max NEW " 495 "bcopy thresh %d " 496 "pkt_len %d pack_len %d(next)", 497 bcopy_thresh, 498 pkt_len, pack_len)); 499 } 500 last_bidx = i; 501 } 502 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 503 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 504 hdrp = (p_tx_pkt_header_t)kaddr; 505 header_set = B_TRUE; 506 NXGE_DEBUG_MSG((nxgep, TX_CTL, 507 "==> nxge_start(7_x2): " 508 "pkt_len %d pack_len %d (new hdrp $%p)", 509 pkt_len, pack_len, hdrp)); 510 } 511 tx_msg_p->flags.dma_type = USE_BCOPY; 512 kaddr += boff; 513 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 514 "USE BCOPY: before bcopy " 515 "DESC IOADDR $%p entry %d " 516 "bcopy packets %d " 517 "bcopy kaddr $%p " 518 "bcopy ioaddr (SAD) $%p " 519 "bcopy clen %d " 520 "bcopy boff %d", 521 DMA_COMMON_IOADDR(desc_area), i, 522 tdc_stats->tx_hdr_pkts, 523 kaddr, 524 dma_ioaddr, 525 clen, 526 boff)); 527 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 528 "1USE BCOPY: ")); 529 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 530 "2USE BCOPY: ")); 531 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 532 "last USE BCOPY: copy from b_rptr $%p " 533 "to KADDR $%p (len %d offset %d", 534 b_rptr, kaddr, len, boff)); 535 536 bcopy(b_rptr, kaddr, len); 537 538 #ifdef NXGE_DEBUG 539 dump_len = (len > 128) ? 128: len; 540 NXGE_DEBUG_MSG((nxgep, TX_CTL, 541 "==> nxge_start: dump packets " 542 "(After BCOPY len %d)" 543 "(b_rptr $%p): %s", len, nmp->b_rptr, 544 nxge_dump_packet((char *)nmp->b_rptr, 545 dump_len))); 546 #endif 547 548 dma_handle = tx_msg_p->buf_dma_handle; 549 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 550 (void) ddi_dma_sync(dma_handle, 551 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 552 DDI_DMA_SYNC_FORDEV); 553 clen = len + boff; 554 tdc_stats->tx_hdr_pkts++; 555 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 556 "USE BCOPY: " 557 "DESC IOADDR $%p entry %d " 558 "bcopy packets %d " 559 "bcopy kaddr $%p " 560 "bcopy ioaddr (SAD) $%p " 561 "bcopy clen %d " 562 "bcopy boff %d", 563 DMA_COMMON_IOADDR(desc_area), 564 i, 565 tdc_stats->tx_hdr_pkts, 566 kaddr, 567 dma_ioaddr, 568 clen, 569 boff)); 570 } else { 571 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 572 "USE DVMA: len %d", len)); 573 tx_msg_p->flags.dma_type = USE_DMA; 574 dma_flags = DDI_DMA_WRITE; 575 if (len < nxge_dma_stream_thresh) { 576 dma_flags |= DDI_DMA_CONSISTENT; 577 } else { 578 dma_flags |= DDI_DMA_STREAMING; 579 } 580 581 dma_handle = tx_msg_p->dma_handle; 582 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 583 (caddr_t)b_rptr, len, dma_flags, 584 DDI_DMA_DONTWAIT, NULL, 585 &dma_cookie, &ncookies); 586 if (status == DDI_DMA_MAPPED) { 587 dma_ioaddr = dma_cookie.dmac_laddress; 588 len = (int)dma_cookie.dmac_size; 589 clen = (uint32_t)dma_cookie.dmac_size; 590 NXGE_DEBUG_MSG((nxgep, TX_CTL, 591 "==> nxge_start(12_1): " 592 "USE DVMA: len %d clen %d " 593 "ngathers %d", 594 len, clen, 595 ngathers)); 596 #if defined(__i386) 597 npi_desc_handle.regp = (uint32_t)tx_desc_p; 598 #else 599 npi_desc_handle.regp = (uint64_t)tx_desc_p; 600 #endif 601 while (ncookies > 1) { 602 ngathers++; 603 /* 604 * this is the fix for multiple 605 * cookies, which are basically 606 * a descriptor entry, we don't set 607 * SOP bit as well as related fields 608 */ 609 610 (void) npi_txdma_desc_gather_set( 611 npi_desc_handle, 612 &tx_desc, 613 (ngathers -1), 614 mark_mode, 615 ngathers, 616 dma_ioaddr, 617 clen); 618 619 tx_msg_p->tx_msg_size = clen; 620 NXGE_DEBUG_MSG((nxgep, TX_CTL, 621 "==> nxge_start: DMA " 622 "ncookie %d " 623 "ngathers %d " 624 "dma_ioaddr $%p len %d" 625 "desc $%p descp $%p (%d)", 626 ncookies, 627 ngathers, 628 dma_ioaddr, clen, 629 *tx_desc_p, tx_desc_p, i)); 630 631 ddi_dma_nextcookie(dma_handle, 632 &dma_cookie); 633 dma_ioaddr = 634 dma_cookie.dmac_laddress; 635 636 len = (int)dma_cookie.dmac_size; 637 clen = (uint32_t)dma_cookie.dmac_size; 638 NXGE_DEBUG_MSG((nxgep, TX_CTL, 639 "==> nxge_start(12_2): " 640 "USE DVMA: len %d clen %d ", 641 len, clen)); 642 643 i = TXDMA_DESC_NEXT_INDEX(i, 1, 644 tx_ring_p->tx_wrap_mask); 645 tx_desc_p = &tx_desc_ring_vp[i]; 646 647 #if defined(__i386) 648 npi_desc_handle.regp = 649 (uint32_t)tx_desc_p; 650 #else 651 npi_desc_handle.regp = 652 (uint64_t)tx_desc_p; 653 #endif 654 tx_msg_p = &tx_msg_ring[i]; 655 tx_msg_p->flags.dma_type = USE_NONE; 656 tx_desc.value = 0; 657 658 ncookies--; 659 } 660 tdc_stats->tx_ddi_pkts++; 661 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 662 "DMA: ddi packets %d", 663 tdc_stats->tx_ddi_pkts)); 664 } else { 665 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 666 "dma mapping failed for %d " 667 "bytes addr $%p flags %x (%d)", 668 len, b_rptr, status, status)); 669 good_packet = B_FALSE; 670 tdc_stats->tx_dma_bind_fail++; 671 tx_msg_p->flags.dma_type = USE_NONE; 672 if (is_lso) { 673 mp = nmp; 674 goto nxge_start_fail_lso; 675 } else { 676 goto nxge_start_fail2; 677 } 678 } 679 } /* ddi dvma */ 680 681 if (is_lso) { 682 nmp_lso_save = nmp; 683 } 684 nmp = nmp->b_cont; 685 nxge_start_control_header_only: 686 #if defined(__i386) 687 npi_desc_handle.regp = (uint32_t)tx_desc_p; 688 #else 689 npi_desc_handle.regp = (uint64_t)tx_desc_p; 690 #endif 691 ngathers++; 692 693 if (ngathers == 1) { 694 #ifdef NXGE_DEBUG 695 save_desc_p = &sop_tx_desc; 696 #endif 697 sop_tx_desc_p = &sop_tx_desc; 698 sop_tx_desc_p->value = 0; 699 sop_tx_desc_p->bits.hdw.tr_len = clen; 700 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 701 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 702 } else { 703 #ifdef NXGE_DEBUG 704 save_desc_p = &tx_desc; 705 #endif 706 tmp_desc_p = &tx_desc; 707 tmp_desc_p->value = 0; 708 tmp_desc_p->bits.hdw.tr_len = clen; 709 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 710 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 711 712 tx_desc_p->value = tmp_desc_p->value; 713 } 714 715 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 716 "Desc_entry %d ngathers %d " 717 "desc_vp $%p tx_desc_p $%p " 718 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 719 "dma_ioaddr (SAD) $%p mark %d", 720 i, ngathers, 721 tx_desc_ring_vp, tx_desc_p, 722 len, clen, pkt_len, pack_len, nmblks, 723 dma_ioaddr, mark_mode)); 724 725 #ifdef NXGE_DEBUG 726 npi_desc_handle.nxgep = nxgep; 727 npi_desc_handle.function.function = nxgep->function_num; 728 npi_desc_handle.function.instance = nxgep->instance; 729 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 730 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 731 TX_PKT_DESC_TR_LEN_SHIFT); 732 733 734 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 735 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 736 "mark %d sop %d\n", 737 save_desc_p->value, 738 sad, 739 save_desc_p->bits.hdw.tr_len, 740 xfer_len, 741 save_desc_p->bits.hdw.num_ptr, 742 save_desc_p->bits.hdw.mark, 743 save_desc_p->bits.hdw.sop)); 744 745 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 746 #endif 747 748 tx_msg_p->tx_msg_size = clen; 749 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 750 if (ngathers > nxge_tx_max_gathers) { 751 good_packet = B_FALSE; 752 hcksum_retrieve(mp, NULL, NULL, &start_offset, 753 &stuff_offset, &end_offset, &value, 754 &cksum_flags); 755 756 NXGE_DEBUG_MSG((NULL, TX_CTL, 757 "==> nxge_start(14): pull msg - " 758 "len %d pkt_len %d ngathers %d", 759 len, pkt_len, ngathers)); 760 /* Pull all message blocks from b_cont */ 761 if (is_lso) { 762 mp = nmp_lso_save; 763 goto nxge_start_fail_lso; 764 } 765 if ((msgpullup(mp, -1)) == NULL) { 766 goto nxge_start_fail2; 767 } 768 goto nxge_start_fail2; 769 } 770 } /* while (nmp) */ 771 772 tx_msg_p->tx_message = mp; 773 tx_desc_p = &tx_desc_ring_vp[sop_index]; 774 #if defined(__i386) 775 npi_desc_handle.regp = (uint32_t)tx_desc_p; 776 #else 777 npi_desc_handle.regp = (uint64_t)tx_desc_p; 778 #endif 779 780 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 781 pkthdrp->reserved = 0; 782 hdrp->value = 0; 783 bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 784 785 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 786 tdc_stats->tx_jumbo_pkts++; 787 } 788 789 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 790 if (pkt_len < min_len) { 791 /* Assume we use bcopy to premapped buffers */ 792 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 793 NXGE_DEBUG_MSG((NULL, TX_CTL, 794 "==> nxge_start(14-1): < (msg_min + 16)" 795 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 796 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 797 bzero((kaddr + pkt_len), (min_len - pkt_len)); 798 pkt_len = tx_msg_p->tx_msg_size = min_len; 799 800 sop_tx_desc_p->bits.hdw.tr_len = min_len; 801 802 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 803 tx_desc_p->value = sop_tx_desc_p->value; 804 805 NXGE_DEBUG_MSG((NULL, TX_CTL, 806 "==> nxge_start(14-2): < msg_min - " 807 "len %d pkt_len %d min_len %d ngathers %d", 808 len, pkt_len, min_len, ngathers)); 809 } 810 811 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 812 cksum_flags)); 813 { 814 uint64_t tmp_len; 815 816 /* pkt_len already includes 16 + paddings!! */ 817 /* Update the control header length */ 818 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 819 tmp_len = hdrp->value | 820 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 821 822 NXGE_DEBUG_MSG((nxgep, TX_CTL, 823 "==> nxge_start(15_x1): setting SOP " 824 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 825 "0x%llx hdrp->value 0x%llx", 826 tot_xfer_len, tot_xfer_len, pkt_len, 827 tmp_len, hdrp->value)); 828 #if defined(_BIG_ENDIAN) 829 hdrp->value = ddi_swap64(tmp_len); 830 #else 831 hdrp->value = tmp_len; 832 #endif 833 NXGE_DEBUG_MSG((nxgep, 834 TX_CTL, "==> nxge_start(15_x2): setting SOP " 835 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 836 "tmp_len 0x%llx hdrp->value 0x%llx", 837 tot_xfer_len, pkt_len, 838 tmp_len, hdrp->value)); 839 } 840 841 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 842 "wr_index %d " 843 "tot_xfer_len (%d) pkt_len %d npads %d", 844 sop_index, 845 tot_xfer_len, pkt_len, 846 npads)); 847 848 sop_tx_desc_p->bits.hdw.sop = 1; 849 sop_tx_desc_p->bits.hdw.mark = mark_mode; 850 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 851 852 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 853 854 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 855 856 #ifdef NXGE_DEBUG 857 npi_desc_handle.nxgep = nxgep; 858 npi_desc_handle.function.function = nxgep->function_num; 859 npi_desc_handle.function.instance = nxgep->instance; 860 861 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 862 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 863 save_desc_p->value, 864 sad, 865 save_desc_p->bits.hdw.tr_len, 866 xfer_len, 867 save_desc_p->bits.hdw.num_ptr, 868 save_desc_p->bits.hdw.mark, 869 save_desc_p->bits.hdw.sop)); 870 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 871 872 dump_len = (pkt_len > 128) ? 128: pkt_len; 873 NXGE_DEBUG_MSG((nxgep, TX_CTL, 874 "==> nxge_start: dump packets(17) (after sop set, len " 875 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 876 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 877 (char *)hdrp, 878 nxge_dump_packet((char *)hdrp, dump_len))); 879 NXGE_DEBUG_MSG((nxgep, TX_CTL, 880 "==> nxge_start(18): TX desc sync: sop_index %d", 881 sop_index)); 882 #endif 883 884 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 885 (void) ddi_dma_sync(tx_desc_dma_handle, 886 sop_index * sizeof (tx_desc_t), 887 ngathers * sizeof (tx_desc_t), 888 DDI_DMA_SYNC_FORDEV); 889 890 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 891 "cs_off = 0x%02X cs_s_off = 0x%02X " 892 "pkt_len %d ngathers %d sop_index %d\n", 893 stuff_offset, start_offset, 894 pkt_len, ngathers, sop_index)); 895 } else { /* more than one descriptor and wrap around */ 896 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 897 (void) ddi_dma_sync(tx_desc_dma_handle, 898 sop_index * sizeof (tx_desc_t), 899 nsdescs * sizeof (tx_desc_t), 900 DDI_DMA_SYNC_FORDEV); 901 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 902 "cs_off = 0x%02X cs_s_off = 0x%02X " 903 "pkt_len %d ngathers %d sop_index %d\n", 904 stuff_offset, start_offset, 905 pkt_len, ngathers, sop_index)); 906 907 (void) ddi_dma_sync(tx_desc_dma_handle, 908 0, 909 (ngathers - nsdescs) * sizeof (tx_desc_t), 910 DDI_DMA_SYNC_FORDEV); 911 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 912 "cs_off = 0x%02X cs_s_off = 0x%02X " 913 "pkt_len %d ngathers %d sop_index %d\n", 914 stuff_offset, start_offset, 915 pkt_len, ngathers, sop_index)); 916 } 917 918 tail_index = tx_ring_p->wr_index; 919 tail_wrap = tx_ring_p->wr_index_wrap; 920 921 tx_ring_p->wr_index = i; 922 if (tx_ring_p->wr_index <= tail_index) { 923 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 924 B_FALSE : B_TRUE); 925 } 926 927 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 928 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 929 tx_ring_p->tdc, 930 tx_ring_p->wr_index, 931 tx_ring_p->wr_index_wrap, 932 ngathers, 933 tx_ring_p->descs_pending)); 934 935 if (is_lso) { 936 lso_ngathers += ngathers; 937 if (mp_chain != NULL) { 938 mp = mp_chain; 939 mp_chain = mp_chain->b_next; 940 mp->b_next = NULL; 941 if (nxge_lso_kick_cnt == lso_ngathers) { 942 tx_ring_p->descs_pending += lso_ngathers; 943 { 944 tx_ring_kick_t kick; 945 946 kick.value = 0; 947 kick.bits.ldw.wrap = 948 tx_ring_p->wr_index_wrap; 949 kick.bits.ldw.tail = 950 (uint16_t)tx_ring_p->wr_index; 951 952 /* Kick the Transmit kick register */ 953 TXDMA_REG_WRITE64( 954 NXGE_DEV_NPI_HANDLE(nxgep), 955 TX_RING_KICK_REG, 956 (uint8_t)tx_ring_p->tdc, 957 kick.value); 958 tdc_stats->tx_starts++; 959 960 NXGE_DEBUG_MSG((nxgep, TX_CTL, 961 "==> nxge_start: more LSO: " 962 "LSO_CNT %d", 963 lso_ngathers)); 964 } 965 lso_ngathers = 0; 966 ngathers = 0; 967 cur_index_lso = sop_index = tx_ring_p->wr_index; 968 lso_tail_wrap = tx_ring_p->wr_index_wrap; 969 } 970 NXGE_DEBUG_MSG((nxgep, TX_CTL, 971 "==> nxge_start: lso again: " 972 "lso_gathers %d ngathers %d cur_index_lso %d " 973 "wr_index %d sop_index %d", 974 lso_ngathers, ngathers, cur_index_lso, 975 tx_ring_p->wr_index, sop_index)); 976 977 NXGE_DEBUG_MSG((nxgep, TX_CTL, 978 "==> nxge_start: next : count %d", 979 lso_ngathers)); 980 lso_again = B_TRUE; 981 goto start_again; 982 } 983 ngathers = lso_ngathers; 984 } 985 986 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 987 988 { 989 tx_ring_kick_t kick; 990 991 kick.value = 0; 992 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 993 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 994 995 /* Kick start the Transmit kick register */ 996 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 997 TX_RING_KICK_REG, 998 (uint8_t)tx_ring_p->tdc, 999 kick.value); 1000 } 1001 1002 tx_ring_p->descs_pending += ngathers; 1003 tdc_stats->tx_starts++; 1004 1005 if (isLDOMservice(nxgep)) { 1006 tx_ring_p->tx_ring_busy = B_FALSE; 1007 if (tx_ring_p->tx_ring_offline) { 1008 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1009 NXGE_TX_RING_OFFLINED); 1010 } 1011 } 1012 1013 MUTEX_EXIT(&tx_ring_p->lock); 1014 1015 nxge_txdma_freemsg_task(tx_ring_p); 1016 1017 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1018 1019 return (status); 1020 1021 nxge_start_fail_lso: 1022 status = 0; 1023 good_packet = B_FALSE; 1024 if (mp != NULL) { 1025 freemsg(mp); 1026 } 1027 if (mp_chain != NULL) { 1028 freemsg(mp_chain); 1029 } 1030 if (!lso_again && !ngathers) { 1031 if (isLDOMservice(nxgep)) { 1032 tx_ring_p->tx_ring_busy = B_FALSE; 1033 if (tx_ring_p->tx_ring_offline) { 1034 (void) atomic_swap_32( 1035 &tx_ring_p->tx_ring_offline, 1036 NXGE_TX_RING_OFFLINED); 1037 } 1038 } 1039 1040 MUTEX_EXIT(&tx_ring_p->lock); 1041 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1042 "==> nxge_start: lso exit (nothing changed)")); 1043 goto nxge_start_fail1; 1044 } 1045 1046 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1047 "==> nxge_start (channel %d): before lso " 1048 "lso_gathers %d ngathers %d cur_index_lso %d " 1049 "wr_index %d sop_index %d lso_again %d", 1050 tx_ring_p->tdc, 1051 lso_ngathers, ngathers, cur_index_lso, 1052 tx_ring_p->wr_index, sop_index, lso_again)); 1053 1054 if (lso_again) { 1055 lso_ngathers += ngathers; 1056 ngathers = lso_ngathers; 1057 sop_index = cur_index_lso; 1058 tx_ring_p->wr_index = sop_index; 1059 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1060 } 1061 1062 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1063 "==> nxge_start (channel %d): after lso " 1064 "lso_gathers %d ngathers %d cur_index_lso %d " 1065 "wr_index %d sop_index %d lso_again %d", 1066 tx_ring_p->tdc, 1067 lso_ngathers, ngathers, cur_index_lso, 1068 tx_ring_p->wr_index, sop_index, lso_again)); 1069 1070 nxge_start_fail2: 1071 if (good_packet == B_FALSE) { 1072 cur_index = sop_index; 1073 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1074 for (i = 0; i < ngathers; i++) { 1075 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1076 #if defined(__i386) 1077 npi_handle.regp = (uint32_t)tx_desc_p; 1078 #else 1079 npi_handle.regp = (uint64_t)tx_desc_p; 1080 #endif 1081 tx_msg_p = &tx_msg_ring[cur_index]; 1082 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1083 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1084 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1085 "tx_desc_p = %X index = %d", 1086 tx_desc_p, tx_ring_p->rd_index)); 1087 (void) dvma_unload(tx_msg_p->dvma_handle, 1088 0, -1); 1089 tx_msg_p->dvma_handle = NULL; 1090 if (tx_ring_p->dvma_wr_index == 1091 tx_ring_p->dvma_wrap_mask) 1092 tx_ring_p->dvma_wr_index = 0; 1093 else 1094 tx_ring_p->dvma_wr_index++; 1095 tx_ring_p->dvma_pending--; 1096 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1097 if (ddi_dma_unbind_handle( 1098 tx_msg_p->dma_handle)) { 1099 cmn_err(CE_WARN, "!nxge_start: " 1100 "ddi_dma_unbind_handle failed"); 1101 } 1102 } 1103 tx_msg_p->flags.dma_type = USE_NONE; 1104 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1105 tx_ring_p->tx_wrap_mask); 1106 1107 } 1108 1109 nxgep->resched_needed = B_TRUE; 1110 } 1111 1112 if (isLDOMservice(nxgep)) { 1113 tx_ring_p->tx_ring_busy = B_FALSE; 1114 if (tx_ring_p->tx_ring_offline) { 1115 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1116 NXGE_TX_RING_OFFLINED); 1117 } 1118 } 1119 1120 MUTEX_EXIT(&tx_ring_p->lock); 1121 1122 nxge_start_fail1: 1123 /* Add FMA to check the access handle nxge_hregh */ 1124 1125 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1126 1127 return (status); 1128 } 1129 1130 int 1131 nxge_serial_tx(mblk_t *mp, void *arg) 1132 { 1133 p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 1134 p_nxge_t nxgep = tx_ring_p->nxgep; 1135 int status = 0; 1136 1137 if (isLDOMservice(nxgep)) { 1138 if (tx_ring_p->tx_ring_offline) { 1139 freemsg(mp); 1140 return (status); 1141 } 1142 } 1143 1144 status = nxge_start(nxgep, tx_ring_p, mp); 1145 return (status); 1146 } 1147 1148 boolean_t 1149 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 1150 { 1151 p_tx_ring_t *tx_rings; 1152 uint8_t ring_index; 1153 p_tx_ring_t tx_ring_p; 1154 nxge_grp_t *group; 1155 1156 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 1157 1158 ASSERT(mp->b_next == NULL); 1159 1160 group = nxgep->tx_set.group[0]; /* The default group */ 1161 ring_index = nxge_tx_lb_ring_1(mp, group->count, hp); 1162 1163 tx_rings = nxgep->tx_rings->rings; 1164 tx_ring_p = tx_rings[group->legend[ring_index]]; 1165 1166 if (isLDOMservice(nxgep)) { 1167 if (tx_ring_p->tx_ring_offline) { 1168 /* 1169 * OFFLINE means that it is in the process of being 1170 * shared - that is, it has been claimed by the HIO 1171 * code, but hasn't been unlinked from <group> yet. 1172 * So in this case use the first TDC, which always 1173 * belongs to the service domain and can't be shared. 1174 */ 1175 ring_index = 0; 1176 tx_ring_p = tx_rings[group->legend[ring_index]]; 1177 } 1178 } 1179 1180 NXGE_DEBUG_MSG((nxgep, TX_CTL, "count %d, tx_rings[%d] = %p", 1181 (int)group->count, group->legend[ring_index], tx_ring_p)); 1182 1183 switch (nxge_tx_scheme) { 1184 case NXGE_USE_START: 1185 if (nxge_start(nxgep, tx_ring_p, mp)) { 1186 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 1187 "ring index %d", ring_index)); 1188 return (B_FALSE); 1189 } 1190 break; 1191 1192 case NXGE_USE_SERIAL: 1193 default: 1194 nxge_serialize_enter(tx_ring_p->serial, mp); 1195 break; 1196 } 1197 1198 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 1199 ring_index)); 1200 1201 return (B_TRUE); 1202 } 1203 1204 /* 1205 * nxge_m_tx() - send a chain of packets 1206 */ 1207 mblk_t * 1208 nxge_m_tx(void *arg, mblk_t *mp) 1209 { 1210 p_nxge_t nxgep = (p_nxge_t)arg; 1211 mblk_t *next; 1212 mac_tx_hint_t hint; 1213 1214 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_m_tx")); 1215 1216 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 1217 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 1218 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1219 "==> nxge_m_tx: hardware not initialized")); 1220 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1221 "<== nxge_m_tx")); 1222 freemsgchain(mp); 1223 mp = NULL; 1224 return (mp); 1225 } 1226 1227 hint.hash = NULL; 1228 hint.vid = 0; 1229 hint.sap = 0; 1230 1231 while (mp != NULL) { 1232 next = mp->b_next; 1233 mp->b_next = NULL; 1234 1235 /* 1236 * Until Nemo tx resource works, the mac driver 1237 * does the load balancing based on TCP port, 1238 * or CPU. For debugging, we use a system 1239 * configurable parameter. 1240 */ 1241 if (!nxge_send(nxgep, mp, &hint)) { 1242 mp->b_next = next; 1243 break; 1244 } 1245 1246 mp = next; 1247 1248 NXGE_DEBUG_MSG((NULL, TX_CTL, 1249 "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 1250 mp, next)); 1251 } 1252 1253 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_m_tx")); 1254 return (mp); 1255 } 1256 1257 int 1258 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 1259 { 1260 uint8_t ring_index = 0; 1261 uint8_t *tcp_port; 1262 p_mblk_t nmp; 1263 size_t mblk_len; 1264 size_t iph_len; 1265 size_t hdrs_size; 1266 uint8_t hdrs_buf[sizeof (struct ether_vlan_header) + 1267 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 1268 /* 1269 * allocate space big enough to cover 1270 * the max ip header length and the first 1271 * 4 bytes of the TCP/IP header. 1272 */ 1273 1274 boolean_t qos = B_FALSE; 1275 ushort_t eth_type; 1276 size_t eth_hdr_size; 1277 1278 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 1279 1280 if (hp->vid) { 1281 qos = B_TRUE; 1282 } 1283 switch (nxge_tx_lb_policy) { 1284 case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 1285 default: 1286 tcp_port = mp->b_rptr; 1287 eth_type = ntohs(((struct ether_header *)tcp_port)->ether_type); 1288 if (eth_type == VLAN_ETHERTYPE) { 1289 eth_type = ntohs(((struct ether_vlan_header *) 1290 tcp_port)->ether_type); 1291 eth_hdr_size = sizeof (struct ether_vlan_header); 1292 } else { 1293 eth_hdr_size = sizeof (struct ether_header); 1294 } 1295 1296 if (!nxge_no_tx_lb && !qos && eth_type == ETHERTYPE_IP) { 1297 nmp = mp; 1298 mblk_len = MBLKL(nmp); 1299 tcp_port = NULL; 1300 if (mblk_len > eth_hdr_size + sizeof (uint8_t)) { 1301 tcp_port = nmp->b_rptr + eth_hdr_size; 1302 mblk_len -= eth_hdr_size; 1303 iph_len = ((*tcp_port) & 0x0f) << 2; 1304 if (mblk_len > (iph_len + sizeof (uint32_t))) { 1305 tcp_port = nmp->b_rptr; 1306 } else { 1307 tcp_port = NULL; 1308 } 1309 } 1310 if (tcp_port == NULL) { 1311 hdrs_size = 0; 1312 while ((nmp) && (hdrs_size < 1313 sizeof (hdrs_buf))) { 1314 mblk_len = MBLKL(nmp); 1315 if (mblk_len >= 1316 (sizeof (hdrs_buf) - hdrs_size)) 1317 mblk_len = sizeof (hdrs_buf) - 1318 hdrs_size; 1319 bcopy(nmp->b_rptr, 1320 &hdrs_buf[hdrs_size], mblk_len); 1321 hdrs_size += mblk_len; 1322 nmp = nmp->b_cont; 1323 } 1324 tcp_port = hdrs_buf; 1325 } 1326 tcp_port += eth_hdr_size; 1327 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1328 switch (tcp_port[9]) { 1329 case IPPROTO_TCP: 1330 case IPPROTO_UDP: 1331 case IPPROTO_ESP: 1332 tcp_port += ((*tcp_port) & 0x0f) << 2; 1333 ring_index = 1334 ((tcp_port[0] ^ 1335 tcp_port[1] ^ 1336 tcp_port[2] ^ 1337 tcp_port[3]) % maxtdcs); 1338 break; 1339 1340 case IPPROTO_AH: 1341 /* SPI starts at the 4th byte */ 1342 tcp_port += ((*tcp_port) & 0x0f) << 2; 1343 ring_index = 1344 ((tcp_port[4] ^ 1345 tcp_port[5] ^ 1346 tcp_port[6] ^ 1347 tcp_port[7]) % maxtdcs); 1348 break; 1349 1350 default: 1351 ring_index = tcp_port[19] % maxtdcs; 1352 break; 1353 } 1354 } else { /* fragmented packet */ 1355 ring_index = tcp_port[19] % maxtdcs; 1356 } 1357 } else { 1358 ring_index = mp->b_band % maxtdcs; 1359 } 1360 break; 1361 1362 case NXGE_TX_LB_HASH: 1363 if (hp->hash) { 1364 #if defined(__i386) 1365 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1366 #else 1367 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1368 #endif 1369 } else { 1370 ring_index = mp->b_band % maxtdcs; 1371 } 1372 break; 1373 1374 case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 1375 tcp_port = mp->b_rptr; 1376 ring_index = tcp_port[5] % maxtdcs; 1377 break; 1378 } 1379 1380 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 1381 1382 return (ring_index); 1383 } 1384 1385 uint_t 1386 nxge_reschedule(caddr_t arg) 1387 { 1388 p_nxge_t nxgep; 1389 1390 nxgep = (p_nxge_t)arg; 1391 1392 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 1393 1394 if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1395 nxgep->resched_needed) { 1396 if (!isLDOMguest(nxgep)) 1397 mac_tx_update(nxgep->mach); 1398 #if defined(sun4v) 1399 else { /* isLDOMguest(nxgep) */ 1400 nxge_hio_data_t *nhd = (nxge_hio_data_t *) 1401 nxgep->nxge_hw_p->hio; 1402 nx_vio_fp_t *vio = &nhd->hio.vio; 1403 1404 /* Call back vnet. */ 1405 if (vio->cb.vio_net_tx_update) { 1406 (*vio->cb.vio_net_tx_update) 1407 (nxgep->hio_vr->vhp); 1408 } 1409 } 1410 #endif 1411 nxgep->resched_needed = B_FALSE; 1412 nxgep->resched_running = B_FALSE; 1413 } 1414 1415 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 1416 return (DDI_INTR_CLAIMED); 1417 } 1418 1419 1420 /* Software LSO starts here */ 1421 static void 1422 nxge_hcksum_retrieve(mblk_t *mp, 1423 uint32_t *start, uint32_t *stuff, uint32_t *end, 1424 uint32_t *value, uint32_t *flags) 1425 { 1426 if (mp->b_datap->db_type == M_DATA) { 1427 if (flags != NULL) { 1428 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1429 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1430 HCK_FULLCKSUM_OK); 1431 if ((*flags & (HCK_PARTIALCKSUM | 1432 HCK_FULLCKSUM)) != 0) { 1433 if (value != NULL) 1434 *value = (uint32_t)DB_CKSUM16(mp); 1435 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1436 if (start != NULL) 1437 *start = 1438 (uint32_t)DB_CKSUMSTART(mp); 1439 if (stuff != NULL) 1440 *stuff = 1441 (uint32_t)DB_CKSUMSTUFF(mp); 1442 if (end != NULL) 1443 *end = 1444 (uint32_t)DB_CKSUMEND(mp); 1445 } 1446 } 1447 } 1448 } 1449 } 1450 1451 static void 1452 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1453 { 1454 ASSERT(DB_TYPE(mp) == M_DATA); 1455 1456 *mss = 0; 1457 if (flags != NULL) { 1458 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1459 if ((*flags != 0) && (mss != NULL)) { 1460 *mss = (uint32_t)DB_LSOMSS(mp); 1461 } 1462 NXGE_DEBUG_MSG((NULL, TX_CTL, 1463 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1464 *mss, *flags)); 1465 } 1466 1467 NXGE_DEBUG_MSG((NULL, TX_CTL, 1468 "<== nxge_lso_info_get: mss %d", *mss)); 1469 } 1470 1471 /* 1472 * Do Soft LSO on the oversized packet. 1473 * 1474 * 1. Create a chain of message for headers. 1475 * 2. Fill up header messages with proper information. 1476 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1477 * each new message with necessary adjustments. 1478 * * Unchange the ethernet header for DIX frames. (by default) 1479 * * IP Total Length field is updated to MSS or less(only for the last one). 1480 * * IP Identification value is incremented by one for each packet. 1481 * * TCP sequence Number is recalculated according to the payload length. 1482 * * Set FIN and/or PSH flags for the *last* packet if applied. 1483 * * TCP partial Checksum 1484 * 4. Update LSO information in the first message header. 1485 * 5. Release the original message header. 1486 */ 1487 static mblk_t * 1488 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1489 { 1490 uint32_t hckflags; 1491 int pktlen; 1492 int hdrlen; 1493 int segnum; 1494 int i; 1495 struct ether_vlan_header *evh; 1496 int ehlen, iphlen, tcphlen; 1497 struct ip *oiph, *niph; 1498 struct tcphdr *otcph, *ntcph; 1499 int available, len, left; 1500 uint16_t ip_id; 1501 uint32_t tcp_seq; 1502 #ifdef __sparc 1503 uint32_t tcp_seq_tmp; 1504 #endif 1505 mblk_t *datamp; 1506 uchar_t *rptr; 1507 mblk_t *nmp; 1508 mblk_t *cmp; 1509 mblk_t *mp_chain; 1510 boolean_t do_cleanup = B_FALSE; 1511 t_uscalar_t start_offset = 0; 1512 t_uscalar_t stuff_offset = 0; 1513 t_uscalar_t value = 0; 1514 uint16_t l4_len; 1515 ipaddr_t src, dst; 1516 uint32_t cksum, sum, l4cksum; 1517 1518 NXGE_DEBUG_MSG((NULL, TX_CTL, 1519 "==> nxge_do_softlso")); 1520 /* 1521 * check the length of LSO packet payload and calculate the number of 1522 * segments to be generated. 1523 */ 1524 pktlen = msgsize(mp); 1525 evh = (struct ether_vlan_header *)mp->b_rptr; 1526 1527 /* VLAN? */ 1528 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1529 ehlen = sizeof (struct ether_vlan_header); 1530 else 1531 ehlen = sizeof (struct ether_header); 1532 oiph = (struct ip *)(mp->b_rptr + ehlen); 1533 iphlen = oiph->ip_hl * 4; 1534 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1535 tcphlen = otcph->th_off * 4; 1536 1537 l4_len = pktlen - ehlen - iphlen; 1538 1539 NXGE_DEBUG_MSG((NULL, TX_CTL, 1540 "==> nxge_do_softlso: mss %d oiph $%p " 1541 "original ip_sum oiph->ip_sum 0x%x " 1542 "original tcp_sum otcph->th_sum 0x%x " 1543 "oiph->ip_len %d pktlen %d ehlen %d " 1544 "l4_len %d (0x%x) ip_len - iphlen %d ", 1545 mss, 1546 oiph, 1547 oiph->ip_sum, 1548 otcph->th_sum, 1549 ntohs(oiph->ip_len), pktlen, 1550 ehlen, 1551 l4_len, 1552 l4_len, 1553 ntohs(oiph->ip_len) - iphlen)); 1554 1555 /* IPv4 + TCP */ 1556 if (!(oiph->ip_v == IPV4_VERSION)) { 1557 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1558 "<== nxge_do_softlso: not IPV4 " 1559 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1560 ntohs(oiph->ip_len), pktlen, ehlen, 1561 tcphlen)); 1562 freemsg(mp); 1563 return (NULL); 1564 } 1565 1566 if (!(oiph->ip_p == IPPROTO_TCP)) { 1567 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1568 "<== nxge_do_softlso: not TCP " 1569 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1570 ntohs(oiph->ip_len), pktlen, ehlen, 1571 tcphlen)); 1572 freemsg(mp); 1573 return (NULL); 1574 } 1575 1576 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1577 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1578 "<== nxge_do_softlso: len not matched " 1579 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1580 ntohs(oiph->ip_len), pktlen, ehlen, 1581 tcphlen)); 1582 freemsg(mp); 1583 return (NULL); 1584 } 1585 1586 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1587 tcphlen = otcph->th_off * 4; 1588 1589 /* TCP flags can not include URG, RST, or SYN */ 1590 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1591 1592 hdrlen = ehlen + iphlen + tcphlen; 1593 1594 VERIFY(MBLKL(mp) >= hdrlen); 1595 1596 if (MBLKL(mp) > hdrlen) { 1597 datamp = mp; 1598 rptr = mp->b_rptr + hdrlen; 1599 } else { /* = */ 1600 datamp = mp->b_cont; 1601 rptr = datamp->b_rptr; 1602 } 1603 1604 NXGE_DEBUG_MSG((NULL, TX_CTL, 1605 "nxge_do_softlso: otcph $%p pktlen: %d, " 1606 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1607 "mblkl(mp): %d, mblkl(datamp): %d", 1608 otcph, 1609 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1610 (int)MBLKL(mp), (int)MBLKL(datamp))); 1611 1612 hckflags = 0; 1613 nxge_hcksum_retrieve(mp, 1614 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1615 1616 dst = oiph->ip_dst.s_addr; 1617 src = oiph->ip_src.s_addr; 1618 1619 cksum = (dst >> 16) + (dst & 0xFFFF) + 1620 (src >> 16) + (src & 0xFFFF); 1621 l4cksum = cksum + IP_TCP_CSUM_COMP; 1622 1623 sum = l4_len + l4cksum; 1624 sum = (sum & 0xFFFF) + (sum >> 16); 1625 1626 NXGE_DEBUG_MSG((NULL, TX_CTL, 1627 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1628 "hckflags 0x%x start_offset %d stuff_offset %d " 1629 "value (original) 0x%x th_sum 0x%x " 1630 "pktlen %d l4_len %d (0x%x) " 1631 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1632 dst, src, 1633 (sum & 0xffff), (~sum & 0xffff), 1634 hckflags, start_offset, stuff_offset, 1635 value, otcph->th_sum, 1636 pktlen, 1637 l4_len, 1638 l4_len, 1639 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1640 (int)MBLKL(datamp), 1641 nxge_dump_packet((char *)evh, 12))); 1642 1643 /* 1644 * Start to process. 1645 */ 1646 available = pktlen - hdrlen; 1647 segnum = (available - 1) / mss + 1; 1648 1649 NXGE_DEBUG_MSG((NULL, TX_CTL, 1650 "==> nxge_do_softlso: pktlen %d " 1651 "MBLKL(mp): %d, MBLKL(datamp): %d " 1652 "available %d mss %d segnum %d", 1653 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1654 available, 1655 mss, 1656 segnum)); 1657 1658 VERIFY(segnum >= 2); 1659 1660 /* 1661 * Try to pre-allocate all header messages 1662 */ 1663 mp_chain = NULL; 1664 for (i = 0; i < segnum; i++) { 1665 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1666 /* Clean up the mp_chain */ 1667 while (mp_chain != NULL) { 1668 nmp = mp_chain; 1669 mp_chain = mp_chain->b_next; 1670 freemsg(nmp); 1671 } 1672 NXGE_DEBUG_MSG((NULL, TX_CTL, 1673 "<== nxge_do_softlso: " 1674 "Could not allocate enough messages for headers!")); 1675 freemsg(mp); 1676 return (NULL); 1677 } 1678 nmp->b_next = mp_chain; 1679 mp_chain = nmp; 1680 1681 NXGE_DEBUG_MSG((NULL, TX_CTL, 1682 "==> nxge_do_softlso: " 1683 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1684 mp, nmp, mp_chain, mp_chain->b_next)); 1685 } 1686 1687 NXGE_DEBUG_MSG((NULL, TX_CTL, 1688 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1689 mp, nmp, mp_chain)); 1690 1691 /* 1692 * Associate payload with new packets 1693 */ 1694 cmp = mp_chain; 1695 left = available; 1696 while (cmp != NULL) { 1697 nmp = dupb(datamp); 1698 if (nmp == NULL) { 1699 do_cleanup = B_TRUE; 1700 NXGE_DEBUG_MSG((NULL, TX_CTL, 1701 "==>nxge_do_softlso: " 1702 "Can not dupb(datamp), have to do clean up")); 1703 goto cleanup_allocated_msgs; 1704 } 1705 1706 NXGE_DEBUG_MSG((NULL, TX_CTL, 1707 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1708 "dupb nmp $%p len %d left %d msd %d ", 1709 mp, cmp, nmp, len, left, mss)); 1710 1711 cmp->b_cont = nmp; 1712 nmp->b_rptr = rptr; 1713 len = (left < mss) ? left : mss; 1714 left -= len; 1715 1716 NXGE_DEBUG_MSG((NULL, TX_CTL, 1717 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1718 "dupb nmp $%p len %d left %d mss %d ", 1719 mp, cmp, nmp, len, left, mss)); 1720 NXGE_DEBUG_MSG((NULL, TX_CTL, 1721 "nxge_do_softlso: before available: %d, " 1722 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1723 available, left, len, segnum, (int)MBLKL(nmp))); 1724 1725 len -= MBLKL(nmp); 1726 NXGE_DEBUG_MSG((NULL, TX_CTL, 1727 "nxge_do_softlso: after available: %d, " 1728 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1729 available, left, len, segnum, (int)MBLKL(nmp))); 1730 1731 while (len > 0) { 1732 mblk_t *mmp = NULL; 1733 1734 NXGE_DEBUG_MSG((NULL, TX_CTL, 1735 "nxge_do_softlso: (4) len > 0 available: %d, " 1736 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1737 available, left, len, segnum, (int)MBLKL(nmp))); 1738 1739 if (datamp->b_cont != NULL) { 1740 datamp = datamp->b_cont; 1741 rptr = datamp->b_rptr; 1742 mmp = dupb(datamp); 1743 if (mmp == NULL) { 1744 do_cleanup = B_TRUE; 1745 NXGE_DEBUG_MSG((NULL, TX_CTL, 1746 "==> nxge_do_softlso: " 1747 "Can not dupb(datamp) (1), :" 1748 "have to do clean up")); 1749 NXGE_DEBUG_MSG((NULL, TX_CTL, 1750 "==> nxge_do_softlso: " 1751 "available: %d, left: %d, " 1752 "len: %d, MBLKL(nmp): %d", 1753 available, left, len, 1754 (int)MBLKL(nmp))); 1755 goto cleanup_allocated_msgs; 1756 } 1757 } else { 1758 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1759 "==> nxge_do_softlso: " 1760 "(1)available: %d, left: %d, " 1761 "len: %d, MBLKL(nmp): %d", 1762 available, left, len, 1763 (int)MBLKL(nmp))); 1764 cmn_err(CE_PANIC, 1765 "==> nxge_do_softlso: " 1766 "Pointers must have been corrupted!\n" 1767 "datamp: $%p, nmp: $%p, rptr: $%p", 1768 (void *)datamp, 1769 (void *)nmp, 1770 (void *)rptr); 1771 } 1772 nmp->b_cont = mmp; 1773 nmp = mmp; 1774 len -= MBLKL(nmp); 1775 } 1776 if (len < 0) { 1777 nmp->b_wptr += len; 1778 rptr = nmp->b_wptr; 1779 NXGE_DEBUG_MSG((NULL, TX_CTL, 1780 "(5) len < 0 (less than 0)" 1781 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1782 available, left, len, (int)MBLKL(nmp))); 1783 1784 } else if (len == 0) { 1785 if (datamp->b_cont != NULL) { 1786 NXGE_DEBUG_MSG((NULL, TX_CTL, 1787 "(5) len == 0" 1788 "available: %d, left: %d, len: %d, " 1789 "MBLKL(nmp): %d", 1790 available, left, len, (int)MBLKL(nmp))); 1791 datamp = datamp->b_cont; 1792 rptr = datamp->b_rptr; 1793 } else { 1794 NXGE_DEBUG_MSG((NULL, TX_CTL, 1795 "(6)available b_cont == NULL : %d, " 1796 "left: %d, len: %d, MBLKL(nmp): %d", 1797 available, left, len, (int)MBLKL(nmp))); 1798 1799 VERIFY(cmp->b_next == NULL); 1800 VERIFY(left == 0); 1801 break; /* Done! */ 1802 } 1803 } 1804 cmp = cmp->b_next; 1805 1806 NXGE_DEBUG_MSG((NULL, TX_CTL, 1807 "(7) do_softlso: " 1808 "next mp in mp_chain available len != 0 : %d, " 1809 "left: %d, len: %d, MBLKL(nmp): %d", 1810 available, left, len, (int)MBLKL(nmp))); 1811 } 1812 1813 /* 1814 * From now, start to fill up all headers for the first message 1815 * Hardware checksum flags need to be updated separately for FULLCKSUM 1816 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1817 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1818 * required fields need to be updated properly. 1819 */ 1820 nmp = mp_chain; 1821 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1822 nmp->b_wptr = nmp->b_rptr + hdrlen; 1823 niph = (struct ip *)(nmp->b_rptr + ehlen); 1824 niph->ip_len = htons(mss + iphlen + tcphlen); 1825 ip_id = ntohs(niph->ip_id); 1826 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1827 #ifdef __sparc 1828 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1829 tcp_seq = ntohl(tcp_seq_tmp); 1830 #else 1831 tcp_seq = ntohl(ntcph->th_seq); 1832 #endif 1833 1834 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1835 1836 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1837 DB_CKSUMSTART(nmp) = start_offset; 1838 DB_CKSUMSTUFF(nmp) = stuff_offset; 1839 1840 /* calculate IP checksum and TCP pseudo header checksum */ 1841 niph->ip_sum = 0; 1842 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1843 1844 l4_len = mss + tcphlen; 1845 sum = htons(l4_len) + l4cksum; 1846 sum = (sum & 0xFFFF) + (sum >> 16); 1847 ntcph->th_sum = (sum & 0xffff); 1848 1849 NXGE_DEBUG_MSG((NULL, TX_CTL, 1850 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1851 "mss %d pktlen %d l4_len %d (0x%x) " 1852 "MBLKL(mp): %d, MBLKL(datamp): %d " 1853 "ip_sum 0x%x " 1854 "th_sum 0x%x sum 0x%x ) " 1855 "dump first ip->tcp %s", 1856 nmp, mp_chain, 1857 mss, 1858 pktlen, 1859 l4_len, 1860 l4_len, 1861 (int)MBLKL(mp), (int)MBLKL(datamp), 1862 niph->ip_sum, 1863 ntcph->th_sum, 1864 sum, 1865 nxge_dump_packet((char *)niph, 52))); 1866 1867 cmp = nmp; 1868 while ((nmp = nmp->b_next)->b_next != NULL) { 1869 NXGE_DEBUG_MSG((NULL, TX_CTL, 1870 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1871 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1872 nmp->b_wptr = nmp->b_rptr + hdrlen; 1873 niph = (struct ip *)(nmp->b_rptr + ehlen); 1874 niph->ip_id = htons(++ip_id); 1875 niph->ip_len = htons(mss + iphlen + tcphlen); 1876 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1877 tcp_seq += mss; 1878 1879 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1880 1881 #ifdef __sparc 1882 tcp_seq_tmp = htonl(tcp_seq); 1883 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1884 #else 1885 ntcph->th_seq = htonl(tcp_seq); 1886 #endif 1887 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1888 DB_CKSUMSTART(nmp) = start_offset; 1889 DB_CKSUMSTUFF(nmp) = stuff_offset; 1890 1891 /* calculate IP checksum and TCP pseudo header checksum */ 1892 niph->ip_sum = 0; 1893 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1894 ntcph->th_sum = (sum & 0xffff); 1895 1896 NXGE_DEBUG_MSG((NULL, TX_CTL, 1897 "==> nxge_do_softlso: middle ip_sum 0x%x " 1898 "th_sum 0x%x " 1899 " mp $%p (mp_chain $%p) pktlen %d " 1900 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1901 niph->ip_sum, 1902 ntcph->th_sum, 1903 nmp, mp_chain, 1904 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1905 } 1906 1907 /* Last segment */ 1908 /* 1909 * Set FIN and/or PSH flags if present only in the last packet. 1910 * The ip_len could be different from prior packets. 1911 */ 1912 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1913 nmp->b_wptr = nmp->b_rptr + hdrlen; 1914 niph = (struct ip *)(nmp->b_rptr + ehlen); 1915 niph->ip_id = htons(++ip_id); 1916 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1917 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1918 tcp_seq += mss; 1919 #ifdef __sparc 1920 tcp_seq_tmp = htonl(tcp_seq); 1921 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1922 #else 1923 ntcph->th_seq = htonl(tcp_seq); 1924 #endif 1925 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1926 1927 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1928 DB_CKSUMSTART(nmp) = start_offset; 1929 DB_CKSUMSTUFF(nmp) = stuff_offset; 1930 1931 /* calculate IP checksum and TCP pseudo header checksum */ 1932 niph->ip_sum = 0; 1933 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1934 1935 l4_len = ntohs(niph->ip_len) - iphlen; 1936 sum = htons(l4_len) + l4cksum; 1937 sum = (sum & 0xFFFF) + (sum >> 16); 1938 ntcph->th_sum = (sum & 0xffff); 1939 1940 NXGE_DEBUG_MSG((NULL, TX_CTL, 1941 "==> nxge_do_softlso: last next " 1942 "niph->ip_sum 0x%x " 1943 "ntcph->th_sum 0x%x sum 0x%x " 1944 "dump last ip->tcp %s " 1945 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1946 "l4_len %d (0x%x) " 1947 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1948 niph->ip_sum, 1949 ntcph->th_sum, sum, 1950 nxge_dump_packet((char *)niph, 52), 1951 cmp, nmp, mp_chain, 1952 pktlen, pktlen, 1953 l4_len, 1954 l4_len, 1955 (int)MBLKL(mp), (int)MBLKL(datamp))); 1956 1957 cleanup_allocated_msgs: 1958 if (do_cleanup) { 1959 NXGE_DEBUG_MSG((NULL, TX_CTL, 1960 "==> nxge_do_softlso: " 1961 "Failed allocating messages, " 1962 "have to clean up and fail!")); 1963 while (mp_chain != NULL) { 1964 nmp = mp_chain; 1965 mp_chain = mp_chain->b_next; 1966 freemsg(nmp); 1967 } 1968 } 1969 /* 1970 * We're done here, so just free the original message and return the 1971 * new message chain, that could be NULL if failed, back to the caller. 1972 */ 1973 freemsg(mp); 1974 1975 NXGE_DEBUG_MSG((NULL, TX_CTL, 1976 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1977 return (mp_chain); 1978 } 1979 1980 /* 1981 * Will be called before NIC driver do further operation on the message. 1982 * The input message may include LSO information, if so, go to softlso logic 1983 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1984 * The return could be the same non-LSO message or a message chain for LSO case. 1985 * 1986 * The driver needs to call this function per packet and process the whole chain 1987 * if applied. 1988 */ 1989 static mblk_t * 1990 nxge_lso_eliminate(mblk_t *mp) 1991 { 1992 uint32_t lsoflags; 1993 uint32_t mss; 1994 1995 NXGE_DEBUG_MSG((NULL, TX_CTL, 1996 "==>nxge_lso_eliminate:")); 1997 nxge_lso_info_get(mp, &mss, &lsoflags); 1998 1999 if (lsoflags & HW_LSO) { 2000 mblk_t *nmp; 2001 2002 NXGE_DEBUG_MSG((NULL, TX_CTL, 2003 "==>nxge_lso_eliminate:" 2004 "HW_LSO:mss %d mp $%p", 2005 mss, mp)); 2006 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 2007 NXGE_DEBUG_MSG((NULL, TX_CTL, 2008 "<== nxge_lso_eliminate: " 2009 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 2010 nmp, mss, mp)); 2011 return (nmp); 2012 } else { 2013 NXGE_DEBUG_MSG((NULL, TX_CTL, 2014 "<== nxge_lso_eliminate_ " 2015 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 2016 nmp, mss, mp)); 2017 return (NULL); 2018 } 2019 } 2020 2021 NXGE_DEBUG_MSG((NULL, TX_CTL, 2022 "<== nxge_lso_eliminate")); 2023 return (mp); 2024 } 2025 2026 static uint32_t 2027 nxge_csgen(uint16_t *adr, int len) 2028 { 2029 int i, odd; 2030 uint32_t sum = 0; 2031 uint32_t c = 0; 2032 2033 odd = len % 2; 2034 for (i = 0; i < (len / 2); i++) { 2035 sum += (adr[i] & 0xffff); 2036 } 2037 if (odd) { 2038 sum += adr[len / 2] & 0xff00; 2039 } 2040 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 2041 sum &= 0xffff; 2042 sum += c; 2043 } 2044 return (~sum & 0xffff); 2045 } 2046