1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/nxge/nxge_impl.h> 29 30 extern uint32_t nxge_reclaim_pending; 31 extern uint32_t nxge_bcopy_thresh; 32 extern uint32_t nxge_dvma_thresh; 33 extern uint32_t nxge_dma_stream_thresh; 34 extern uint32_t nxge_tx_minfree; 35 extern uint32_t nxge_tx_intr_thres; 36 extern uint32_t nxge_tx_max_gathers; 37 extern uint32_t nxge_tx_tiny_pack; 38 extern uint32_t nxge_tx_use_bcopy; 39 extern uint32_t nxge_tx_lb_policy; 40 extern uint32_t nxge_no_tx_lb; 41 extern nxge_tx_mode_t nxge_tx_scheme; 42 43 typedef struct _mac_tx_hint { 44 uint16_t sap; 45 uint16_t vid; 46 void *hash; 47 } mac_tx_hint_t, *p_mac_tx_hint_t; 48 49 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 50 51 int 52 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 53 { 54 int status = 0; 55 p_tx_desc_t tx_desc_ring_vp; 56 npi_handle_t npi_desc_handle; 57 nxge_os_dma_handle_t tx_desc_dma_handle; 58 p_tx_desc_t tx_desc_p; 59 p_tx_msg_t tx_msg_ring; 60 p_tx_msg_t tx_msg_p; 61 tx_desc_t tx_desc, *tmp_desc_p; 62 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 63 p_tx_pkt_header_t hdrp; 64 p_tx_pkt_hdr_all_t pkthdrp; 65 uint8_t npads = 0; 66 uint64_t dma_ioaddr; 67 uint32_t dma_flags; 68 int last_bidx; 69 uint8_t *b_rptr; 70 caddr_t kaddr; 71 uint32_t nmblks; 72 uint32_t ngathers; 73 uint32_t clen; 74 int len; 75 uint32_t pkt_len, pack_len, min_len; 76 uint32_t bcopy_thresh; 77 int i, cur_index, sop_index; 78 uint16_t tail_index; 79 boolean_t tail_wrap = B_FALSE; 80 nxge_dma_common_t desc_area; 81 nxge_os_dma_handle_t dma_handle; 82 ddi_dma_cookie_t dma_cookie; 83 npi_handle_t npi_handle; 84 p_mblk_t nmp; 85 p_mblk_t t_mp; 86 uint32_t ncookies; 87 boolean_t good_packet; 88 boolean_t mark_mode = B_FALSE; 89 p_nxge_stats_t statsp; 90 p_nxge_tx_ring_stats_t tdc_stats; 91 t_uscalar_t start_offset = 0; 92 t_uscalar_t stuff_offset = 0; 93 t_uscalar_t end_offset = 0; 94 t_uscalar_t value = 0; 95 t_uscalar_t cksum_flags = 0; 96 boolean_t cksum_on = B_FALSE; 97 uint32_t boff = 0; 98 uint64_t tot_xfer_len = 0, tmp_len = 0; 99 boolean_t header_set = B_FALSE; 100 #ifdef NXGE_DEBUG 101 p_tx_desc_t tx_desc_ring_pp; 102 p_tx_desc_t tx_desc_pp; 103 tx_desc_t *save_desc_p; 104 int dump_len; 105 int sad_len; 106 uint64_t sad; 107 int xfer_len; 108 uint32_t msgsize; 109 #endif 110 111 NXGE_DEBUG_MSG((nxgep, TX_CTL, 112 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 113 NXGE_DEBUG_MSG((nxgep, TX_CTL, 114 "==> nxge_start: Starting tdc %d desc pending %d", 115 tx_ring_p->tdc, tx_ring_p->descs_pending)); 116 117 statsp = nxgep->statsp; 118 119 if (nxgep->statsp->port_stats.lb_mode == nxge_lb_normal) { 120 if (!statsp->mac_stats.link_up) { 121 freemsg(mp); 122 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 123 "link not up or LB mode")); 124 goto nxge_start_fail1; 125 } 126 } 127 128 hcksum_retrieve(mp, NULL, NULL, &start_offset, 129 &stuff_offset, &end_offset, &value, &cksum_flags); 130 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 131 start_offset += sizeof (ether_header_t); 132 stuff_offset += sizeof (ether_header_t); 133 } else { 134 start_offset += sizeof (struct ether_vlan_header); 135 stuff_offset += sizeof (struct ether_vlan_header); 136 } 137 138 if (cksum_flags & HCK_PARTIALCKSUM) { 139 NXGE_DEBUG_MSG((nxgep, TX_CTL, 140 "==> nxge_start: cksum_flags 0x%x (partial checksum) ", 141 cksum_flags)); 142 cksum_on = B_TRUE; 143 } 144 145 #ifdef NXGE_DEBUG 146 if (tx_ring_p->descs_pending) { 147 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 148 "desc pending %d ", tx_ring_p->descs_pending)); 149 } 150 151 dump_len = (int)(MBLKL(mp)); 152 dump_len = (dump_len > 128) ? 128: dump_len; 153 154 NXGE_DEBUG_MSG((nxgep, TX_CTL, 155 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 156 "(Before header reserve: ORIGINAL LEN %d)", 157 tx_ring_p->tdc, 158 mp->b_rptr, 159 dump_len)); 160 161 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 162 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 163 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 164 #endif 165 166 MUTEX_ENTER(&tx_ring_p->lock); 167 tdc_stats = tx_ring_p->tdc_stats; 168 mark_mode = (tx_ring_p->descs_pending && 169 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 170 < nxge_tx_minfree)); 171 172 NXGE_DEBUG_MSG((nxgep, TX_CTL, 173 "TX Descriptor ring is channel %d mark mode %d", 174 tx_ring_p->tdc, mark_mode)); 175 176 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { 177 NXGE_DEBUG_MSG((nxgep, TX_CTL, 178 "TX Descriptor ring is full: channel %d", 179 tx_ring_p->tdc)); 180 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 181 tdc_stats->tx_no_desc++; 182 MUTEX_EXIT(&tx_ring_p->lock); 183 if (nxgep->resched_needed && !nxgep->resched_running) { 184 nxgep->resched_running = B_TRUE; 185 ddi_trigger_softintr(nxgep->resched_id); 186 } 187 status = 1; 188 goto nxge_start_fail1; 189 } 190 191 nmp = mp; 192 i = sop_index = tx_ring_p->wr_index; 193 nmblks = 0; 194 ngathers = 0; 195 pkt_len = 0; 196 pack_len = 0; 197 clen = 0; 198 last_bidx = -1; 199 good_packet = B_TRUE; 200 201 desc_area = tx_ring_p->tdc_desc; 202 npi_handle = desc_area.npi_handle; 203 npi_desc_handle.regh = (nxge_os_acc_handle_t) 204 DMA_COMMON_ACC_HANDLE(desc_area); 205 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 206 #ifdef NXGE_DEBUG 207 tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area); 208 #endif 209 tx_desc_dma_handle = (nxge_os_dma_handle_t) 210 DMA_COMMON_HANDLE(desc_area); 211 tx_msg_ring = tx_ring_p->tx_msg_ring; 212 213 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 214 sop_index, i)); 215 216 #ifdef NXGE_DEBUG 217 msgsize = msgdsize(nmp); 218 NXGE_DEBUG_MSG((nxgep, TX_CTL, 219 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 220 sop_index, i, msgsize)); 221 #endif 222 /* 223 * The first 16 bytes of the premapped buffer are reserved 224 * for header. No padding will be used. 225 */ 226 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 227 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 228 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 229 } else { 230 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 231 } 232 while (nmp) { 233 good_packet = B_TRUE; 234 b_rptr = nmp->b_rptr; 235 len = MBLKL(nmp); 236 if (len <= 0) { 237 nmp = nmp->b_cont; 238 continue; 239 } 240 nmblks++; 241 242 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 243 "len %d pkt_len %d pack_len %d", 244 nmblks, len, pkt_len, pack_len)); 245 /* 246 * Hardware limits the transfer length to 4K for NIU and 247 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 248 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 249 * If len is longer than the limit, then we break nmp into 250 * two chunks: Make the first chunk equal to the limit and 251 * the second chunk for the remaining data. If the second 252 * chunk is still larger than the limit, then it will be 253 * broken into two in the next pass. 254 */ 255 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 256 if ((t_mp = dupb(nmp)) != NULL) { 257 nmp->b_wptr = nmp->b_rptr + 258 (TX_MAX_TRANSFER_LENGTH 259 - TX_PKT_HEADER_SIZE); 260 t_mp->b_rptr = nmp->b_wptr; 261 t_mp->b_cont = nmp->b_cont; 262 nmp->b_cont = t_mp; 263 len = MBLKL(nmp); 264 } else { 265 good_packet = B_FALSE; 266 goto nxge_start_fail2; 267 } 268 } 269 tx_desc.value = 0; 270 tx_desc_p = &tx_desc_ring_vp[i]; 271 #ifdef NXGE_DEBUG 272 tx_desc_pp = &tx_desc_ring_pp[i]; 273 #endif 274 tx_msg_p = &tx_msg_ring[i]; 275 #if defined(__i386) 276 npi_desc_handle.regp = (uint32_t)tx_desc_p; 277 #else 278 npi_desc_handle.regp = (uint64_t)tx_desc_p; 279 #endif 280 if (!header_set && 281 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 282 (len >= bcopy_thresh))) { 283 header_set = B_TRUE; 284 bcopy_thresh += TX_PKT_HEADER_SIZE; 285 boff = 0; 286 pack_len = 0; 287 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 288 hdrp = (p_tx_pkt_header_t)kaddr; 289 clen = pkt_len; 290 dma_handle = tx_msg_p->buf_dma_handle; 291 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 292 (void) ddi_dma_sync(dma_handle, 293 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 294 DDI_DMA_SYNC_FORDEV); 295 296 tx_msg_p->flags.dma_type = USE_BCOPY; 297 goto nxge_start_control_header_only; 298 } 299 300 pkt_len += len; 301 pack_len += len; 302 303 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 304 "desc entry %d " 305 "DESC IOADDR $%p " 306 "desc_vp $%p tx_desc_p $%p " 307 "desc_pp $%p tx_desc_pp $%p " 308 "len %d pkt_len %d pack_len %d", 309 i, 310 DMA_COMMON_IOADDR(desc_area), 311 tx_desc_ring_vp, tx_desc_p, 312 tx_desc_ring_pp, tx_desc_pp, 313 len, pkt_len, pack_len)); 314 315 if (len < bcopy_thresh) { 316 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 317 "USE BCOPY: ")); 318 if (nxge_tx_tiny_pack) { 319 uint32_t blst = 320 TXDMA_DESC_NEXT_INDEX(i, -1, 321 tx_ring_p->tx_wrap_mask); 322 NXGE_DEBUG_MSG((nxgep, TX_CTL, 323 "==> nxge_start(5): pack")); 324 if ((pack_len <= bcopy_thresh) && 325 (last_bidx == blst)) { 326 NXGE_DEBUG_MSG((nxgep, TX_CTL, 327 "==> nxge_start: pack(6) " 328 "(pkt_len %d pack_len %d)", 329 pkt_len, pack_len)); 330 i = blst; 331 tx_desc_p = &tx_desc_ring_vp[i]; 332 #ifdef NXGE_DEBUG 333 tx_desc_pp = &tx_desc_ring_pp[i]; 334 #endif 335 tx_msg_p = &tx_msg_ring[i]; 336 boff = pack_len - len; 337 ngathers--; 338 } else if (pack_len > bcopy_thresh && 339 header_set) { 340 pack_len = len; 341 boff = 0; 342 bcopy_thresh = nxge_bcopy_thresh; 343 NXGE_DEBUG_MSG((nxgep, TX_CTL, 344 "==> nxge_start(7): > max NEW " 345 "bcopy thresh %d " 346 "pkt_len %d pack_len %d(next)", 347 bcopy_thresh, 348 pkt_len, pack_len)); 349 } 350 last_bidx = i; 351 } 352 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 353 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 354 hdrp = (p_tx_pkt_header_t)kaddr; 355 header_set = B_TRUE; 356 NXGE_DEBUG_MSG((nxgep, TX_CTL, 357 "==> nxge_start(7_x2): " 358 "pkt_len %d pack_len %d (new hdrp $%p)", 359 pkt_len, pack_len, hdrp)); 360 } 361 tx_msg_p->flags.dma_type = USE_BCOPY; 362 kaddr += boff; 363 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 364 "USE BCOPY: before bcopy " 365 "DESC IOADDR $%p entry %d " 366 "bcopy packets %d " 367 "bcopy kaddr $%p " 368 "bcopy ioaddr (SAD) $%p " 369 "bcopy clen %d " 370 "bcopy boff %d", 371 DMA_COMMON_IOADDR(desc_area), i, 372 tdc_stats->tx_hdr_pkts, 373 kaddr, 374 dma_ioaddr, 375 clen, 376 boff)); 377 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 378 "1USE BCOPY: ")); 379 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 380 "2USE BCOPY: ")); 381 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 382 "last USE BCOPY: copy from b_rptr $%p " 383 "to KADDR $%p (len %d offset %d", 384 b_rptr, kaddr, len, boff)); 385 386 bcopy(b_rptr, kaddr, len); 387 388 #ifdef NXGE_DEBUG 389 dump_len = (len > 128) ? 128: len; 390 NXGE_DEBUG_MSG((nxgep, TX_CTL, 391 "==> nxge_start: dump packets " 392 "(After BCOPY len %d)" 393 "(b_rptr $%p): %s", len, nmp->b_rptr, 394 nxge_dump_packet((char *)nmp->b_rptr, 395 dump_len))); 396 #endif 397 398 dma_handle = tx_msg_p->buf_dma_handle; 399 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 400 (void) ddi_dma_sync(dma_handle, 401 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 402 DDI_DMA_SYNC_FORDEV); 403 clen = len + boff; 404 tdc_stats->tx_hdr_pkts++; 405 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 406 "USE BCOPY: " 407 "DESC IOADDR $%p entry %d " 408 "bcopy packets %d " 409 "bcopy kaddr $%p " 410 "bcopy ioaddr (SAD) $%p " 411 "bcopy clen %d " 412 "bcopy boff %d", 413 DMA_COMMON_IOADDR(desc_area), 414 i, 415 tdc_stats->tx_hdr_pkts, 416 kaddr, 417 dma_ioaddr, 418 clen, 419 boff)); 420 } else { 421 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 422 "USE DVMA: len %d", len)); 423 tx_msg_p->flags.dma_type = USE_DMA; 424 dma_flags = DDI_DMA_WRITE; 425 if (len < nxge_dma_stream_thresh) { 426 dma_flags |= DDI_DMA_CONSISTENT; 427 } else { 428 dma_flags |= DDI_DMA_STREAMING; 429 } 430 431 dma_handle = tx_msg_p->dma_handle; 432 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 433 (caddr_t)b_rptr, len, dma_flags, 434 DDI_DMA_DONTWAIT, NULL, 435 &dma_cookie, &ncookies); 436 if (status == DDI_DMA_MAPPED) { 437 dma_ioaddr = dma_cookie.dmac_laddress; 438 len = (int)dma_cookie.dmac_size; 439 clen = (uint32_t)dma_cookie.dmac_size; 440 NXGE_DEBUG_MSG((nxgep, TX_CTL, 441 "==> nxge_start(12_1): " 442 "USE DVMA: len %d clen %d " 443 "ngathers %d", 444 len, clen, 445 ngathers)); 446 #if defined(__i386) 447 npi_desc_handle.regp = (uint32_t)tx_desc_p; 448 #else 449 npi_desc_handle.regp = (uint64_t)tx_desc_p; 450 #endif 451 while (ncookies > 1) { 452 ngathers++; 453 /* 454 * this is the fix for multiple 455 * cookies, which are basicaly 456 * a descriptor entry, we don't set 457 * SOP bit as well as related fields 458 */ 459 460 (void) npi_txdma_desc_gather_set( 461 npi_desc_handle, 462 &tx_desc, 463 (ngathers -1), 464 mark_mode, 465 ngathers, 466 dma_ioaddr, 467 clen); 468 469 tx_msg_p->tx_msg_size = clen; 470 NXGE_DEBUG_MSG((nxgep, TX_CTL, 471 "==> nxge_start: DMA " 472 "ncookie %d " 473 "ngathers %d " 474 "dma_ioaddr $%p len %d" 475 "desc $%p descp $%p (%d)", 476 ncookies, 477 ngathers, 478 dma_ioaddr, clen, 479 *tx_desc_p, tx_desc_p, i)); 480 481 ddi_dma_nextcookie(dma_handle, 482 &dma_cookie); 483 dma_ioaddr = 484 dma_cookie.dmac_laddress; 485 486 len = (int)dma_cookie.dmac_size; 487 clen = (uint32_t)dma_cookie.dmac_size; 488 NXGE_DEBUG_MSG((nxgep, TX_CTL, 489 "==> nxge_start(12_2): " 490 "USE DVMA: len %d clen %d ", 491 len, clen)); 492 493 i = TXDMA_DESC_NEXT_INDEX(i, 1, 494 tx_ring_p->tx_wrap_mask); 495 tx_desc_p = &tx_desc_ring_vp[i]; 496 497 npi_desc_handle.regp = 498 #if defined(__i386) 499 (uint32_t)tx_desc_p; 500 #else 501 (uint64_t)tx_desc_p; 502 #endif 503 tx_msg_p = &tx_msg_ring[i]; 504 tx_msg_p->flags.dma_type = USE_NONE; 505 tx_desc.value = 0; 506 507 ncookies--; 508 } 509 tdc_stats->tx_ddi_pkts++; 510 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 511 "DMA: ddi packets %d", 512 tdc_stats->tx_ddi_pkts)); 513 } else { 514 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 515 "dma mapping failed for %d " 516 "bytes addr $%p flags %x (%d)", 517 len, b_rptr, status, status)); 518 good_packet = B_FALSE; 519 tdc_stats->tx_dma_bind_fail++; 520 tx_msg_p->flags.dma_type = USE_NONE; 521 goto nxge_start_fail2; 522 } 523 } /* ddi dvma */ 524 525 nmp = nmp->b_cont; 526 nxge_start_control_header_only: 527 #if defined(__i386) 528 npi_desc_handle.regp = (uint32_t)tx_desc_p; 529 #else 530 npi_desc_handle.regp = (uint64_t)tx_desc_p; 531 #endif 532 ngathers++; 533 534 if (ngathers == 1) { 535 #ifdef NXGE_DEBUG 536 save_desc_p = &sop_tx_desc; 537 #endif 538 sop_tx_desc_p = &sop_tx_desc; 539 sop_tx_desc_p->value = 0; 540 sop_tx_desc_p->bits.hdw.tr_len = clen; 541 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 542 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 543 } else { 544 #ifdef NXGE_DEBUG 545 save_desc_p = &tx_desc; 546 #endif 547 tmp_desc_p = &tx_desc; 548 tmp_desc_p->value = 0; 549 tmp_desc_p->bits.hdw.tr_len = clen; 550 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 551 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 552 553 tx_desc_p->value = tmp_desc_p->value; 554 } 555 556 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 557 "Desc_entry %d ngathers %d " 558 "desc_vp $%p tx_desc_p $%p " 559 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 560 "dma_ioaddr (SAD) $%p mark %d", 561 i, ngathers, 562 tx_desc_ring_vp, tx_desc_p, 563 len, clen, pkt_len, pack_len, nmblks, 564 dma_ioaddr, mark_mode)); 565 566 #ifdef NXGE_DEBUG 567 npi_desc_handle.nxgep = nxgep; 568 npi_desc_handle.function.function = nxgep->function_num; 569 npi_desc_handle.function.instance = nxgep->instance; 570 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 571 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 572 TX_PKT_DESC_TR_LEN_SHIFT); 573 574 575 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 576 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 577 "mark %d sop %d\n", 578 save_desc_p->value, 579 sad, 580 save_desc_p->bits.hdw.tr_len, 581 xfer_len, 582 save_desc_p->bits.hdw.num_ptr, 583 save_desc_p->bits.hdw.mark, 584 save_desc_p->bits.hdw.sop)); 585 586 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 587 #endif 588 589 tx_msg_p->tx_msg_size = clen; 590 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 591 if (ngathers > nxge_tx_max_gathers) { 592 good_packet = B_FALSE; 593 hcksum_retrieve(mp, NULL, NULL, &start_offset, 594 &stuff_offset, &end_offset, &value, 595 &cksum_flags); 596 597 NXGE_DEBUG_MSG((NULL, TX_CTL, 598 "==> nxge_start(14): pull msg - " 599 "len %d pkt_len %d ngathers %d", 600 len, pkt_len, ngathers)); 601 /* Pull all message blocks from b_cont */ 602 if ((msgpullup(mp, -1)) == NULL) { 603 goto nxge_start_fail2; 604 } 605 goto nxge_start_fail2; 606 } 607 } /* while (nmp) */ 608 609 tx_msg_p->tx_message = mp; 610 tx_desc_p = &tx_desc_ring_vp[sop_index]; 611 #if defined(__i386) 612 npi_desc_handle.regp = (uint32_t)tx_desc_p; 613 #else 614 npi_desc_handle.regp = (uint64_t)tx_desc_p; 615 #endif 616 617 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 618 pkthdrp->reserved = 0; 619 hdrp->value = 0; 620 (void) nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 621 (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp); 622 623 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 624 tdc_stats->tx_jumbo_pkts++; 625 } 626 627 min_len = (nxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2)); 628 if (pkt_len < min_len) { 629 /* Assume we use bcopy to premapped buffers */ 630 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 631 NXGE_DEBUG_MSG((NULL, TX_CTL, 632 "==> nxge_start(14-1): < (msg_min + 16)" 633 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 634 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 635 bzero((kaddr + pkt_len), (min_len - pkt_len)); 636 pkt_len = tx_msg_p->tx_msg_size = min_len; 637 638 sop_tx_desc_p->bits.hdw.tr_len = min_len; 639 640 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 641 tx_desc_p->value = sop_tx_desc_p->value; 642 643 NXGE_DEBUG_MSG((NULL, TX_CTL, 644 "==> nxge_start(14-2): < msg_min - " 645 "len %d pkt_len %d min_len %d ngathers %d", 646 len, pkt_len, min_len, ngathers)); 647 } 648 649 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 650 cksum_flags)); 651 if (cksum_flags & HCK_PARTIALCKSUM) { 652 NXGE_DEBUG_MSG((nxgep, TX_CTL, 653 "==> nxge_start: cksum_flags 0x%x (partial checksum) ", 654 cksum_flags)); 655 cksum_on = B_TRUE; 656 NXGE_DEBUG_MSG((nxgep, TX_CTL, 657 "==> nxge_start: from IP cksum_flags 0x%x " 658 "(partial checksum) " 659 "start_offset %d stuff_offset %d", 660 cksum_flags, start_offset, stuff_offset)); 661 tmp_len = (uint64_t)(start_offset >> 1); 662 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT); 663 tmp_len = (uint64_t)(stuff_offset >> 1); 664 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT); 665 666 NXGE_DEBUG_MSG((nxgep, TX_CTL, 667 "==> nxge_start: from IP cksum_flags 0x%x " 668 "(partial checksum) " 669 "after SHIFT start_offset %d stuff_offset %d", 670 cksum_flags, start_offset, stuff_offset)); 671 } 672 { 673 uint64_t tmp_len; 674 675 /* pkt_len already includes 16 + paddings!! */ 676 /* Update the control header length */ 677 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 678 tmp_len = hdrp->value | 679 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 680 681 NXGE_DEBUG_MSG((nxgep, TX_CTL, 682 "==> nxge_start(15_x1): setting SOP " 683 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 684 "0x%llx hdrp->value 0x%llx", 685 tot_xfer_len, tot_xfer_len, pkt_len, 686 tmp_len, hdrp->value)); 687 #if defined(_BIG_ENDIAN) 688 hdrp->value = ddi_swap64(tmp_len); 689 #else 690 hdrp->value = tmp_len; 691 #endif 692 NXGE_DEBUG_MSG((nxgep, 693 TX_CTL, "==> nxge_start(15_x2): setting SOP " 694 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 695 "tmp_len 0x%llx hdrp->value 0x%llx", 696 tot_xfer_len, pkt_len, 697 tmp_len, hdrp->value)); 698 } 699 700 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 701 "wr_index %d " 702 "tot_xfer_len (%d) pkt_len %d npads %d", 703 sop_index, 704 tot_xfer_len, pkt_len, 705 npads)); 706 707 sop_tx_desc_p->bits.hdw.sop = 1; 708 sop_tx_desc_p->bits.hdw.mark = mark_mode; 709 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 710 711 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 712 713 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 714 715 #ifdef NXGE_DEBUG 716 npi_desc_handle.nxgep = nxgep; 717 npi_desc_handle.function.function = nxgep->function_num; 718 npi_desc_handle.function.instance = nxgep->instance; 719 720 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 721 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 722 save_desc_p->value, 723 sad, 724 save_desc_p->bits.hdw.tr_len, 725 xfer_len, 726 save_desc_p->bits.hdw.num_ptr, 727 save_desc_p->bits.hdw.mark, 728 save_desc_p->bits.hdw.sop)); 729 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 730 731 dump_len = (pkt_len > 128) ? 128: pkt_len; 732 NXGE_DEBUG_MSG((nxgep, TX_CTL, 733 "==> nxge_start: dump packets(17) (after sop set, len " 734 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 735 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 736 (char *)hdrp, 737 nxge_dump_packet((char *)hdrp, dump_len))); 738 NXGE_DEBUG_MSG((nxgep, TX_CTL, 739 "==> nxge_start(18): TX desc sync: sop_index %d", 740 sop_index)); 741 #endif 742 743 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 744 (void) ddi_dma_sync(tx_desc_dma_handle, 745 sop_index * sizeof (tx_desc_t), 746 ngathers * sizeof (tx_desc_t), 747 DDI_DMA_SYNC_FORDEV); 748 749 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 750 "cs_off = 0x%02X cs_s_off = 0x%02X " 751 "pkt_len %d ngathers %d sop_index %d\n", 752 stuff_offset, start_offset, 753 pkt_len, ngathers, sop_index)); 754 } else { /* more than one descriptor and wrap around */ 755 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 756 (void) ddi_dma_sync(tx_desc_dma_handle, 757 sop_index * sizeof (tx_desc_t), 758 nsdescs * sizeof (tx_desc_t), 759 DDI_DMA_SYNC_FORDEV); 760 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 761 "cs_off = 0x%02X cs_s_off = 0x%02X " 762 "pkt_len %d ngathers %d sop_index %d\n", 763 stuff_offset, start_offset, 764 pkt_len, ngathers, sop_index)); 765 766 (void) ddi_dma_sync(tx_desc_dma_handle, 767 0, 768 (ngathers - nsdescs) * sizeof (tx_desc_t), 769 DDI_DMA_SYNC_FORDEV); 770 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 771 "cs_off = 0x%02X cs_s_off = 0x%02X " 772 "pkt_len %d ngathers %d sop_index %d\n", 773 stuff_offset, start_offset, 774 pkt_len, ngathers, sop_index)); 775 } 776 777 tail_index = tx_ring_p->wr_index; 778 tail_wrap = tx_ring_p->wr_index_wrap; 779 780 tx_ring_p->wr_index = i; 781 if (tx_ring_p->wr_index <= tail_index) { 782 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 783 B_FALSE : B_TRUE); 784 } 785 786 tx_ring_p->descs_pending += ngathers; 787 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 788 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 789 tx_ring_p->tdc, 790 tx_ring_p->wr_index, 791 tx_ring_p->wr_index_wrap, 792 ngathers, 793 tx_ring_p->descs_pending)); 794 795 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 796 797 { 798 tx_ring_kick_t kick; 799 800 kick.value = 0; 801 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 802 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 803 804 /* Kick start the Transmit kick register */ 805 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 806 TX_RING_KICK_REG, 807 (uint8_t)tx_ring_p->tdc, 808 kick.value); 809 } 810 811 tdc_stats->tx_starts++; 812 813 MUTEX_EXIT(&tx_ring_p->lock); 814 815 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 816 817 return (status); 818 819 nxge_start_fail2: 820 if (good_packet == B_FALSE) { 821 cur_index = sop_index; 822 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 823 for (i = 0; i < ngathers; i++) { 824 tx_desc_p = &tx_desc_ring_vp[cur_index]; 825 #if defined(__i386) 826 npi_handle.regp = (uint32_t)tx_desc_p; 827 #else 828 npi_handle.regp = (uint64_t)tx_desc_p; 829 #endif 830 tx_msg_p = &tx_msg_ring[cur_index]; 831 (void) npi_txdma_desc_set_zero(npi_handle, 1); 832 if (tx_msg_p->flags.dma_type == USE_DVMA) { 833 NXGE_DEBUG_MSG((nxgep, TX_CTL, 834 "tx_desc_p = %X index = %d", 835 tx_desc_p, tx_ring_p->rd_index)); 836 (void) dvma_unload(tx_msg_p->dvma_handle, 837 0, -1); 838 tx_msg_p->dvma_handle = NULL; 839 if (tx_ring_p->dvma_wr_index == 840 tx_ring_p->dvma_wrap_mask) 841 tx_ring_p->dvma_wr_index = 0; 842 else 843 tx_ring_p->dvma_wr_index++; 844 tx_ring_p->dvma_pending--; 845 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 846 if (ddi_dma_unbind_handle( 847 tx_msg_p->dma_handle)) { 848 cmn_err(CE_WARN, "!nxge_start: " 849 "ddi_dma_unbind_handle failed"); 850 } 851 } 852 tx_msg_p->flags.dma_type = USE_NONE; 853 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 854 tx_ring_p->tx_wrap_mask); 855 856 } 857 858 nxgep->resched_needed = B_TRUE; 859 } 860 861 MUTEX_EXIT(&tx_ring_p->lock); 862 863 nxge_start_fail1: 864 /* Add FMA to check the access handle nxge_hregh */ 865 866 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 867 868 return (status); 869 } 870 871 int 872 nxge_serial_tx(mblk_t *mp, void *arg) 873 { 874 p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 875 p_nxge_t nxgep = tx_ring_p->nxgep; 876 877 return (nxge_start(nxgep, tx_ring_p, mp)); 878 } 879 880 boolean_t 881 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 882 { 883 p_tx_ring_t *tx_rings; 884 uint8_t ring_index; 885 p_tx_ring_t tx_ring_p; 886 887 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 888 889 ASSERT(mp->b_next == NULL); 890 891 ring_index = nxge_tx_lb_ring_1(mp, nxgep->max_tdcs, hp); 892 tx_rings = nxgep->tx_rings->rings; 893 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: tx_rings $%p", 894 tx_rings)); 895 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: max_tdcs %d " 896 "ring_index %d", nxgep->max_tdcs, ring_index)); 897 898 switch (nxge_tx_scheme) { 899 case NXGE_USE_START: 900 if (nxge_start(nxgep, tx_rings[ring_index], mp)) { 901 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 902 "ring index %d", ring_index)); 903 return (B_FALSE); 904 } 905 break; 906 907 case NXGE_USE_SERIAL: 908 default: 909 tx_ring_p = tx_rings[ring_index]; 910 nxge_serialize_enter(tx_ring_p->serial, mp); 911 break; 912 } 913 914 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 915 ring_index)); 916 917 return (B_TRUE); 918 } 919 920 /* 921 * nxge_m_tx() - send a chain of packets 922 */ 923 mblk_t * 924 nxge_m_tx(void *arg, mblk_t *mp) 925 { 926 p_nxge_t nxgep = (p_nxge_t)arg; 927 mblk_t *next; 928 mac_tx_hint_t hint; 929 930 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 931 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 932 "==> nxge_m_tx: hardware not initialized")); 933 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 934 "<== nxge_m_tx")); 935 return (mp); 936 } 937 938 hint.hash = NULL; 939 hint.vid = 0; 940 hint.sap = 0; 941 942 while (mp != NULL) { 943 next = mp->b_next; 944 mp->b_next = NULL; 945 946 /* 947 * Until Nemo tx resource works, the mac driver 948 * does the load balancing based on TCP port, 949 * or CPU. For debugging, we use a system 950 * configurable parameter. 951 */ 952 if (!nxge_send(nxgep, mp, &hint)) { 953 mp->b_next = next; 954 break; 955 } 956 957 mp = next; 958 } 959 960 return (mp); 961 } 962 963 int 964 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 965 { 966 uint8_t ring_index = 0; 967 uint8_t *tcp_port; 968 p_mblk_t nmp; 969 size_t mblk_len; 970 size_t iph_len; 971 size_t hdrs_size; 972 uint8_t hdrs_buf[sizeof (struct ether_header) + 973 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 974 /* 975 * allocate space big enough to cover 976 * the max ip header length and the first 977 * 4 bytes of the TCP/IP header. 978 */ 979 980 boolean_t qos = B_FALSE; 981 982 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 983 984 if (hp->vid) { 985 qos = B_TRUE; 986 } 987 switch (nxge_tx_lb_policy) { 988 case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 989 default: 990 tcp_port = mp->b_rptr; 991 if (!nxge_no_tx_lb && !qos && 992 (ntohs(((p_ether_header_t)tcp_port)->ether_type) 993 == ETHERTYPE_IP)) { 994 nmp = mp; 995 mblk_len = MBLKL(nmp); 996 tcp_port = NULL; 997 if (mblk_len > sizeof (struct ether_header) + 998 sizeof (uint8_t)) { 999 tcp_port = nmp->b_rptr + 1000 sizeof (struct ether_header); 1001 mblk_len -= sizeof (struct ether_header); 1002 iph_len = ((*tcp_port) & 0x0f) << 2; 1003 if (mblk_len > (iph_len + sizeof (uint32_t))) { 1004 tcp_port = nmp->b_rptr; 1005 } else { 1006 tcp_port = NULL; 1007 } 1008 } 1009 if (tcp_port == NULL) { 1010 hdrs_size = 0; 1011 ((p_ether_header_t)hdrs_buf)->ether_type = 0; 1012 while ((nmp) && (hdrs_size < 1013 sizeof (hdrs_buf))) { 1014 mblk_len = MBLKL(nmp); 1015 if (mblk_len >= 1016 (sizeof (hdrs_buf) - hdrs_size)) 1017 mblk_len = sizeof (hdrs_buf) - 1018 hdrs_size; 1019 bcopy(nmp->b_rptr, 1020 &hdrs_buf[hdrs_size], mblk_len); 1021 hdrs_size += mblk_len; 1022 nmp = nmp->b_cont; 1023 } 1024 tcp_port = hdrs_buf; 1025 } 1026 tcp_port += sizeof (ether_header_t); 1027 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1028 switch (tcp_port[9]) { 1029 case IPPROTO_TCP: 1030 case IPPROTO_UDP: 1031 case IPPROTO_ESP: 1032 tcp_port += ((*tcp_port) & 0x0f) << 2; 1033 ring_index = 1034 ((tcp_port[0] ^ 1035 tcp_port[1] ^ 1036 tcp_port[2] ^ 1037 tcp_port[3]) % maxtdcs); 1038 break; 1039 1040 case IPPROTO_AH: 1041 /* SPI starts at the 4th byte */ 1042 tcp_port += ((*tcp_port) & 0x0f) << 2; 1043 ring_index = 1044 ((tcp_port[4] ^ 1045 tcp_port[5] ^ 1046 tcp_port[6] ^ 1047 tcp_port[7]) % maxtdcs); 1048 break; 1049 1050 default: 1051 ring_index = tcp_port[19] % maxtdcs; 1052 break; 1053 } 1054 } else { /* fragmented packet */ 1055 ring_index = tcp_port[19] % maxtdcs; 1056 } 1057 } else { 1058 ring_index = mp->b_band % maxtdcs; 1059 } 1060 break; 1061 1062 case NXGE_TX_LB_HASH: 1063 if (hp->hash) { 1064 #if defined(__i386) 1065 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1066 #else 1067 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1068 #endif 1069 } else { 1070 ring_index = mp->b_band % maxtdcs; 1071 } 1072 break; 1073 1074 case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 1075 tcp_port = mp->b_rptr; 1076 ring_index = tcp_port[5] % maxtdcs; 1077 break; 1078 } 1079 1080 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 1081 1082 return (ring_index); 1083 } 1084 1085 uint_t 1086 nxge_reschedule(caddr_t arg) 1087 { 1088 p_nxge_t nxgep; 1089 1090 nxgep = (p_nxge_t)arg; 1091 1092 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 1093 1094 if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1095 nxgep->resched_needed) { 1096 mac_tx_update(nxgep->mach); 1097 nxgep->resched_needed = B_FALSE; 1098 nxgep->resched_running = B_FALSE; 1099 } 1100 1101 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 1102 return (DDI_INTR_CLAIMED); 1103 } 1104