1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <hxge_impl.h> 29 30 extern uint32_t hxge_reclaim_pending; 31 extern uint32_t hxge_bcopy_thresh; 32 extern uint32_t hxge_dvma_thresh; 33 extern uint32_t hxge_dma_stream_thresh; 34 extern uint32_t hxge_tx_minfree; 35 extern uint32_t hxge_tx_intr_thres; 36 extern uint32_t hxge_tx_max_gathers; 37 extern uint32_t hxge_tx_tiny_pack; 38 extern uint32_t hxge_tx_use_bcopy; 39 extern uint32_t hxge_tx_lb_policy; 40 extern uint32_t hxge_no_tx_lb; 41 42 typedef struct _mac_tx_hint { 43 uint16_t sap; 44 uint16_t vid; 45 void *hash; 46 } mac_tx_hint_t, *p_mac_tx_hint_t; 47 48 int hxge_tx_lb_ring(p_mblk_t, uint32_t, p_mac_tx_hint_t); 49 50 int 51 hxge_start(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 52 { 53 int status = 0; 54 p_tx_desc_t tx_desc_ring_vp; 55 hpi_handle_t hpi_desc_handle; 56 hxge_os_dma_handle_t tx_desc_dma_handle; 57 p_tx_desc_t tx_desc_p; 58 p_tx_msg_t tx_msg_ring; 59 p_tx_msg_t tx_msg_p; 60 tx_desc_t tx_desc, *tmp_desc_p; 61 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 62 p_tx_pkt_header_t hdrp; 63 p_tx_pkt_hdr_all_t pkthdrp; 64 uint8_t npads = 0; 65 uint64_t dma_ioaddr; 66 uint32_t dma_flags; 67 int last_bidx; 68 uint8_t *b_rptr; 69 caddr_t kaddr; 70 uint32_t nmblks; 71 uint32_t ngathers; 72 uint32_t clen; 73 int len; 74 uint32_t pkt_len, pack_len, min_len; 75 uint32_t bcopy_thresh; 76 int i, cur_index, sop_index; 77 uint16_t tail_index; 78 boolean_t tail_wrap = B_FALSE; 79 hxge_dma_common_t desc_area; 80 hxge_os_dma_handle_t dma_handle; 81 ddi_dma_cookie_t dma_cookie; 82 hpi_handle_t hpi_handle; 83 p_mblk_t nmp; 84 p_mblk_t t_mp; 85 uint32_t ncookies; 86 boolean_t good_packet; 87 boolean_t mark_mode = B_FALSE; 88 p_hxge_stats_t statsp; 89 p_hxge_tx_ring_stats_t tdc_stats; 90 t_uscalar_t start_offset = 0; 91 t_uscalar_t stuff_offset = 0; 92 t_uscalar_t end_offset = 0; 93 t_uscalar_t value = 0; 94 t_uscalar_t cksum_flags = 0; 95 boolean_t cksum_on = B_FALSE; 96 uint32_t boff = 0; 97 uint64_t tot_xfer_len = 0, tmp_len = 0; 98 boolean_t header_set = B_FALSE; 99 tdc_tdr_kick_t kick; 100 #ifdef HXGE_DEBUG 101 p_tx_desc_t tx_desc_ring_pp; 102 p_tx_desc_t tx_desc_pp; 103 tx_desc_t *save_desc_p; 104 int dump_len; 105 int sad_len; 106 uint64_t sad; 107 int xfer_len; 108 uint32_t msgsize; 109 #endif 110 111 HXGE_DEBUG_MSG((hxgep, TX_CTL, 112 "==> hxge_start: tx dma channel %d", tx_ring_p->tdc)); 113 HXGE_DEBUG_MSG((hxgep, TX_CTL, 114 "==> hxge_start: Starting tdc %d desc pending %d", 115 tx_ring_p->tdc, tx_ring_p->descs_pending)); 116 117 statsp = hxgep->statsp; 118 119 if (hxgep->statsp->port_stats.lb_mode == hxge_lb_normal) { 120 if (!statsp->mac_stats.link_up) { 121 freemsg(mp); 122 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: " 123 "link not up or LB mode")); 124 goto hxge_start_fail1; 125 } 126 } 127 128 hcksum_retrieve(mp, NULL, NULL, &start_offset, 129 &stuff_offset, &end_offset, &value, &cksum_flags); 130 if (!HXGE_IS_VLAN_PACKET(mp->b_rptr)) { 131 start_offset += sizeof (ether_header_t); 132 stuff_offset += sizeof (ether_header_t); 133 } else { 134 start_offset += sizeof (struct ether_vlan_header); 135 stuff_offset += sizeof (struct ether_vlan_header); 136 } 137 138 if (cksum_flags & HCK_PARTIALCKSUM) { 139 HXGE_DEBUG_MSG((hxgep, TX_CTL, 140 "==> hxge_start: mp $%p len %d " 141 "cksum_flags 0x%x (partial checksum) ", 142 mp, MBLKL(mp), cksum_flags)); 143 cksum_on = B_TRUE; 144 } 145 146 MUTEX_ENTER(&tx_ring_p->lock); 147 start_again: 148 ngathers = 0; 149 sop_index = tx_ring_p->wr_index; 150 #ifdef HXGE_DEBUG 151 if (tx_ring_p->descs_pending) { 152 HXGE_DEBUG_MSG((hxgep, TX_CTL, 153 "==> hxge_start: desc pending %d ", 154 tx_ring_p->descs_pending)); 155 } 156 157 dump_len = (int)(MBLKL(mp)); 158 dump_len = (dump_len > 128) ? 128: dump_len; 159 160 HXGE_DEBUG_MSG((hxgep, TX_CTL, 161 "==> hxge_start: tdc %d: dumping ...: b_rptr $%p " 162 "(Before header reserve: ORIGINAL LEN %d)", 163 tx_ring_p->tdc, mp->b_rptr, dump_len)); 164 165 HXGE_DEBUG_MSG((hxgep, TX_CTL, 166 "==> hxge_start: dump packets (IP ORIGINAL b_rptr $%p): %s", 167 mp->b_rptr, hxge_dump_packet((char *)mp->b_rptr, dump_len))); 168 #endif 169 170 tdc_stats = tx_ring_p->tdc_stats; 171 mark_mode = (tx_ring_p->descs_pending && 172 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) < 173 hxge_tx_minfree)); 174 175 HXGE_DEBUG_MSG((hxgep, TX_CTL, 176 "TX Descriptor ring is channel %d mark mode %d", 177 tx_ring_p->tdc, mark_mode)); 178 179 if (!hxge_txdma_reclaim(hxgep, tx_ring_p, hxge_tx_minfree)) { 180 HXGE_DEBUG_MSG((hxgep, TX_CTL, 181 "TX Descriptor ring is full: channel %d", tx_ring_p->tdc)); 182 HXGE_DEBUG_MSG((hxgep, TX_CTL, 183 "TX Descriptor ring is full: channel %d", tx_ring_p->tdc)); 184 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 185 tdc_stats->tx_no_desc++; 186 MUTEX_EXIT(&tx_ring_p->lock); 187 if (hxgep->resched_needed && !hxgep->resched_running) { 188 hxgep->resched_running = B_TRUE; 189 ddi_trigger_softintr(hxgep->resched_id); 190 } 191 status = 1; 192 goto hxge_start_fail1; 193 } 194 195 nmp = mp; 196 i = sop_index = tx_ring_p->wr_index; 197 nmblks = 0; 198 ngathers = 0; 199 pkt_len = 0; 200 pack_len = 0; 201 clen = 0; 202 last_bidx = -1; 203 good_packet = B_TRUE; 204 205 desc_area = tx_ring_p->tdc_desc; 206 hpi_handle = desc_area.hpi_handle; 207 hpi_desc_handle.regh = (hxge_os_acc_handle_t) 208 DMA_COMMON_ACC_HANDLE(desc_area); 209 hpi_desc_handle.hxgep = hxgep; 210 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 211 #ifdef HXGE_DEBUG 212 #if defined(__i386) 213 tx_desc_ring_pp = (p_tx_desc_t)(uint32_t)DMA_COMMON_IOADDR(desc_area); 214 #else 215 tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area); 216 #endif 217 #endif 218 tx_desc_dma_handle = (hxge_os_dma_handle_t)DMA_COMMON_HANDLE(desc_area); 219 tx_msg_ring = tx_ring_p->tx_msg_ring; 220 221 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: wr_index %d i %d", 222 sop_index, i)); 223 224 #ifdef HXGE_DEBUG 225 msgsize = msgdsize(nmp); 226 HXGE_DEBUG_MSG((hxgep, TX_CTL, 227 "==> hxge_start(1): wr_index %d i %d msgdsize %d", 228 sop_index, i, msgsize)); 229 #endif 230 /* 231 * The first 16 bytes of the premapped buffer are reserved 232 * for header. No padding will be used. 233 */ 234 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 235 if (hxge_tx_use_bcopy) { 236 bcopy_thresh = (hxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 237 } else { 238 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 239 } 240 while (nmp) { 241 good_packet = B_TRUE; 242 b_rptr = nmp->b_rptr; 243 len = MBLKL(nmp); 244 if (len <= 0) { 245 nmp = nmp->b_cont; 246 continue; 247 } 248 nmblks++; 249 250 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(1): nmblks %d " 251 "len %d pkt_len %d pack_len %d", 252 nmblks, len, pkt_len, pack_len)); 253 /* 254 * Hardware limits the transfer length to 4K. 255 * If len is more than 4K, we need to break 256 * nmp into two chunks: Make first chunk smaller 257 * than 4K. The second chunk will be broken into 258 * less than 4K (if needed) during the next pass. 259 */ 260 if (len > (TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE)) { 261 if ((t_mp = dupb(nmp)) != NULL) { 262 nmp->b_wptr = nmp->b_rptr + 263 (TX_MAX_TRANSFER_LENGTH - 264 TX_PKT_HEADER_SIZE); 265 t_mp->b_rptr = nmp->b_wptr; 266 t_mp->b_cont = nmp->b_cont; 267 nmp->b_cont = t_mp; 268 len = MBLKL(nmp); 269 } else { 270 good_packet = B_FALSE; 271 goto hxge_start_fail2; 272 } 273 } 274 tx_desc.value = 0; 275 tx_desc_p = &tx_desc_ring_vp[i]; 276 #ifdef HXGE_DEBUG 277 tx_desc_pp = &tx_desc_ring_pp[i]; 278 #endif 279 tx_msg_p = &tx_msg_ring[i]; 280 #if defined(__i386) 281 hpi_desc_handle.regp = (uint32_t)tx_desc_p; 282 #else 283 hpi_desc_handle.regp = (uint64_t)tx_desc_p; 284 #endif 285 if (!header_set && 286 ((!hxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 287 (len >= bcopy_thresh))) { 288 header_set = B_TRUE; 289 bcopy_thresh += TX_PKT_HEADER_SIZE; 290 boff = 0; 291 pack_len = 0; 292 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 293 hdrp = (p_tx_pkt_header_t)kaddr; 294 clen = pkt_len; 295 dma_handle = tx_msg_p->buf_dma_handle; 296 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 297 (void) ddi_dma_sync(dma_handle, 298 i * hxge_bcopy_thresh, hxge_bcopy_thresh, 299 DDI_DMA_SYNC_FORDEV); 300 301 tx_msg_p->flags.dma_type = USE_BCOPY; 302 goto hxge_start_control_header_only; 303 } 304 305 pkt_len += len; 306 pack_len += len; 307 308 HXGE_DEBUG_MSG((hxgep, TX_CTL, 309 "==> hxge_start(3): desc entry %d DESC IOADDR $%p " 310 "desc_vp $%p tx_desc_p $%p desc_pp $%p tx_desc_pp $%p " 311 "len %d pkt_len %d pack_len %d", 312 i, 313 DMA_COMMON_IOADDR(desc_area), 314 tx_desc_ring_vp, tx_desc_p, 315 tx_desc_ring_pp, tx_desc_pp, 316 len, pkt_len, pack_len)); 317 318 if (len < bcopy_thresh) { 319 HXGE_DEBUG_MSG((hxgep, TX_CTL, 320 "==> hxge_start(4): USE BCOPY: ")); 321 if (hxge_tx_tiny_pack) { 322 uint32_t blst = TXDMA_DESC_NEXT_INDEX(i, -1, 323 tx_ring_p->tx_wrap_mask); 324 HXGE_DEBUG_MSG((hxgep, TX_CTL, 325 "==> hxge_start(5): pack")); 326 if ((pack_len <= bcopy_thresh) && 327 (last_bidx == blst)) { 328 HXGE_DEBUG_MSG((hxgep, TX_CTL, 329 "==> hxge_start: pack(6) " 330 "(pkt_len %d pack_len %d)", 331 pkt_len, pack_len)); 332 i = blst; 333 tx_desc_p = &tx_desc_ring_vp[i]; 334 #ifdef HXGE_DEBUG 335 tx_desc_pp = &tx_desc_ring_pp[i]; 336 #endif 337 tx_msg_p = &tx_msg_ring[i]; 338 boff = pack_len - len; 339 ngathers--; 340 } else if (pack_len > bcopy_thresh && 341 header_set) { 342 pack_len = len; 343 boff = 0; 344 bcopy_thresh = hxge_bcopy_thresh; 345 HXGE_DEBUG_MSG((hxgep, TX_CTL, 346 "==> hxge_start(7): > max NEW " 347 "bcopy thresh %d " 348 "pkt_len %d pack_len %d(next)", 349 bcopy_thresh, pkt_len, pack_len)); 350 } 351 last_bidx = i; 352 } 353 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 354 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 355 hdrp = (p_tx_pkt_header_t)kaddr; 356 header_set = B_TRUE; 357 HXGE_DEBUG_MSG((hxgep, TX_CTL, 358 "==> hxge_start(7_x2): " 359 "pkt_len %d pack_len %d (new hdrp $%p)", 360 pkt_len, pack_len, hdrp)); 361 } 362 tx_msg_p->flags.dma_type = USE_BCOPY; 363 kaddr += boff; 364 HXGE_DEBUG_MSG((hxgep, TX_CTL, 365 "==> hxge_start(8): USE BCOPY: before bcopy " 366 "DESC IOADDR $%p entry %d bcopy packets %d " 367 "bcopy kaddr $%p bcopy ioaddr (SAD) $%p " 368 "bcopy clen %d bcopy boff %d", 369 DMA_COMMON_IOADDR(desc_area), i, 370 tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr, 371 clen, boff)); 372 HXGE_DEBUG_MSG((hxgep, TX_CTL, 373 "==> hxge_start: 1USE BCOPY: ")); 374 HXGE_DEBUG_MSG((hxgep, TX_CTL, 375 "==> hxge_start: 2USE BCOPY: ")); 376 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: " 377 "last USE BCOPY: copy from b_rptr $%p " 378 "to KADDR $%p (len %d offset %d", 379 b_rptr, kaddr, len, boff)); 380 bcopy(b_rptr, kaddr, len); 381 #ifdef HXGE_DEBUG 382 dump_len = (len > 128) ? 128: len; 383 HXGE_DEBUG_MSG((hxgep, TX_CTL, 384 "==> hxge_start: dump packets " 385 "(After BCOPY len %d)" 386 "(b_rptr $%p): %s", len, nmp->b_rptr, 387 hxge_dump_packet((char *)nmp->b_rptr, 388 dump_len))); 389 #endif 390 dma_handle = tx_msg_p->buf_dma_handle; 391 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 392 (void) ddi_dma_sync(dma_handle, 393 i * hxge_bcopy_thresh, hxge_bcopy_thresh, 394 DDI_DMA_SYNC_FORDEV); 395 clen = len + boff; 396 tdc_stats->tx_hdr_pkts++; 397 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(9): " 398 "USE BCOPY: DESC IOADDR $%p entry %d " 399 "bcopy packets %d bcopy kaddr $%p " 400 "bcopy ioaddr (SAD) $%p bcopy clen %d " 401 "bcopy boff %d", 402 DMA_COMMON_IOADDR(desc_area), i, 403 tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr, 404 clen, boff)); 405 } else { 406 HXGE_DEBUG_MSG((hxgep, TX_CTL, 407 "==> hxge_start(12): USE DVMA: len %d", len)); 408 tx_msg_p->flags.dma_type = USE_DMA; 409 dma_flags = DDI_DMA_WRITE; 410 if (len < hxge_dma_stream_thresh) { 411 dma_flags |= DDI_DMA_CONSISTENT; 412 } else { 413 dma_flags |= DDI_DMA_STREAMING; 414 } 415 416 dma_handle = tx_msg_p->dma_handle; 417 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 418 (caddr_t)b_rptr, len, dma_flags, 419 DDI_DMA_DONTWAIT, NULL, 420 &dma_cookie, &ncookies); 421 if (status == DDI_DMA_MAPPED) { 422 dma_ioaddr = dma_cookie.dmac_laddress; 423 len = (int)dma_cookie.dmac_size; 424 clen = (uint32_t)dma_cookie.dmac_size; 425 HXGE_DEBUG_MSG((hxgep, TX_CTL, 426 "==> hxge_start(12_1): " 427 "USE DVMA: len %d clen %d ngathers %d", 428 len, clen, ngathers)); 429 #if defined(__i386) 430 hpi_desc_handle.regp = (uint32_t)tx_desc_p; 431 #else 432 hpi_desc_handle.regp = (uint64_t)tx_desc_p; 433 #endif 434 while (ncookies > 1) { 435 ngathers++; 436 /* 437 * this is the fix for multiple 438 * cookies, which are basically 439 * a descriptor entry, we don't set 440 * SOP bit as well as related fields 441 */ 442 443 (void) hpi_txdma_desc_gather_set( 444 hpi_desc_handle, &tx_desc, 445 (ngathers -1), mark_mode, 446 ngathers, dma_ioaddr, clen); 447 tx_msg_p->tx_msg_size = clen; 448 HXGE_DEBUG_MSG((hxgep, TX_CTL, 449 "==> hxge_start: DMA " 450 "ncookie %d ngathers %d " 451 "dma_ioaddr $%p len %d" 452 "desc $%p descp $%p (%d)", 453 ncookies, ngathers, 454 dma_ioaddr, clen, 455 *tx_desc_p, tx_desc_p, i)); 456 457 ddi_dma_nextcookie(dma_handle, 458 &dma_cookie); 459 dma_ioaddr = dma_cookie.dmac_laddress; 460 461 len = (int)dma_cookie.dmac_size; 462 clen = (uint32_t)dma_cookie.dmac_size; 463 HXGE_DEBUG_MSG((hxgep, TX_CTL, 464 "==> hxge_start(12_2): " 465 "USE DVMA: len %d clen %d ", 466 len, clen)); 467 468 i = TXDMA_DESC_NEXT_INDEX(i, 1, 469 tx_ring_p->tx_wrap_mask); 470 tx_desc_p = &tx_desc_ring_vp[i]; 471 472 hpi_desc_handle.regp = 473 #if defined(__i386) 474 (uint32_t)tx_desc_p; 475 #else 476 (uint64_t)tx_desc_p; 477 #endif 478 tx_msg_p = &tx_msg_ring[i]; 479 tx_msg_p->flags.dma_type = USE_NONE; 480 tx_desc.value = 0; 481 ncookies--; 482 } 483 tdc_stats->tx_ddi_pkts++; 484 HXGE_DEBUG_MSG((hxgep, TX_CTL, 485 "==> hxge_start: DMA: ddi packets %d", 486 tdc_stats->tx_ddi_pkts)); 487 } else { 488 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 489 "dma mapping failed for %d " 490 "bytes addr $%p flags %x (%d)", 491 len, b_rptr, status, status)); 492 good_packet = B_FALSE; 493 tdc_stats->tx_dma_bind_fail++; 494 tx_msg_p->flags.dma_type = USE_NONE; 495 goto hxge_start_fail2; 496 } 497 } /* ddi dvma */ 498 499 nmp = nmp->b_cont; 500 hxge_start_control_header_only: 501 #if defined(__i386) 502 hpi_desc_handle.regp = (uint32_t)tx_desc_p; 503 #else 504 hpi_desc_handle.regp = (uint64_t)tx_desc_p; 505 #endif 506 ngathers++; 507 508 if (ngathers == 1) { 509 #ifdef HXGE_DEBUG 510 save_desc_p = &sop_tx_desc; 511 #endif 512 sop_tx_desc_p = &sop_tx_desc; 513 sop_tx_desc_p->value = 0; 514 sop_tx_desc_p->bits.tr_len = clen; 515 sop_tx_desc_p->bits.sad = dma_ioaddr >> 32; 516 sop_tx_desc_p->bits.sad_l = dma_ioaddr & 0xffffffff; 517 } else { 518 #ifdef HXGE_DEBUG 519 save_desc_p = &tx_desc; 520 #endif 521 tmp_desc_p = &tx_desc; 522 tmp_desc_p->value = 0; 523 tmp_desc_p->bits.tr_len = clen; 524 tmp_desc_p->bits.sad = dma_ioaddr >> 32; 525 tmp_desc_p->bits.sad_l = dma_ioaddr & 0xffffffff; 526 527 tx_desc_p->value = tmp_desc_p->value; 528 } 529 530 HXGE_DEBUG_MSG((hxgep, TX_CTL, 531 "==> hxge_start(13): Desc_entry %d ngathers %d " 532 "desc_vp $%p tx_desc_p $%p " 533 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 534 "dma_ioaddr (SAD) $%p mark %d", 535 i, ngathers, tx_desc_ring_vp, tx_desc_p, 536 len, clen, pkt_len, pack_len, nmblks, 537 dma_ioaddr, mark_mode)); 538 539 #ifdef HXGE_DEBUG 540 hpi_desc_handle.hxgep = hxgep; 541 hpi_desc_handle.function.function = 0; 542 hpi_desc_handle.function.instance = hxgep->instance; 543 sad = save_desc_p->bits.sad; 544 sad = (sad << 32) | save_desc_p->bits.sad_l; 545 xfer_len = save_desc_p->bits.tr_len; 546 547 HXGE_DEBUG_MSG((hxgep, TX_CTL, "\n\t: value 0x%llx\n" 548 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 549 "mark %d sop %d\n", 550 save_desc_p->value, sad, save_desc_p->bits.tr_len, 551 xfer_len, save_desc_p->bits.num_ptr, 552 save_desc_p->bits.mark, save_desc_p->bits.sop)); 553 554 hpi_txdma_dump_desc_one(hpi_desc_handle, NULL, i); 555 #endif 556 557 tx_msg_p->tx_msg_size = clen; 558 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 559 if (ngathers > hxge_tx_max_gathers) { 560 good_packet = B_FALSE; 561 hcksum_retrieve(mp, NULL, NULL, &start_offset, 562 &stuff_offset, &end_offset, &value, &cksum_flags); 563 564 HXGE_DEBUG_MSG((NULL, TX_CTL, 565 "==> hxge_start(14): pull msg - " 566 "len %d pkt_len %d ngathers %d", 567 len, pkt_len, ngathers)); 568 /* Pull all message blocks from b_cont */ 569 if ((msgpullup(mp, -1)) == NULL) { 570 goto hxge_start_fail2; 571 } 572 goto hxge_start_fail2; 573 } 574 } /* while (nmp) */ 575 576 tx_msg_p->tx_message = mp; 577 tx_desc_p = &tx_desc_ring_vp[sop_index]; 578 #if defined(__i386) 579 hpi_desc_handle.regp = (uint32_t)tx_desc_p; 580 #else 581 hpi_desc_handle.regp = (uint64_t)tx_desc_p; 582 #endif 583 584 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 585 pkthdrp->reserved = 0; 586 hdrp->value = 0; 587 (void) hxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 588 (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp); 589 590 /* 591 * Hardware header should not be counted as part of the frame 592 * when determining the frame size 593 */ 594 if ((pkt_len - TX_PKT_HEADER_SIZE) > (STD_FRAME_SIZE - ETHERFCSL)) { 595 tdc_stats->tx_jumbo_pkts++; 596 } 597 598 min_len = (hxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2)); 599 if (pkt_len < min_len) { 600 /* Assume we use bcopy to premapped buffers */ 601 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 602 HXGE_DEBUG_MSG((NULL, TX_CTL, 603 "==> hxge_start(14-1): < (msg_min + 16)" 604 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 605 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 606 bzero((kaddr + pkt_len), (min_len - pkt_len)); 607 pkt_len = tx_msg_p->tx_msg_size = min_len; 608 609 sop_tx_desc_p->bits.tr_len = min_len; 610 611 HXGE_MEM_PIO_WRITE64(hpi_desc_handle, sop_tx_desc_p->value); 612 tx_desc_p->value = sop_tx_desc_p->value; 613 614 HXGE_DEBUG_MSG((NULL, TX_CTL, 615 "==> hxge_start(14-2): < msg_min - " 616 "len %d pkt_len %d min_len %d ngathers %d", 617 len, pkt_len, min_len, ngathers)); 618 } 619 620 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: cksum_flags 0x%x ", 621 cksum_flags)); 622 if (cksum_flags & HCK_PARTIALCKSUM) { 623 HXGE_DEBUG_MSG((hxgep, TX_CTL, 624 "==> hxge_start: cksum_flags 0x%x (partial checksum) ", 625 cksum_flags)); 626 cksum_on = B_TRUE; 627 HXGE_DEBUG_MSG((hxgep, TX_CTL, 628 "==> hxge_start: from IP cksum_flags 0x%x " 629 "(partial checksum) " 630 "start_offset %d stuff_offset %d", 631 cksum_flags, start_offset, stuff_offset)); 632 tmp_len = (uint64_t)(start_offset >> 1); 633 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT); 634 tmp_len = (uint64_t)(stuff_offset >> 1); 635 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT); 636 637 HXGE_DEBUG_MSG((hxgep, TX_CTL, 638 "==> hxge_start: from IP cksum_flags 0x%x " 639 "(partial checksum) " 640 "after SHIFT start_offset %d stuff_offset %d", 641 cksum_flags, start_offset, stuff_offset)); 642 } 643 644 /* 645 * pkt_len already includes 16 + paddings!! 646 * Update the control header length 647 */ 648 649 /* 650 * Note that Hydra is different from Neptune where 651 * tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 652 */ 653 tot_xfer_len = pkt_len; 654 tmp_len = hdrp->value | 655 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 656 657 HXGE_DEBUG_MSG((hxgep, TX_CTL, 658 "==> hxge_start(15_x1): setting SOP " 659 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 660 "0x%llx hdrp->value 0x%llx", 661 tot_xfer_len, tot_xfer_len, pkt_len, tmp_len, hdrp->value)); 662 #if defined(_BIG_ENDIAN) 663 hdrp->value = ddi_swap64(tmp_len); 664 #else 665 hdrp->value = tmp_len; 666 #endif 667 HXGE_DEBUG_MSG((hxgep, 668 TX_CTL, "==> hxge_start(15_x2): setting SOP " 669 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 670 "tmp_len 0x%llx hdrp->value 0x%llx", 671 tot_xfer_len, pkt_len, tmp_len, hdrp->value)); 672 673 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(15): setting SOP " 674 "wr_index %d tot_xfer_len (%d) pkt_len %d npads %d", 675 sop_index, tot_xfer_len, pkt_len, npads)); 676 677 sop_tx_desc_p->bits.sop = 1; 678 sop_tx_desc_p->bits.mark = mark_mode; 679 sop_tx_desc_p->bits.num_ptr = ngathers; 680 681 if (mark_mode) 682 tdc_stats->tx_marks++; 683 684 HXGE_MEM_PIO_WRITE64(hpi_desc_handle, sop_tx_desc_p->value); 685 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(16): set SOP done")); 686 687 #ifdef HXGE_DEBUG 688 hpi_desc_handle.hxgep = hxgep; 689 hpi_desc_handle.function.function = 0; 690 hpi_desc_handle.function.instance = hxgep->instance; 691 692 HXGE_DEBUG_MSG((hxgep, TX_CTL, "\n\t: value 0x%llx\n" 693 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 694 save_desc_p->value, sad, save_desc_p->bits.tr_len, 695 xfer_len, save_desc_p->bits.num_ptr, save_desc_p->bits.mark, 696 save_desc_p->bits.sop)); 697 (void) hpi_txdma_dump_desc_one(hpi_desc_handle, NULL, sop_index); 698 699 dump_len = (pkt_len > 128) ? 128: pkt_len; 700 HXGE_DEBUG_MSG((hxgep, TX_CTL, 701 "==> hxge_start: dump packets(17) (after sop set, len " 702 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 703 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 704 (char *)hdrp, hxge_dump_packet((char *)hdrp, dump_len))); 705 HXGE_DEBUG_MSG((hxgep, TX_CTL, 706 "==> hxge_start(18): TX desc sync: sop_index %d", sop_index)); 707 #endif 708 709 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 710 (void) ddi_dma_sync(tx_desc_dma_handle, 711 sop_index * sizeof (tx_desc_t), 712 ngathers * sizeof (tx_desc_t), DDI_DMA_SYNC_FORDEV); 713 714 HXGE_DEBUG_MSG((hxgep, TX_CTL, "hxge_start(19): sync 1 " 715 "cs_off = 0x%02X cs_s_off = 0x%02X " 716 "pkt_len %d ngathers %d sop_index %d\n", 717 stuff_offset, start_offset, 718 pkt_len, ngathers, sop_index)); 719 } else { /* more than one descriptor and wrap around */ 720 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 721 (void) ddi_dma_sync(tx_desc_dma_handle, 722 sop_index * sizeof (tx_desc_t), 723 nsdescs * sizeof (tx_desc_t), DDI_DMA_SYNC_FORDEV); 724 HXGE_DEBUG_MSG((hxgep, TX_CTL, "hxge_start(20): sync 1 " 725 "cs_off = 0x%02X cs_s_off = 0x%02X " 726 "pkt_len %d ngathers %d sop_index %d\n", 727 stuff_offset, start_offset, pkt_len, ngathers, sop_index)); 728 729 (void) ddi_dma_sync(tx_desc_dma_handle, 0, 730 (ngathers - nsdescs) * sizeof (tx_desc_t), 731 DDI_DMA_SYNC_FORDEV); 732 HXGE_DEBUG_MSG((hxgep, TX_CTL, "hxge_start(21): sync 2 " 733 "cs_off = 0x%02X cs_s_off = 0x%02X " 734 "pkt_len %d ngathers %d sop_index %d\n", 735 stuff_offset, start_offset, 736 pkt_len, ngathers, sop_index)); 737 } 738 739 tail_index = tx_ring_p->wr_index; 740 tail_wrap = tx_ring_p->wr_index_wrap; 741 742 tx_ring_p->wr_index = i; 743 if (tx_ring_p->wr_index <= tail_index) { 744 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 745 B_FALSE : B_TRUE); 746 } 747 748 tx_ring_p->descs_pending += ngathers; 749 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: TX kick: " 750 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 751 tx_ring_p->tdc, tx_ring_p->wr_index, tx_ring_p->wr_index_wrap, 752 ngathers, tx_ring_p->descs_pending)); 753 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: TX KICKING: ")); 754 755 kick.value = 0; 756 kick.bits.wrap = tx_ring_p->wr_index_wrap; 757 kick.bits.tail = (uint16_t)tx_ring_p->wr_index; 758 759 /* Kick start the Transmit kick register */ 760 TXDMA_REG_WRITE64(HXGE_DEV_HPI_HANDLE(hxgep), 761 TDC_TDR_KICK, (uint8_t)tx_ring_p->tdc, kick.value); 762 tdc_stats->tx_starts++; 763 MUTEX_EXIT(&tx_ring_p->lock); 764 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_start")); 765 return (status); 766 767 hxge_start_fail2: 768 if (good_packet == B_FALSE) { 769 cur_index = sop_index; 770 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: clean up")); 771 for (i = 0; i < ngathers; i++) { 772 tx_desc_p = &tx_desc_ring_vp[cur_index]; 773 #if defined(__i386) 774 hpi_handle.regp = (uint32_t)tx_desc_p; 775 #else 776 hpi_handle.regp = (uint64_t)tx_desc_p; 777 #endif 778 tx_msg_p = &tx_msg_ring[cur_index]; 779 (void) hpi_txdma_desc_set_zero(hpi_handle, 1); 780 if (tx_msg_p->flags.dma_type == USE_DVMA) { 781 HXGE_DEBUG_MSG((hxgep, TX_CTL, 782 "tx_desc_p = %X index = %d", 783 tx_desc_p, tx_ring_p->rd_index)); 784 (void) dvma_unload(tx_msg_p->dvma_handle, 785 0, -1); 786 tx_msg_p->dvma_handle = NULL; 787 if (tx_ring_p->dvma_wr_index == 788 tx_ring_p->dvma_wrap_mask) 789 tx_ring_p->dvma_wr_index = 0; 790 else 791 tx_ring_p->dvma_wr_index++; 792 tx_ring_p->dvma_pending--; 793 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 794 if (ddi_dma_unbind_handle( 795 tx_msg_p->dma_handle)) { 796 cmn_err(CE_WARN, "hxge_start: " 797 "ddi_dma_unbind_handle failed"); 798 } 799 } 800 tx_msg_p->flags.dma_type = USE_NONE; 801 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 802 tx_ring_p->tx_wrap_mask); 803 804 } 805 806 hxgep->resched_needed = B_TRUE; 807 } 808 809 MUTEX_EXIT(&tx_ring_p->lock); 810 811 hxge_start_fail1: 812 /* Add FMA to check the access handle hxge_hregh */ 813 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_start")); 814 return (status); 815 } 816 817 boolean_t 818 hxge_send(p_hxge_t hxgep, mblk_t *mp, p_mac_tx_hint_t hp) 819 { 820 p_tx_ring_t *tx_rings; 821 uint8_t ring_index; 822 823 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_send")); 824 825 ASSERT(mp->b_next == NULL); 826 827 ring_index = hxge_tx_lb_ring(mp, hxgep->max_tdcs, hp); 828 tx_rings = hxgep->tx_rings->rings; 829 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_tx_msg: tx_rings $%p", 830 tx_rings)); 831 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_tx_msg: max_tdcs %d " 832 "ring_index %d", hxgep->max_tdcs, ring_index)); 833 834 if (hxge_start(hxgep, tx_rings[ring_index], mp)) { 835 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_send: failed " 836 "ring index %d", ring_index)); 837 return (B_FALSE); 838 } 839 840 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_send: ring index %d", 841 ring_index)); 842 return (B_TRUE); 843 } 844 845 /* 846 * hxge_m_tx() - send a chain of packets 847 */ 848 mblk_t * 849 hxge_m_tx(void *arg, mblk_t *mp) 850 { 851 p_hxge_t hxgep = (p_hxge_t)arg; 852 mblk_t *next; 853 mac_tx_hint_t hint; 854 855 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) { 856 HXGE_DEBUG_MSG((hxgep, DDI_CTL, 857 "==> hxge_m_tx: hardware not initialized")); 858 HXGE_DEBUG_MSG((hxgep, DDI_CTL, "<== hxge_m_tx")); 859 return (mp); 860 } 861 862 hint.hash = NULL; 863 hint.vid = 0; 864 hint.sap = 0; 865 866 while (mp != NULL) { 867 next = mp->b_next; 868 mp->b_next = NULL; 869 870 /* 871 * Until Nemo tx resource works, the mac driver 872 * does the load balancing based on TCP port, 873 * or CPU. For debugging, we use a system 874 * configurable parameter. 875 */ 876 if (!hxge_send(hxgep, mp, &hint)) { 877 mp->b_next = next; 878 break; 879 } 880 881 mp = next; 882 883 HXGE_DEBUG_MSG((NULL, TX_CTL, 884 "==> hxge_m_tx: (go back to loop) mp $%p next $%p", 885 mp, next)); 886 } 887 return (mp); 888 } 889 890 int 891 hxge_tx_lb_ring(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 892 { 893 uint8_t ring_index = 0; 894 uint8_t *tcp_port; 895 p_mblk_t nmp; 896 size_t mblk_len; 897 size_t iph_len; 898 size_t hdrs_size; 899 uint8_t hdrs_buf[sizeof (struct ether_header) + 900 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 901 902 /* 903 * allocate space big enough to cover 904 * the max ip header length and the first 905 * 4 bytes of the TCP/IP header. 906 */ 907 boolean_t qos = B_FALSE; 908 909 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_lb_ring")); 910 911 if (hp->vid) { 912 qos = B_TRUE; 913 } 914 switch (hxge_tx_lb_policy) { 915 case HXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 916 default: 917 tcp_port = mp->b_rptr; 918 if (!hxge_no_tx_lb && !qos && 919 (ntohs(((p_ether_header_t)tcp_port)->ether_type) == 920 ETHERTYPE_IP)) { 921 nmp = mp; 922 mblk_len = MBLKL(nmp); 923 tcp_port = NULL; 924 if (mblk_len > sizeof (struct ether_header) + 925 sizeof (uint8_t)) { 926 tcp_port = nmp->b_rptr + 927 sizeof (struct ether_header); 928 mblk_len -= sizeof (struct ether_header); 929 iph_len = ((*tcp_port) & 0x0f) << 2; 930 if (mblk_len > (iph_len + sizeof (uint32_t))) { 931 tcp_port = nmp->b_rptr; 932 } else { 933 tcp_port = NULL; 934 } 935 } 936 if (tcp_port == NULL) { 937 hdrs_size = 0; 938 ((p_ether_header_t)hdrs_buf)->ether_type = 0; 939 while ((nmp) && (hdrs_size < 940 sizeof (hdrs_buf))) { 941 mblk_len = MBLKL(nmp); 942 if (mblk_len >= 943 (sizeof (hdrs_buf) - hdrs_size)) 944 mblk_len = sizeof (hdrs_buf) - 945 hdrs_size; 946 bcopy(nmp->b_rptr, 947 &hdrs_buf[hdrs_size], mblk_len); 948 hdrs_size += mblk_len; 949 nmp = nmp->b_cont; 950 } 951 tcp_port = hdrs_buf; 952 } 953 tcp_port += sizeof (ether_header_t); 954 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 955 switch (tcp_port[9]) { 956 case IPPROTO_TCP: 957 case IPPROTO_UDP: 958 case IPPROTO_ESP: 959 tcp_port += ((*tcp_port) & 0x0f) << 2; 960 ring_index = ((tcp_port[0] ^ 961 tcp_port[1] ^ 962 tcp_port[2] ^ 963 tcp_port[3]) % maxtdcs); 964 break; 965 966 case IPPROTO_AH: 967 /* SPI starts at the 4th byte */ 968 tcp_port += ((*tcp_port) & 0x0f) << 2; 969 ring_index = ((tcp_port[4] ^ 970 tcp_port[5] ^ 971 tcp_port[6] ^ 972 tcp_port[7]) % maxtdcs); 973 break; 974 975 default: 976 ring_index = tcp_port[19] % maxtdcs; 977 break; 978 } 979 } else { /* fragmented packet */ 980 ring_index = tcp_port[19] % maxtdcs; 981 } 982 } else { 983 ring_index = mp->b_band % maxtdcs; 984 } 985 break; 986 987 case HXGE_TX_LB_HASH: 988 if (hp->hash) { 989 #if defined(__i386) 990 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 991 #else 992 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 993 #endif 994 } else { 995 ring_index = mp->b_band % maxtdcs; 996 } 997 break; 998 999 case HXGE_TX_LB_DEST_MAC: 1000 /* Use destination MAC address */ 1001 tcp_port = mp->b_rptr; 1002 ring_index = tcp_port[5] % maxtdcs; 1003 break; 1004 } 1005 HXGE_DEBUG_MSG((NULL, TX_CTL, "<== hxge_tx_lb_ring")); 1006 return (ring_index); 1007 } 1008 1009 uint_t 1010 hxge_reschedule(caddr_t arg) 1011 { 1012 p_hxge_t hxgep; 1013 1014 hxgep = (p_hxge_t)arg; 1015 1016 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_reschedule")); 1017 1018 if (hxgep->hxge_mac_state == HXGE_MAC_STARTED && 1019 hxgep->resched_needed) { 1020 mac_tx_update(hxgep->mach); 1021 hxgep->resched_needed = B_FALSE; 1022 hxgep->resched_running = B_FALSE; 1023 } 1024 1025 HXGE_DEBUG_MSG((NULL, TX_CTL, "<== hxge_reschedule")); 1026 return (DDI_INTR_CLAIMED); 1027 } 1028