1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <hxge_impl.h> 27 28 extern uint32_t hxge_reclaim_pending; 29 extern uint32_t hxge_bcopy_thresh; 30 extern uint32_t hxge_dvma_thresh; 31 extern uint32_t hxge_dma_stream_thresh; 32 extern uint32_t hxge_tx_minfree; 33 extern uint32_t hxge_tx_intr_thres; 34 extern uint32_t hxge_tx_max_gathers; 35 extern uint32_t hxge_tx_tiny_pack; 36 extern uint32_t hxge_tx_use_bcopy; 37 38 static int hxge_start(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp); 39 40 void 41 hxge_tx_ring_task(void *arg) 42 { 43 p_tx_ring_t ring = (p_tx_ring_t)arg; 44 45 MUTEX_ENTER(&ring->lock); 46 (void) hxge_txdma_reclaim(ring->hxgep, ring, 0); 47 MUTEX_EXIT(&ring->lock); 48 49 mac_tx_ring_update(ring->hxgep->mach, ring->ring_handle); 50 } 51 52 static void 53 hxge_tx_ring_dispatch(p_tx_ring_t ring) 54 { 55 /* 56 * Kick the ring task to reclaim some buffers. 57 */ 58 (void) ddi_taskq_dispatch(ring->taskq, 59 hxge_tx_ring_task, (void *)ring, DDI_SLEEP); 60 } 61 62 mblk_t * 63 hxge_tx_ring_send(void *arg, mblk_t *mp) 64 { 65 p_hxge_ring_handle_t rhp = (p_hxge_ring_handle_t)arg; 66 p_hxge_t hxgep; 67 p_tx_ring_t tx_ring_p; 68 int status; 69 70 ASSERT(rhp != NULL); 71 ASSERT((rhp->index >= 0) && (rhp->index < HXGE_MAX_TDCS)); 72 73 hxgep = rhp->hxgep; 74 tx_ring_p = hxgep->tx_rings->rings[rhp->index]; 75 ASSERT(hxgep == tx_ring_p->hxgep); 76 77 status = hxge_start(hxgep, tx_ring_p, mp); 78 if (status != 0) { 79 hxge_tx_ring_dispatch(tx_ring_p); 80 return (mp); 81 } 82 83 return ((mblk_t *)NULL); 84 } 85 86 static int 87 hxge_start(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 88 { 89 int status = 0; 90 p_tx_desc_t tx_desc_ring_vp; 91 hpi_handle_t hpi_desc_handle; 92 hxge_os_dma_handle_t tx_desc_dma_handle; 93 p_tx_desc_t tx_desc_p; 94 p_tx_msg_t tx_msg_ring; 95 p_tx_msg_t tx_msg_p; 96 tx_desc_t tx_desc, *tmp_desc_p; 97 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 98 p_tx_pkt_header_t hdrp; 99 p_tx_pkt_hdr_all_t pkthdrp; 100 uint8_t npads = 0; 101 uint64_t dma_ioaddr; 102 uint32_t dma_flags; 103 int last_bidx; 104 uint8_t *b_rptr; 105 caddr_t kaddr; 106 uint32_t nmblks; 107 uint32_t ngathers; 108 uint32_t clen; 109 int len; 110 uint32_t pkt_len, pack_len, min_len; 111 uint32_t bcopy_thresh; 112 int i, cur_index, sop_index; 113 uint16_t tail_index; 114 boolean_t tail_wrap = B_FALSE; 115 hxge_dma_common_t desc_area; 116 hxge_os_dma_handle_t dma_handle; 117 ddi_dma_cookie_t dma_cookie; 118 hpi_handle_t hpi_handle; 119 p_mblk_t nmp; 120 p_mblk_t t_mp; 121 uint32_t ncookies; 122 boolean_t good_packet; 123 boolean_t mark_mode = B_FALSE; 124 p_hxge_stats_t statsp; 125 p_hxge_tx_ring_stats_t tdc_stats; 126 t_uscalar_t start_offset = 0; 127 t_uscalar_t stuff_offset = 0; 128 t_uscalar_t end_offset = 0; 129 t_uscalar_t value = 0; 130 t_uscalar_t cksum_flags = 0; 131 boolean_t cksum_on = B_FALSE; 132 uint32_t boff = 0; 133 uint64_t tot_xfer_len = 0, tmp_len = 0; 134 boolean_t header_set = B_FALSE; 135 tdc_tdr_kick_t kick; 136 uint32_t offset; 137 #ifdef HXGE_DEBUG 138 p_tx_desc_t tx_desc_ring_pp; 139 p_tx_desc_t tx_desc_pp; 140 tx_desc_t *save_desc_p; 141 int dump_len; 142 int sad_len; 143 uint64_t sad; 144 int xfer_len; 145 uint32_t msgsize; 146 #endif 147 148 HXGE_DEBUG_MSG((hxgep, TX_CTL, 149 "==> hxge_start: tx dma channel %d", tx_ring_p->tdc)); 150 HXGE_DEBUG_MSG((hxgep, TX_CTL, 151 "==> hxge_start: Starting tdc %d desc pending %d", 152 tx_ring_p->tdc, tx_ring_p->descs_pending)); 153 154 statsp = hxgep->statsp; 155 156 if (hxgep->statsp->port_stats.lb_mode == hxge_lb_normal) { 157 if (!statsp->mac_stats.link_up) { 158 freemsg(mp); 159 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: " 160 "link not up or LB mode")); 161 goto hxge_start_fail1; 162 } 163 } 164 165 hcksum_retrieve(mp, NULL, NULL, &start_offset, 166 &stuff_offset, &end_offset, &value, &cksum_flags); 167 if (!HXGE_IS_VLAN_PACKET(mp->b_rptr)) { 168 start_offset += sizeof (ether_header_t); 169 stuff_offset += sizeof (ether_header_t); 170 } else { 171 start_offset += sizeof (struct ether_vlan_header); 172 stuff_offset += sizeof (struct ether_vlan_header); 173 } 174 175 if (cksum_flags & HCK_PARTIALCKSUM) { 176 HXGE_DEBUG_MSG((hxgep, TX_CTL, 177 "==> hxge_start: mp $%p len %d " 178 "cksum_flags 0x%x (partial checksum) ", 179 mp, MBLKL(mp), cksum_flags)); 180 cksum_on = B_TRUE; 181 } 182 183 MUTEX_ENTER(&tx_ring_p->lock); 184 start_again: 185 ngathers = 0; 186 sop_index = tx_ring_p->wr_index; 187 #ifdef HXGE_DEBUG 188 if (tx_ring_p->descs_pending) { 189 HXGE_DEBUG_MSG((hxgep, TX_CTL, 190 "==> hxge_start: desc pending %d ", 191 tx_ring_p->descs_pending)); 192 } 193 194 dump_len = (int)(MBLKL(mp)); 195 dump_len = (dump_len > 128) ? 128: dump_len; 196 197 HXGE_DEBUG_MSG((hxgep, TX_CTL, 198 "==> hxge_start: tdc %d: dumping ...: b_rptr $%p " 199 "(Before header reserve: ORIGINAL LEN %d)", 200 tx_ring_p->tdc, mp->b_rptr, dump_len)); 201 202 HXGE_DEBUG_MSG((hxgep, TX_CTL, 203 "==> hxge_start: dump packets (IP ORIGINAL b_rptr $%p): %s", 204 mp->b_rptr, hxge_dump_packet((char *)mp->b_rptr, dump_len))); 205 #endif 206 207 tdc_stats = tx_ring_p->tdc_stats; 208 mark_mode = (tx_ring_p->descs_pending && 209 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) < 210 hxge_tx_minfree)); 211 212 HXGE_DEBUG_MSG((hxgep, TX_CTL, 213 "TX Descriptor ring is channel %d mark mode %d", 214 tx_ring_p->tdc, mark_mode)); 215 216 if (!hxge_txdma_reclaim(hxgep, tx_ring_p, hxge_tx_minfree)) { 217 HXGE_DEBUG_MSG((hxgep, TX_CTL, 218 "TX Descriptor ring is full: channel %d", tx_ring_p->tdc)); 219 HXGE_DEBUG_MSG((hxgep, TX_CTL, 220 "TX Descriptor ring is full: channel %d", tx_ring_p->tdc)); 221 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 222 tdc_stats->tx_no_desc++; 223 MUTEX_EXIT(&tx_ring_p->lock); 224 status = 1; 225 goto hxge_start_fail1; 226 } 227 228 nmp = mp; 229 i = sop_index = tx_ring_p->wr_index; 230 nmblks = 0; 231 ngathers = 0; 232 pkt_len = 0; 233 pack_len = 0; 234 clen = 0; 235 last_bidx = -1; 236 good_packet = B_TRUE; 237 238 desc_area = tx_ring_p->tdc_desc; 239 hpi_handle = desc_area.hpi_handle; 240 hpi_desc_handle.regh = (hxge_os_acc_handle_t) 241 DMA_COMMON_ACC_HANDLE(desc_area); 242 hpi_desc_handle.hxgep = hxgep; 243 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 244 #ifdef HXGE_DEBUG 245 #if defined(__i386) 246 tx_desc_ring_pp = (p_tx_desc_t)(uint32_t)DMA_COMMON_IOADDR(desc_area); 247 #else 248 tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area); 249 #endif 250 #endif 251 tx_desc_dma_handle = (hxge_os_dma_handle_t)DMA_COMMON_HANDLE(desc_area); 252 tx_msg_ring = tx_ring_p->tx_msg_ring; 253 254 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: wr_index %d i %d", 255 sop_index, i)); 256 257 #ifdef HXGE_DEBUG 258 msgsize = msgdsize(nmp); 259 HXGE_DEBUG_MSG((hxgep, TX_CTL, 260 "==> hxge_start(1): wr_index %d i %d msgdsize %d", 261 sop_index, i, msgsize)); 262 #endif 263 /* 264 * The first 16 bytes of the premapped buffer are reserved 265 * for header. No padding will be used. 266 */ 267 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 268 if (hxge_tx_use_bcopy) { 269 bcopy_thresh = (hxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 270 } else { 271 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 272 } 273 while (nmp) { 274 good_packet = B_TRUE; 275 b_rptr = nmp->b_rptr; 276 len = MBLKL(nmp); 277 if (len <= 0) { 278 nmp = nmp->b_cont; 279 continue; 280 } 281 nmblks++; 282 283 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(1): nmblks %d " 284 "len %d pkt_len %d pack_len %d", 285 nmblks, len, pkt_len, pack_len)); 286 /* 287 * Hardware limits the transfer length to 4K. 288 * If len is more than 4K, we need to break 289 * nmp into two chunks: Make first chunk smaller 290 * than 4K. The second chunk will be broken into 291 * less than 4K (if needed) during the next pass. 292 */ 293 if (len > (TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE)) { 294 if ((t_mp = dupb(nmp)) != NULL) { 295 nmp->b_wptr = nmp->b_rptr + 296 (TX_MAX_TRANSFER_LENGTH - 297 TX_PKT_HEADER_SIZE); 298 t_mp->b_rptr = nmp->b_wptr; 299 t_mp->b_cont = nmp->b_cont; 300 nmp->b_cont = t_mp; 301 len = MBLKL(nmp); 302 } else { 303 good_packet = B_FALSE; 304 goto hxge_start_fail2; 305 } 306 } 307 tx_desc.value = 0; 308 tx_desc_p = &tx_desc_ring_vp[i]; 309 #ifdef HXGE_DEBUG 310 tx_desc_pp = &tx_desc_ring_pp[i]; 311 #endif 312 tx_msg_p = &tx_msg_ring[i]; 313 #if defined(__i386) 314 hpi_desc_handle.regp = (uint32_t)tx_desc_p; 315 #else 316 hpi_desc_handle.regp = (uint64_t)tx_desc_p; 317 #endif 318 if (!header_set && 319 ((!hxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 320 (len >= bcopy_thresh))) { 321 header_set = B_TRUE; 322 bcopy_thresh += TX_PKT_HEADER_SIZE; 323 boff = 0; 324 pack_len = 0; 325 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 326 hdrp = (p_tx_pkt_header_t)kaddr; 327 clen = pkt_len; 328 dma_handle = tx_msg_p->buf_dma_handle; 329 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 330 offset = tx_msg_p->offset_index * hxge_bcopy_thresh; 331 (void) ddi_dma_sync(dma_handle, 332 offset, hxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV); 333 334 tx_msg_p->flags.dma_type = USE_BCOPY; 335 goto hxge_start_control_header_only; 336 } 337 338 pkt_len += len; 339 pack_len += len; 340 341 HXGE_DEBUG_MSG((hxgep, TX_CTL, 342 "==> hxge_start(3): desc entry %d DESC IOADDR $%p " 343 "desc_vp $%p tx_desc_p $%p desc_pp $%p tx_desc_pp $%p " 344 "len %d pkt_len %d pack_len %d", 345 i, 346 DMA_COMMON_IOADDR(desc_area), 347 tx_desc_ring_vp, tx_desc_p, 348 tx_desc_ring_pp, tx_desc_pp, 349 len, pkt_len, pack_len)); 350 351 if (len < bcopy_thresh) { 352 HXGE_DEBUG_MSG((hxgep, TX_CTL, 353 "==> hxge_start(4): USE BCOPY: ")); 354 if (hxge_tx_tiny_pack) { 355 uint32_t blst = TXDMA_DESC_NEXT_INDEX(i, -1, 356 tx_ring_p->tx_wrap_mask); 357 HXGE_DEBUG_MSG((hxgep, TX_CTL, 358 "==> hxge_start(5): pack")); 359 if ((pack_len <= bcopy_thresh) && 360 (last_bidx == blst)) { 361 HXGE_DEBUG_MSG((hxgep, TX_CTL, 362 "==> hxge_start: pack(6) " 363 "(pkt_len %d pack_len %d)", 364 pkt_len, pack_len)); 365 i = blst; 366 tx_desc_p = &tx_desc_ring_vp[i]; 367 #ifdef HXGE_DEBUG 368 tx_desc_pp = &tx_desc_ring_pp[i]; 369 #endif 370 tx_msg_p = &tx_msg_ring[i]; 371 boff = pack_len - len; 372 ngathers--; 373 } else if (pack_len > bcopy_thresh && 374 header_set) { 375 pack_len = len; 376 boff = 0; 377 bcopy_thresh = hxge_bcopy_thresh; 378 HXGE_DEBUG_MSG((hxgep, TX_CTL, 379 "==> hxge_start(7): > max NEW " 380 "bcopy thresh %d " 381 "pkt_len %d pack_len %d(next)", 382 bcopy_thresh, pkt_len, pack_len)); 383 } 384 last_bidx = i; 385 } 386 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 387 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 388 hdrp = (p_tx_pkt_header_t)kaddr; 389 header_set = B_TRUE; 390 HXGE_DEBUG_MSG((hxgep, TX_CTL, 391 "==> hxge_start(7_x2): " 392 "pkt_len %d pack_len %d (new hdrp $%p)", 393 pkt_len, pack_len, hdrp)); 394 } 395 tx_msg_p->flags.dma_type = USE_BCOPY; 396 kaddr += boff; 397 HXGE_DEBUG_MSG((hxgep, TX_CTL, 398 "==> hxge_start(8): USE BCOPY: before bcopy " 399 "DESC IOADDR $%p entry %d bcopy packets %d " 400 "bcopy kaddr $%p bcopy ioaddr (SAD) $%p " 401 "bcopy clen %d bcopy boff %d", 402 DMA_COMMON_IOADDR(desc_area), i, 403 tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr, 404 clen, boff)); 405 HXGE_DEBUG_MSG((hxgep, TX_CTL, 406 "==> hxge_start: 1USE BCOPY: ")); 407 HXGE_DEBUG_MSG((hxgep, TX_CTL, 408 "==> hxge_start: 2USE BCOPY: ")); 409 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: " 410 "last USE BCOPY: copy from b_rptr $%p " 411 "to KADDR $%p (len %d offset %d", 412 b_rptr, kaddr, len, boff)); 413 bcopy(b_rptr, kaddr, len); 414 #ifdef HXGE_DEBUG 415 dump_len = (len > 128) ? 128: len; 416 HXGE_DEBUG_MSG((hxgep, TX_CTL, 417 "==> hxge_start: dump packets " 418 "(After BCOPY len %d)" 419 "(b_rptr $%p): %s", len, nmp->b_rptr, 420 hxge_dump_packet((char *)nmp->b_rptr, 421 dump_len))); 422 #endif 423 dma_handle = tx_msg_p->buf_dma_handle; 424 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 425 offset = tx_msg_p->offset_index * hxge_bcopy_thresh; 426 (void) ddi_dma_sync(dma_handle, 427 offset, hxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV); 428 clen = len + boff; 429 tdc_stats->tx_hdr_pkts++; 430 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(9): " 431 "USE BCOPY: DESC IOADDR $%p entry %d " 432 "bcopy packets %d bcopy kaddr $%p " 433 "bcopy ioaddr (SAD) $%p bcopy clen %d " 434 "bcopy boff %d", 435 DMA_COMMON_IOADDR(desc_area), i, 436 tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr, 437 clen, boff)); 438 } else { 439 HXGE_DEBUG_MSG((hxgep, TX_CTL, 440 "==> hxge_start(12): USE DVMA: len %d", len)); 441 tx_msg_p->flags.dma_type = USE_DMA; 442 dma_flags = DDI_DMA_WRITE; 443 if (len < hxge_dma_stream_thresh) { 444 dma_flags |= DDI_DMA_CONSISTENT; 445 } else { 446 dma_flags |= DDI_DMA_STREAMING; 447 } 448 449 dma_handle = tx_msg_p->dma_handle; 450 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 451 (caddr_t)b_rptr, len, dma_flags, 452 DDI_DMA_DONTWAIT, NULL, 453 &dma_cookie, &ncookies); 454 if (status == DDI_DMA_MAPPED) { 455 dma_ioaddr = dma_cookie.dmac_laddress; 456 len = (int)dma_cookie.dmac_size; 457 clen = (uint32_t)dma_cookie.dmac_size; 458 HXGE_DEBUG_MSG((hxgep, TX_CTL, 459 "==> hxge_start(12_1): " 460 "USE DVMA: len %d clen %d ngathers %d", 461 len, clen, ngathers)); 462 #if defined(__i386) 463 hpi_desc_handle.regp = (uint32_t)tx_desc_p; 464 #else 465 hpi_desc_handle.regp = (uint64_t)tx_desc_p; 466 #endif 467 while (ncookies > 1) { 468 ngathers++; 469 /* 470 * this is the fix for multiple 471 * cookies, which are basically 472 * a descriptor entry, we don't set 473 * SOP bit as well as related fields 474 */ 475 476 (void) hpi_txdma_desc_gather_set( 477 hpi_desc_handle, &tx_desc, 478 (ngathers -1), mark_mode, 479 ngathers, dma_ioaddr, clen); 480 tx_msg_p->tx_msg_size = clen; 481 HXGE_DEBUG_MSG((hxgep, TX_CTL, 482 "==> hxge_start: DMA " 483 "ncookie %d ngathers %d " 484 "dma_ioaddr $%p len %d" 485 "desc $%p descp $%p (%d)", 486 ncookies, ngathers, 487 dma_ioaddr, clen, 488 *tx_desc_p, tx_desc_p, i)); 489 490 ddi_dma_nextcookie(dma_handle, 491 &dma_cookie); 492 dma_ioaddr = dma_cookie.dmac_laddress; 493 494 len = (int)dma_cookie.dmac_size; 495 clen = (uint32_t)dma_cookie.dmac_size; 496 HXGE_DEBUG_MSG((hxgep, TX_CTL, 497 "==> hxge_start(12_2): " 498 "USE DVMA: len %d clen %d ", 499 len, clen)); 500 501 i = TXDMA_DESC_NEXT_INDEX(i, 1, 502 tx_ring_p->tx_wrap_mask); 503 tx_desc_p = &tx_desc_ring_vp[i]; 504 505 hpi_desc_handle.regp = 506 #if defined(__i386) 507 (uint32_t)tx_desc_p; 508 #else 509 (uint64_t)tx_desc_p; 510 #endif 511 tx_msg_p = &tx_msg_ring[i]; 512 tx_msg_p->flags.dma_type = USE_NONE; 513 tx_desc.value = 0; 514 ncookies--; 515 } 516 tdc_stats->tx_ddi_pkts++; 517 HXGE_DEBUG_MSG((hxgep, TX_CTL, 518 "==> hxge_start: DMA: ddi packets %d", 519 tdc_stats->tx_ddi_pkts)); 520 } else { 521 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 522 "dma mapping failed for %d " 523 "bytes addr $%p flags %x (%d)", 524 len, b_rptr, status, status)); 525 good_packet = B_FALSE; 526 tdc_stats->tx_dma_bind_fail++; 527 tx_msg_p->flags.dma_type = USE_NONE; 528 goto hxge_start_fail2; 529 } 530 } /* ddi dvma */ 531 532 nmp = nmp->b_cont; 533 hxge_start_control_header_only: 534 #if defined(__i386) 535 hpi_desc_handle.regp = (uint32_t)tx_desc_p; 536 #else 537 hpi_desc_handle.regp = (uint64_t)tx_desc_p; 538 #endif 539 ngathers++; 540 541 if (ngathers == 1) { 542 #ifdef HXGE_DEBUG 543 save_desc_p = &sop_tx_desc; 544 #endif 545 sop_tx_desc_p = &sop_tx_desc; 546 sop_tx_desc_p->value = 0; 547 sop_tx_desc_p->bits.tr_len = clen; 548 sop_tx_desc_p->bits.sad = dma_ioaddr >> 32; 549 sop_tx_desc_p->bits.sad_l = dma_ioaddr & 0xffffffff; 550 } else { 551 #ifdef HXGE_DEBUG 552 save_desc_p = &tx_desc; 553 #endif 554 tmp_desc_p = &tx_desc; 555 tmp_desc_p->value = 0; 556 tmp_desc_p->bits.tr_len = clen; 557 tmp_desc_p->bits.sad = dma_ioaddr >> 32; 558 tmp_desc_p->bits.sad_l = dma_ioaddr & 0xffffffff; 559 560 tx_desc_p->value = tmp_desc_p->value; 561 } 562 563 HXGE_DEBUG_MSG((hxgep, TX_CTL, 564 "==> hxge_start(13): Desc_entry %d ngathers %d " 565 "desc_vp $%p tx_desc_p $%p " 566 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 567 "dma_ioaddr (SAD) $%p mark %d", 568 i, ngathers, tx_desc_ring_vp, tx_desc_p, 569 len, clen, pkt_len, pack_len, nmblks, 570 dma_ioaddr, mark_mode)); 571 572 #ifdef HXGE_DEBUG 573 hpi_desc_handle.hxgep = hxgep; 574 hpi_desc_handle.function.function = 0; 575 hpi_desc_handle.function.instance = hxgep->instance; 576 sad = save_desc_p->bits.sad; 577 sad = (sad << 32) | save_desc_p->bits.sad_l; 578 xfer_len = save_desc_p->bits.tr_len; 579 580 HXGE_DEBUG_MSG((hxgep, TX_CTL, "\n\t: value 0x%llx\n" 581 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 582 "mark %d sop %d\n", 583 save_desc_p->value, sad, save_desc_p->bits.tr_len, 584 xfer_len, save_desc_p->bits.num_ptr, 585 save_desc_p->bits.mark, save_desc_p->bits.sop)); 586 587 hpi_txdma_dump_desc_one(hpi_desc_handle, NULL, i); 588 #endif 589 590 tx_msg_p->tx_msg_size = clen; 591 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 592 if (ngathers > hxge_tx_max_gathers) { 593 good_packet = B_FALSE; 594 hcksum_retrieve(mp, NULL, NULL, &start_offset, 595 &stuff_offset, &end_offset, &value, &cksum_flags); 596 597 HXGE_DEBUG_MSG((NULL, TX_CTL, 598 "==> hxge_start(14): pull msg - " 599 "len %d pkt_len %d ngathers %d", 600 len, pkt_len, ngathers)); 601 /* Pull all message blocks from b_cont */ 602 if ((msgpullup(mp, -1)) == NULL) { 603 goto hxge_start_fail2; 604 } 605 goto hxge_start_fail2; 606 } 607 } /* while (nmp) */ 608 609 tx_msg_p->tx_message = mp; 610 tx_desc_p = &tx_desc_ring_vp[sop_index]; 611 #if defined(__i386) 612 hpi_desc_handle.regp = (uint32_t)tx_desc_p; 613 #else 614 hpi_desc_handle.regp = (uint64_t)tx_desc_p; 615 #endif 616 617 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 618 pkthdrp->reserved = 0; 619 hdrp->value = 0; 620 (void) hxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 621 (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp); 622 623 /* 624 * Hardware header should not be counted as part of the frame 625 * when determining the frame size 626 */ 627 if ((pkt_len - TX_PKT_HEADER_SIZE) > (STD_FRAME_SIZE - ETHERFCSL)) { 628 tdc_stats->tx_jumbo_pkts++; 629 } 630 631 min_len = (hxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2)); 632 if (pkt_len < min_len) { 633 /* Assume we use bcopy to premapped buffers */ 634 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 635 HXGE_DEBUG_MSG((NULL, TX_CTL, 636 "==> hxge_start(14-1): < (msg_min + 16)" 637 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 638 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 639 bzero((kaddr + pkt_len), (min_len - pkt_len)); 640 pkt_len = tx_msg_p->tx_msg_size = min_len; 641 642 sop_tx_desc_p->bits.tr_len = min_len; 643 644 HXGE_MEM_PIO_WRITE64(hpi_desc_handle, sop_tx_desc_p->value); 645 tx_desc_p->value = sop_tx_desc_p->value; 646 647 HXGE_DEBUG_MSG((NULL, TX_CTL, 648 "==> hxge_start(14-2): < msg_min - " 649 "len %d pkt_len %d min_len %d ngathers %d", 650 len, pkt_len, min_len, ngathers)); 651 } 652 653 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: cksum_flags 0x%x ", 654 cksum_flags)); 655 if (cksum_flags & HCK_PARTIALCKSUM) { 656 HXGE_DEBUG_MSG((hxgep, TX_CTL, 657 "==> hxge_start: cksum_flags 0x%x (partial checksum) ", 658 cksum_flags)); 659 cksum_on = B_TRUE; 660 HXGE_DEBUG_MSG((hxgep, TX_CTL, 661 "==> hxge_start: from IP cksum_flags 0x%x " 662 "(partial checksum) " 663 "start_offset %d stuff_offset %d", 664 cksum_flags, start_offset, stuff_offset)); 665 tmp_len = (uint64_t)(start_offset >> 1); 666 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT); 667 tmp_len = (uint64_t)(stuff_offset >> 1); 668 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT); 669 670 HXGE_DEBUG_MSG((hxgep, TX_CTL, 671 "==> hxge_start: from IP cksum_flags 0x%x " 672 "(partial checksum) " 673 "after SHIFT start_offset %d stuff_offset %d", 674 cksum_flags, start_offset, stuff_offset)); 675 } 676 677 /* 678 * pkt_len already includes 16 + paddings!! 679 * Update the control header length 680 */ 681 682 /* 683 * Note that Hydra is different from Neptune where 684 * tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 685 */ 686 tot_xfer_len = pkt_len; 687 tmp_len = hdrp->value | 688 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 689 690 HXGE_DEBUG_MSG((hxgep, TX_CTL, 691 "==> hxge_start(15_x1): setting SOP " 692 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 693 "0x%llx hdrp->value 0x%llx", 694 tot_xfer_len, tot_xfer_len, pkt_len, tmp_len, hdrp->value)); 695 #if defined(_BIG_ENDIAN) 696 hdrp->value = ddi_swap64(tmp_len); 697 #else 698 hdrp->value = tmp_len; 699 #endif 700 HXGE_DEBUG_MSG((hxgep, 701 TX_CTL, "==> hxge_start(15_x2): setting SOP " 702 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 703 "tmp_len 0x%llx hdrp->value 0x%llx", 704 tot_xfer_len, pkt_len, tmp_len, hdrp->value)); 705 706 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(15): setting SOP " 707 "wr_index %d tot_xfer_len (%d) pkt_len %d npads %d", 708 sop_index, tot_xfer_len, pkt_len, npads)); 709 710 sop_tx_desc_p->bits.sop = 1; 711 sop_tx_desc_p->bits.mark = mark_mode; 712 sop_tx_desc_p->bits.num_ptr = ngathers; 713 714 if (mark_mode) 715 tdc_stats->tx_marks++; 716 717 HXGE_MEM_PIO_WRITE64(hpi_desc_handle, sop_tx_desc_p->value); 718 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start(16): set SOP done")); 719 720 #ifdef HXGE_DEBUG 721 hpi_desc_handle.hxgep = hxgep; 722 hpi_desc_handle.function.function = 0; 723 hpi_desc_handle.function.instance = hxgep->instance; 724 725 HXGE_DEBUG_MSG((hxgep, TX_CTL, "\n\t: value 0x%llx\n" 726 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 727 save_desc_p->value, sad, save_desc_p->bits.tr_len, 728 xfer_len, save_desc_p->bits.num_ptr, save_desc_p->bits.mark, 729 save_desc_p->bits.sop)); 730 (void) hpi_txdma_dump_desc_one(hpi_desc_handle, NULL, sop_index); 731 732 dump_len = (pkt_len > 128) ? 128: pkt_len; 733 HXGE_DEBUG_MSG((hxgep, TX_CTL, 734 "==> hxge_start: dump packets(17) (after sop set, len " 735 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 736 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 737 (char *)hdrp, hxge_dump_packet((char *)hdrp, dump_len))); 738 HXGE_DEBUG_MSG((hxgep, TX_CTL, 739 "==> hxge_start(18): TX desc sync: sop_index %d", sop_index)); 740 #endif 741 742 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 743 (void) ddi_dma_sync(tx_desc_dma_handle, 744 sop_index * sizeof (tx_desc_t), 745 ngathers * sizeof (tx_desc_t), DDI_DMA_SYNC_FORDEV); 746 747 HXGE_DEBUG_MSG((hxgep, TX_CTL, "hxge_start(19): sync 1 " 748 "cs_off = 0x%02X cs_s_off = 0x%02X " 749 "pkt_len %d ngathers %d sop_index %d\n", 750 stuff_offset, start_offset, 751 pkt_len, ngathers, sop_index)); 752 } else { /* more than one descriptor and wrap around */ 753 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 754 (void) ddi_dma_sync(tx_desc_dma_handle, 755 sop_index * sizeof (tx_desc_t), 756 nsdescs * sizeof (tx_desc_t), DDI_DMA_SYNC_FORDEV); 757 HXGE_DEBUG_MSG((hxgep, TX_CTL, "hxge_start(20): sync 1 " 758 "cs_off = 0x%02X cs_s_off = 0x%02X " 759 "pkt_len %d ngathers %d sop_index %d\n", 760 stuff_offset, start_offset, pkt_len, ngathers, sop_index)); 761 762 (void) ddi_dma_sync(tx_desc_dma_handle, 0, 763 (ngathers - nsdescs) * sizeof (tx_desc_t), 764 DDI_DMA_SYNC_FORDEV); 765 HXGE_DEBUG_MSG((hxgep, TX_CTL, "hxge_start(21): sync 2 " 766 "cs_off = 0x%02X cs_s_off = 0x%02X " 767 "pkt_len %d ngathers %d sop_index %d\n", 768 stuff_offset, start_offset, 769 pkt_len, ngathers, sop_index)); 770 } 771 772 tail_index = tx_ring_p->wr_index; 773 tail_wrap = tx_ring_p->wr_index_wrap; 774 775 tx_ring_p->wr_index = i; 776 if (tx_ring_p->wr_index <= tail_index) { 777 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 778 B_FALSE : B_TRUE); 779 } 780 781 tx_ring_p->descs_pending += ngathers; 782 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: TX kick: " 783 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 784 tx_ring_p->tdc, tx_ring_p->wr_index, tx_ring_p->wr_index_wrap, 785 ngathers, tx_ring_p->descs_pending)); 786 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: TX KICKING: ")); 787 788 kick.value = 0; 789 kick.bits.wrap = tx_ring_p->wr_index_wrap; 790 kick.bits.tail = (uint16_t)tx_ring_p->wr_index; 791 792 /* Kick start the Transmit kick register */ 793 TXDMA_REG_WRITE64(HXGE_DEV_HPI_HANDLE(hxgep), 794 TDC_TDR_KICK, (uint8_t)tx_ring_p->tdc, kick.value); 795 tdc_stats->tx_starts++; 796 MUTEX_EXIT(&tx_ring_p->lock); 797 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_start")); 798 return (status); 799 800 hxge_start_fail2: 801 if (good_packet == B_FALSE) { 802 cur_index = sop_index; 803 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_start: clean up")); 804 for (i = 0; i < ngathers; i++) { 805 tx_desc_p = &tx_desc_ring_vp[cur_index]; 806 #if defined(__i386) 807 hpi_handle.regp = (uint32_t)tx_desc_p; 808 #else 809 hpi_handle.regp = (uint64_t)tx_desc_p; 810 #endif 811 tx_msg_p = &tx_msg_ring[cur_index]; 812 (void) hpi_txdma_desc_set_zero(hpi_handle, 1); 813 if (tx_msg_p->flags.dma_type == USE_DVMA) { 814 HXGE_DEBUG_MSG((hxgep, TX_CTL, 815 "tx_desc_p = %X index = %d", 816 tx_desc_p, tx_ring_p->rd_index)); 817 (void) dvma_unload(tx_msg_p->dvma_handle, 818 0, -1); 819 tx_msg_p->dvma_handle = NULL; 820 if (tx_ring_p->dvma_wr_index == 821 tx_ring_p->dvma_wrap_mask) 822 tx_ring_p->dvma_wr_index = 0; 823 else 824 tx_ring_p->dvma_wr_index++; 825 tx_ring_p->dvma_pending--; 826 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 827 if (ddi_dma_unbind_handle( 828 tx_msg_p->dma_handle)) { 829 cmn_err(CE_WARN, "hxge_start: " 830 "ddi_dma_unbind_handle failed"); 831 } 832 } 833 tx_msg_p->flags.dma_type = USE_NONE; 834 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 835 tx_ring_p->tx_wrap_mask); 836 837 } 838 } 839 840 MUTEX_EXIT(&tx_ring_p->lock); 841 842 hxge_start_fail1: 843 /* Add FMA to check the access handle hxge_hregh */ 844 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_start")); 845 return (status); 846 } 847