/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" #include <sys/nxge/nxge_impl.h> /* Software LSO required header files */ #include <netinet/tcp.h> #include <inet/ip_impl.h> #include <inet/tcp.h> static mblk_t *nxge_lso_eliminate(mblk_t *); static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); static void nxge_hcksum_retrieve(mblk_t *, uint32_t *, uint32_t *, uint32_t *, uint32_t *, uint32_t *); static uint32_t nxge_csgen(uint16_t *, int); extern uint32_t nxge_reclaim_pending; extern uint32_t nxge_bcopy_thresh; extern uint32_t nxge_dvma_thresh; extern uint32_t nxge_dma_stream_thresh; extern uint32_t nxge_tx_minfree; extern uint32_t nxge_tx_intr_thres; extern uint32_t nxge_tx_max_gathers; extern uint32_t nxge_tx_tiny_pack; extern uint32_t nxge_tx_use_bcopy; extern uint32_t nxge_tx_lb_policy; extern uint32_t nxge_no_tx_lb; extern nxge_tx_mode_t nxge_tx_scheme; extern uint32_t nxge_lso_enable; uint32_t nxge_lso_kick_cnt = 2; typedef struct _mac_tx_hint { uint16_t sap; uint16_t vid; void *hash; } mac_tx_hint_t, *p_mac_tx_hint_t; int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); int nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) { int status = 0; p_tx_desc_t tx_desc_ring_vp; npi_handle_t npi_desc_handle; nxge_os_dma_handle_t tx_desc_dma_handle; p_tx_desc_t tx_desc_p; p_tx_msg_t tx_msg_ring; p_tx_msg_t tx_msg_p; tx_desc_t tx_desc, *tmp_desc_p; tx_desc_t sop_tx_desc, *sop_tx_desc_p; p_tx_pkt_header_t hdrp; p_tx_pkt_hdr_all_t pkthdrp; uint8_t npads = 0; uint64_t dma_ioaddr; uint32_t dma_flags; int last_bidx; uint8_t *b_rptr; caddr_t kaddr; uint32_t nmblks; uint32_t ngathers; uint32_t clen; int len; uint32_t pkt_len, pack_len, min_len; uint32_t bcopy_thresh; int i, cur_index, sop_index; uint16_t tail_index; boolean_t tail_wrap = B_FALSE; nxge_dma_common_t desc_area; nxge_os_dma_handle_t dma_handle; ddi_dma_cookie_t dma_cookie; npi_handle_t npi_handle; p_mblk_t nmp; p_mblk_t t_mp; uint32_t ncookies; boolean_t good_packet; boolean_t mark_mode = B_FALSE; p_nxge_stats_t statsp; p_nxge_tx_ring_stats_t tdc_stats; t_uscalar_t start_offset = 0; t_uscalar_t stuff_offset = 0; t_uscalar_t end_offset = 0; t_uscalar_t value = 0; t_uscalar_t cksum_flags = 0; boolean_t cksum_on = B_FALSE; uint32_t boff = 0; uint64_t tot_xfer_len = 0, tmp_len = 0; boolean_t header_set = B_FALSE; #ifdef NXGE_DEBUG p_tx_desc_t tx_desc_ring_pp; p_tx_desc_t tx_desc_pp; tx_desc_t *save_desc_p; int dump_len; int sad_len; uint64_t sad; int xfer_len; uint32_t msgsize; #endif p_mblk_t mp_chain = NULL; boolean_t is_lso = B_FALSE; boolean_t lso_again; int cur_index_lso; p_mblk_t nmp_lso_save; uint32_t lso_ngathers; boolean_t lso_tail_wrap = B_FALSE; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: Starting tdc %d desc pending %d", tx_ring_p->tdc, tx_ring_p->descs_pending)); statsp = nxgep->statsp; if (nxgep->statsp->port_stats.lb_mode == nxge_lb_normal) { if (!statsp->mac_stats.link_up) { freemsg(mp); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " "link not up or LB mode")); goto nxge_start_fail1; } } if (nxge_lso_enable) { mp_chain = nxge_lso_eliminate(mp); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(0): LSO mp $%p mp_chain $%p", mp, mp_chain)); if (mp_chain == NULL) { NXGE_ERROR_MSG((nxgep, TX_CTL, "==> nxge_send(0): NULL mp_chain $%p != mp $%p", mp_chain, mp)); goto nxge_start_fail1; } if (mp_chain != mp) { NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", mp_chain, mp)); is_lso = B_TRUE; mp = mp_chain; mp_chain = mp_chain->b_next; mp->b_next = NULL; } } hcksum_retrieve(mp, NULL, NULL, &start_offset, &stuff_offset, &end_offset, &value, &cksum_flags); if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { start_offset += sizeof (ether_header_t); stuff_offset += sizeof (ether_header_t); } else { start_offset += sizeof (struct ether_vlan_header); stuff_offset += sizeof (struct ether_vlan_header); } if (cksum_flags & HCK_PARTIALCKSUM) { NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: mp $%p len %d " "cksum_flags 0x%x (partial checksum) ", mp, MBLKL(mp), cksum_flags)); cksum_on = B_TRUE; } lso_again = B_FALSE; lso_ngathers = 0; MUTEX_ENTER(&tx_ring_p->lock); cur_index_lso = tx_ring_p->wr_index; lso_tail_wrap = tx_ring_p->wr_index_wrap; start_again: ngathers = 0; sop_index = tx_ring_p->wr_index; #ifdef NXGE_DEBUG if (tx_ring_p->descs_pending) { NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " "desc pending %d ", tx_ring_p->descs_pending)); } dump_len = (int)(MBLKL(mp)); dump_len = (dump_len > 128) ? 128: dump_len; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " "(Before header reserve: ORIGINAL LEN %d)", tx_ring_p->tdc, mp->b_rptr, dump_len)); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, nxge_dump_packet((char *)mp->b_rptr, dump_len))); #endif tdc_stats = tx_ring_p->tdc_stats; mark_mode = (tx_ring_p->descs_pending && ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) < nxge_tx_minfree)); NXGE_DEBUG_MSG((nxgep, TX_CTL, "TX Descriptor ring is channel %d mark mode %d", tx_ring_p->tdc, mark_mode)); if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { NXGE_DEBUG_MSG((nxgep, TX_CTL, "TX Descriptor ring is full: channel %d", tx_ring_p->tdc)); NXGE_DEBUG_MSG((nxgep, TX_CTL, "TX Descriptor ring is full: channel %d", tx_ring_p->tdc)); if (is_lso) { /* free the current mp and mp_chain if not FULL */ tdc_stats->tx_no_desc++; NXGE_DEBUG_MSG((nxgep, TX_CTL, "LSO packet: TX Descriptor ring is full: " "channel %d", tx_ring_p->tdc)); goto nxge_start_fail_lso; } else { cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); tdc_stats->tx_no_desc++; MUTEX_EXIT(&tx_ring_p->lock); if (nxgep->resched_needed && !nxgep->resched_running) { nxgep->resched_running = B_TRUE; ddi_trigger_softintr(nxgep->resched_id); } status = 1; goto nxge_start_fail1; } } nmp = mp; i = sop_index = tx_ring_p->wr_index; nmblks = 0; ngathers = 0; pkt_len = 0; pack_len = 0; clen = 0; last_bidx = -1; good_packet = B_TRUE; desc_area = tx_ring_p->tdc_desc; npi_handle = desc_area.npi_handle; npi_desc_handle.regh = (nxge_os_acc_handle_t) DMA_COMMON_ACC_HANDLE(desc_area); tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); #ifdef NXGE_DEBUG tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area); #endif tx_desc_dma_handle = (nxge_os_dma_handle_t) DMA_COMMON_HANDLE(desc_area); tx_msg_ring = tx_ring_p->tx_msg_ring; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", sop_index, i)); #ifdef NXGE_DEBUG msgsize = msgdsize(nmp); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): wr_index %d i %d msgdsize %d", sop_index, i, msgsize)); #endif /* * The first 16 bytes of the premapped buffer are reserved * for header. No padding will be used. */ pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); } else { bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); } while (nmp) { good_packet = B_TRUE; b_rptr = nmp->b_rptr; len = MBLKL(nmp); if (len <= 0) { nmp = nmp->b_cont; continue; } nmblks++; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " "len %d pkt_len %d pack_len %d", nmblks, len, pkt_len, pack_len)); /* * Hardware limits the transfer length to 4K for NIU and * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just * use TX_MAX_TRANSFER_LENGTH as the limit for both. * If len is longer than the limit, then we break nmp into * two chunks: Make the first chunk equal to the limit and * the second chunk for the remaining data. If the second * chunk is still larger than the limit, then it will be * broken into two in the next pass. */ if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { if ((t_mp = dupb(nmp)) != NULL) { nmp->b_wptr = nmp->b_rptr + (TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE); t_mp->b_rptr = nmp->b_wptr; t_mp->b_cont = nmp->b_cont; nmp->b_cont = t_mp; len = MBLKL(nmp); } else { if (is_lso) { NXGE_DEBUG_MSG((nxgep, TX_CTL, "LSO packet: dupb failed: " "channel %d", tx_ring_p->tdc)); mp = nmp; goto nxge_start_fail_lso; } else { good_packet = B_FALSE; goto nxge_start_fail2; } } } tx_desc.value = 0; tx_desc_p = &tx_desc_ring_vp[i]; #ifdef NXGE_DEBUG tx_desc_pp = &tx_desc_ring_pp[i]; #endif tx_msg_p = &tx_msg_ring[i]; #if defined(__i386) npi_desc_handle.regp = (uint32_t)tx_desc_p; #else npi_desc_handle.regp = (uint64_t)tx_desc_p; #endif if (!header_set && ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || (len >= bcopy_thresh))) { header_set = B_TRUE; bcopy_thresh += TX_PKT_HEADER_SIZE; boff = 0; pack_len = 0; kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); hdrp = (p_tx_pkt_header_t)kaddr; clen = pkt_len; dma_handle = tx_msg_p->buf_dma_handle; dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); (void) ddi_dma_sync(dma_handle, i * nxge_bcopy_thresh, nxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV); tx_msg_p->flags.dma_type = USE_BCOPY; goto nxge_start_control_header_only; } pkt_len += len; pack_len += len; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " "desc entry %d " "DESC IOADDR $%p " "desc_vp $%p tx_desc_p $%p " "desc_pp $%p tx_desc_pp $%p " "len %d pkt_len %d pack_len %d", i, DMA_COMMON_IOADDR(desc_area), tx_desc_ring_vp, tx_desc_p, tx_desc_ring_pp, tx_desc_pp, len, pkt_len, pack_len)); if (len < bcopy_thresh) { NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " "USE BCOPY: ")); if (nxge_tx_tiny_pack) { uint32_t blst = TXDMA_DESC_NEXT_INDEX(i, -1, tx_ring_p->tx_wrap_mask); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(5): pack")); if ((pack_len <= bcopy_thresh) && (last_bidx == blst)) { NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: pack(6) " "(pkt_len %d pack_len %d)", pkt_len, pack_len)); i = blst; tx_desc_p = &tx_desc_ring_vp[i]; #ifdef NXGE_DEBUG tx_desc_pp = &tx_desc_ring_pp[i]; #endif tx_msg_p = &tx_msg_ring[i]; boff = pack_len - len; ngathers--; } else if (pack_len > bcopy_thresh && header_set) { pack_len = len; boff = 0; bcopy_thresh = nxge_bcopy_thresh; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(7): > max NEW " "bcopy thresh %d " "pkt_len %d pack_len %d(next)", bcopy_thresh, pkt_len, pack_len)); } last_bidx = i; } kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { hdrp = (p_tx_pkt_header_t)kaddr; header_set = B_TRUE; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(7_x2): " "pkt_len %d pack_len %d (new hdrp $%p)", pkt_len, pack_len, hdrp)); } tx_msg_p->flags.dma_type = USE_BCOPY; kaddr += boff; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " "USE BCOPY: before bcopy " "DESC IOADDR $%p entry %d " "bcopy packets %d " "bcopy kaddr $%p " "bcopy ioaddr (SAD) $%p " "bcopy clen %d " "bcopy boff %d", DMA_COMMON_IOADDR(desc_area), i, tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr, clen, boff)); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " "1USE BCOPY: ")); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " "2USE BCOPY: ")); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " "last USE BCOPY: copy from b_rptr $%p " "to KADDR $%p (len %d offset %d", b_rptr, kaddr, len, boff)); bcopy(b_rptr, kaddr, len); #ifdef NXGE_DEBUG dump_len = (len > 128) ? 128: len; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " "(After BCOPY len %d)" "(b_rptr $%p): %s", len, nmp->b_rptr, nxge_dump_packet((char *)nmp->b_rptr, dump_len))); #endif dma_handle = tx_msg_p->buf_dma_handle; dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); (void) ddi_dma_sync(dma_handle, i * nxge_bcopy_thresh, nxge_bcopy_thresh, DDI_DMA_SYNC_FORDEV); clen = len + boff; tdc_stats->tx_hdr_pkts++; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " "USE BCOPY: " "DESC IOADDR $%p entry %d " "bcopy packets %d " "bcopy kaddr $%p " "bcopy ioaddr (SAD) $%p " "bcopy clen %d " "bcopy boff %d", DMA_COMMON_IOADDR(desc_area), i, tdc_stats->tx_hdr_pkts, kaddr, dma_ioaddr, clen, boff)); } else { NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " "USE DVMA: len %d", len)); tx_msg_p->flags.dma_type = USE_DMA; dma_flags = DDI_DMA_WRITE; if (len < nxge_dma_stream_thresh) { dma_flags |= DDI_DMA_CONSISTENT; } else { dma_flags |= DDI_DMA_STREAMING; } dma_handle = tx_msg_p->dma_handle; status = ddi_dma_addr_bind_handle(dma_handle, NULL, (caddr_t)b_rptr, len, dma_flags, DDI_DMA_DONTWAIT, NULL, &dma_cookie, &ncookies); if (status == DDI_DMA_MAPPED) { dma_ioaddr = dma_cookie.dmac_laddress; len = (int)dma_cookie.dmac_size; clen = (uint32_t)dma_cookie.dmac_size; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12_1): " "USE DVMA: len %d clen %d " "ngathers %d", len, clen, ngathers)); #if defined(__i386) npi_desc_handle.regp = (uint32_t)tx_desc_p; #else npi_desc_handle.regp = (uint64_t)tx_desc_p; #endif while (ncookies > 1) { ngathers++; /* * this is the fix for multiple * cookies, which are basically * a descriptor entry, we don't set * SOP bit as well as related fields */ (void) npi_txdma_desc_gather_set( npi_desc_handle, &tx_desc, (ngathers -1), mark_mode, ngathers, dma_ioaddr, clen); tx_msg_p->tx_msg_size = clen; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: DMA " "ncookie %d " "ngathers %d " "dma_ioaddr $%p len %d" "desc $%p descp $%p (%d)", ncookies, ngathers, dma_ioaddr, clen, *tx_desc_p, tx_desc_p, i)); ddi_dma_nextcookie(dma_handle, &dma_cookie); dma_ioaddr = dma_cookie.dmac_laddress; len = (int)dma_cookie.dmac_size; clen = (uint32_t)dma_cookie.dmac_size; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12_2): " "USE DVMA: len %d clen %d ", len, clen)); i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); tx_desc_p = &tx_desc_ring_vp[i]; npi_desc_handle.regp = #if defined(__i386) (uint32_t)tx_desc_p; #else (uint64_t)tx_desc_p; #endif tx_msg_p = &tx_msg_ring[i]; tx_msg_p->flags.dma_type = USE_NONE; tx_desc.value = 0; ncookies--; } tdc_stats->tx_ddi_pkts++; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" "DMA: ddi packets %d", tdc_stats->tx_ddi_pkts)); } else { NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "dma mapping failed for %d " "bytes addr $%p flags %x (%d)", len, b_rptr, status, status)); good_packet = B_FALSE; tdc_stats->tx_dma_bind_fail++; tx_msg_p->flags.dma_type = USE_NONE; if (is_lso) { mp = nmp; goto nxge_start_fail_lso; } else { goto nxge_start_fail2; } } } /* ddi dvma */ if (is_lso) { nmp_lso_save = nmp; } nmp = nmp->b_cont; nxge_start_control_header_only: #if defined(__i386) npi_desc_handle.regp = (uint32_t)tx_desc_p; #else npi_desc_handle.regp = (uint64_t)tx_desc_p; #endif ngathers++; if (ngathers == 1) { #ifdef NXGE_DEBUG save_desc_p = &sop_tx_desc; #endif sop_tx_desc_p = &sop_tx_desc; sop_tx_desc_p->value = 0; sop_tx_desc_p->bits.hdw.tr_len = clen; sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; } else { #ifdef NXGE_DEBUG save_desc_p = &tx_desc; #endif tmp_desc_p = &tx_desc; tmp_desc_p->value = 0; tmp_desc_p->bits.hdw.tr_len = clen; tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; tx_desc_p->value = tmp_desc_p->value; } NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " "Desc_entry %d ngathers %d " "desc_vp $%p tx_desc_p $%p " "len %d clen %d pkt_len %d pack_len %d nmblks %d " "dma_ioaddr (SAD) $%p mark %d", i, ngathers, tx_desc_ring_vp, tx_desc_p, len, clen, pkt_len, pack_len, nmblks, dma_ioaddr, mark_mode)); #ifdef NXGE_DEBUG npi_desc_handle.nxgep = nxgep; npi_desc_handle.function.function = nxgep->function_num; npi_desc_handle.function.instance = nxgep->instance; sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> TX_PKT_DESC_TR_LEN_SHIFT); NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" "mark %d sop %d\n", save_desc_p->value, sad, save_desc_p->bits.hdw.tr_len, xfer_len, save_desc_p->bits.hdw.num_ptr, save_desc_p->bits.hdw.mark, save_desc_p->bits.hdw.sop)); npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); #endif tx_msg_p->tx_msg_size = clen; i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); if (ngathers > nxge_tx_max_gathers) { good_packet = B_FALSE; hcksum_retrieve(mp, NULL, NULL, &start_offset, &stuff_offset, &end_offset, &value, &cksum_flags); NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_start(14): pull msg - " "len %d pkt_len %d ngathers %d", len, pkt_len, ngathers)); /* Pull all message blocks from b_cont */ if (is_lso) { mp = nmp_lso_save; goto nxge_start_fail_lso; } if ((msgpullup(mp, -1)) == NULL) { goto nxge_start_fail2; } goto nxge_start_fail2; } } /* while (nmp) */ tx_msg_p->tx_message = mp; tx_desc_p = &tx_desc_ring_vp[sop_index]; #if defined(__i386) npi_desc_handle.regp = (uint32_t)tx_desc_p; #else npi_desc_handle.regp = (uint64_t)tx_desc_p; #endif pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; pkthdrp->reserved = 0; hdrp->value = 0; (void) nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp); if (pkt_len > NXGE_MTU_DEFAULT_MAX) { tdc_stats->tx_jumbo_pkts++; } min_len = (nxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2)); if (pkt_len < min_len) { /* Assume we use bcopy to premapped buffers */ kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_start(14-1): < (msg_min + 16)" "len %d pkt_len %d min_len %d bzero %d ngathers %d", len, pkt_len, min_len, (min_len - pkt_len), ngathers)); bzero((kaddr + pkt_len), (min_len - pkt_len)); pkt_len = tx_msg_p->tx_msg_size = min_len; sop_tx_desc_p->bits.hdw.tr_len = min_len; NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); tx_desc_p->value = sop_tx_desc_p->value; NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_start(14-2): < msg_min - " "len %d pkt_len %d min_len %d ngathers %d", len, pkt_len, min_len, ngathers)); } NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", cksum_flags)); if (cksum_flags & HCK_PARTIALCKSUM) { NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x (partial checksum) ", cksum_flags)); cksum_on = B_TRUE; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: from IP cksum_flags 0x%x " "(partial checksum) " "start_offset %d stuff_offset %d", cksum_flags, start_offset, stuff_offset)); tmp_len = (uint64_t)(start_offset >> 1); hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT); tmp_len = (uint64_t)(stuff_offset >> 1); hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: from IP cksum_flags 0x%x " "(partial checksum) " "after SHIFT start_offset %d stuff_offset %d", cksum_flags, start_offset, stuff_offset)); } { uint64_t tmp_len; /* pkt_len already includes 16 + paddings!! */ /* Update the control header length */ tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); tmp_len = hdrp->value | (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15_x1): setting SOP " "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " "0x%llx hdrp->value 0x%llx", tot_xfer_len, tot_xfer_len, pkt_len, tmp_len, hdrp->value)); #if defined(_BIG_ENDIAN) hdrp->value = ddi_swap64(tmp_len); #else hdrp->value = tmp_len; #endif NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15_x2): setting SOP " "after SWAP: tot_xfer_len 0x%llx pkt_len %d " "tmp_len 0x%llx hdrp->value 0x%llx", tot_xfer_len, pkt_len, tmp_len, hdrp->value)); } NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " "wr_index %d " "tot_xfer_len (%d) pkt_len %d npads %d", sop_index, tot_xfer_len, pkt_len, npads)); sop_tx_desc_p->bits.hdw.sop = 1; sop_tx_desc_p->bits.hdw.mark = mark_mode; sop_tx_desc_p->bits.hdw.num_ptr = ngathers; NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); #ifdef NXGE_DEBUG npi_desc_handle.nxgep = nxgep; npi_desc_handle.function.function = nxgep->function_num; npi_desc_handle.function.instance = nxgep->instance; NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", save_desc_p->value, sad, save_desc_p->bits.hdw.tr_len, xfer_len, save_desc_p->bits.hdw.num_ptr, save_desc_p->bits.hdw.mark, save_desc_p->bits.hdw.sop)); (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); dump_len = (pkt_len > 128) ? 128: pkt_len; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets(17) (after sop set, len " " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, (char *)hdrp, nxge_dump_packet((char *)hdrp, dump_len))); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(18): TX desc sync: sop_index %d", sop_index)); #endif if ((ngathers == 1) || tx_ring_p->wr_index < i) { (void) ddi_dma_sync(tx_desc_dma_handle, sop_index * sizeof (tx_desc_t), ngathers * sizeof (tx_desc_t), DDI_DMA_SYNC_FORDEV); NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " "cs_off = 0x%02X cs_s_off = 0x%02X " "pkt_len %d ngathers %d sop_index %d\n", stuff_offset, start_offset, pkt_len, ngathers, sop_index)); } else { /* more than one descriptor and wrap around */ uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; (void) ddi_dma_sync(tx_desc_dma_handle, sop_index * sizeof (tx_desc_t), nsdescs * sizeof (tx_desc_t), DDI_DMA_SYNC_FORDEV); NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " "cs_off = 0x%02X cs_s_off = 0x%02X " "pkt_len %d ngathers %d sop_index %d\n", stuff_offset, start_offset, pkt_len, ngathers, sop_index)); (void) ddi_dma_sync(tx_desc_dma_handle, 0, (ngathers - nsdescs) * sizeof (tx_desc_t), DDI_DMA_SYNC_FORDEV); NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " "cs_off = 0x%02X cs_s_off = 0x%02X " "pkt_len %d ngathers %d sop_index %d\n", stuff_offset, start_offset, pkt_len, ngathers, sop_index)); } tail_index = tx_ring_p->wr_index; tail_wrap = tx_ring_p->wr_index_wrap; tx_ring_p->wr_index = i; if (tx_ring_p->wr_index <= tail_index) { tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? B_FALSE : B_TRUE); } tx_ring_p->descs_pending += ngathers; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", tx_ring_p->tdc, tx_ring_p->wr_index, tx_ring_p->wr_index_wrap, ngathers, tx_ring_p->descs_pending)); if (is_lso) { lso_ngathers += ngathers; if (mp_chain != NULL) { mp = mp_chain; mp_chain = mp_chain->b_next; mp->b_next = NULL; if (nxge_lso_kick_cnt == lso_ngathers) { { tx_ring_kick_t kick; kick.value = 0; kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; /* Kick the Transmit kick register */ TXDMA_REG_WRITE64( NXGE_DEV_NPI_HANDLE(nxgep), TX_RING_KICK_REG, (uint8_t)tx_ring_p->tdc, kick.value); tdc_stats->tx_starts++; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: more LSO: " "LSO_CNT %d", lso_gathers)); } lso_ngathers = 0; ngathers = 0; cur_index_lso = sop_index = tx_ring_p->wr_index; lso_tail_wrap = tx_ring_p->wr_index_wrap; } NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: lso again: " "lso_gathers %d ngathers %d cur_index_lso %d " "wr_index %d sop_index %d", lso_ngathers, ngathers, cur_index_lso, tx_ring_p->wr_index, sop_index)); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: next : count %d", lso_gathers)); lso_again = B_TRUE; goto start_again; } } NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); { tx_ring_kick_t kick; kick.value = 0; kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; /* Kick start the Transmit kick register */ TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), TX_RING_KICK_REG, (uint8_t)tx_ring_p->tdc, kick.value); } tdc_stats->tx_starts++; MUTEX_EXIT(&tx_ring_p->lock); NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); return (status); nxge_start_fail_lso: status = 0; good_packet = B_FALSE; if (mp != NULL) { freemsg(mp); } if (mp_chain != NULL) { freemsg(mp_chain); } if (!lso_again && !ngathers) { MUTEX_EXIT(&tx_ring_p->lock); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: lso exit (nothing changed)")); goto nxge_start_fail1; } NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start (channel %d): before lso " "lso_gathers %d ngathers %d cur_index_lso %d " "wr_index %d sop_index %d lso_again %d", tx_ring_p->tdc, lso_ngathers, ngathers, cur_index_lso, tx_ring_p->wr_index, sop_index, lso_again)); if (lso_again) { lso_ngathers += ngathers; ngathers = lso_ngathers; sop_index = cur_index_lso; tx_ring_p->wr_index = sop_index; tx_ring_p->wr_index_wrap = lso_tail_wrap; } NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start (channel %d): after lso " "lso_gathers %d ngathers %d cur_index_lso %d " "wr_index %d sop_index %d lso_again %d", tx_ring_p->tdc, lso_ngathers, ngathers, cur_index_lso, tx_ring_p->wr_index, sop_index, lso_again)); nxge_start_fail2: if (good_packet == B_FALSE) { cur_index = sop_index; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); for (i = 0; i < ngathers; i++) { tx_desc_p = &tx_desc_ring_vp[cur_index]; #if defined(__i386) npi_handle.regp = (uint32_t)tx_desc_p; #else npi_handle.regp = (uint64_t)tx_desc_p; #endif tx_msg_p = &tx_msg_ring[cur_index]; (void) npi_txdma_desc_set_zero(npi_handle, 1); if (tx_msg_p->flags.dma_type == USE_DVMA) { NXGE_DEBUG_MSG((nxgep, TX_CTL, "tx_desc_p = %X index = %d", tx_desc_p, tx_ring_p->rd_index)); (void) dvma_unload(tx_msg_p->dvma_handle, 0, -1); tx_msg_p->dvma_handle = NULL; if (tx_ring_p->dvma_wr_index == tx_ring_p->dvma_wrap_mask) tx_ring_p->dvma_wr_index = 0; else tx_ring_p->dvma_wr_index++; tx_ring_p->dvma_pending--; } else if (tx_msg_p->flags.dma_type == USE_DMA) { if (ddi_dma_unbind_handle( tx_msg_p->dma_handle)) { cmn_err(CE_WARN, "!nxge_start: " "ddi_dma_unbind_handle failed"); } } tx_msg_p->flags.dma_type = USE_NONE; cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, tx_ring_p->tx_wrap_mask); } nxgep->resched_needed = B_TRUE; } MUTEX_EXIT(&tx_ring_p->lock); nxge_start_fail1: /* Add FMA to check the access handle nxge_hregh */ NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); return (status); } int nxge_serial_tx(mblk_t *mp, void *arg) { p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; p_nxge_t nxgep = tx_ring_p->nxgep; return (nxge_start(nxgep, tx_ring_p, mp)); } boolean_t nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) { p_tx_ring_t *tx_rings; uint8_t ring_index; p_tx_ring_t tx_ring_p; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); ASSERT(mp->b_next == NULL); ring_index = nxge_tx_lb_ring_1(mp, nxgep->max_tdcs, hp); tx_rings = nxgep->tx_rings->rings; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: tx_rings $%p", tx_rings)); NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: max_tdcs %d " "ring_index %d", nxgep->max_tdcs, ring_index)); switch (nxge_tx_scheme) { case NXGE_USE_START: if (nxge_start(nxgep, tx_rings[ring_index], mp)) { NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " "ring index %d", ring_index)); return (B_FALSE); } break; case NXGE_USE_SERIAL: default: tx_ring_p = tx_rings[ring_index]; nxge_serialize_enter(tx_ring_p->serial, mp); break; } NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", ring_index)); return (B_TRUE); } /* * nxge_m_tx() - send a chain of packets */ mblk_t * nxge_m_tx(void *arg, mblk_t *mp) { p_nxge_t nxgep = (p_nxge_t)arg; mblk_t *next; mac_tx_hint_t hint; if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_m_tx: hardware not initialized")); NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_m_tx")); return (mp); } hint.hash = NULL; hint.vid = 0; hint.sap = 0; while (mp != NULL) { next = mp->b_next; mp->b_next = NULL; /* * Until Nemo tx resource works, the mac driver * does the load balancing based on TCP port, * or CPU. For debugging, we use a system * configurable parameter. */ if (!nxge_send(nxgep, mp, &hint)) { mp->b_next = next; break; } mp = next; NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_m_tx: (go back to loop) mp $%p next $%p", mp, next)); } return (mp); } int nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) { uint8_t ring_index = 0; uint8_t *tcp_port; p_mblk_t nmp; size_t mblk_len; size_t iph_len; size_t hdrs_size; uint8_t hdrs_buf[sizeof (struct ether_header) + IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; /* * allocate space big enough to cover * the max ip header length and the first * 4 bytes of the TCP/IP header. */ boolean_t qos = B_FALSE; NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); if (hp->vid) { qos = B_TRUE; } switch (nxge_tx_lb_policy) { case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ default: tcp_port = mp->b_rptr; if (!nxge_no_tx_lb && !qos && (ntohs(((p_ether_header_t)tcp_port)->ether_type) == ETHERTYPE_IP)) { nmp = mp; mblk_len = MBLKL(nmp); tcp_port = NULL; if (mblk_len > sizeof (struct ether_header) + sizeof (uint8_t)) { tcp_port = nmp->b_rptr + sizeof (struct ether_header); mblk_len -= sizeof (struct ether_header); iph_len = ((*tcp_port) & 0x0f) << 2; if (mblk_len > (iph_len + sizeof (uint32_t))) { tcp_port = nmp->b_rptr; } else { tcp_port = NULL; } } if (tcp_port == NULL) { hdrs_size = 0; ((p_ether_header_t)hdrs_buf)->ether_type = 0; while ((nmp) && (hdrs_size < sizeof (hdrs_buf))) { mblk_len = MBLKL(nmp); if (mblk_len >= (sizeof (hdrs_buf) - hdrs_size)) mblk_len = sizeof (hdrs_buf) - hdrs_size; bcopy(nmp->b_rptr, &hdrs_buf[hdrs_size], mblk_len); hdrs_size += mblk_len; nmp = nmp->b_cont; } tcp_port = hdrs_buf; } tcp_port += sizeof (ether_header_t); if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { switch (tcp_port[9]) { case IPPROTO_TCP: case IPPROTO_UDP: case IPPROTO_ESP: tcp_port += ((*tcp_port) & 0x0f) << 2; ring_index = ((tcp_port[0] ^ tcp_port[1] ^ tcp_port[2] ^ tcp_port[3]) % maxtdcs); break; case IPPROTO_AH: /* SPI starts at the 4th byte */ tcp_port += ((*tcp_port) & 0x0f) << 2; ring_index = ((tcp_port[4] ^ tcp_port[5] ^ tcp_port[6] ^ tcp_port[7]) % maxtdcs); break; default: ring_index = tcp_port[19] % maxtdcs; break; } } else { /* fragmented packet */ ring_index = tcp_port[19] % maxtdcs; } } else { ring_index = mp->b_band % maxtdcs; } break; case NXGE_TX_LB_HASH: if (hp->hash) { #if defined(__i386) ring_index = ((uint32_t)(hp->hash) % maxtdcs); #else ring_index = ((uint64_t)(hp->hash) % maxtdcs); #endif } else { ring_index = mp->b_band % maxtdcs; } break; case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ tcp_port = mp->b_rptr; ring_index = tcp_port[5] % maxtdcs; break; } NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); return (ring_index); } uint_t nxge_reschedule(caddr_t arg) { p_nxge_t nxgep; nxgep = (p_nxge_t)arg; NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && nxgep->resched_needed) { mac_tx_update(nxgep->mach); nxgep->resched_needed = B_FALSE; nxgep->resched_running = B_FALSE; } NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); return (DDI_INTR_CLAIMED); } /* Software LSO starts here */ static void nxge_hcksum_retrieve(mblk_t *mp, uint32_t *start, uint32_t *stuff, uint32_t *end, uint32_t *value, uint32_t *flags) { if (mp->b_datap->db_type == M_DATA) { if (flags != NULL) { *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | HCK_PARTIALCKSUM | HCK_FULLCKSUM | HCK_FULLCKSUM_OK); if ((*flags & (HCK_PARTIALCKSUM | HCK_FULLCKSUM)) != 0) { if (value != NULL) *value = (uint32_t)DB_CKSUM16(mp); if ((*flags & HCK_PARTIALCKSUM) != 0) { if (start != NULL) *start = (uint32_t)DB_CKSUMSTART(mp); if (stuff != NULL) *stuff = (uint32_t)DB_CKSUMSTUFF(mp); if (end != NULL) *end = (uint32_t)DB_CKSUMEND(mp); } } } } } static void nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) { ASSERT(DB_TYPE(mp) == M_DATA); *mss = 0; if (flags != NULL) { *flags = DB_CKSUMFLAGS(mp) & HW_LSO; if ((*flags != 0) && (mss != NULL)) { *mss = (uint32_t)DB_LSOMSS(mp); } NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", *mss, *flags)); } NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_lso_info_get: mss %d", *mss)); } /* * Do Soft LSO on the oversized packet. * * 1. Create a chain of message for headers. * 2. Fill up header messages with proper information. * 3. Copy Eithernet, IP, and TCP headers from the original message to * each new message with necessary adjustments. * * Unchange the ethernet header for DIX frames. (by default) * * IP Total Length field is updated to MSS or less(only for the last one). * * IP Identification value is incremented by one for each packet. * * TCP sequence Number is recalculated according to the payload length. * * Set FIN and/or PSH flags for the *last* packet if applied. * * TCP partial Checksum * 4. Update LSO information in the first message header. * 5. Release the original message header. */ static mblk_t * nxge_do_softlso(mblk_t *mp, uint32_t mss) { uint32_t hckflags; int pktlen; int hdrlen; int segnum; int i; struct ether_vlan_header *evh; int ehlen, iphlen, tcphlen; struct ip *oiph, *niph; struct tcphdr *otcph, *ntcph; int available, len, left; uint16_t ip_id; uint32_t tcp_seq; #ifdef __sparc uint32_t tcp_seq_tmp; #endif mblk_t *datamp; uchar_t *rptr; mblk_t *nmp; mblk_t *cmp; mblk_t *mp_chain; boolean_t do_cleanup = B_FALSE; t_uscalar_t start_offset = 0; t_uscalar_t stuff_offset = 0; t_uscalar_t value = 0; uint16_t l4_len; ipaddr_t src, dst; uint32_t cksum, sum, l4cksum; NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso")); /* * check the length of LSO packet payload and calculate the number of * segments to be generated. */ pktlen = msgsize(mp); evh = (struct ether_vlan_header *)mp->b_rptr; /* VLAN? */ if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) ehlen = sizeof (struct ether_vlan_header); else ehlen = sizeof (struct ether_header); oiph = (struct ip *)(mp->b_rptr + ehlen); iphlen = oiph->ip_hl * 4; otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); tcphlen = otcph->th_off * 4; l4_len = pktlen - ehlen - iphlen; NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: mss %d oiph $%p " "original ip_sum oiph->ip_sum 0x%x " "original tcp_sum otcph->th_sum 0x%x " "oiph->ip_len %d pktlen %d ehlen %d " "l4_len %d (0x%x) ip_len - iphlen %d ", mss, oiph, oiph->ip_sum, otcph->th_sum, ntohs(oiph->ip_len), pktlen, ehlen, l4_len, l4_len, ntohs(oiph->ip_len) - iphlen)); /* IPv4 + TCP */ if (!(oiph->ip_v == IPV4_VERSION)) { NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, "<== nxge_do_softlso: not IPV4 " "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", ntohs(oiph->ip_len), pktlen, ehlen, tcphlen)); freemsg(mp); return (NULL); } if (!(oiph->ip_p == IPPROTO_TCP)) { NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, "<== nxge_do_softlso: not TCP " "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", ntohs(oiph->ip_len), pktlen, ehlen, tcphlen)); freemsg(mp); return (NULL); } if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, "<== nxge_do_softlso: len not matched " "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", ntohs(oiph->ip_len), pktlen, ehlen, tcphlen)); freemsg(mp); return (NULL); } otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); tcphlen = otcph->th_off * 4; /* TCP flags can not include URG, RST, or SYN */ VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); hdrlen = ehlen + iphlen + tcphlen; VERIFY(MBLKL(mp) >= hdrlen); if (MBLKL(mp) > hdrlen) { datamp = mp; rptr = mp->b_rptr + hdrlen; } else { /* = */ datamp = mp->b_cont; rptr = datamp->b_rptr; } NXGE_DEBUG_MSG((NULL, TX_CTL, "nxge_do_softlso: otcph $%p pktlen: %d, " "hdrlen %d ehlen %d iphlen %d tcphlen %d " "mblkl(mp): %d, mblkl(datamp): %d", otcph, pktlen, hdrlen, ehlen, iphlen, tcphlen, (int)MBLKL(mp), (int)MBLKL(datamp))); hckflags = 0; nxge_hcksum_retrieve(mp, &start_offset, &stuff_offset, &value, NULL, &hckflags); dst = oiph->ip_dst.s_addr; src = oiph->ip_src.s_addr; cksum = (dst >> 16) + (dst & 0xFFFF) + (src >> 16) + (src & 0xFFFF); l4cksum = cksum + IP_TCP_CSUM_COMP; sum = l4_len + l4cksum; sum = (sum & 0xFFFF) + (sum >> 16); NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " "hckflags 0x%x start_offset %d stuff_offset %d " "value (original) 0x%x th_sum 0x%x " "pktlen %d l4_len %d (0x%x) " "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", dst, src, (sum & 0xffff), (~sum & 0xffff), hckflags, start_offset, stuff_offset, value, otcph->th_sum, pktlen, l4_len, l4_len, ntohs(oiph->ip_len) - (int)MBLKL(mp), (int)MBLKL(datamp), nxge_dump_packet((char *)evh, 12))); /* * Start to process. */ available = pktlen - hdrlen; segnum = (available - 1) / mss + 1; NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: pktlen %d " "MBLKL(mp): %d, MBLKL(datamp): %d " "available %d mss %d segnum %d", pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), available, mss, segnum)); VERIFY(segnum >= 2); /* * Try to pre-allocate all header messages */ mp_chain = NULL; for (i = 0; i < segnum; i++) { if ((nmp = allocb(hdrlen, 0)) == NULL) { /* Clean up the mp_chain */ while (mp_chain != NULL) { nmp = mp_chain; mp_chain = mp_chain->b_next; freemsg(nmp); } NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_do_softlso: " "Could not allocate enough messages for headers!")); freemsg(mp); return (NULL); } nmp->b_next = mp_chain; mp_chain = nmp; NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: " "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", mp, nmp, mp_chain, mp_chain->b_next)); } NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", mp, nmp, mp_chain)); /* * Associate payload with new packets */ cmp = mp_chain; left = available; while (cmp != NULL) { nmp = dupb(datamp); if (nmp == NULL) { do_cleanup = B_TRUE; NXGE_DEBUG_MSG((NULL, TX_CTL, "==>nxge_do_softlso: " "Can not dupb(datamp), have to do clean up")); goto cleanup_allocated_msgs; } NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " "dupb nmp $%p len %d left %d msd %d ", mp, cmp, nmp, len, left, mss)); cmp->b_cont = nmp; nmp->b_rptr = rptr; len = (left < mss) ? left : mss; left -= len; NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " "dupb nmp $%p len %d left %d mss %d ", mp, cmp, nmp, len, left, mss)); NXGE_DEBUG_MSG((NULL, TX_CTL, "nxge_do_softlso: before available: %d, " "left: %d, len: %d, segnum: %d MBLK(nmp): %d", available, left, len, segnum, (int)MBLKL(nmp))); len -= MBLKL(nmp); NXGE_DEBUG_MSG((NULL, TX_CTL, "nxge_do_softlso: after available: %d, " "left: %d, len: %d, segnum: %d MBLK(nmp): %d", available, left, len, segnum, (int)MBLKL(nmp))); while (len > 0) { mblk_t *mmp = NULL; NXGE_DEBUG_MSG((NULL, TX_CTL, "nxge_do_softlso: (4) len > 0 available: %d, " "left: %d, len: %d, segnum: %d MBLK(nmp): %d", available, left, len, segnum, (int)MBLKL(nmp))); if (datamp->b_cont != NULL) { datamp = datamp->b_cont; rptr = datamp->b_rptr; mmp = dupb(datamp); if (mmp == NULL) { do_cleanup = B_TRUE; NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: " "Can not dupb(datamp) (1), : "have to do clean up")); NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: " "available: %d, left: %d, " "len: %d, MBLKL(nmp): %d", available, left, len, (int)MBLKL(nmp))); goto cleanup_allocated_msgs; } } else { NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, "==> nxge_do_softlso: " "(1)available: %d, left: %d, " "len: %d, MBLKL(nmp): %d", available, left, len, (int)MBLKL(nmp))); cmn_err(CE_PANIC, "==> nxge_do_softlso: " "Pointers must have been corrupted!\n" "datamp: $%p, nmp: $%p, rptr: $%p", (void *)datamp, (void *)nmp, (void *)rptr); } nmp->b_cont = mmp; nmp = mmp; len -= MBLKL(nmp); } if (len < 0) { nmp->b_wptr += len; rptr = nmp->b_wptr; NXGE_DEBUG_MSG((NULL, TX_CTL, "(5) len < 0 (less than 0)" "available: %d, left: %d, len: %d, MBLKL(nmp): %d", available, left, len, (int)MBLKL(nmp))); } else if (len == 0) { if (datamp->b_cont != NULL) { NXGE_DEBUG_MSG((NULL, TX_CTL, "(5) len == 0" "available: %d, left: %d, len: %d, " "MBLKL(nmp): %d", available, left, len, (int)MBLKL(nmp))); datamp = datamp->b_cont; rptr = datamp->b_rptr; } else { NXGE_DEBUG_MSG((NULL, TX_CTL, "(6)available b_cont == NULL : %d, " "left: %d, len: %d, MBLKL(nmp): %d", available, left, len, (int)MBLKL(nmp))); VERIFY(cmp->b_next == NULL); VERIFY(left == 0); break; /* Done! */ } } cmp = cmp->b_next; NXGE_DEBUG_MSG((NULL, TX_CTL, "(7) do_softlso: " "next mp in mp_chain available len != 0 : %d, " "left: %d, len: %d, MBLKL(nmp): %d", available, left, len, (int)MBLKL(nmp))); } /* * From now, start to fill up all headers for the first message * Hardware checksum flags need to be updated separately for FULLCKSUM * and PARTIALCKSUM cases. For full checksum, copy the original flags * into every new packet is enough. But for HCK_PARTIALCKSUM, all * required fields need to be updated properly. */ nmp = mp_chain; bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); nmp->b_wptr = nmp->b_rptr + hdrlen; niph = (struct ip *)(nmp->b_rptr + ehlen); niph->ip_len = htons(mss + iphlen + tcphlen); ip_id = ntohs(niph->ip_id); ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); #ifdef __sparc bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); tcp_seq = ntohl(tcp_seq_tmp); #else tcp_seq = ntohl(ntcph->th_seq); #endif ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; DB_CKSUMSTART(nmp) = start_offset; DB_CKSUMSTUFF(nmp) = stuff_offset; /* calculate IP checksum and TCP pseudo header checksum */ niph->ip_sum = 0; niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); l4_len = mss + tcphlen; sum = htons(l4_len) + l4cksum; sum = (sum & 0xFFFF) + (sum >> 16); ntcph->th_sum = (sum & 0xffff); NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " "mss %d pktlen %d l4_len %d (0x%x) " "MBLKL(mp): %d, MBLKL(datamp): %d " "ip_sum 0x%x " "th_sum 0x%x sum 0x%x ) " "dump first ip->tcp %s", nmp, mp_chain, mss, pktlen, l4_len, l4_len, (int)MBLKL(mp), (int)MBLKL(datamp), niph->ip_sum, ntcph->th_sum, sum, nxge_dump_packet((char *)niph, 52))); cmp = nmp; while ((nmp = nmp->b_next)->b_next != NULL) { NXGE_DEBUG_MSG((NULL, TX_CTL, "==>nxge_do_softlso: middle l4_len %d ", l4_len)); bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); nmp->b_wptr = nmp->b_rptr + hdrlen; niph = (struct ip *)(nmp->b_rptr + ehlen); niph->ip_id = htons(++ip_id); niph->ip_len = htons(mss + iphlen + tcphlen); ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); tcp_seq += mss; ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); #ifdef __sparc tcp_seq_tmp = htonl(tcp_seq); bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); #else ntcph->th_seq = htonl(tcp_seq); #endif DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; DB_CKSUMSTART(nmp) = start_offset; DB_CKSUMSTUFF(nmp) = stuff_offset; /* calculate IP checksum and TCP pseudo header checksum */ niph->ip_sum = 0; niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); ntcph->th_sum = (sum & 0xffff); NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: middle ip_sum 0x%x " "th_sum 0x%x " " mp $%p (mp_chain $%p) pktlen %d " "MBLKL(mp): %d, MBLKL(datamp): %d ", niph->ip_sum, ntcph->th_sum, nmp, mp_chain, pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); } /* Last segment */ /* * Set FIN and/or PSH flags if present only in the last packet. * The ip_len could be different from prior packets. */ bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); nmp->b_wptr = nmp->b_rptr + hdrlen; niph = (struct ip *)(nmp->b_rptr + ehlen); niph->ip_id = htons(++ip_id); niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); tcp_seq += mss; #ifdef __sparc tcp_seq_tmp = htonl(tcp_seq); bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); #else ntcph->th_seq = htonl(tcp_seq); #endif ntcph->th_flags = (otcph->th_flags & ~TH_URG); DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; DB_CKSUMSTART(nmp) = start_offset; DB_CKSUMSTUFF(nmp) = stuff_offset; /* calculate IP checksum and TCP pseudo header checksum */ niph->ip_sum = 0; niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); l4_len = ntohs(niph->ip_len) - iphlen; sum = htons(l4_len) + l4cksum; sum = (sum & 0xFFFF) + (sum >> 16); ntcph->th_sum = (sum & 0xffff); NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: last next " "niph->ip_sum 0x%x " "ntcph->th_sum 0x%x sum 0x%x " "dump last ip->tcp %s " "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " "l4_len %d (0x%x) " "MBLKL(mp): %d, MBLKL(datamp): %d ", niph->ip_sum, ntcph->th_sum, sum, nxge_dump_packet((char *)niph, 52), cmp, nmp, mp_chain, pktlen, pktlen, l4_len, l4_len, (int)MBLKL(mp), (int)MBLKL(datamp))); cleanup_allocated_msgs: if (do_cleanup) { NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_do_softlso: " "Failed allocating messages, " "have to clean up and fail!")); while (mp_chain != NULL) { nmp = mp_chain; mp_chain = mp_chain->b_next; freemsg(nmp); } } /* * We're done here, so just free the original message and return the * new message chain, that could be NULL if failed, back to the caller. */ freemsg(mp); NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_do_softlso:mp_chain $%p", mp_chain)); return (mp_chain); } /* * Will be called before NIC driver do further operation on the message. * The input message may include LSO information, if so, go to softlso logic * to eliminate the oversized LSO packet for the incapable underlying h/w. * The return could be the same non-LSO message or a message chain for LSO case. * * The driver needs to call this function per packet and process the whole chain * if applied. */ static mblk_t * nxge_lso_eliminate(mblk_t *mp) { uint32_t lsoflags; uint32_t mss; NXGE_DEBUG_MSG((NULL, TX_CTL, "==>nxge_lso_eliminate:")); nxge_lso_info_get(mp, &mss, &lsoflags); if (lsoflags & HW_LSO) { mblk_t *nmp; NXGE_DEBUG_MSG((NULL, TX_CTL, "==>nxge_lso_eliminate:" "HW_LSO:mss %d mp $%p", mss, mp)); if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_lso_eliminate: " "LSO: nmp not NULL nmp $%p mss %d mp $%p", nmp, mss, mp)); return (nmp); } else { NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_lso_eliminate_ " "LSO: failed nmp NULL nmp $%p mss %d mp $%p", nmp, mss, mp)); return (NULL); } } NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_lso_eliminate")); return (mp); } static uint32_t nxge_csgen(uint16_t *adr, int len) { int i, odd; uint32_t sum = 0; uint32_t c = 0; odd = len % 2; for (i = 0; i < (len / 2); i++) { sum += (adr[i] & 0xffff); } if (odd) { sum += adr[len / 2] & 0xff00; } while ((c = ((sum & 0xffff0000) >> 16)) != 0) { sum &= 0xffff; sum += c; } return (~sum & 0xffff); }