xref: /titanic_51/usr/src/uts/common/io/bnxe/bnxe_tx.c (revision d14abf155341d55053c76eeec58b787a456b753b)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2014 QLogic Corporation
 * The contents of this file are subject to the terms of the
 * QLogic End User License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the License at
 * http://www.qlogic.com/Resources/Documents/DriverDownloadHelp/
 * QLogic_End_User_Software_License.txt
 * See the License for the specific language governing permissions
 * and limitations under the License.
 */

/*
 * Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved.
 */

#include "bnxe.h"

ddi_dma_attr_t bnxeTxDmaAttrib =
{
    DMA_ATTR_V0,                /* dma_attr_version */
    0,                          /* dma_attr_addr_lo */
    0xffffffffffffffff,         /* dma_attr_addr_hi */
    0xffffffffffffffff,         /* dma_attr_count_max */
    BNXE_DMA_ALIGNMENT,         /* dma_attr_align */
    0xffffffff,                 /* dma_attr_burstsizes */
    1,                          /* dma_attr_minxfer */
    0xffffffffffffffff,         /* dma_attr_maxxfer */
    0xffffffffffffffff,         /* dma_attr_seg */
    BNXE_MAX_DMA_SGLLEN,        /* dma_attr_sgllen */
    1,                          /* dma_attr_granular */
    0,                          /* dma_attr_flags */
};

ddi_dma_attr_t bnxeTxCbDmaAttrib =
{
    DMA_ATTR_V0,                /* dma_attr_version */
    0,                          /* dma_attr_addr_lo */
    0xffffffffffffffff,         /* dma_attr_addr_hi */
    0xffffffffffffffff,         /* dma_attr_count_max */
    BNXE_DMA_ALIGNMENT,         /* dma_attr_align */
    0xffffffff,                 /* dma_attr_burstsizes */
    1,                          /* dma_attr_minxfer */
    0xffffffffffffffff,         /* dma_attr_maxxfer */
    0xffffffffffffffff,         /* dma_attr_seg */
    1,                          /* dma_attr_sgllen */
    1,                          /* dma_attr_granular */
    0,                          /* dma_attr_flags */
};


static um_txpacket_t * BnxeTxPktAlloc(um_device_t * pUM, size_t size);


static inline void BnxeTxPktUnmap(um_txpacket_t * pTxPkt)
{
    int i;

    for (i = 0; i < pTxPkt->num_handles; i++)
    {
        ddi_dma_unbind_handle(pTxPkt->dmaHandles[i]);
    }

    pTxPkt->num_handles = 0;
}


static void BnxeTxPktsFree(um_txpacket_t * pTxPkt)
{
    int i;

    if (pTxPkt->num_handles > 0)
    {
        BnxeTxPktUnmap(pTxPkt);
    }

    if (pTxPkt->pMblk != NULL)
    {
        freemsg(pTxPkt->pMblk);
    }

    for (i = 0; i < BNXE_MAX_DMA_HANDLES_PER_PKT; i++)
    {
        ddi_dma_free_handle(&pTxPkt->dmaHandles[i]);
    }

    pTxPkt->pMblk         = NULL;
    pTxPkt->num_handles   = 0;
    pTxPkt->frag_list.cnt = 0;

    ddi_dma_unbind_handle(pTxPkt->cbDmaHandle);
    ddi_dma_mem_free(&pTxPkt->cbDmaAccHandle);
    ddi_dma_free_handle(&pTxPkt->cbDmaHandle);
    kmem_free(pTxPkt, sizeof(um_txpacket_t));
}


static void BnxeTxPktsFreeList(s_list_t * pPktList)
{
    um_txpacket_t * pTxPkt;

    while (!s_list_is_empty(pPktList))
    {
        pTxPkt = (um_txpacket_t *)s_list_pop_head(pPktList);
        BnxeTxPktsFree(pTxPkt);
    }
}


/*
 * Free the mblk and all frag mappings used by each packet in the list
 * and then put the entire list on the free queue for immediate use.
 */
void BnxeTxPktsReclaim(um_device_t * pUM,
                       int           idx,
                       s_list_t *    pPktList)
{
    um_txpacket_t * pTxPkt;

    if (s_list_entry_cnt(pPktList) == 0)
    {
        return;
    }

    for (pTxPkt = (um_txpacket_t *)s_list_peek_head(pPktList);
         pTxPkt;
         pTxPkt = (um_txpacket_t *)s_list_next_entry(&pTxPkt->lm_pkt.link))
    {
        if (pTxPkt->num_handles > 0)
        {
            BnxeTxPktUnmap(pTxPkt);
        }

        if (pTxPkt->pMblk != NULL)
        {
            freemsg(pTxPkt->pMblk);
            pTxPkt->pMblk = NULL;
        }
    }

    BNXE_LOCK_ENTER_FREETX(pUM, idx);
    s_list_add_tail(&pUM->txq[idx].freeTxDescQ, pPktList);
    BNXE_LOCK_EXIT_FREETX(pUM, idx);
}


/* Must be called with TX lock held!!! */
static int BnxeTxSendWaitingPkt(um_device_t * pUM,
                                int           idx)
{
    TxQueue *       pTxQ = &pUM->txq[idx];
    lm_device_t *   pLM = &pUM->lm_dev;
    lm_tx_chain_t * pLmTxChain;
    um_txpacket_t * pTxPkt;
    int rc;

    pLmTxChain = &pLM->tx_info.chain[idx];

    while (s_list_entry_cnt(&pTxQ->waitTxDescQ))
    {
        pTxPkt = (um_txpacket_t *)s_list_peek_head(&pTxQ->waitTxDescQ);

        if (pTxPkt->frag_list.cnt + 2 > pLmTxChain->bd_chain.bd_left)
        {
            return BNXE_TX_DEFERPKT;
        }

        pTxPkt = (um_txpacket_t *)s_list_pop_head(&pTxQ->waitTxDescQ);

        rc = lm_send_packet(pLM, idx, &pTxPkt->lm_pkt, &pTxPkt->frag_list);

        if (pUM->fmCapabilities &&
            BnxeCheckAccHandle(pLM->vars.reg_handle[BAR_0]) != DDI_FM_OK)
        {
            ddi_fm_service_impact(pUM->pDev, DDI_SERVICE_DEGRADED);
        }

        if (rc != LM_STATUS_SUCCESS)
        {
            /*
             * Send failed (probably not enough BDs available)...
             * Put the packet back at the head of the wait queue.
             */
            pTxQ->txFailed++;
            s_list_push_head(&pTxQ->waitTxDescQ, &pTxPkt->lm_pkt.link);
            return BNXE_TX_DEFERPKT;
        }
    }

    return BNXE_TX_GOODXMIT;
}


void BnxeTxRingProcess(um_device_t * pUM,
                       int           idx)
{
    TxQueue *       pTxQ = &pUM->txq[idx];
    lm_device_t *   pLM = &pUM->lm_dev;
    lm_tx_chain_t * pLmTxChain;
    s_list_t        tmpList;
    u32_t           pktsTxed;
    int rc;

    s_list_clear(&tmpList);

    BNXE_LOCK_ENTER_TX(pUM, idx);

    pktsTxed = lm_get_packets_sent(&pUM->lm_dev, idx, &tmpList);

    if (pUM->fmCapabilities &&
        BnxeCheckAccHandle(pUM->lm_dev.vars.reg_handle[BAR_0]) != DDI_FM_OK)
    {
        ddi_fm_service_impact(pUM->pDev, DDI_SERVICE_DEGRADED);
    }

    if ((pktsTxed + s_list_entry_cnt(&pTxQ->sentTxQ)) >=
        pUM->devParams.maxTxFree)
    {
        s_list_add_tail(&tmpList, &pTxQ->sentTxQ);
        s_list_clear(&pTxQ->sentTxQ);
    }
    else
    {
        s_list_add_tail(&pTxQ->sentTxQ, &tmpList);
        s_list_clear(&tmpList);
    }

    BNXE_LOCK_EXIT_TX(pUM, idx);

    if (s_list_entry_cnt(&tmpList))
    {
        BnxeTxPktsReclaim(pUM, idx, &tmpList);
    }

    if (pTxQ->noTxCredits == 0)
    {
        /* no need to notify the stack */
        return;
    }

    pLmTxChain = &pUM->lm_dev.tx_info.chain[idx];

    if (pTxQ->noTxCredits & BNXE_TX_RESOURCES_NO_CREDIT)
    {
        BNXE_LOCK_ENTER_TX(pUM, idx);
        rc = BnxeTxSendWaitingPkt(pUM, idx);
        BNXE_LOCK_EXIT_TX(pUM, idx);

        if ((rc == BNXE_TX_GOODXMIT) &&
            (pLmTxChain->bd_chain.bd_left >= BNXE_MAX_DMA_FRAGS_PER_PKT))
        {
            atomic_and_32(&pTxQ->noTxCredits, ~BNXE_TX_RESOURCES_NO_CREDIT);
        }
    }

    if ((pTxQ->noTxCredits & BNXE_TX_RESOURCES_NO_DESC) &&
        (s_list_entry_cnt(&pTxQ->freeTxDescQ) > pTxQ->thresh_pdwm))
    {
        atomic_and_32(&pTxQ->noTxCredits, ~BNXE_TX_RESOURCES_NO_DESC);
    }

    if (pTxQ->noTxCredits == 0)
    {
        if (idx == FCOE_CID(pLM))
        {
            BnxeLogInfo(pUM, "FCoE tx credit ok, no upcall!");
        }
        else
        {
            /* notify the stack that tx resources are now available */
#if defined(BNXE_RINGS) && (defined(__S11) || defined(__S12))
            mac_tx_ring_update(pUM->pMac, pTxQ->ringHandle);
#else
            mac_tx_update(pUM->pMac);
#endif
        }
    }
}


static inline int BnxeTxPktMapFrag(um_device_t *   pUM,
                                   um_txpacket_t * pTxPkt,
                                   mblk_t *        pMblk)
{
    ddi_dma_handle_t dmaHandle;
    ddi_dma_cookie_t cookie;
    lm_frag_t *      pFrag;
    boolean_t        partial;
    u32_t            bindLen;
    u32_t            count;
    int rc, i;

    if (pTxPkt->num_handles == BNXE_MAX_DMA_HANDLES_PER_PKT)
    {
        return BNXE_TX_RESOURCES_NO_OS_DMA_RES;
    }

    if (pTxPkt->frag_list.cnt >= BNXE_MAX_DMA_FRAGS_PER_PKT)
    {
        return BNXE_TX_RESOURCES_TOO_MANY_FRAGS;
    }

    dmaHandle = pTxPkt->dmaHandles[pTxPkt->num_handles];

    if ((rc = ddi_dma_addr_bind_handle(dmaHandle,
                                       NULL,
                                       (caddr_t)pMblk->b_rptr,
                                       (pMblk->b_wptr - pMblk->b_rptr),
                                       (DDI_DMA_WRITE | DDI_DMA_STREAMING),
                                       DDI_DMA_DONTWAIT,
                                       NULL,
                                       &cookie,
                                       &count)) != DDI_DMA_MAPPED)
    {
        BnxeLogWarn(pUM, "Failed to bind DMA address for tx packet (%d)", rc);
        return BNXE_TX_RESOURCES_NO_OS_DMA_RES;
    }

    /*
     * ddi_dma_addr_bind_handle() correctly returns an error if the physical
     * fragment count exceeds the maximum fragment count specified in the
     * ddi_dma_attrib structure for the current pMblk.  However, a packet can
     * span multiple mblk's.  The purpose of the check below is to make sure we
     * do not overflow our fragment count limit based on what has already been
     * mapped from this packet.
     */
    partial = ((pTxPkt->frag_list.cnt + count) >
               (pMblk->b_cont ? BNXE_MAX_DMA_FRAGS_PER_PKT - 1
                              : BNXE_MAX_DMA_FRAGS_PER_PKT));
    if (partial)
    {
        /*
         * Going to try a partial dma so (re)set count to the remaining number
         * of dma fragments that are available leaving one fragment at the end.
         */
        count = (BNXE_MAX_DMA_FRAGS_PER_PKT - 1 - pTxPkt->frag_list.cnt);
        if (count == 0)
        {
            /*
             * No more dma fragments are available.  This fragment was not
             * mapped and will be copied into the copy buffer along with the
             * rest of the packet data.
             */
            ddi_dma_unbind_handle(dmaHandle);
            return BNXE_TX_RESOURCES_TOO_MANY_FRAGS;
        }
    }

    pFrag = &pTxPkt->frag_list.frag_arr[pTxPkt->frag_list.cnt];
    pTxPkt->frag_list.cnt += count;

    /* map "count" dma fragments */

    bindLen = 0;
    for (i = 0; i < (count - 1); i++)
    {
        pFrag->addr.as_u64 = cookie.dmac_laddress;
        bindLen += pFrag->size = cookie.dmac_size;

        pFrag++;

        ddi_dma_nextcookie(dmaHandle, &cookie);
    }

    pFrag->addr.as_u64 = cookie.dmac_laddress;
    bindLen += pFrag->size = cookie.dmac_size;

    pTxPkt->num_handles++;

    if (partial)
    {
        /*
         * Move the mblk's read pointer past the data that was bound to a DMA
         * fragment.  Any remaining data will get copied into the copy buffer.
         */
        pMblk->b_rptr += bindLen;
        return BNXE_TX_RESOURCES_TOO_MANY_FRAGS;
    }

    return 0;
}


static int BnxeTxPktCopy(um_device_t *   pUM,
                         TxQueue *       pTxQ,
                         um_txpacket_t * pTxPkt)
{
    lm_frag_t * pCopyFrag = NULL;
    size_t      msgSize;
    size_t      copySize = 0;
    size_t      pktLen = 0;
    boolean_t   tryMap = B_TRUE;
    mblk_t *    pMblk;
    caddr_t     pTmp;
    int rc;

    /* Walk the chain to get the total pkt length... */
    for (pMblk = pTxPkt->pMblk; pMblk; pMblk = pMblk->b_cont)
    {
        pktLen += MBLKL(pMblk);
    }

    /*
     * If the packet length is under the tx copy threshold then copy
     * the all data into the copy buffer.
     */
    if (pktLen < pUM->devParams.txCopyThreshold)
    {
        ASSERT(pktLen <= pTxPkt->cbLength);

        pTmp = pTxPkt->pCbBuf;

        for (pMblk = pTxPkt->pMblk; pMblk; pMblk = pMblk->b_cont)
        {
            if ((msgSize = MBLKL(pMblk)) == 0)
            {
                continue;
            }

            bcopy(pMblk->b_rptr, pTmp, msgSize);
            pTmp += msgSize;
        }

        pCopyFrag              = &pTxPkt->frag_list.frag_arr[0];
        pCopyFrag->addr.as_u64 = pTxPkt->cbPhysAddr.as_u64;
        pCopyFrag->size        = pktLen;
        pTxPkt->frag_list.cnt++;

        copySize = pktLen;
        pTxQ->txCopied++;

        /* Done! */
        goto _BnxeTxPktCopy_DMA_SYNC_COPY_BUFFER;
    }

    /* Try to DMA map all the blocks... */

    for (pMblk = pTxPkt->pMblk; pMblk; pMblk = pMblk->b_cont)
    {
        if ((msgSize = MBLKL(pMblk)) == 0)
        {
            continue;
        }

        if (tryMap)
        {
            if (BnxeTxPktMapFrag(pUM, pTxPkt, pMblk) == 0)
            {
                /*
                 * The fragment was successfully mapped now move on to the
                 * next one.  Here we set pCopyFrag to NULL which represents
                 * a break of continuous data in the copy buffer.  If the
                 * packet header was copied the first fragment points to the
                 * beginning of the copy buffer.  Since this block was mapped
                 * any future blocks that have to be copied must be handled by
                 * a new fragment even though the fragment is pointed to the
                 * copied data in the copy buffer.
                 */
                pCopyFrag = NULL;
                continue;
            }
            else
            {
                /*
                 * The frament was not mapped or was partially mapped.  In
                 * either case we will no longer try to map the remaining
                 * blocks.  All remaining packet data is copied.
                 */
                tryMap = B_FALSE;
                msgSize = MBLKL(pMblk); /* new msgSize with partial binding */
            }
        }

#if 0
        if ((copySize + msgSize) > pTxPkt->cbLength)
        {
            /* remaining packet is too large (length more than copy buffer) */
            BnxeTxPktUnmap(pTxPkt);
            return -1;
        }
#else
        ASSERT((copySize + msgSize) <= pTxPkt->cbLength);
#endif

        bcopy(pMblk->b_rptr, (pTxPkt->pCbBuf + copySize), msgSize);

        /*
         * If pCopyFrag is already specified then simply update the copy size.
         * If not then set pCopyFrag to the next available fragment.
         */
        if (pCopyFrag)
        {
            pCopyFrag->size += msgSize;
        }
        else
        {
            ASSERT((pTxPkt->frag_list.cnt + 1) <= BNXE_MAX_DMA_FRAGS_PER_PKT);
            pCopyFrag              = &pTxPkt->frag_list.frag_arr[pTxPkt->frag_list.cnt++];
            pCopyFrag->size        = msgSize;
            pCopyFrag->addr.as_u64 = pTxPkt->cbPhysAddr.as_u64 + copySize;
        }

        /* update count of bytes in the copy buffer needed for DMA sync */
        copySize += msgSize;
    }

_BnxeTxPktCopy_DMA_SYNC_COPY_BUFFER:

    if (copySize > 0)
    {
        /* DMA sync the copy buffer before sending */

        rc = ddi_dma_sync(pTxPkt->cbDmaHandle, 0, copySize,
                          DDI_DMA_SYNC_FORDEV);

        if (pUM->fmCapabilities &&
            BnxeCheckDmaHandle(pTxPkt->cbDmaHandle) != DDI_FM_OK)
        {
            ddi_fm_service_impact(pUM->pDev, DDI_SERVICE_DEGRADED);
        }

        if (rc != DDI_SUCCESS)
        {
            BnxeLogWarn(pUM, "(%d) Failed to dma sync tx copy (%p / %d)",
                        rc, pTxPkt, copySize);
        }
    }

    if (pTxPkt->num_handles == 0)
    {
        freemsg(pTxPkt->pMblk);
        pTxPkt->pMblk = NULL;
    }

    return 0;
}


/* this code is derived from that shown in RFC 1071 Section 4.1 */
static inline u16_t BnxeCalcCksum(void * start,
                                  u32_t  len,
                                  u16_t  prev_sum)
{
    u16_t * pword;
    u32_t   sum = 0;

    pword = (u16_t *)start;

    for ( ; len > 1; len -= 2, pword++)
    {
        /* the inner loop */
        sum += *pword;
    }

    /* add left-over byte, if any */
    if (len)
    {
        sum += (u16_t)(*((u8_t *)pword));
    }

    sum += prev_sum;

    /* fold 32-bit sum to 16 bits */
    while (sum >> 16)
    {
        sum = ((sum & 0xffff) + (sum >> 16));
    }

    return (u16_t)sum;
}


/*
 * Everest1 (i.e. 57710, 57711, 57711E) does not natively support UDP checksums
 * and does not know anything about the UDP header and where the checksum field
 * is located.  It only knows about TCP.  Therefore we "lie" to the hardware for
 * outgoing UDP packets w/ checksum offload.  Since the checksum field offset
 * for TCP is 16 bytes and for UDP it is 6 bytes we pass a pointer to the
 * hardware that is 10 bytes less than the start of the UDP header.  This allows
 * the hardware to write the checksum in the correct spot.  But the hardware
 * will compute a checksum which includes the last 10 bytes of the IP header.
 * To correct this we tweak the stack computed pseudo checksum by folding in the
 * calculation of the inverse checksum for those final 10 bytes of the IP
 * header.  This allows the correct checksum to be computed by the hardware.
 */

#define TCP_CS_OFFSET           16
#define UDP_CS_OFFSET           6
#define UDP_TCP_CS_OFFSET_DIFF  (TCP_CS_OFFSET - UDP_CS_OFFSET)

static inline u16_t BnxeUdpPseudoCsum(um_device_t * pUM,
                                      u8_t *        pUdpHdr,
                                      u8_t *        pIpHdr,
                                      u8_t          ipHdrLen)
{
    u32_t sum32;
    u16_t sum16;
    u16_t pseudo_cs;

    ASSERT(ipHdrLen >= UDP_TCP_CS_OFFSET_DIFF);

    /* calc cksum on last UDP_TCP_CS_OFFSET_DIFF bytes of ip header */
    sum16 = BnxeCalcCksum(&pIpHdr[ipHdrLen - UDP_TCP_CS_OFFSET_DIFF],
                          UDP_TCP_CS_OFFSET_DIFF, 0);

    /* substruct the calculated cksum from the udp pseudo cksum */
    pseudo_cs = (*((u16_t *)&pUdpHdr[6]));
    sum16     = ~sum16;
    sum32     = (pseudo_cs + sum16);

    /* fold 32-bit sum to 16 bits */
    while (sum32 >> 16)
    {
        sum32 = ((sum32 & 0xffff) + (sum32 >> 16));
    }

    return ntohs((u16_t)sum32);
}


static inline u16_t BnxeGetVlanTag(mblk_t * pMblk)
{
    ASSERT(MBLKL(pMblk) >= sizeof(struct ether_vlan_header));
    return GLD_VTAG_VID(ntohs(((struct ether_vlan_header *)pMblk->b_rptr)->ether_tci));
}


static inline int BnxeGetHdrInfo(um_device_t *   pUM,
                                 um_txpacket_t * pTxPkt)
{
    mblk_t *      pMblk;
    size_t        msgSize;
    uint32_t      csStart;
    uint32_t      csStuff;
    uint32_t      csFlags;
    uint32_t      lso;
    u8_t *        pL2Hdr;
    uint32_t      l2HdrLen;
    u8_t *        pL3Hdr;
    u32_t         l3HdrLen;
    u8_t *        pL4Hdr;
    u32_t         l4HdrLen;

    pMblk = pTxPkt->pMblk;
    msgSize = MBLKL(pMblk);

    /* At least the MAC header... */
#if 0
    if (msgSize < sizeof(struct ether_header))
    {
        BnxeLogWarn(pUM, "Invalid initial segment size in packet!");
        return -1;
    }
#else
    ASSERT(msgSize >= sizeof(struct ether_header));
#endif

    mac_hcksum_get(pMblk, &csStart, &csStuff, NULL, NULL, &csFlags);

    lso = DB_LSOFLAGS(pMblk) & HW_LSO;

    /* get the Ethernet header */
    pL2Hdr = (u8_t *)pMblk->b_rptr;

    /* grab the destination mac addr */
    memcpy(pTxPkt->tx_info.dst_mac_addr, pL2Hdr, 6);

    if (lso)
    {
        pTxPkt->tx_info.flags |= LM_TX_FLAG_TCP_LSO_FRAME;

        pTxPkt->tx_info.lso_mss = (u16_t)DB_LSOMSS(pMblk);
    }
    else if (!csFlags)
    {
        /* no offload requested, just check for VLAN */

        if (((struct ether_header *)pMblk->b_rptr)->ether_type ==
            htons(ETHERTYPE_VLAN))
        {
            pTxPkt->tx_info.vlan_tag = BnxeGetVlanTag(pMblk);
            pTxPkt->tx_info.flags |= LM_TX_FLAG_VLAN_TAG_EXISTS;
        }

        return 0;
    }

    if (((struct ether_header *)pL2Hdr)->ether_type == htons(ETHERTYPE_VLAN))
    {
        l2HdrLen = sizeof(struct ether_vlan_header);

        pTxPkt->tx_info.vlan_tag = BnxeGetVlanTag(pMblk);
        pTxPkt->tx_info.flags |= LM_TX_FLAG_VLAN_TAG_EXISTS;
    }
    else
    {
        l2HdrLen = sizeof(struct ether_header);
    }

    if (csFlags & HCK_IPV4_HDRCKSUM)
    {
        pTxPkt->tx_info.flags |= LM_TX_FLAG_COMPUTE_IP_CKSUM;
    }

    if (csFlags & HCK_PARTIALCKSUM)
    {
        pTxPkt->tx_info.flags |= LM_TX_FLAG_COMPUTE_TCP_UDP_CKSUM;

        l3HdrLen = csStart;
        l4HdrLen = (l2HdrLen + csStuff + sizeof(u16_t));

        /*
         * For TCP, here we ignore the urgent pointer and size of the
         * options.  We'll get that info later.
         */
    }
    else if (lso)
    {
        /* Solaris doesn't do LSO if there is option in the IP header. */
        l3HdrLen = sizeof(struct ip);
        l4HdrLen = (l2HdrLen + l3HdrLen + sizeof(struct tcphdr));
    }
    else
    {
        return 0;
    }

    if (msgSize >= l4HdrLen)
    {
        /* the header is in the first block */
        pL3Hdr = (pL2Hdr + l2HdrLen);
    }
    else
    {
        if ((msgSize <= l2HdrLen) && pMblk->b_cont &&
            ((msgSize + MBLKL(pMblk->b_cont)) >= l4HdrLen))
        {
            /* the header is in the second block */
            pL3Hdr = pMblk->b_cont->b_rptr + (l2HdrLen - msgSize);
        }
        else
        {
            /* do a pullup to make sure headers are in the first block */
            pUM->txMsgPullUp++;

            if ((pMblk = msgpullup(pMblk, l4HdrLen)) == NULL)
            {
                return -1;
            }

            freemsg(pTxPkt->pMblk);
            pTxPkt->pMblk = pMblk;

            pL3Hdr = (pMblk->b_rptr + l2HdrLen);
        }
    }

    /* must be IPv4 or IPv6 */
    ASSERT((pL3Hdr[0] & 0xf0) == 0x60 || (pL3Hdr[0] & 0xf0) == 0x40);

    if ((pL3Hdr[0] & 0xf0) == 0x60)
    {
        pTxPkt->tx_info.flags |= LM_TX_FLAG_IPV6_PACKET;
    }

    if (lso || ((csStuff - csStart) == TCP_CS_OFFSET))
    {
        /* get the TCP header */
        pL4Hdr   = (pL3Hdr + l3HdrLen);
        l4HdrLen = ((pL4Hdr[12] & 0xf0) >> 2);

        pTxPkt->tx_info.cs_any_offset     = 0;
        pTxPkt->tx_info.tcp_nonce_sum_bit = (pL4Hdr[12] & 0x1);
        pTxPkt->tx_info.tcp_pseudo_csum   = ntohs(*((u16_t *)&pL4Hdr[TCP_CS_OFFSET]));

        if (lso)
        {
            pTxPkt->tx_info.lso_ipid         = ntohs(*((u16_t *)&pL3Hdr[4]));
            pTxPkt->tx_info.lso_tcp_send_seq = ntohl(*((u32_t *)&pL4Hdr[4]));
            pTxPkt->tx_info.lso_tcp_flags    = pL4Hdr[13];
        }
    }
    else
    {
        ASSERT((csStuff - csStart) == UDP_CS_OFFSET);

        /* get the UDP header */
        pL4Hdr = pL3Hdr + l3HdrLen;

        l4HdrLen = sizeof(struct udphdr);

        pTxPkt->tx_info.cs_any_offset     = UDP_TCP_CS_OFFSET_DIFF;
        pTxPkt->tx_info.tcp_nonce_sum_bit = 0;
        pTxPkt->tx_info.tcp_pseudo_csum   =
            CHIP_IS_E1x(((lm_device_t *)pUM)) ?
                BnxeUdpPseudoCsum(pUM, pL4Hdr, pL3Hdr, l3HdrLen) :
                ntohs(*((u16_t *)&pL4Hdr[UDP_CS_OFFSET]));
    }

    pTxPkt->tx_info.lso_ip_hdr_len  = l3HdrLen;
    pTxPkt->tx_info.lso_tcp_hdr_len = l4HdrLen;

    return 0;
}


int BnxeTxSendMblk(um_device_t * pUM,
                   int           idx,
                   mblk_t *      pMblk,
                   u32_t         flags,
                   u16_t         vlan_tag)
{
    lm_device_t *   pLM = &pUM->lm_dev;
    TxQueue *       pTxQ = &pUM->txq[idx];
    lm_tx_chain_t * pLmTxChain;
    um_txpacket_t * pTxPkt;
    s_list_t        tmpList;
    u32_t           numPkts;
    int rc;

    BNXE_LOCK_ENTER_FREETX(pUM, idx);

    pTxPkt = (um_txpacket_t *)s_list_pop_head(&pTxQ->freeTxDescQ);

    if (pTxQ->txLowWater > s_list_entry_cnt(&pTxQ->freeTxDescQ))
    {
        pTxQ->txLowWater = s_list_entry_cnt(&pTxQ->freeTxDescQ);
    }

    BNXE_LOCK_EXIT_FREETX(pUM, idx);

    /* try to recycle if no more packet available */
    if (pTxPkt == NULL)
    {
        pTxQ->txRecycle++;

        s_list_clear(&tmpList);

        BNXE_LOCK_ENTER_TX(pUM, idx);
        numPkts = lm_get_packets_sent(pLM, idx, &tmpList);
        BNXE_LOCK_EXIT_TX(pUM, idx);

        if (pUM->fmCapabilities &&
            BnxeCheckAccHandle(pLM->vars.reg_handle[BAR_0]) != DDI_FM_OK)
        {
            ddi_fm_service_impact(pUM->pDev, DDI_SERVICE_DEGRADED);
        }

        if (!numPkts)
        {
            atomic_or_32(&pTxQ->noTxCredits, BNXE_TX_RESOURCES_NO_DESC);
            pTxQ->txBlocked++;
            return BNXE_TX_HDWRFULL;
        }

        /* steal the first packet from the list before reclaiming */

        pTxPkt = (um_txpacket_t *)s_list_pop_head(&tmpList);

        if (pTxPkt->num_handles)
        {
            BnxeTxPktUnmap(pTxPkt);
        }

        if (pTxPkt->pMblk)
        {
            freemsg(pTxPkt->pMblk);
            pTxPkt->pMblk = NULL;
        }

        BnxeTxPktsReclaim(pUM, idx, &tmpList);
    }

    pTxPkt->lm_pkt.link.next = NULL;

    pTxPkt->tx_info.flags    = 0;
    pTxPkt->tx_info.vlan_tag = 0;
    pTxPkt->frag_list.cnt    = 0;
    pTxPkt->pMblk            = pMblk;

#if 0
    BnxeDumpPkt(pUM,
                (BNXE_FCOE(pUM) && (idx == FCOE_CID(&pUM->lm_dev))) ?
                    "-> FCoE L2 TX ->" : "-> L2 TX ->",
                pMblk, B_TRUE);
#endif

    if (idx == FCOE_CID(pLM))
    {
        if (flags & PRV_TX_VLAN_TAG)
        {
            pTxPkt->tx_info.vlan_tag = vlan_tag;
            pTxPkt->tx_info.flags |= LM_TX_FLAG_INSERT_VLAN_TAG;
        }
    }
    else if (BnxeGetHdrInfo(pUM, pTxPkt))
    {
        goto BnxeTxSendMblk_fail;
    }

    if (BnxeTxPktCopy(pUM, pTxQ, pTxPkt))
    {
        goto BnxeTxSendMblk_fail;
    }

    /* Now try to send the packet... */

    pLmTxChain = &pLM->tx_info.chain[idx];

    BNXE_LOCK_ENTER_TX(pUM, idx);

    /* Try to reclaim sent packets if available BDs is lower than threshold */
    if (pLmTxChain->bd_chain.bd_left < BNXE_MAX_DMA_FRAGS_PER_PKT + 2)
    {
        pTxQ->txRecycle++;

        s_list_clear(&tmpList);

        numPkts = lm_get_packets_sent(pLM, idx, &tmpList);

        if (pUM->fmCapabilities &&
            BnxeCheckAccHandle(pLM->vars.reg_handle[BAR_0]) != DDI_FM_OK)
        {
            ddi_fm_service_impact(pUM->pDev, DDI_SERVICE_DEGRADED);
        }

        if (numPkts)
        {
            BnxeTxPktsReclaim(pUM, idx, &tmpList);
        }
    }

    /*
     * If there are no packets currently waiting to be sent and there are enough
     * BDs available to satisfy this packet then send it now.
     */
    if (s_list_is_empty(&pTxQ->waitTxDescQ) &&
        (pLmTxChain->bd_chain.bd_left >= pTxPkt->frag_list.cnt + 2))
    {
        rc = lm_send_packet(pLM, idx, &pTxPkt->lm_pkt, &pTxPkt->frag_list);

        if (pUM->fmCapabilities &&
            BnxeCheckAccHandle(pLM->vars.reg_handle[BAR_0]) != DDI_FM_OK)
        {
            ddi_fm_service_impact(pUM->pDev, DDI_SERVICE_DEGRADED);
        }

        if (pUM->fmCapabilities &&
            BnxeCheckAccHandle(pLM->vars.reg_handle[BAR_1]) != DDI_FM_OK)
        {
            ddi_fm_service_impact(pUM->pDev, DDI_SERVICE_DEGRADED);
        }

        if (rc == LM_STATUS_SUCCESS)
        {
            /* send completely successfully */
            BNXE_LOCK_EXIT_TX(pUM, idx);
            return BNXE_TX_GOODXMIT;
        }

        /*
         * Send failed (probably not enough BDs available)...
         * Continue on with putting this packet on the wait queue.
         */
        pTxQ->txFailed++;
    }

#if 0
    BnxeLogWarn(pUM, "WAIT TX DESCQ %lu %d %d",
                s_list_entry_cnt(&pTxQ->waitTxDescQ),
                pLmTxChain->bd_chain.bd_left, pTxPkt->frag_list.cnt);
#endif

    /*
     * If we got here then there are other packets waiting to be sent or there
     * aren't enough BDs available.  In either case put this packet at the end
     * of the waiting queue.
     */
    s_list_push_tail(&pTxQ->waitTxDescQ, &pTxPkt->lm_pkt.link);

    pTxQ->txWait++;

    /*
     * If there appears to be a sufficient number of BDs available then make a
     * quick attempt to send as many waiting packets as possible.
     */
    if ((pLmTxChain->bd_chain.bd_left >= BNXE_MAX_DMA_FRAGS_PER_PKT) &&
        (BnxeTxSendWaitingPkt(pUM, idx) == BNXE_TX_GOODXMIT))
    {
        BNXE_LOCK_EXIT_TX(pUM, idx);
        return BNXE_TX_GOODXMIT;
    }

    /* Couldn't send anything! */
    atomic_or_32(&pTxQ->noTxCredits, BNXE_TX_RESOURCES_NO_CREDIT);
    pTxQ->txBlocked++;

    BNXE_LOCK_EXIT_TX(pUM, idx);

    return BNXE_TX_DEFERPKT;

BnxeTxSendMblk_fail:

    pTxQ->txDiscards++;

    ASSERT(pTxPkt != NULL);

    if (pTxPkt->pMblk)
    {
        freemsg(pTxPkt->pMblk);
        pTxPkt->pMblk = NULL;
    }

    BNXE_LOCK_ENTER_FREETX(pUM, idx);
    s_list_push_tail(&pTxQ->freeTxDescQ, &pTxPkt->lm_pkt.link);
    BNXE_LOCK_EXIT_FREETX(pUM, idx);

    /*
     * Yes GOODXMIT since mblk was free'd here and this triggers caller to
     * try and send the next packet in its chain.
     */
    return BNXE_TX_GOODXMIT;
}


static void BnxeTxPktsAbortIdx(um_device_t * pUM,
                               int           idx)
{
    s_list_t tmpList;

    BNXE_LOCK_ENTER_TX(pUM, idx);
    lm_abort(&pUM->lm_dev, ABORT_OP_INDICATE_TX_CHAIN, idx);
    tmpList = pUM->txq[idx].waitTxDescQ;
    s_list_clear(&pUM->txq[idx].waitTxDescQ);
    BNXE_LOCK_EXIT_TX(pUM, idx);

    BnxeTxPktsReclaim(pUM, idx, &tmpList);
}


void BnxeTxPktsAbort(um_device_t * pUM,
                     int           cliIdx)
{
    int idx;

    switch (cliIdx)
    {
    case LM_CLI_IDX_FCOE:

        BnxeTxPktsAbortIdx(pUM, FCOE_CID(&pUM->lm_dev));
        break;

    case LM_CLI_IDX_NDIS:

        LM_FOREACH_TSS_IDX(&pUM->lm_dev, idx)
        {
            BnxeTxPktsAbortIdx(pUM, idx);
        }

        break;

    default:

        BnxeLogWarn(pUM, "ERROR: Invalid cliIdx for BnxeTxPktsAbort (%d)", cliIdx);
        break;
    }
}


static um_txpacket_t * BnxeTxPktAlloc(um_device_t * pUM,
                                      size_t        size)
{
    um_txpacket_t *   pTxPkt;
    ddi_dma_cookie_t  cookie;
    u32_t             count;
    size_t            length;
    int rc, j;

    if ((pTxPkt = kmem_zalloc(sizeof(um_txpacket_t), KM_NOSLEEP)) == NULL)
    {
        return NULL;
    }

    pTxPkt->lm_pkt.l2pkt_tx_info = &pTxPkt->tx_info;

    if ((rc = ddi_dma_alloc_handle(pUM->pDev,
                                   &bnxeTxCbDmaAttrib,
                                   DDI_DMA_DONTWAIT,
                                   NULL,
                                   &pTxPkt->cbDmaHandle)) != DDI_SUCCESS)
    {
        BnxeLogWarn(pUM, "Failed to alloc DMA handle for Tx Desc (%d)", rc);
        kmem_free(pTxPkt, sizeof(um_txpacket_t));
        return NULL;
    }

    if ((rc = ddi_dma_mem_alloc(pTxPkt->cbDmaHandle,
                                size,
                                &bnxeAccessAttribBUF,
                                DDI_DMA_STREAMING,
                                DDI_DMA_DONTWAIT,
                                NULL,
                                &pTxPkt->pCbBuf,
                                &length,
                                &pTxPkt->cbDmaAccHandle)) != DDI_SUCCESS)
    {
        BnxeLogWarn(pUM, "Failed to alloc DMA memory for Tx Desc (%d)", rc);
        ddi_dma_free_handle(&pTxPkt->cbDmaHandle);
        kmem_free(pTxPkt, sizeof(um_txpacket_t));
        return NULL;
    }

    if ((rc = ddi_dma_addr_bind_handle(pTxPkt->cbDmaHandle,
                                       NULL,
                                       pTxPkt->pCbBuf,
                                       length,
                                       DDI_DMA_WRITE | DDI_DMA_STREAMING,
                                       DDI_DMA_DONTWAIT,
                                       NULL,
                                       &cookie,
                                       &count)) != DDI_DMA_MAPPED)
    {
        BnxeLogWarn(pUM, "Failed to bind DMA address for Tx Desc (%d)", rc);
        ddi_dma_mem_free(&pTxPkt->cbDmaAccHandle);
        ddi_dma_free_handle(&pTxPkt->cbDmaHandle);
        kmem_free(pTxPkt, sizeof(um_txpacket_t));
        return NULL;
    }

    pTxPkt->cbPhysAddr.as_u64 = cookie.dmac_laddress;

    for (j = 0; j < BNXE_MAX_DMA_HANDLES_PER_PKT; j++)
    {
        if ((rc = ddi_dma_alloc_handle(pUM->pDev,
                                       &bnxeTxDmaAttrib,
                                       DDI_DMA_DONTWAIT,
                                       NULL,
                                       &pTxPkt->dmaHandles[j])) !=
            DDI_SUCCESS)
        {
            BnxeLogWarn(pUM, "Failed to alloc DMA handles for Tx Pkt %d (%d)",
                        j, rc);

            for(--j; j >= 0; j--) /* unwind */
            {
                ddi_dma_free_handle(&pTxPkt->dmaHandles[j]);
            }

            ddi_dma_unbind_handle(pTxPkt->cbDmaHandle);
            ddi_dma_mem_free(&pTxPkt->cbDmaAccHandle);
            ddi_dma_free_handle(&pTxPkt->cbDmaHandle);
            kmem_free(pTxPkt, sizeof(um_txpacket_t));
            return NULL;
        }
    }

    ASSERT(pTxPkt->pMblk == NULL);
    ASSERT(pTxPkt->num_handles == 0);
    ASSERT(pTxPkt->frag_list.cnt == 0);
    pTxPkt->cbLength = size;

    return pTxPkt;
}


static int BnxeTxPktsInitIdx(um_device_t * pUM,
                             int           idx)
{
    lm_device_t *   pLM = &pUM->lm_dev;
    TxQueue *       pTxQ;
    um_txpacket_t * pTxPkt;
    s_list_t        tmpList;
    int i;

    pTxQ = &pUM->txq[idx];

    s_list_clear(&pTxQ->sentTxQ);
    s_list_clear(&pTxQ->freeTxDescQ);
    s_list_clear(&pTxQ->waitTxDescQ);

    pTxQ->desc_cnt    = pUM->devParams.numTxDesc[LM_CHAIN_IDX_CLI(pLM, idx)];
    pTxQ->txLowWater  = pUM->devParams.numTxDesc[LM_CHAIN_IDX_CLI(pLM, idx)];
    pTxQ->thresh_pdwm = BNXE_PDWM_THRESHOLD;
    pTxQ->txFailed    = 0;
    pTxQ->txDiscards  = 0;
    pTxQ->txRecycle   = 0;
    pTxQ->txCopied    = 0;
    pTxQ->txBlocked   = 0;
    pTxQ->txWait      = 0;

    if (pUM->devParams.lsoEnable)
    {
        for (i = 0; i < pTxQ->desc_cnt; i++)
        {
            pTxPkt = BnxeTxPktAlloc(pUM,
                                    (BNXE_IP_MAXLEN +
                                     sizeof(struct ether_vlan_header)));
            if (pTxPkt == NULL)
            {
                BnxeLogWarn(pUM, "Failed to allocate all Tx Descs for LSO (%d/%d allocated), LSO is disabled",
                            i, pTxQ->desc_cnt);

                /* free existing in freeTxDescQ... */

                BNXE_LOCK_ENTER_FREETX(pUM, idx);
                tmpList = pTxQ->freeTxDescQ;
                s_list_clear(&pTxQ->freeTxDescQ);
                BNXE_LOCK_EXIT_FREETX(pUM, idx);

                BnxeTxPktsFreeList(&tmpList);

                pUM->devParams.lsoEnable = 0; /* Disabling LSO! */

                break;
            }

            BNXE_LOCK_ENTER_FREETX(pUM, idx);
            s_list_push_tail(&pTxQ->freeTxDescQ, &pTxPkt->lm_pkt.link);
            BNXE_LOCK_EXIT_FREETX(pUM, idx);
        }
    }

    if (!pUM->devParams.lsoEnable)
    {
        for (i = 0; i < pTxQ->desc_cnt; i++)
        {
            pTxPkt = BnxeTxPktAlloc(pUM,
                                    (pUM->devParams.mtu[LM_CHAIN_IDX_CLI(pLM, idx)] +
                                     sizeof(struct ether_vlan_header)));
            if (pTxPkt == NULL)
            {
                BnxeLogWarn(pUM, "Failed to allocate all Tx Descs (%d/%d allocated)",
                            i, pTxQ->desc_cnt);

                /* free existing in freeTxDescQ... */

                BNXE_LOCK_ENTER_FREETX(pUM, idx);
                tmpList = pTxQ->freeTxDescQ;
                s_list_clear(&pTxQ->freeTxDescQ);
                BNXE_LOCK_EXIT_FREETX(pUM, idx);

                BnxeTxPktsFreeList(&tmpList);

                return -1;
            }

            BNXE_LOCK_ENTER_FREETX(pUM, idx);
            s_list_push_tail(&pTxQ->freeTxDescQ, &pTxPkt->lm_pkt.link);
            BNXE_LOCK_EXIT_FREETX(pUM, idx);
        }
    }

    return 0;
}


int BnxeTxPktsInit(um_device_t * pUM,
                   int           cliIdx)
{
    int idx, rc;

    switch (cliIdx)
    {
    case LM_CLI_IDX_FCOE:

        rc = BnxeTxPktsInitIdx(pUM, FCOE_CID(&pUM->lm_dev));
        break;

    case LM_CLI_IDX_NDIS:

        LM_FOREACH_TSS_IDX(&pUM->lm_dev, idx)
        {
            if ((rc = BnxeTxPktsInitIdx(pUM, idx)) < 0)
            {
                break;
            }
        }

        break;

    default:

        BnxeLogWarn(pUM, "ERROR: Invalid cliIdx for BnxeTxPktsFini (%d)", cliIdx);
        rc = -1;
        break;
    }

    return rc;
}


static void BnxeTxPktsFiniIdx(um_device_t * pUM,
                              int           idx)
{
    lm_device_t * pLM = &pUM->lm_dev;
    TxQueue *     pTxQ;
    s_list_t      tmpList;

    pTxQ = &pUM->txq[idx];

    BNXE_LOCK_ENTER_FREETX(pUM, idx);
    tmpList = pTxQ->freeTxDescQ;
    s_list_clear(&pTxQ->freeTxDescQ);
    BNXE_LOCK_EXIT_FREETX(pUM, idx);

    BNXE_LOCK_ENTER_TX(pUM, idx);
    s_list_add_tail(&tmpList, &pTxQ->sentTxQ);
    s_list_clear(&pTxQ->sentTxQ);
    BNXE_LOCK_EXIT_TX(pUM, idx);

    /* there could be more than originally allocated but less is bad */
    if (s_list_entry_cnt(&tmpList) <
        pUM->devParams.numTxDesc[LM_CHAIN_IDX_CLI(pLM, idx)])
    {
        BnxeLogWarn(pUM, "Missing TX descriptors (%lu / %d) (TxFail: %d)",
                    s_list_entry_cnt(&tmpList), pUM->devParams.numTxDesc,
                    pTxQ->txFailed);
    }

    BnxeTxPktsFreeList(&tmpList);
}


void BnxeTxPktsFini(um_device_t * pUM,
                    int           cliIdx)
{
    int idx;

    switch (cliIdx)
    {
    case LM_CLI_IDX_FCOE:

        BnxeTxPktsFiniIdx(pUM, FCOE_CID(&pUM->lm_dev));
        break;

    case LM_CLI_IDX_NDIS:

        LM_FOREACH_TSS_IDX(&pUM->lm_dev, idx)
        {
            BnxeTxPktsFiniIdx(pUM, idx);
        }

        break;

    default:

        BnxeLogWarn(pUM, "ERROR: Invalid cliIdx for BnxeTxPktsFini (%d)", cliIdx);
        break;
    }
}