171d10453SEric Joyner /* SPDX-License-Identifier: BSD-3-Clause */ 2d08b8680SEric Joyner /* Copyright (c) 2021, Intel Corporation 371d10453SEric Joyner * All rights reserved. 471d10453SEric Joyner * 571d10453SEric Joyner * Redistribution and use in source and binary forms, with or without 671d10453SEric Joyner * modification, are permitted provided that the following conditions are met: 771d10453SEric Joyner * 871d10453SEric Joyner * 1. Redistributions of source code must retain the above copyright notice, 971d10453SEric Joyner * this list of conditions and the following disclaimer. 1071d10453SEric Joyner * 1171d10453SEric Joyner * 2. Redistributions in binary form must reproduce the above copyright 1271d10453SEric Joyner * notice, this list of conditions and the following disclaimer in the 1371d10453SEric Joyner * documentation and/or other materials provided with the distribution. 1471d10453SEric Joyner * 1571d10453SEric Joyner * 3. Neither the name of the Intel Corporation nor the names of its 1671d10453SEric Joyner * contributors may be used to endorse or promote products derived from 1771d10453SEric Joyner * this software without specific prior written permission. 1871d10453SEric Joyner * 1971d10453SEric Joyner * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 2071d10453SEric Joyner * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2171d10453SEric Joyner * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2271d10453SEric Joyner * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 2371d10453SEric Joyner * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2471d10453SEric Joyner * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 2571d10453SEric Joyner * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 2671d10453SEric Joyner * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 2771d10453SEric Joyner * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 2871d10453SEric Joyner * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 2971d10453SEric Joyner * POSSIBILITY OF SUCH DAMAGE. 3071d10453SEric Joyner */ 3171d10453SEric Joyner /*$FreeBSD$*/ 3271d10453SEric Joyner 3371d10453SEric Joyner /** 3471d10453SEric Joyner * @file ice_iflib_txrx.c 3571d10453SEric Joyner * @brief iflib Tx/Rx hotpath 3671d10453SEric Joyner * 3771d10453SEric Joyner * Main location for the iflib Tx/Rx hotpath implementation. 3871d10453SEric Joyner * 3971d10453SEric Joyner * Contains the implementation for the iflib function callbacks and the 4071d10453SEric Joyner * if_txrx ops structure. 4171d10453SEric Joyner */ 4271d10453SEric Joyner 4371d10453SEric Joyner #include "ice_iflib.h" 4471d10453SEric Joyner 4571d10453SEric Joyner /* Tx/Rx hotpath utility functions */ 4671d10453SEric Joyner #include "ice_common_txrx.h" 4771d10453SEric Joyner 4871d10453SEric Joyner /* 4971d10453SEric Joyner * iflib txrx method declarations 5071d10453SEric Joyner */ 5171d10453SEric Joyner static int ice_ift_txd_encap(void *arg, if_pkt_info_t pi); 5271d10453SEric Joyner static int ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri); 5371d10453SEric Joyner static void ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); 5471d10453SEric Joyner static int ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear); 5571d10453SEric Joyner static int ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget); 5671d10453SEric Joyner static void ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx, qidx_t pidx); 5771d10453SEric Joyner static void ice_ift_rxd_refill(void *arg, if_rxd_update_t iru); 5856429daeSEric Joyner static qidx_t ice_ift_queue_select(void *arg, struct mbuf *m); 5971d10453SEric Joyner 6071d10453SEric Joyner /* Macro to help extract the NIC mode flexible Rx descriptor fields from the 6171d10453SEric Joyner * advanced 32byte Rx descriptors. 6271d10453SEric Joyner */ 6371d10453SEric Joyner #define RX_FLEX_NIC(desc, field) \ 6471d10453SEric Joyner (((struct ice_32b_rx_flex_desc_nic *)desc)->field) 6571d10453SEric Joyner 6671d10453SEric Joyner /** 6771d10453SEric Joyner * @var ice_txrx 6871d10453SEric Joyner * @brief Tx/Rx operations for the iflib stack 6971d10453SEric Joyner * 7071d10453SEric Joyner * Structure defining the Tx and Rx related operations that iflib can request 7171d10453SEric Joyner * the driver to perform. These are the main entry points for the hot path of 7271d10453SEric Joyner * the transmit and receive paths in the iflib driver. 7371d10453SEric Joyner */ 7471d10453SEric Joyner struct if_txrx ice_txrx = { 7571d10453SEric Joyner .ift_txd_encap = ice_ift_txd_encap, 7671d10453SEric Joyner .ift_txd_flush = ice_ift_txd_flush, 7771d10453SEric Joyner .ift_txd_credits_update = ice_ift_txd_credits_update, 7871d10453SEric Joyner .ift_rxd_available = ice_ift_rxd_available, 7971d10453SEric Joyner .ift_rxd_pkt_get = ice_ift_rxd_pkt_get, 8071d10453SEric Joyner .ift_rxd_refill = ice_ift_rxd_refill, 8171d10453SEric Joyner .ift_rxd_flush = ice_ift_rxd_flush, 8256429daeSEric Joyner .ift_txq_select = ice_ift_queue_select, 8371d10453SEric Joyner }; 8471d10453SEric Joyner 8571d10453SEric Joyner /** 8671d10453SEric Joyner * ice_ift_txd_encap - prepare Tx descriptors for a packet 8771d10453SEric Joyner * @arg: the iflib softc structure pointer 8871d10453SEric Joyner * @pi: packet info 8971d10453SEric Joyner * 9071d10453SEric Joyner * Prepares and encapsulates the given packet into into Tx descriptors, in 9171d10453SEric Joyner * preparation for sending to the transmit engine. Sets the necessary context 9271d10453SEric Joyner * descriptors for TSO and other offloads, and prepares the last descriptor 9371d10453SEric Joyner * for the writeback status. 9471d10453SEric Joyner * 9571d10453SEric Joyner * Return 0 on success, non-zero error code on failure. 9671d10453SEric Joyner */ 9771d10453SEric Joyner static int 9871d10453SEric Joyner ice_ift_txd_encap(void *arg, if_pkt_info_t pi) 9971d10453SEric Joyner { 10071d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)arg; 10171d10453SEric Joyner struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[pi->ipi_qsidx]; 10271d10453SEric Joyner int nsegs = pi->ipi_nsegs; 10371d10453SEric Joyner bus_dma_segment_t *segs = pi->ipi_segs; 10471d10453SEric Joyner struct ice_tx_desc *txd = NULL; 10571d10453SEric Joyner int i, j, mask, pidx_last; 10671d10453SEric Joyner u32 cmd, off; 10771d10453SEric Joyner 10871d10453SEric Joyner cmd = off = 0; 10971d10453SEric Joyner i = pi->ipi_pidx; 11071d10453SEric Joyner 11171d10453SEric Joyner /* Set up the TSO/CSUM offload */ 11271d10453SEric Joyner if (pi->ipi_csum_flags & ICE_CSUM_OFFLOAD) { 11371d10453SEric Joyner /* Set up the TSO context descriptor if required */ 11471d10453SEric Joyner if (pi->ipi_csum_flags & CSUM_TSO) { 11571d10453SEric Joyner if (ice_tso_detect_sparse(pi)) 11671d10453SEric Joyner return (EFBIG); 11771d10453SEric Joyner i = ice_tso_setup(txq, pi); 11871d10453SEric Joyner } 11971d10453SEric Joyner ice_tx_setup_offload(txq, pi, &cmd, &off); 12071d10453SEric Joyner } 12171d10453SEric Joyner if (pi->ipi_mflags & M_VLANTAG) 12271d10453SEric Joyner cmd |= ICE_TX_DESC_CMD_IL2TAG1; 12371d10453SEric Joyner 12471d10453SEric Joyner mask = txq->desc_count - 1; 12571d10453SEric Joyner for (j = 0; j < nsegs; j++) { 12671d10453SEric Joyner bus_size_t seglen; 12771d10453SEric Joyner 12871d10453SEric Joyner txd = &txq->tx_base[i]; 12971d10453SEric Joyner seglen = segs[j].ds_len; 13071d10453SEric Joyner 13171d10453SEric Joyner txd->buf_addr = htole64(segs[j].ds_addr); 13271d10453SEric Joyner txd->cmd_type_offset_bsz = 13371d10453SEric Joyner htole64(ICE_TX_DESC_DTYPE_DATA 13471d10453SEric Joyner | ((u64)cmd << ICE_TXD_QW1_CMD_S) 13571d10453SEric Joyner | ((u64)off << ICE_TXD_QW1_OFFSET_S) 13671d10453SEric Joyner | ((u64)seglen << ICE_TXD_QW1_TX_BUF_SZ_S) 13771d10453SEric Joyner | ((u64)htole16(pi->ipi_vtag) << ICE_TXD_QW1_L2TAG1_S)); 13871d10453SEric Joyner 13971d10453SEric Joyner txq->stats.tx_bytes += seglen; 14071d10453SEric Joyner pidx_last = i; 14171d10453SEric Joyner i = (i+1) & mask; 14271d10453SEric Joyner } 14371d10453SEric Joyner 14471d10453SEric Joyner /* Set the last descriptor for report */ 14571d10453SEric Joyner #define ICE_TXD_CMD (ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS) 14671d10453SEric Joyner txd->cmd_type_offset_bsz |= 14771d10453SEric Joyner htole64(((u64)ICE_TXD_CMD << ICE_TXD_QW1_CMD_S)); 14871d10453SEric Joyner 14971d10453SEric Joyner /* Add to report status array */ 15071d10453SEric Joyner txq->tx_rsq[txq->tx_rs_pidx] = pidx_last; 15171d10453SEric Joyner txq->tx_rs_pidx = (txq->tx_rs_pidx+1) & mask; 15271d10453SEric Joyner MPASS(txq->tx_rs_pidx != txq->tx_rs_cidx); 15371d10453SEric Joyner 15471d10453SEric Joyner pi->ipi_new_pidx = i; 15571d10453SEric Joyner 15671d10453SEric Joyner ++txq->stats.tx_packets; 15771d10453SEric Joyner return (0); 15871d10453SEric Joyner } 15971d10453SEric Joyner 16071d10453SEric Joyner /** 16171d10453SEric Joyner * ice_ift_txd_flush - Flush Tx descriptors to hardware 16271d10453SEric Joyner * @arg: device specific softc pointer 16371d10453SEric Joyner * @txqid: the Tx queue to flush 16471d10453SEric Joyner * @pidx: descriptor index to advance tail to 16571d10453SEric Joyner * 16671d10453SEric Joyner * Advance the Transmit Descriptor Tail (TDT). This indicates to hardware that 16771d10453SEric Joyner * frames are available for transmit. 16871d10453SEric Joyner */ 16971d10453SEric Joyner static void 17071d10453SEric Joyner ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) 17171d10453SEric Joyner { 17271d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)arg; 17371d10453SEric Joyner struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid]; 17471d10453SEric Joyner struct ice_hw *hw = &sc->hw; 17571d10453SEric Joyner 17671d10453SEric Joyner wr32(hw, txq->tail, pidx); 17771d10453SEric Joyner } 17871d10453SEric Joyner 17971d10453SEric Joyner /** 18071d10453SEric Joyner * ice_ift_txd_credits_update - cleanup Tx descriptors 18171d10453SEric Joyner * @arg: device private softc 18271d10453SEric Joyner * @txqid: the Tx queue to update 18371d10453SEric Joyner * @clear: if false, only report, do not actually clean 18471d10453SEric Joyner * 18571d10453SEric Joyner * If clear is false, iflib is asking if we *could* clean up any Tx 18671d10453SEric Joyner * descriptors. 18771d10453SEric Joyner * 18871d10453SEric Joyner * If clear is true, iflib is requesting to cleanup and reclaim used Tx 18971d10453SEric Joyner * descriptors. 19071d10453SEric Joyner */ 19171d10453SEric Joyner static int 19271d10453SEric Joyner ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear) 19371d10453SEric Joyner { 19471d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)arg; 19571d10453SEric Joyner struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid]; 19671d10453SEric Joyner 19771d10453SEric Joyner qidx_t processed = 0; 19871d10453SEric Joyner qidx_t cur, prev, ntxd, rs_cidx; 19971d10453SEric Joyner int32_t delta; 20071d10453SEric Joyner bool is_done; 20171d10453SEric Joyner 20271d10453SEric Joyner rs_cidx = txq->tx_rs_cidx; 20371d10453SEric Joyner if (rs_cidx == txq->tx_rs_pidx) 20471d10453SEric Joyner return (0); 20571d10453SEric Joyner cur = txq->tx_rsq[rs_cidx]; 20671d10453SEric Joyner MPASS(cur != QIDX_INVALID); 20771d10453SEric Joyner is_done = ice_is_tx_desc_done(&txq->tx_base[cur]); 20871d10453SEric Joyner 20971d10453SEric Joyner if (!is_done) 21071d10453SEric Joyner return (0); 21171d10453SEric Joyner else if (clear == false) 21271d10453SEric Joyner return (1); 21371d10453SEric Joyner 21471d10453SEric Joyner prev = txq->tx_cidx_processed; 21571d10453SEric Joyner ntxd = txq->desc_count; 21671d10453SEric Joyner do { 21771d10453SEric Joyner MPASS(prev != cur); 21871d10453SEric Joyner delta = (int32_t)cur - (int32_t)prev; 21971d10453SEric Joyner if (delta < 0) 22071d10453SEric Joyner delta += ntxd; 22171d10453SEric Joyner MPASS(delta > 0); 22271d10453SEric Joyner processed += delta; 22371d10453SEric Joyner prev = cur; 22471d10453SEric Joyner rs_cidx = (rs_cidx + 1) & (ntxd-1); 22571d10453SEric Joyner if (rs_cidx == txq->tx_rs_pidx) 22671d10453SEric Joyner break; 22771d10453SEric Joyner cur = txq->tx_rsq[rs_cidx]; 22871d10453SEric Joyner MPASS(cur != QIDX_INVALID); 22971d10453SEric Joyner is_done = ice_is_tx_desc_done(&txq->tx_base[cur]); 23071d10453SEric Joyner } while (is_done); 23171d10453SEric Joyner 23271d10453SEric Joyner txq->tx_rs_cidx = rs_cidx; 23371d10453SEric Joyner txq->tx_cidx_processed = prev; 23471d10453SEric Joyner 23571d10453SEric Joyner return (processed); 23671d10453SEric Joyner } 23771d10453SEric Joyner 23871d10453SEric Joyner /** 23971d10453SEric Joyner * ice_ift_rxd_available - Return number of available Rx packets 24071d10453SEric Joyner * @arg: device private softc 24171d10453SEric Joyner * @rxqid: the Rx queue id 24271d10453SEric Joyner * @pidx: descriptor start point 24371d10453SEric Joyner * @budget: maximum Rx budget 24471d10453SEric Joyner * 24571d10453SEric Joyner * Determines how many Rx packets are available on the queue, up to a maximum 24671d10453SEric Joyner * of the given budget. 24771d10453SEric Joyner */ 24871d10453SEric Joyner static int 24971d10453SEric Joyner ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget) 25071d10453SEric Joyner { 25171d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)arg; 25271d10453SEric Joyner struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid]; 25371d10453SEric Joyner union ice_32b_rx_flex_desc *rxd; 25471d10453SEric Joyner uint16_t status0; 25571d10453SEric Joyner int cnt, i, nrxd; 25671d10453SEric Joyner 25771d10453SEric Joyner nrxd = rxq->desc_count; 25871d10453SEric Joyner 25971d10453SEric Joyner for (cnt = 0, i = pidx; cnt < nrxd - 1 && cnt < budget;) { 26071d10453SEric Joyner rxd = &rxq->rx_base[i]; 26171d10453SEric Joyner status0 = le16toh(rxd->wb.status_error0); 26271d10453SEric Joyner 26371d10453SEric Joyner if ((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) == 0) 26471d10453SEric Joyner break; 26571d10453SEric Joyner if (++i == nrxd) 26671d10453SEric Joyner i = 0; 26771d10453SEric Joyner if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S)) 26871d10453SEric Joyner cnt++; 26971d10453SEric Joyner } 27071d10453SEric Joyner 27171d10453SEric Joyner return (cnt); 27271d10453SEric Joyner } 27371d10453SEric Joyner 27471d10453SEric Joyner /** 27571d10453SEric Joyner * ice_ift_rxd_pkt_get - Called by iflib to send data to upper layer 27671d10453SEric Joyner * @arg: device specific softc 27771d10453SEric Joyner * @ri: receive packet info 27871d10453SEric Joyner * 27971d10453SEric Joyner * This function is called by iflib, and executes in ithread context. It is 28071d10453SEric Joyner * called by iflib to obtain data which has been DMA'ed into host memory. 28156429daeSEric Joyner * Returns zero on success, and EBADMSG on failure. 28271d10453SEric Joyner */ 28371d10453SEric Joyner static int 28471d10453SEric Joyner ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri) 28571d10453SEric Joyner { 28671d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)arg; 28752f45d8aSVincenzo Maffione if_softc_ctx_t scctx = sc->scctx; 28871d10453SEric Joyner struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[ri->iri_qsidx]; 28971d10453SEric Joyner union ice_32b_rx_flex_desc *cur; 290f7926a6dSVincenzo Maffione u16 status0, plen, ptype; 29171d10453SEric Joyner bool eop; 29271d10453SEric Joyner size_t cidx; 29371d10453SEric Joyner int i; 29471d10453SEric Joyner 29571d10453SEric Joyner cidx = ri->iri_cidx; 29671d10453SEric Joyner i = 0; 29771d10453SEric Joyner do { 29871d10453SEric Joyner /* 5 descriptor receive limit */ 29971d10453SEric Joyner MPASS(i < ICE_MAX_RX_SEGS); 30071d10453SEric Joyner 30171d10453SEric Joyner cur = &rxq->rx_base[cidx]; 30271d10453SEric Joyner status0 = le16toh(cur->wb.status_error0); 30371d10453SEric Joyner plen = le16toh(cur->wb.pkt_len) & 30471d10453SEric Joyner ICE_RX_FLX_DESC_PKT_LEN_M; 30571d10453SEric Joyner 30671d10453SEric Joyner /* we should never be called without a valid descriptor */ 30771d10453SEric Joyner MPASS((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) != 0); 30871d10453SEric Joyner 30971d10453SEric Joyner ri->iri_len += plen; 31071d10453SEric Joyner 31171d10453SEric Joyner cur->wb.status_error0 = 0; 31271d10453SEric Joyner eop = (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S)); 31371d10453SEric Joyner 31471d10453SEric Joyner ri->iri_frags[i].irf_flid = 0; 31571d10453SEric Joyner ri->iri_frags[i].irf_idx = cidx; 31671d10453SEric Joyner ri->iri_frags[i].irf_len = plen; 31771d10453SEric Joyner if (++cidx == rxq->desc_count) 31871d10453SEric Joyner cidx = 0; 31971d10453SEric Joyner i++; 32071d10453SEric Joyner } while (!eop); 32171d10453SEric Joyner 32256429daeSEric Joyner /* End of Packet reached; cur is eop/last descriptor */ 32371d10453SEric Joyner 32456429daeSEric Joyner /* Make sure packets with bad L2 values are discarded. 32556429daeSEric Joyner * This bit is only valid in the last descriptor. 32656429daeSEric Joyner */ 32756429daeSEric Joyner if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S)) { 32856429daeSEric Joyner rxq->stats.desc_errs++; 32956429daeSEric Joyner return (EBADMSG); 33056429daeSEric Joyner } 33156429daeSEric Joyner 33256429daeSEric Joyner /* Get VLAN tag information if one is in descriptor */ 333f7926a6dSVincenzo Maffione if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) { 334f7926a6dSVincenzo Maffione ri->iri_vtag = le16toh(cur->wb.l2tag1); 33571d10453SEric Joyner ri->iri_flags |= M_VLANTAG; 336f7926a6dSVincenzo Maffione } 33756429daeSEric Joyner 33856429daeSEric Joyner /* Capture soft statistics for this Rx queue */ 33956429daeSEric Joyner rxq->stats.rx_packets++; 34056429daeSEric Joyner rxq->stats.rx_bytes += ri->iri_len; 34156429daeSEric Joyner 34256429daeSEric Joyner /* Get packet type and set checksum flags */ 34356429daeSEric Joyner ptype = le16toh(cur->wb.ptype_flex_flags0) & 34456429daeSEric Joyner ICE_RX_FLEX_DESC_PTYPE_M; 34556429daeSEric Joyner if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0) 34656429daeSEric Joyner ice_rx_checksum(rxq, &ri->iri_csum_flags, 34756429daeSEric Joyner &ri->iri_csum_data, status0, ptype); 34856429daeSEric Joyner 34956429daeSEric Joyner /* Set remaining iflib RX descriptor info fields */ 35056429daeSEric Joyner ri->iri_flowid = le32toh(RX_FLEX_NIC(&cur->wb, rss_hash)); 35156429daeSEric Joyner ri->iri_rsstype = ice_ptype_to_hash(ptype); 352f7926a6dSVincenzo Maffione ri->iri_nfrags = i; 35371d10453SEric Joyner return (0); 35471d10453SEric Joyner } 35571d10453SEric Joyner 35671d10453SEric Joyner /** 35771d10453SEric Joyner * ice_ift_rxd_refill - Prepare Rx descriptors for re-use by hardware 35871d10453SEric Joyner * @arg: device specific softc structure 35971d10453SEric Joyner * @iru: the Rx descriptor update structure 36071d10453SEric Joyner * 36171d10453SEric Joyner * Update the Rx descriptor indices for a given queue, assigning new physical 36271d10453SEric Joyner * addresses to the descriptors, preparing them for re-use by the hardware. 36371d10453SEric Joyner */ 36471d10453SEric Joyner static void 36571d10453SEric Joyner ice_ift_rxd_refill(void *arg, if_rxd_update_t iru) 36671d10453SEric Joyner { 36771d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)arg; 36871d10453SEric Joyner struct ice_rx_queue *rxq; 36971d10453SEric Joyner uint32_t next_pidx; 37071d10453SEric Joyner int i; 37171d10453SEric Joyner uint64_t *paddrs; 37271d10453SEric Joyner uint32_t pidx; 37371d10453SEric Joyner uint16_t qsidx, count; 37471d10453SEric Joyner 37571d10453SEric Joyner paddrs = iru->iru_paddrs; 37671d10453SEric Joyner pidx = iru->iru_pidx; 37771d10453SEric Joyner qsidx = iru->iru_qsidx; 37871d10453SEric Joyner count = iru->iru_count; 37971d10453SEric Joyner 38071d10453SEric Joyner rxq = &(sc->pf_vsi.rx_queues[qsidx]); 38171d10453SEric Joyner 38271d10453SEric Joyner for (i = 0, next_pidx = pidx; i < count; i++) { 38371d10453SEric Joyner rxq->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]); 38471d10453SEric Joyner if (++next_pidx == (uint32_t)rxq->desc_count) 38571d10453SEric Joyner next_pidx = 0; 38671d10453SEric Joyner } 38771d10453SEric Joyner } 38871d10453SEric Joyner 38971d10453SEric Joyner /** 39071d10453SEric Joyner * ice_ift_rxd_flush - Flush Rx descriptors to hardware 39171d10453SEric Joyner * @arg: device specific softc pointer 39271d10453SEric Joyner * @rxqid: the Rx queue to flush 39371d10453SEric Joyner * @flidx: unused parameter 39471d10453SEric Joyner * @pidx: descriptor index to advance tail to 39571d10453SEric Joyner * 39671d10453SEric Joyner * Advance the Receive Descriptor Tail (RDT). This indicates to hardware that 39771d10453SEric Joyner * software is done with the descriptor and it can be recycled. 39871d10453SEric Joyner */ 39971d10453SEric Joyner static void 40071d10453SEric Joyner ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx __unused, 40171d10453SEric Joyner qidx_t pidx) 40271d10453SEric Joyner { 40371d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)arg; 40471d10453SEric Joyner struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid]; 40571d10453SEric Joyner struct ice_hw *hw = &sc->hw; 40671d10453SEric Joyner 40771d10453SEric Joyner wr32(hw, rxq->tail, pidx); 40871d10453SEric Joyner } 40956429daeSEric Joyner 41056429daeSEric Joyner static qidx_t 41156429daeSEric Joyner ice_ift_queue_select(void *arg, struct mbuf *m) 41256429daeSEric Joyner { 41356429daeSEric Joyner struct ice_softc *sc = (struct ice_softc *)arg; 41456429daeSEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 41556429daeSEric Joyner u16 tc_base_queue, tc_qcount; 41656429daeSEric Joyner u8 up, tc; 41756429daeSEric Joyner 418*61d83041SEric Joyner #ifdef ALTQ 419*61d83041SEric Joyner /* Included to match default iflib behavior */ 42056429daeSEric Joyner /* Only go out on default queue if ALTQ is enabled */ 421*61d83041SEric Joyner struct ifnet *ifp = (struct ifnet *)iflib_get_ifp(sc->ctx); 42256429daeSEric Joyner if (ALTQ_IS_ENABLED(&ifp->if_snd)) 42356429daeSEric Joyner return (0); 424*61d83041SEric Joyner #endif 42556429daeSEric Joyner 42656429daeSEric Joyner if (!ice_test_state(&sc->state, ICE_STATE_MULTIPLE_TCS)) { 42756429daeSEric Joyner if (M_HASHTYPE_GET(m)) { 42856429daeSEric Joyner /* Default iflib queue selection method */ 42956429daeSEric Joyner return (m->m_pkthdr.flowid % sc->pf_vsi.num_tx_queues); 43056429daeSEric Joyner } else 43156429daeSEric Joyner return (0); 43256429daeSEric Joyner } 43356429daeSEric Joyner 43456429daeSEric Joyner /* Use default TC unless overridden */ 43556429daeSEric Joyner tc = 0; /* XXX: Get default TC for traffic if >1 TC? */ 43656429daeSEric Joyner 43756429daeSEric Joyner if (m->m_flags & M_VLANTAG) { 43856429daeSEric Joyner up = EVL_PRIOFTAG(m->m_pkthdr.ether_vtag); 43956429daeSEric Joyner tc = sc->hw.port_info->qos_cfg.local_dcbx_cfg.etscfg.prio_table[up]; 44056429daeSEric Joyner } 44156429daeSEric Joyner 44256429daeSEric Joyner tc_base_queue = vsi->tc_info[tc].qoffset; 44356429daeSEric Joyner tc_qcount = vsi->tc_info[tc].qcount_tx; 44456429daeSEric Joyner 44556429daeSEric Joyner if (M_HASHTYPE_GET(m)) 44656429daeSEric Joyner return ((m->m_pkthdr.flowid % tc_qcount) + tc_base_queue); 44756429daeSEric Joyner else 44856429daeSEric Joyner return (tc_base_queue); 44956429daeSEric Joyner } 450