1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright (c) 2023, Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Intel Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /** 33 * @file ice_iflib_txrx.c 34 * @brief iflib Tx/Rx hotpath 35 * 36 * Main location for the iflib Tx/Rx hotpath implementation. 37 * 38 * Contains the implementation for the iflib function callbacks and the 39 * if_txrx ops structure. 40 */ 41 42 #include "ice_iflib.h" 43 44 /* Tx/Rx hotpath utility functions */ 45 #include "ice_common_txrx.h" 46 47 /* 48 * iflib txrx method declarations 49 */ 50 static int ice_ift_txd_encap(void *arg, if_pkt_info_t pi); 51 static int ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri); 52 static void ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); 53 static int ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear); 54 static int ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget); 55 static void ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx, qidx_t pidx); 56 static void ice_ift_rxd_refill(void *arg, if_rxd_update_t iru); 57 static qidx_t ice_ift_queue_select(void *arg, struct mbuf *m, if_pkt_info_t pi); 58 59 /* Macro to help extract the NIC mode flexible Rx descriptor fields from the 60 * advanced 32byte Rx descriptors. 61 */ 62 #define RX_FLEX_NIC(desc, field) \ 63 (((struct ice_32b_rx_flex_desc_nic *)desc)->field) 64 65 /** 66 * @var ice_txrx 67 * @brief Tx/Rx operations for the iflib stack 68 * 69 * Structure defining the Tx and Rx related operations that iflib can request 70 * the driver to perform. These are the main entry points for the hot path of 71 * the transmit and receive paths in the iflib driver. 72 */ 73 struct if_txrx ice_txrx = { 74 .ift_txd_encap = ice_ift_txd_encap, 75 .ift_txd_flush = ice_ift_txd_flush, 76 .ift_txd_credits_update = ice_ift_txd_credits_update, 77 .ift_rxd_available = ice_ift_rxd_available, 78 .ift_rxd_pkt_get = ice_ift_rxd_pkt_get, 79 .ift_rxd_refill = ice_ift_rxd_refill, 80 .ift_rxd_flush = ice_ift_rxd_flush, 81 .ift_txq_select_v2 = ice_ift_queue_select, 82 }; 83 84 /** 85 * ice_ift_txd_encap - prepare Tx descriptors for a packet 86 * @arg: the iflib softc structure pointer 87 * @pi: packet info 88 * 89 * Prepares and encapsulates the given packet into into Tx descriptors, in 90 * preparation for sending to the transmit engine. Sets the necessary context 91 * descriptors for TSO and other offloads, and prepares the last descriptor 92 * for the writeback status. 93 * 94 * Return 0 on success, non-zero error code on failure. 95 */ 96 static int 97 ice_ift_txd_encap(void *arg, if_pkt_info_t pi) 98 { 99 struct ice_softc *sc = (struct ice_softc *)arg; 100 struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[pi->ipi_qsidx]; 101 int nsegs = pi->ipi_nsegs; 102 bus_dma_segment_t *segs = pi->ipi_segs; 103 struct ice_tx_desc *txd = NULL; 104 int i, j, mask, pidx_last; 105 u32 cmd, off; 106 107 cmd = off = 0; 108 i = pi->ipi_pidx; 109 110 /* Set up the TSO/CSUM offload */ 111 if (pi->ipi_csum_flags & ICE_CSUM_OFFLOAD) { 112 /* Set up the TSO context descriptor if required */ 113 if (pi->ipi_csum_flags & CSUM_TSO) { 114 if (ice_tso_detect_sparse(pi)) 115 return (EFBIG); 116 i = ice_tso_setup(txq, pi); 117 } 118 ice_tx_setup_offload(txq, pi, &cmd, &off); 119 } 120 if (pi->ipi_mflags & M_VLANTAG) 121 cmd |= ICE_TX_DESC_CMD_IL2TAG1; 122 123 mask = txq->desc_count - 1; 124 for (j = 0; j < nsegs; j++) { 125 bus_size_t seglen; 126 127 txd = &txq->tx_base[i]; 128 seglen = segs[j].ds_len; 129 130 txd->buf_addr = htole64(segs[j].ds_addr); 131 txd->cmd_type_offset_bsz = 132 htole64(ICE_TX_DESC_DTYPE_DATA 133 | ((u64)cmd << ICE_TXD_QW1_CMD_S) 134 | ((u64)off << ICE_TXD_QW1_OFFSET_S) 135 | ((u64)seglen << ICE_TXD_QW1_TX_BUF_SZ_S) 136 | ((u64)htole16(pi->ipi_vtag) << ICE_TXD_QW1_L2TAG1_S)); 137 138 txq->stats.tx_bytes += seglen; 139 pidx_last = i; 140 i = (i+1) & mask; 141 } 142 143 /* Set the last descriptor for report */ 144 #define ICE_TXD_CMD (ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS) 145 txd->cmd_type_offset_bsz |= 146 htole64(((u64)ICE_TXD_CMD << ICE_TXD_QW1_CMD_S)); 147 148 /* Add to report status array */ 149 txq->tx_rsq[txq->tx_rs_pidx] = pidx_last; 150 txq->tx_rs_pidx = (txq->tx_rs_pidx+1) & mask; 151 MPASS(txq->tx_rs_pidx != txq->tx_rs_cidx); 152 153 pi->ipi_new_pidx = i; 154 155 ++txq->stats.tx_packets; 156 return (0); 157 } 158 159 /** 160 * ice_ift_txd_flush - Flush Tx descriptors to hardware 161 * @arg: device specific softc pointer 162 * @txqid: the Tx queue to flush 163 * @pidx: descriptor index to advance tail to 164 * 165 * Advance the Transmit Descriptor Tail (TDT). This indicates to hardware that 166 * frames are available for transmit. 167 */ 168 static void 169 ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) 170 { 171 struct ice_softc *sc = (struct ice_softc *)arg; 172 struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid]; 173 struct ice_hw *hw = &sc->hw; 174 175 wr32(hw, txq->tail, pidx); 176 } 177 178 /** 179 * ice_ift_txd_credits_update - cleanup Tx descriptors 180 * @arg: device private softc 181 * @txqid: the Tx queue to update 182 * @clear: if false, only report, do not actually clean 183 * 184 * If clear is false, iflib is asking if we *could* clean up any Tx 185 * descriptors. 186 * 187 * If clear is true, iflib is requesting to cleanup and reclaim used Tx 188 * descriptors. 189 */ 190 static int 191 ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear) 192 { 193 struct ice_softc *sc = (struct ice_softc *)arg; 194 struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid]; 195 196 qidx_t processed = 0; 197 qidx_t cur, prev, ntxd, rs_cidx; 198 int32_t delta; 199 bool is_done; 200 201 rs_cidx = txq->tx_rs_cidx; 202 if (rs_cidx == txq->tx_rs_pidx) 203 return (0); 204 cur = txq->tx_rsq[rs_cidx]; 205 MPASS(cur != QIDX_INVALID); 206 is_done = ice_is_tx_desc_done(&txq->tx_base[cur]); 207 208 if (!is_done) 209 return (0); 210 else if (clear == false) 211 return (1); 212 213 prev = txq->tx_cidx_processed; 214 ntxd = txq->desc_count; 215 do { 216 MPASS(prev != cur); 217 delta = (int32_t)cur - (int32_t)prev; 218 if (delta < 0) 219 delta += ntxd; 220 MPASS(delta > 0); 221 processed += delta; 222 prev = cur; 223 rs_cidx = (rs_cidx + 1) & (ntxd-1); 224 if (rs_cidx == txq->tx_rs_pidx) 225 break; 226 cur = txq->tx_rsq[rs_cidx]; 227 MPASS(cur != QIDX_INVALID); 228 is_done = ice_is_tx_desc_done(&txq->tx_base[cur]); 229 } while (is_done); 230 231 txq->tx_rs_cidx = rs_cidx; 232 txq->tx_cidx_processed = prev; 233 234 return (processed); 235 } 236 237 /** 238 * ice_ift_rxd_available - Return number of available Rx packets 239 * @arg: device private softc 240 * @rxqid: the Rx queue id 241 * @pidx: descriptor start point 242 * @budget: maximum Rx budget 243 * 244 * Determines how many Rx packets are available on the queue, up to a maximum 245 * of the given budget. 246 */ 247 static int 248 ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget) 249 { 250 struct ice_softc *sc = (struct ice_softc *)arg; 251 struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid]; 252 union ice_32b_rx_flex_desc *rxd; 253 uint16_t status0; 254 int cnt, i, nrxd; 255 256 nrxd = rxq->desc_count; 257 258 for (cnt = 0, i = pidx; cnt < nrxd - 1 && cnt < budget;) { 259 rxd = &rxq->rx_base[i]; 260 status0 = le16toh(rxd->wb.status_error0); 261 262 if ((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) == 0) 263 break; 264 if (++i == nrxd) 265 i = 0; 266 if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S)) 267 cnt++; 268 } 269 270 return (cnt); 271 } 272 273 /** 274 * ice_ift_rxd_pkt_get - Called by iflib to send data to upper layer 275 * @arg: device specific softc 276 * @ri: receive packet info 277 * 278 * This function is called by iflib, and executes in ithread context. It is 279 * called by iflib to obtain data which has been DMA'ed into host memory. 280 * Returns zero on success, and EBADMSG on failure. 281 */ 282 static int 283 ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri) 284 { 285 struct ice_softc *sc = (struct ice_softc *)arg; 286 if_softc_ctx_t scctx = sc->scctx; 287 struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[ri->iri_qsidx]; 288 union ice_32b_rx_flex_desc *cur; 289 u16 status0, plen, ptype; 290 bool eop; 291 size_t cidx; 292 int i; 293 294 cidx = ri->iri_cidx; 295 i = 0; 296 do { 297 /* 5 descriptor receive limit */ 298 MPASS(i < ICE_MAX_RX_SEGS); 299 300 cur = &rxq->rx_base[cidx]; 301 status0 = le16toh(cur->wb.status_error0); 302 plen = le16toh(cur->wb.pkt_len) & 303 ICE_RX_FLX_DESC_PKT_LEN_M; 304 305 /* we should never be called without a valid descriptor */ 306 MPASS((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) != 0); 307 308 ri->iri_len += plen; 309 310 cur->wb.status_error0 = 0; 311 eop = (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S)); 312 313 ri->iri_frags[i].irf_flid = 0; 314 ri->iri_frags[i].irf_idx = cidx; 315 ri->iri_frags[i].irf_len = plen; 316 if (++cidx == rxq->desc_count) 317 cidx = 0; 318 i++; 319 } while (!eop); 320 321 /* End of Packet reached; cur is eop/last descriptor */ 322 323 /* Make sure packets with bad L2 values are discarded. 324 * This bit is only valid in the last descriptor. 325 */ 326 if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S)) { 327 rxq->stats.desc_errs++; 328 return (EBADMSG); 329 } 330 331 /* Get VLAN tag information if one is in descriptor */ 332 if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) { 333 ri->iri_vtag = le16toh(cur->wb.l2tag1); 334 ri->iri_flags |= M_VLANTAG; 335 } 336 337 /* Capture soft statistics for this Rx queue */ 338 rxq->stats.rx_packets++; 339 rxq->stats.rx_bytes += ri->iri_len; 340 341 /* Get packet type and set checksum flags */ 342 ptype = le16toh(cur->wb.ptype_flex_flags0) & 343 ICE_RX_FLEX_DESC_PTYPE_M; 344 if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0) 345 ice_rx_checksum(rxq, &ri->iri_csum_flags, 346 &ri->iri_csum_data, status0, ptype); 347 348 /* Set remaining iflib RX descriptor info fields */ 349 ri->iri_flowid = le32toh(RX_FLEX_NIC(&cur->wb, rss_hash)); 350 ri->iri_rsstype = ice_ptype_to_hash(ptype); 351 ri->iri_nfrags = i; 352 return (0); 353 } 354 355 /** 356 * ice_ift_rxd_refill - Prepare Rx descriptors for re-use by hardware 357 * @arg: device specific softc structure 358 * @iru: the Rx descriptor update structure 359 * 360 * Update the Rx descriptor indices for a given queue, assigning new physical 361 * addresses to the descriptors, preparing them for re-use by the hardware. 362 */ 363 static void 364 ice_ift_rxd_refill(void *arg, if_rxd_update_t iru) 365 { 366 struct ice_softc *sc = (struct ice_softc *)arg; 367 struct ice_rx_queue *rxq; 368 uint32_t next_pidx; 369 int i; 370 uint64_t *paddrs; 371 uint32_t pidx; 372 uint16_t qsidx, count; 373 374 paddrs = iru->iru_paddrs; 375 pidx = iru->iru_pidx; 376 qsidx = iru->iru_qsidx; 377 count = iru->iru_count; 378 379 rxq = &(sc->pf_vsi.rx_queues[qsidx]); 380 381 for (i = 0, next_pidx = pidx; i < count; i++) { 382 rxq->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]); 383 if (++next_pidx == (uint32_t)rxq->desc_count) 384 next_pidx = 0; 385 } 386 } 387 388 /** 389 * ice_ift_rxd_flush - Flush Rx descriptors to hardware 390 * @arg: device specific softc pointer 391 * @rxqid: the Rx queue to flush 392 * @flidx: unused parameter 393 * @pidx: descriptor index to advance tail to 394 * 395 * Advance the Receive Descriptor Tail (RDT). This indicates to hardware that 396 * software is done with the descriptor and it can be recycled. 397 */ 398 static void 399 ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx __unused, 400 qidx_t pidx) 401 { 402 struct ice_softc *sc = (struct ice_softc *)arg; 403 struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid]; 404 struct ice_hw *hw = &sc->hw; 405 406 wr32(hw, rxq->tail, pidx); 407 } 408 409 static qidx_t 410 ice_ift_queue_select(void *arg, struct mbuf *m, if_pkt_info_t pi) 411 { 412 struct ice_softc *sc = (struct ice_softc *)arg; 413 struct ice_dcbx_cfg *local_dcbx_cfg; 414 struct ice_vsi *vsi = &sc->pf_vsi; 415 u16 tc_base_queue, tc_qcount; 416 u8 up, tc; 417 418 #ifdef ALTQ 419 /* Included to match default iflib behavior */ 420 /* Only go out on default queue if ALTQ is enabled */ 421 struct ifnet *ifp = (struct ifnet *)iflib_get_ifp(sc->ctx); 422 if (if_altq_is_enabled(ifp)) 423 return (0); 424 #endif 425 426 if (!ice_test_state(&sc->state, ICE_STATE_MULTIPLE_TCS)) { 427 if (M_HASHTYPE_GET(m)) { 428 /* Default iflib queue selection method */ 429 return (m->m_pkthdr.flowid % sc->pf_vsi.num_tx_queues); 430 } else 431 return (0); 432 } 433 434 /* Use default TC unless overridden later */ 435 tc = 0; /* XXX: Get default TC for traffic if >1 TC? */ 436 437 local_dcbx_cfg = &sc->hw.port_info->qos_cfg.local_dcbx_cfg; 438 439 #if defined(INET) || defined(INET6) 440 if ((local_dcbx_cfg->pfc_mode == ICE_QOS_MODE_DSCP) && 441 (pi->ipi_flags & (IPI_TX_IPV4 | IPI_TX_IPV6))) { 442 u8 dscp_val = pi->ipi_ip_tos >> 2; 443 tc = local_dcbx_cfg->dscp_map[dscp_val]; 444 } else 445 #endif /* defined(INET) || defined(INET6) */ 446 if (m->m_flags & M_VLANTAG) { /* ICE_QOS_MODE_VLAN */ 447 up = EVL_PRIOFTAG(m->m_pkthdr.ether_vtag); 448 tc = local_dcbx_cfg->etscfg.prio_table[up]; 449 } 450 451 tc_base_queue = vsi->tc_info[tc].qoffset; 452 tc_qcount = vsi->tc_info[tc].qcount_tx; 453 454 if (M_HASHTYPE_GET(m)) 455 return ((m->m_pkthdr.flowid % tc_qcount) + tc_base_queue); 456 else 457 return (tc_base_queue); 458 } 459