1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright (c) 2023, Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Intel Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 /*$FreeBSD$*/ 32 33 /** 34 * @file ice_iflib_txrx.c 35 * @brief iflib Tx/Rx hotpath 36 * 37 * Main location for the iflib Tx/Rx hotpath implementation. 38 * 39 * Contains the implementation for the iflib function callbacks and the 40 * if_txrx ops structure. 41 */ 42 43 #include "ice_iflib.h" 44 45 /* Tx/Rx hotpath utility functions */ 46 #include "ice_common_txrx.h" 47 48 /* 49 * iflib txrx method declarations 50 */ 51 static int ice_ift_txd_encap(void *arg, if_pkt_info_t pi); 52 static int ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri); 53 static void ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); 54 static int ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear); 55 static int ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget); 56 static void ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx, qidx_t pidx); 57 static void ice_ift_rxd_refill(void *arg, if_rxd_update_t iru); 58 static qidx_t ice_ift_queue_select(void *arg, struct mbuf *m, if_pkt_info_t pi); 59 60 /* Macro to help extract the NIC mode flexible Rx descriptor fields from the 61 * advanced 32byte Rx descriptors. 62 */ 63 #define RX_FLEX_NIC(desc, field) \ 64 (((struct ice_32b_rx_flex_desc_nic *)desc)->field) 65 66 /** 67 * @var ice_txrx 68 * @brief Tx/Rx operations for the iflib stack 69 * 70 * Structure defining the Tx and Rx related operations that iflib can request 71 * the driver to perform. These are the main entry points for the hot path of 72 * the transmit and receive paths in the iflib driver. 73 */ 74 struct if_txrx ice_txrx = { 75 .ift_txd_encap = ice_ift_txd_encap, 76 .ift_txd_flush = ice_ift_txd_flush, 77 .ift_txd_credits_update = ice_ift_txd_credits_update, 78 .ift_rxd_available = ice_ift_rxd_available, 79 .ift_rxd_pkt_get = ice_ift_rxd_pkt_get, 80 .ift_rxd_refill = ice_ift_rxd_refill, 81 .ift_rxd_flush = ice_ift_rxd_flush, 82 .ift_txq_select_v2 = ice_ift_queue_select, 83 }; 84 85 /** 86 * ice_ift_txd_encap - prepare Tx descriptors for a packet 87 * @arg: the iflib softc structure pointer 88 * @pi: packet info 89 * 90 * Prepares and encapsulates the given packet into into Tx descriptors, in 91 * preparation for sending to the transmit engine. Sets the necessary context 92 * descriptors for TSO and other offloads, and prepares the last descriptor 93 * for the writeback status. 94 * 95 * Return 0 on success, non-zero error code on failure. 96 */ 97 static int 98 ice_ift_txd_encap(void *arg, if_pkt_info_t pi) 99 { 100 struct ice_softc *sc = (struct ice_softc *)arg; 101 struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[pi->ipi_qsidx]; 102 int nsegs = pi->ipi_nsegs; 103 bus_dma_segment_t *segs = pi->ipi_segs; 104 struct ice_tx_desc *txd = NULL; 105 int i, j, mask, pidx_last; 106 u32 cmd, off; 107 108 cmd = off = 0; 109 i = pi->ipi_pidx; 110 111 /* Set up the TSO/CSUM offload */ 112 if (pi->ipi_csum_flags & ICE_CSUM_OFFLOAD) { 113 /* Set up the TSO context descriptor if required */ 114 if (pi->ipi_csum_flags & CSUM_TSO) { 115 if (ice_tso_detect_sparse(pi)) 116 return (EFBIG); 117 i = ice_tso_setup(txq, pi); 118 } 119 ice_tx_setup_offload(txq, pi, &cmd, &off); 120 } 121 if (pi->ipi_mflags & M_VLANTAG) 122 cmd |= ICE_TX_DESC_CMD_IL2TAG1; 123 124 mask = txq->desc_count - 1; 125 for (j = 0; j < nsegs; j++) { 126 bus_size_t seglen; 127 128 txd = &txq->tx_base[i]; 129 seglen = segs[j].ds_len; 130 131 txd->buf_addr = htole64(segs[j].ds_addr); 132 txd->cmd_type_offset_bsz = 133 htole64(ICE_TX_DESC_DTYPE_DATA 134 | ((u64)cmd << ICE_TXD_QW1_CMD_S) 135 | ((u64)off << ICE_TXD_QW1_OFFSET_S) 136 | ((u64)seglen << ICE_TXD_QW1_TX_BUF_SZ_S) 137 | ((u64)htole16(pi->ipi_vtag) << ICE_TXD_QW1_L2TAG1_S)); 138 139 txq->stats.tx_bytes += seglen; 140 pidx_last = i; 141 i = (i+1) & mask; 142 } 143 144 /* Set the last descriptor for report */ 145 #define ICE_TXD_CMD (ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS) 146 txd->cmd_type_offset_bsz |= 147 htole64(((u64)ICE_TXD_CMD << ICE_TXD_QW1_CMD_S)); 148 149 /* Add to report status array */ 150 txq->tx_rsq[txq->tx_rs_pidx] = pidx_last; 151 txq->tx_rs_pidx = (txq->tx_rs_pidx+1) & mask; 152 MPASS(txq->tx_rs_pidx != txq->tx_rs_cidx); 153 154 pi->ipi_new_pidx = i; 155 156 ++txq->stats.tx_packets; 157 return (0); 158 } 159 160 /** 161 * ice_ift_txd_flush - Flush Tx descriptors to hardware 162 * @arg: device specific softc pointer 163 * @txqid: the Tx queue to flush 164 * @pidx: descriptor index to advance tail to 165 * 166 * Advance the Transmit Descriptor Tail (TDT). This indicates to hardware that 167 * frames are available for transmit. 168 */ 169 static void 170 ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) 171 { 172 struct ice_softc *sc = (struct ice_softc *)arg; 173 struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid]; 174 struct ice_hw *hw = &sc->hw; 175 176 wr32(hw, txq->tail, pidx); 177 } 178 179 /** 180 * ice_ift_txd_credits_update - cleanup Tx descriptors 181 * @arg: device private softc 182 * @txqid: the Tx queue to update 183 * @clear: if false, only report, do not actually clean 184 * 185 * If clear is false, iflib is asking if we *could* clean up any Tx 186 * descriptors. 187 * 188 * If clear is true, iflib is requesting to cleanup and reclaim used Tx 189 * descriptors. 190 */ 191 static int 192 ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear) 193 { 194 struct ice_softc *sc = (struct ice_softc *)arg; 195 struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid]; 196 197 qidx_t processed = 0; 198 qidx_t cur, prev, ntxd, rs_cidx; 199 int32_t delta; 200 bool is_done; 201 202 rs_cidx = txq->tx_rs_cidx; 203 if (rs_cidx == txq->tx_rs_pidx) 204 return (0); 205 cur = txq->tx_rsq[rs_cidx]; 206 MPASS(cur != QIDX_INVALID); 207 is_done = ice_is_tx_desc_done(&txq->tx_base[cur]); 208 209 if (!is_done) 210 return (0); 211 else if (clear == false) 212 return (1); 213 214 prev = txq->tx_cidx_processed; 215 ntxd = txq->desc_count; 216 do { 217 MPASS(prev != cur); 218 delta = (int32_t)cur - (int32_t)prev; 219 if (delta < 0) 220 delta += ntxd; 221 MPASS(delta > 0); 222 processed += delta; 223 prev = cur; 224 rs_cidx = (rs_cidx + 1) & (ntxd-1); 225 if (rs_cidx == txq->tx_rs_pidx) 226 break; 227 cur = txq->tx_rsq[rs_cidx]; 228 MPASS(cur != QIDX_INVALID); 229 is_done = ice_is_tx_desc_done(&txq->tx_base[cur]); 230 } while (is_done); 231 232 txq->tx_rs_cidx = rs_cidx; 233 txq->tx_cidx_processed = prev; 234 235 return (processed); 236 } 237 238 /** 239 * ice_ift_rxd_available - Return number of available Rx packets 240 * @arg: device private softc 241 * @rxqid: the Rx queue id 242 * @pidx: descriptor start point 243 * @budget: maximum Rx budget 244 * 245 * Determines how many Rx packets are available on the queue, up to a maximum 246 * of the given budget. 247 */ 248 static int 249 ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget) 250 { 251 struct ice_softc *sc = (struct ice_softc *)arg; 252 struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid]; 253 union ice_32b_rx_flex_desc *rxd; 254 uint16_t status0; 255 int cnt, i, nrxd; 256 257 nrxd = rxq->desc_count; 258 259 for (cnt = 0, i = pidx; cnt < nrxd - 1 && cnt < budget;) { 260 rxd = &rxq->rx_base[i]; 261 status0 = le16toh(rxd->wb.status_error0); 262 263 if ((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) == 0) 264 break; 265 if (++i == nrxd) 266 i = 0; 267 if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S)) 268 cnt++; 269 } 270 271 return (cnt); 272 } 273 274 /** 275 * ice_ift_rxd_pkt_get - Called by iflib to send data to upper layer 276 * @arg: device specific softc 277 * @ri: receive packet info 278 * 279 * This function is called by iflib, and executes in ithread context. It is 280 * called by iflib to obtain data which has been DMA'ed into host memory. 281 * Returns zero on success, and EBADMSG on failure. 282 */ 283 static int 284 ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri) 285 { 286 struct ice_softc *sc = (struct ice_softc *)arg; 287 if_softc_ctx_t scctx = sc->scctx; 288 struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[ri->iri_qsidx]; 289 union ice_32b_rx_flex_desc *cur; 290 u16 status0, plen, ptype; 291 bool eop; 292 size_t cidx; 293 int i; 294 295 cidx = ri->iri_cidx; 296 i = 0; 297 do { 298 /* 5 descriptor receive limit */ 299 MPASS(i < ICE_MAX_RX_SEGS); 300 301 cur = &rxq->rx_base[cidx]; 302 status0 = le16toh(cur->wb.status_error0); 303 plen = le16toh(cur->wb.pkt_len) & 304 ICE_RX_FLX_DESC_PKT_LEN_M; 305 306 /* we should never be called without a valid descriptor */ 307 MPASS((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) != 0); 308 309 ri->iri_len += plen; 310 311 cur->wb.status_error0 = 0; 312 eop = (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S)); 313 314 ri->iri_frags[i].irf_flid = 0; 315 ri->iri_frags[i].irf_idx = cidx; 316 ri->iri_frags[i].irf_len = plen; 317 if (++cidx == rxq->desc_count) 318 cidx = 0; 319 i++; 320 } while (!eop); 321 322 /* End of Packet reached; cur is eop/last descriptor */ 323 324 /* Make sure packets with bad L2 values are discarded. 325 * This bit is only valid in the last descriptor. 326 */ 327 if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S)) { 328 rxq->stats.desc_errs++; 329 return (EBADMSG); 330 } 331 332 /* Get VLAN tag information if one is in descriptor */ 333 if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) { 334 ri->iri_vtag = le16toh(cur->wb.l2tag1); 335 ri->iri_flags |= M_VLANTAG; 336 } 337 338 /* Capture soft statistics for this Rx queue */ 339 rxq->stats.rx_packets++; 340 rxq->stats.rx_bytes += ri->iri_len; 341 342 /* Get packet type and set checksum flags */ 343 ptype = le16toh(cur->wb.ptype_flex_flags0) & 344 ICE_RX_FLEX_DESC_PTYPE_M; 345 if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0) 346 ice_rx_checksum(rxq, &ri->iri_csum_flags, 347 &ri->iri_csum_data, status0, ptype); 348 349 /* Set remaining iflib RX descriptor info fields */ 350 ri->iri_flowid = le32toh(RX_FLEX_NIC(&cur->wb, rss_hash)); 351 ri->iri_rsstype = ice_ptype_to_hash(ptype); 352 ri->iri_nfrags = i; 353 return (0); 354 } 355 356 /** 357 * ice_ift_rxd_refill - Prepare Rx descriptors for re-use by hardware 358 * @arg: device specific softc structure 359 * @iru: the Rx descriptor update structure 360 * 361 * Update the Rx descriptor indices for a given queue, assigning new physical 362 * addresses to the descriptors, preparing them for re-use by the hardware. 363 */ 364 static void 365 ice_ift_rxd_refill(void *arg, if_rxd_update_t iru) 366 { 367 struct ice_softc *sc = (struct ice_softc *)arg; 368 struct ice_rx_queue *rxq; 369 uint32_t next_pidx; 370 int i; 371 uint64_t *paddrs; 372 uint32_t pidx; 373 uint16_t qsidx, count; 374 375 paddrs = iru->iru_paddrs; 376 pidx = iru->iru_pidx; 377 qsidx = iru->iru_qsidx; 378 count = iru->iru_count; 379 380 rxq = &(sc->pf_vsi.rx_queues[qsidx]); 381 382 for (i = 0, next_pidx = pidx; i < count; i++) { 383 rxq->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]); 384 if (++next_pidx == (uint32_t)rxq->desc_count) 385 next_pidx = 0; 386 } 387 } 388 389 /** 390 * ice_ift_rxd_flush - Flush Rx descriptors to hardware 391 * @arg: device specific softc pointer 392 * @rxqid: the Rx queue to flush 393 * @flidx: unused parameter 394 * @pidx: descriptor index to advance tail to 395 * 396 * Advance the Receive Descriptor Tail (RDT). This indicates to hardware that 397 * software is done with the descriptor and it can be recycled. 398 */ 399 static void 400 ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx __unused, 401 qidx_t pidx) 402 { 403 struct ice_softc *sc = (struct ice_softc *)arg; 404 struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid]; 405 struct ice_hw *hw = &sc->hw; 406 407 wr32(hw, rxq->tail, pidx); 408 } 409 410 static qidx_t 411 ice_ift_queue_select(void *arg, struct mbuf *m, if_pkt_info_t pi) 412 { 413 struct ice_softc *sc = (struct ice_softc *)arg; 414 struct ice_dcbx_cfg *local_dcbx_cfg; 415 struct ice_vsi *vsi = &sc->pf_vsi; 416 u16 tc_base_queue, tc_qcount; 417 u8 up, tc; 418 419 #ifdef ALTQ 420 /* Included to match default iflib behavior */ 421 /* Only go out on default queue if ALTQ is enabled */ 422 struct ifnet *ifp = (struct ifnet *)iflib_get_ifp(sc->ctx); 423 if (if_altq_is_enabled(ifp)) 424 return (0); 425 #endif 426 427 if (!ice_test_state(&sc->state, ICE_STATE_MULTIPLE_TCS)) { 428 if (M_HASHTYPE_GET(m)) { 429 /* Default iflib queue selection method */ 430 return (m->m_pkthdr.flowid % sc->pf_vsi.num_tx_queues); 431 } else 432 return (0); 433 } 434 435 /* Use default TC unless overridden later */ 436 tc = 0; /* XXX: Get default TC for traffic if >1 TC? */ 437 438 local_dcbx_cfg = &sc->hw.port_info->qos_cfg.local_dcbx_cfg; 439 440 #if defined(INET) || defined(INET6) 441 if ((local_dcbx_cfg->pfc_mode == ICE_QOS_MODE_DSCP) && 442 (pi->ipi_flags & (IPI_TX_IPV4 | IPI_TX_IPV6))) { 443 u8 dscp_val = pi->ipi_ip_tos >> 2; 444 tc = local_dcbx_cfg->dscp_map[dscp_val]; 445 } else 446 #endif /* defined(INET) || defined(INET6) */ 447 if (m->m_flags & M_VLANTAG) { /* ICE_QOS_MODE_VLAN */ 448 up = EVL_PRIOFTAG(m->m_pkthdr.ether_vtag); 449 tc = local_dcbx_cfg->etscfg.prio_table[up]; 450 } 451 452 tc_base_queue = vsi->tc_info[tc].qoffset; 453 tc_qcount = vsi->tc_info[tc].qcount_tx; 454 455 if (M_HASHTYPE_GET(m)) 456 return ((m->m_pkthdr.flowid % tc_qcount) + tc_base_queue); 457 else 458 return (tc_base_queue); 459 } 460