1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (C) 2023 Intel Corporation */ 3 4 #ifndef _IDPF_TXRX_H_ 5 #define _IDPF_TXRX_H_ 6 7 #include <linux/dim.h> 8 #include <linux/net/intel/virtchnl2_lan_desc.h> 9 10 #include <net/libeth/cache.h> 11 #include <net/libeth/types.h> 12 #include <net/netdev_queues.h> 13 #include <net/tcp.h> 14 #include <net/xdp.h> 15 16 #include "idpf_lan_txrx.h" 17 18 #define IDPF_LARGE_MAX_Q 256 19 #define IDPF_MAX_Q 16 20 #define IDPF_MIN_Q 2 21 /* Mailbox Queue */ 22 #define IDPF_MAX_MBXQ 1 23 24 #define IDPF_MIN_TXQ_DESC 128 25 #define IDPF_MIN_RXQ_DESC 64 26 #define IDPF_MIN_TXQ_COMPLQ_DESC 256 27 #define IDPF_MAX_QIDS 256 28 29 /* Number of descriptors in a queue should be a multiple of 32. RX queue 30 * descriptors alone should be a multiple of IDPF_REQ_RXQ_DESC_MULTIPLE 31 * to achieve BufQ descriptors aligned to 32 32 */ 33 #define IDPF_REQ_DESC_MULTIPLE 32 34 #define IDPF_REQ_RXQ_DESC_MULTIPLE (IDPF_MAX_BUFQS_PER_RXQ_GRP * 32) 35 #define IDPF_MIN_TX_DESC_NEEDED (MAX_SKB_FRAGS + 6) 36 #define IDPF_TX_WAKE_THRESH ((u16)IDPF_MIN_TX_DESC_NEEDED * 2) 37 38 #define IDPF_MAX_DESCS 8160 39 #define IDPF_MAX_TXQ_DESC ALIGN_DOWN(IDPF_MAX_DESCS, IDPF_REQ_DESC_MULTIPLE) 40 #define IDPF_MAX_RXQ_DESC ALIGN_DOWN(IDPF_MAX_DESCS, IDPF_REQ_RXQ_DESC_MULTIPLE) 41 #define MIN_SUPPORT_TXDID (\ 42 VIRTCHNL2_TXDID_FLEX_FLOW_SCHED |\ 43 VIRTCHNL2_TXDID_FLEX_TSO_CTX) 44 45 #define IDPF_DFLT_SINGLEQ_TX_Q_GROUPS 1 46 #define IDPF_DFLT_SINGLEQ_RX_Q_GROUPS 1 47 #define IDPF_DFLT_SINGLEQ_TXQ_PER_GROUP 4 48 #define IDPF_DFLT_SINGLEQ_RXQ_PER_GROUP 4 49 50 #define IDPF_COMPLQ_PER_GROUP 1 51 #define IDPF_SINGLE_BUFQ_PER_RXQ_GRP 1 52 #define IDPF_MAX_BUFQS_PER_RXQ_GRP 2 53 #define IDPF_BUFQ2_ENA 1 54 #define IDPF_NUMQ_PER_CHUNK 1 55 56 #define IDPF_DFLT_SPLITQ_TXQ_PER_GROUP 1 57 #define IDPF_DFLT_SPLITQ_RXQ_PER_GROUP 1 58 59 /* Default vector sharing */ 60 #define IDPF_MBX_Q_VEC 1 61 #define IDPF_MIN_Q_VEC 1 62 #define IDPF_MIN_RDMA_VEC 2 63 /* Data vector for NOIRQ queues */ 64 #define IDPF_RESERVED_VECS 1 65 66 #define IDPF_DFLT_TX_Q_DESC_COUNT 512 67 #define IDPF_DFLT_TX_COMPLQ_DESC_COUNT 512 68 #define IDPF_DFLT_RX_Q_DESC_COUNT 512 69 70 /* IMPORTANT: We absolutely _cannot_ have more buffers in the system than a 71 * given RX completion queue has descriptors. This includes _ALL_ buffer 72 * queues. E.g.: If you have two buffer queues of 512 descriptors and buffers, 73 * you have a total of 1024 buffers so your RX queue _must_ have at least that 74 * many descriptors. This macro divides a given number of RX descriptors by 75 * number of buffer queues to calculate how many descriptors each buffer queue 76 * can have without overrunning the RX queue. 77 * 78 * If you give hardware more buffers than completion descriptors what will 79 * happen is that if hardware gets a chance to post more than ring wrap of 80 * descriptors before SW gets an interrupt and overwrites SW head, the gen bit 81 * in the descriptor will be wrong. Any overwritten descriptors' buffers will 82 * be gone forever and SW has no reasonable way to tell that this has happened. 83 * From SW perspective, when we finally get an interrupt, it looks like we're 84 * still waiting for descriptor to be done, stalling forever. 85 */ 86 #define IDPF_RX_BUFQ_DESC_COUNT(RXD, NUM_BUFQ) ((RXD) / (NUM_BUFQ)) 87 88 #define IDPF_RX_BUFQ_WORKING_SET(rxq) ((rxq)->desc_count - 1) 89 90 #define IDPF_RX_BUMP_NTC(rxq, ntc) \ 91 do { \ 92 if (unlikely(++(ntc) == (rxq)->desc_count)) { \ 93 ntc = 0; \ 94 idpf_queue_change(GEN_CHK, rxq); \ 95 } \ 96 } while (0) 97 98 #define IDPF_SINGLEQ_BUMP_RING_IDX(q, idx) \ 99 do { \ 100 if (unlikely(++(idx) == (q)->desc_count)) \ 101 idx = 0; \ 102 } while (0) 103 104 #define IDPF_RX_MAX_BUF_SZ (16384 - 128) 105 #define IDPF_RX_BUF_STRIDE 32 106 #define IDPF_RX_BUF_POST_STRIDE 16 107 #define IDPF_LOW_WATERMARK 64 108 109 #define IDPF_TX_TSO_MIN_MSS 88 110 111 /* Minimum number of descriptors between 2 descriptors with the RE bit set; 112 * only relevant in flow scheduling mode 113 */ 114 #define IDPF_TX_SPLITQ_RE_MIN_GAP 64 115 116 #define IDPF_RFL_BI_GEN_M BIT(16) 117 #define IDPF_RFL_BI_BUFID_M GENMASK(15, 0) 118 119 #define IDPF_RXD_EOF_SPLITQ VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_EOF_M 120 #define IDPF_RXD_EOF_SINGLEQ VIRTCHNL2_RX_BASE_DESC_STATUS_EOF_M 121 122 #define IDPF_DESC_UNUSED(txq) \ 123 ((((txq)->next_to_clean > (txq)->next_to_use) ? 0 : (txq)->desc_count) + \ 124 (txq)->next_to_clean - (txq)->next_to_use - 1) 125 126 #define IDPF_TX_COMPLQ_OVERFLOW_THRESH(txcq) ((txcq)->desc_count >> 1) 127 /* Determine the absolute number of completions pending, i.e. the number of 128 * completions that are expected to arrive on the TX completion queue. 129 */ 130 #define IDPF_TX_COMPLQ_PENDING(txq) \ 131 (((txq)->num_completions_pending >= (txq)->complq->num_completions ? \ 132 0 : U32_MAX) + \ 133 (txq)->num_completions_pending - (txq)->complq->num_completions) 134 135 #define IDPF_TXBUF_NULL U32_MAX 136 137 #define IDPF_TXD_LAST_DESC_CMD (IDPF_TX_DESC_CMD_EOP | IDPF_TX_DESC_CMD_RS) 138 139 #define IDPF_TX_FLAGS_TSO BIT(0) 140 #define IDPF_TX_FLAGS_IPV4 BIT(1) 141 #define IDPF_TX_FLAGS_IPV6 BIT(2) 142 #define IDPF_TX_FLAGS_TUNNEL BIT(3) 143 #define IDPF_TX_FLAGS_TSYN BIT(4) 144 145 struct libeth_rq_napi_stats; 146 147 union idpf_tx_flex_desc { 148 struct idpf_flex_tx_desc q; /* queue based scheduling */ 149 struct idpf_flex_tx_sched_desc flow; /* flow based scheduling */ 150 }; 151 152 #define idpf_tx_buf libeth_sqe 153 154 /** 155 * struct idpf_tx_offload_params - Offload parameters for a given packet 156 * @tx_flags: Feature flags enabled for this packet 157 * @hdr_offsets: Offset parameter for single queue model 158 * @cd_tunneling: Type of tunneling enabled for single queue model 159 * @tso_len: Total length of payload to segment 160 * @mss: Segment size 161 * @tso_segs: Number of segments to be sent 162 * @tso_hdr_len: Length of headers to be duplicated 163 * @td_cmd: Command field to be inserted into descriptor 164 * @desc_ts: Flow scheduling offload timestamp 165 */ 166 struct idpf_tx_offload_params { 167 u32 tx_flags; 168 169 u32 hdr_offsets; 170 u32 cd_tunneling; 171 172 u32 tso_len; 173 u16 mss; 174 u16 tso_segs; 175 u16 tso_hdr_len; 176 177 u16 td_cmd; 178 u8 desc_ts[3]; 179 }; 180 181 /** 182 * struct idpf_tx_splitq_params 183 * @dtype: General descriptor info 184 * @eop_cmd: Type of EOP 185 * @compl_tag: Associated tag for completion 186 * @td_tag: Descriptor tunneling tag 187 * @offload: Offload parameters 188 * @prev_ntu: stored TxQ next_to_use in case of rollback 189 * @prev_refill_ntc: stored refillq next_to_clean in case of packet rollback 190 * @prev_refill_gen: stored refillq generation bit in case of packet rollback 191 */ 192 struct idpf_tx_splitq_params { 193 enum idpf_tx_desc_dtype_value dtype; 194 u16 eop_cmd; 195 union { 196 u16 compl_tag; 197 u16 td_tag; 198 }; 199 200 struct idpf_tx_offload_params offload; 201 202 u16 prev_ntu; 203 u16 prev_refill_ntc; 204 bool prev_refill_gen; 205 }; 206 207 enum idpf_tx_ctx_desc_eipt_offload { 208 IDPF_TX_CTX_EXT_IP_NONE = 0x0, 209 IDPF_TX_CTX_EXT_IP_IPV6 = 0x1, 210 IDPF_TX_CTX_EXT_IP_IPV4_NO_CSUM = 0x2, 211 IDPF_TX_CTX_EXT_IP_IPV4 = 0x3 212 }; 213 214 #define IDPF_TX_COMPLQ_CLEAN_BUDGET 256 215 #define IDPF_TX_MIN_PKT_LEN 17 216 #define IDPF_TX_DESCS_FOR_SKB_DATA_PTR 1 217 #define IDPF_TX_DESCS_PER_CACHE_LINE (L1_CACHE_BYTES / \ 218 sizeof(struct idpf_flex_tx_desc)) 219 #define IDPF_TX_DESCS_FOR_CTX 1 220 /* TX descriptors needed, worst case */ 221 #define IDPF_TX_DESC_NEEDED (MAX_SKB_FRAGS + IDPF_TX_DESCS_FOR_CTX + \ 222 IDPF_TX_DESCS_PER_CACHE_LINE + \ 223 IDPF_TX_DESCS_FOR_SKB_DATA_PTR) 224 225 /* The size limit for a transmit buffer in a descriptor is (16K - 1). 226 * In order to align with the read requests we will align the value to 227 * the nearest 4K which represents our maximum read request size. 228 */ 229 #define IDPF_TX_MAX_READ_REQ_SIZE SZ_4K 230 #define IDPF_TX_MAX_DESC_DATA (SZ_16K - 1) 231 #define IDPF_TX_MAX_DESC_DATA_ALIGNED \ 232 ALIGN_DOWN(IDPF_TX_MAX_DESC_DATA, IDPF_TX_MAX_READ_REQ_SIZE) 233 234 #define idpf_rx_buf libeth_fqe 235 236 #define IDPF_RX_MAX_PTYPE_PROTO_IDS 32 237 #define IDPF_RX_MAX_PTYPE_SZ (sizeof(struct virtchnl2_ptype) + \ 238 (sizeof(u16) * IDPF_RX_MAX_PTYPE_PROTO_IDS)) 239 #define IDPF_RX_PTYPE_HDR_SZ sizeof(struct virtchnl2_get_ptype_info) 240 #define IDPF_RX_MAX_PTYPES_PER_BUF \ 241 DIV_ROUND_DOWN_ULL(LIBIE_CTLQ_MAX_BUF_LEN - IDPF_RX_PTYPE_HDR_SZ, \ 242 IDPF_RX_MAX_PTYPE_SZ) 243 244 #define IDPF_GET_PTYPE_SIZE(p) struct_size((p), proto_id, (p)->proto_id_count) 245 246 #define IDPF_TUN_IP_GRE (\ 247 IDPF_PTYPE_TUNNEL_IP |\ 248 IDPF_PTYPE_TUNNEL_IP_GRENAT) 249 250 #define IDPF_TUN_IP_GRE_MAC (\ 251 IDPF_TUN_IP_GRE |\ 252 IDPF_PTYPE_TUNNEL_IP_GRENAT_MAC) 253 254 #define IDPF_RX_MAX_PTYPE 1024 255 #define IDPF_RX_MAX_BASE_PTYPE 256 256 #define IDPF_INVALID_PTYPE_ID 0xFFFF 257 258 enum idpf_tunnel_state { 259 IDPF_PTYPE_TUNNEL_IP = BIT(0), 260 IDPF_PTYPE_TUNNEL_IP_GRENAT = BIT(1), 261 IDPF_PTYPE_TUNNEL_IP_GRENAT_MAC = BIT(2), 262 }; 263 264 struct idpf_ptype_state { 265 bool outer_ip:1; 266 bool outer_frag:1; 267 u8 tunnel_state:6; 268 }; 269 270 /** 271 * enum idpf_queue_flags_t 272 * @__IDPF_Q_GEN_CHK: Queues operating in splitq mode use a generation bit to 273 * identify new descriptor writebacks on the ring. HW sets 274 * the gen bit to 1 on the first writeback of any given 275 * descriptor. After the ring wraps, HW sets the gen bit of 276 * those descriptors to 0, and continues flipping 277 * 0->1 or 1->0 on each ring wrap. SW maintains its own 278 * gen bit to know what value will indicate writebacks on 279 * the next pass around the ring. E.g. it is initialized 280 * to 1 and knows that reading a gen bit of 1 in any 281 * descriptor on the initial pass of the ring indicates a 282 * writeback. It also flips on every ring wrap. 283 * @__IDPF_Q_RFL_GEN_CHK: Refill queues are SW only, so Q_GEN acts as the HW 284 * bit and Q_RFL_GEN is the SW bit. 285 * @__IDPF_Q_FLOW_SCH_EN: Enable flow scheduling 286 * @__IDPF_Q_SW_MARKER: Used to indicate TX queue marker completions 287 * @__IDPF_Q_CRC_EN: enable CRC offload in singleq mode 288 * @__IDPF_Q_RSC_EN: enable Receive Side Coalescing on Rx (splitq) 289 * @__IDPF_Q_HSPLIT_EN: enable header split on Rx (splitq) 290 * @__IDPF_Q_PTP: indicates whether the Rx timestamping is enabled for the 291 * queue 292 * @__IDPF_Q_NOIRQ: queue is polling-driven and has no interrupt 293 * @__IDPF_Q_XDP: this is an XDP queue 294 * @__IDPF_Q_XSK: the queue has an XSk pool installed 295 * @__IDPF_Q_FLAGS_NBITS: Must be last 296 */ 297 enum idpf_queue_flags_t { 298 __IDPF_Q_GEN_CHK, 299 __IDPF_Q_RFL_GEN_CHK, 300 __IDPF_Q_FLOW_SCH_EN, 301 __IDPF_Q_SW_MARKER, 302 __IDPF_Q_CRC_EN, 303 __IDPF_Q_RSC_EN, 304 __IDPF_Q_HSPLIT_EN, 305 __IDPF_Q_PTP, 306 __IDPF_Q_NOIRQ, 307 __IDPF_Q_XDP, 308 __IDPF_Q_XSK, 309 310 __IDPF_Q_FLAGS_NBITS, 311 }; 312 313 #define idpf_queue_set(f, q) __set_bit(__IDPF_Q_##f, (q)->flags) 314 #define idpf_queue_clear(f, q) __clear_bit(__IDPF_Q_##f, (q)->flags) 315 #define idpf_queue_change(f, q) __change_bit(__IDPF_Q_##f, (q)->flags) 316 #define idpf_queue_has(f, q) test_bit(__IDPF_Q_##f, (q)->flags) 317 318 #define idpf_queue_has_clear(f, q) \ 319 __test_and_clear_bit(__IDPF_Q_##f, (q)->flags) 320 #define idpf_queue_assign(f, q, v) \ 321 __assign_bit(__IDPF_Q_##f, (q)->flags, v) 322 323 /** 324 * struct idpf_vec_regs 325 * @dyn_ctl_reg: Dynamic control interrupt register offset 326 * @itrn_reg: Interrupt Throttling Rate register offset 327 * @itrn_index_spacing: Register spacing between ITR registers of the same 328 * vector 329 */ 330 struct idpf_vec_regs { 331 u32 dyn_ctl_reg; 332 u32 itrn_reg; 333 u32 itrn_index_spacing; 334 }; 335 336 /** 337 * struct idpf_intr_reg 338 * @dyn_ctl: Dynamic control interrupt register 339 * @dyn_ctl_intena_m: Mask for dyn_ctl interrupt enable 340 * @dyn_ctl_intena_msk_m: Mask for dyn_ctl interrupt enable mask 341 * @dyn_ctl_itridx_s: Register bit offset for ITR index 342 * @dyn_ctl_itridx_m: Mask for ITR index 343 * @dyn_ctl_intrvl_s: Register bit offset for ITR interval 344 * @dyn_ctl_wb_on_itr_m: Mask for WB on ITR feature 345 * @dyn_ctl_sw_itridx_ena_m: Mask for SW ITR index 346 * @dyn_ctl_swint_trig_m: Mask for dyn_ctl SW triggered interrupt enable 347 * @rx_itr: RX ITR register 348 * @tx_itr: TX ITR register 349 * @icr_ena: Interrupt cause register offset 350 * @icr_ena_ctlq_m: Mask for ICR 351 */ 352 struct idpf_intr_reg { 353 void __iomem *dyn_ctl; 354 u32 dyn_ctl_intena_m; 355 u32 dyn_ctl_intena_msk_m; 356 u32 dyn_ctl_itridx_s; 357 u32 dyn_ctl_itridx_m; 358 u32 dyn_ctl_intrvl_s; 359 u32 dyn_ctl_wb_on_itr_m; 360 u32 dyn_ctl_sw_itridx_ena_m; 361 u32 dyn_ctl_swint_trig_m; 362 void __iomem *rx_itr; 363 void __iomem *tx_itr; 364 void __iomem *icr_ena; 365 u32 icr_ena_ctlq_m; 366 }; 367 368 /** 369 * struct idpf_q_vector 370 * @vport: Vport back pointer 371 * @num_rxq: Number of RX queues 372 * @num_txq: Number of TX queues 373 * @num_bufq: Number of buffer queues 374 * @num_complq: number of completion queues 375 * @num_xsksq: number of XSk send queues 376 * @rx: Array of RX queues to service 377 * @tx: Array of TX queues to service 378 * @bufq: Array of buffer queues to service 379 * @complq: array of completion queues 380 * @xsksq: array of XSk send queues 381 * @intr_reg: See struct idpf_intr_reg 382 * @csd: XSk wakeup CSD 383 * @total_events: Number of interrupts processed 384 * @wb_on_itr: whether WB on ITR is enabled 385 * @napi: napi handler 386 * @tx_dim: Data for TX net_dim algorithm 387 * @tx_itr_value: TX interrupt throttling rate 388 * @tx_intr_mode: Dynamic ITR or not 389 * @tx_itr_idx: TX ITR index 390 * @rx_dim: Data for RX net_dim algorithm 391 * @rx_itr_value: RX interrupt throttling rate 392 * @rx_intr_mode: Dynamic ITR or not 393 * @rx_itr_idx: RX ITR index 394 * @v_idx: Vector index 395 */ 396 struct idpf_q_vector { 397 __cacheline_group_begin_aligned(read_mostly); 398 struct idpf_vport *vport; 399 400 u16 num_rxq; 401 u16 num_txq; 402 u16 num_bufq; 403 u16 num_complq; 404 u16 num_xsksq; 405 struct idpf_rx_queue **rx; 406 struct idpf_tx_queue **tx; 407 struct idpf_buf_queue **bufq; 408 struct idpf_compl_queue **complq; 409 struct idpf_tx_queue **xsksq; 410 411 struct idpf_intr_reg intr_reg; 412 __cacheline_group_end_aligned(read_mostly); 413 414 __cacheline_group_begin_aligned(read_write); 415 call_single_data_t csd; 416 417 u16 total_events; 418 bool wb_on_itr; 419 420 struct napi_struct napi; 421 422 struct dim tx_dim; 423 u16 tx_itr_value; 424 bool tx_intr_mode; 425 u32 tx_itr_idx; 426 427 struct dim rx_dim; 428 u16 rx_itr_value; 429 bool rx_intr_mode; 430 u32 rx_itr_idx; 431 __cacheline_group_end_aligned(read_write); 432 433 __cacheline_group_begin_aligned(cold); 434 u16 v_idx; 435 436 __cacheline_group_end_aligned(cold); 437 }; 438 libeth_cacheline_set_assert(struct idpf_q_vector, 136, 439 56 + sizeof(struct napi_struct) + 440 2 * sizeof(struct dim), 441 8); 442 443 struct idpf_rx_queue_stats { 444 u64_stats_t packets; 445 u64_stats_t bytes; 446 u64_stats_t rsc_pkts; 447 u64_stats_t hw_csum_err; 448 u64_stats_t hsplit_pkts; 449 u64_stats_t hsplit_buf_ovf; 450 u64_stats_t bad_descs; 451 }; 452 453 struct idpf_tx_queue_stats { 454 u64_stats_t packets; 455 u64_stats_t bytes; 456 u64_stats_t lso_pkts; 457 u64_stats_t linearize; 458 u64_stats_t q_busy; 459 u64_stats_t skb_drops; 460 u64_stats_t dma_map_errs; 461 u64_stats_t tstamp_skipped; 462 }; 463 464 #define IDPF_ITR_DYNAMIC 1 465 #define IDPF_ITR_MAX 0x1FE0 466 #define IDPF_ITR_20K 0x0032 467 #define IDPF_ITR_GRAN_S 1 /* Assume ITR granularity is 2us */ 468 #define IDPF_ITR_MASK 0x1FFE /* ITR register value alignment mask */ 469 #define ITR_REG_ALIGN(setting) ((setting) & IDPF_ITR_MASK) 470 #define IDPF_ITR_IS_DYNAMIC(itr_mode) (itr_mode) 471 #define IDPF_ITR_TX_DEF IDPF_ITR_20K 472 #define IDPF_ITR_RX_DEF IDPF_ITR_20K 473 /* Index used for 'SW ITR' update in DYN_CTL register */ 474 #define IDPF_SW_ITR_UPDATE_IDX 2 475 /* Index used for 'No ITR' update in DYN_CTL register */ 476 #define IDPF_NO_ITR_UPDATE_IDX 3 477 #define IDPF_ITR_IDX_SPACING(spacing, dflt) (spacing ? spacing : dflt) 478 #define IDPF_DIM_DEFAULT_PROFILE_IX 1 479 480 /** 481 * struct idpf_rx_queue - software structure representing a receive queue 482 * @rx: universal receive descriptor array 483 * @single_buf: buffer descriptor array in singleq 484 * @desc_ring: virtual descriptor ring address 485 * @bufq_sets: Pointer to the array of buffer queues in splitq mode 486 * @napi: NAPI instance corresponding to this queue (splitq) 487 * @xdp_prog: attached XDP program 488 * @rx_buf: See struct &libeth_fqe 489 * @pp: Page pool pointer in singleq mode 490 * @tail: Tail offset. Used for both queue models single and split. 491 * @flags: See enum idpf_queue_flags_t 492 * @idx: For RX queue, it is used to index to total RX queue across groups and 493 * used for skb reporting. 494 * @desc_count: Number of descriptors 495 * @num_xdp_txq: total number of XDP Tx queues 496 * @xdpsqs: shortcut for XDP Tx queues array 497 * @rxdids: Supported RX descriptor ids 498 * @truesize: data buffer truesize in singleq 499 * @rx_ptype_lkup: LUT of Rx ptypes 500 * @xdp_rxq: XDP queue info 501 * @next_to_use: Next descriptor to use 502 * @next_to_clean: Next descriptor to clean 503 * @next_to_alloc: RX buffer to allocate at 504 * @xdp: XDP buffer with the current frame 505 * @xsk: current XDP buffer in XSk mode 506 * @pool: XSk pool if installed 507 * @cached_phc_time: Cached PHC time for the Rx queue 508 * @stats_sync: See struct u64_stats_sync 509 * @q_stats: See union idpf_rx_queue_stats 510 * @q_id: Queue id 511 * @size: Length of descriptor ring in bytes 512 * @dma: Physical address of ring 513 * @q_vector: Backreference to associated vector 514 * @rx_buffer_low_watermark: RX buffer low watermark 515 * @rx_hbuf_size: Header buffer size 516 * @rx_buf_size: Buffer size 517 * @rx_max_pkt_size: RX max packet size 518 */ 519 struct idpf_rx_queue { 520 __cacheline_group_begin_aligned(read_mostly); 521 union { 522 union virtchnl2_rx_desc *rx; 523 struct virtchnl2_singleq_rx_buf_desc *single_buf; 524 525 void *desc_ring; 526 }; 527 union { 528 struct { 529 struct idpf_bufq_set *bufq_sets; 530 struct napi_struct *napi; 531 struct bpf_prog __rcu *xdp_prog; 532 }; 533 struct { 534 struct libeth_fqe *rx_buf; 535 struct page_pool *pp; 536 void __iomem *tail; 537 }; 538 }; 539 540 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); 541 u16 idx; 542 u16 desc_count; 543 544 u32 num_xdp_txq; 545 union { 546 struct idpf_tx_queue **xdpsqs; 547 struct { 548 u32 rxdids; 549 u32 truesize; 550 }; 551 }; 552 const struct libeth_rx_pt *rx_ptype_lkup; 553 554 struct xdp_rxq_info xdp_rxq; 555 __cacheline_group_end_aligned(read_mostly); 556 557 __cacheline_group_begin_aligned(read_write); 558 u32 next_to_use; 559 u32 next_to_clean; 560 u32 next_to_alloc; 561 562 union { 563 struct libeth_xdp_buff_stash xdp; 564 struct { 565 struct libeth_xdp_buff *xsk; 566 struct xsk_buff_pool *pool; 567 }; 568 }; 569 u64 cached_phc_time; 570 571 struct u64_stats_sync stats_sync; 572 struct idpf_rx_queue_stats q_stats; 573 __cacheline_group_end_aligned(read_write); 574 575 __cacheline_group_begin_aligned(cold); 576 u32 q_id; 577 u32 size; 578 dma_addr_t dma; 579 580 struct idpf_q_vector *q_vector; 581 582 u16 rx_buffer_low_watermark; 583 u16 rx_hbuf_size; 584 u16 rx_buf_size; 585 u16 rx_max_pkt_size; 586 __cacheline_group_end_aligned(cold); 587 }; 588 libeth_cacheline_set_assert(struct idpf_rx_queue, 589 ALIGN(64, __alignof(struct xdp_rxq_info)) + 590 sizeof(struct xdp_rxq_info), 591 96 + offsetof(struct idpf_rx_queue, q_stats) - 592 offsetofend(struct idpf_rx_queue, cached_phc_time), 593 32); 594 595 /** 596 * struct idpf_tx_queue - software structure representing a transmit queue 597 * @base_tx: base Tx descriptor array 598 * @base_ctx: base Tx context descriptor array 599 * @flex_tx: flex Tx descriptor array 600 * @flex_ctx: flex Tx context descriptor array 601 * @desc_ring: virtual descriptor ring address 602 * @tx_buf: See struct idpf_tx_buf 603 * @txq_grp: See struct idpf_txq_group 604 * @complq: corresponding completion queue in XDP mode 605 * @dev: Device back pointer for DMA mapping 606 * @pool: corresponding XSk pool if installed 607 * @tail: Tail offset. Used for both queue models single and split 608 * @flags: See enum idpf_queue_flags_t 609 * @idx: For TX queue, it is used as index to map between TX queue group and 610 * hot path TX pointers stored in vport. Used in both singleq/splitq. 611 * @desc_count: Number of descriptors 612 * @tx_min_pkt_len: Min supported packet length 613 * @ts_gran_pow2: Txtime timestamp granularity in nanoseconds (log2). 614 * @thresh: XDP queue cleaning threshold 615 * @netdev: &net_device corresponding to this queue 616 * @next_to_use: Next descriptor to use 617 * @next_to_clean: Next descriptor to clean 618 * @last_re: last descriptor index that RE bit was set 619 * @tx_max_bufs: Max buffers that can be transmitted with scatter-gather 620 * @cleaned_bytes: Splitq only, TXQ only: When a TX completion is received on 621 * the TX completion queue, it can be for any TXQ associated 622 * with that completion queue. This means we can clean up to 623 * N TXQs during a single call to clean the completion queue. 624 * cleaned_bytes|pkts tracks the clean stats per TXQ during 625 * that single call to clean the completion queue. By doing so, 626 * we can update BQL with aggregate cleaned stats for each TXQ 627 * only once at the end of the cleaning routine. 628 * @clean_budget: singleq only, queue cleaning budget 629 * @cleaned_pkts: Number of packets cleaned for the above said case 630 * @refillq: Pointer to refill queue 631 * @pending: number of pending descriptors to send in QB 632 * @xdp_tx: number of pending &xdp_buff or &xdp_frame buffers 633 * @timer: timer for XDP Tx queue cleanup 634 * @xdp_lock: lock for XDP Tx queues sharing 635 * @cached_tstamp_caps: Tx timestamp capabilities negotiated with the CP 636 * @tstamp_task: Work that handles Tx timestamp read 637 * @stats_sync: See struct u64_stats_sync 638 * @q_stats: See union idpf_tx_queue_stats 639 * @q_id: Queue id 640 * @size: Length of descriptor ring in bytes 641 * @dma: Physical address of ring 642 * @q_vector: Backreference to associated vector 643 * @buf_pool_size: Total number of idpf_tx_buf 644 * @rel_q_id: relative virtchnl queue index 645 */ 646 struct idpf_tx_queue { 647 __cacheline_group_begin_aligned(read_mostly); 648 union { 649 struct idpf_base_tx_desc *base_tx; 650 struct idpf_base_tx_ctx_desc *base_ctx; 651 union idpf_tx_flex_desc *flex_tx; 652 union idpf_flex_tx_ctx_desc *flex_ctx; 653 654 void *desc_ring; 655 }; 656 struct libeth_sqe *tx_buf; 657 union { 658 struct idpf_txq_group *txq_grp; 659 struct idpf_compl_queue *complq; 660 }; 661 union { 662 struct device *dev; 663 struct xsk_buff_pool *pool; 664 }; 665 void __iomem *tail; 666 667 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); 668 u16 idx; 669 u16 desc_count; 670 671 union { 672 struct { 673 u16 tx_min_pkt_len; 674 u8 ts_gran_pow2; 675 }; 676 u32 thresh; 677 }; 678 679 struct net_device *netdev; 680 __cacheline_group_end_aligned(read_mostly); 681 682 __cacheline_group_begin_aligned(read_write); 683 u32 next_to_use; 684 u32 next_to_clean; 685 686 union { 687 struct { 688 u16 last_re; 689 u16 tx_max_bufs; 690 691 union { 692 u32 cleaned_bytes; 693 u32 clean_budget; 694 }; 695 u16 cleaned_pkts; 696 697 struct idpf_sw_queue *refillq; 698 }; 699 struct { 700 u32 pending; 701 u32 xdp_tx; 702 703 struct libeth_xdpsq_timer *timer; 704 struct libeth_xdpsq_lock xdp_lock; 705 }; 706 }; 707 708 struct idpf_ptp_vport_tx_tstamp_caps *cached_tstamp_caps; 709 struct work_struct *tstamp_task; 710 711 struct u64_stats_sync stats_sync; 712 struct idpf_tx_queue_stats q_stats; 713 __cacheline_group_end_aligned(read_write); 714 715 __cacheline_group_begin_aligned(cold); 716 u32 q_id; 717 u32 size; 718 dma_addr_t dma; 719 720 struct idpf_q_vector *q_vector; 721 722 u32 buf_pool_size; 723 u32 rel_q_id; 724 __cacheline_group_end_aligned(cold); 725 }; 726 libeth_cacheline_set_assert(struct idpf_tx_queue, 64, 727 104 + 728 offsetof(struct idpf_tx_queue, cached_tstamp_caps) - 729 offsetofend(struct idpf_tx_queue, timer) + 730 offsetof(struct idpf_tx_queue, q_stats) - 731 offsetofend(struct idpf_tx_queue, tstamp_task), 732 32); 733 734 /** 735 * struct idpf_buf_queue - software structure representing a buffer queue 736 * @split_buf: buffer descriptor array 737 * @buf: &libeth_fqe for data buffers 738 * @pp: &page_pool for data buffers 739 * @xsk_buf: &xdp_buff for XSk Rx buffers 740 * @pool: &xsk_buff_pool on XSk queues 741 * @hdr_buf: &libeth_fqe for header buffers 742 * @hdr_pp: &page_pool for header buffers 743 * @tail: Tail offset 744 * @flags: See enum idpf_queue_flags_t 745 * @desc_count: Number of descriptors 746 * @thresh: refill threshold in XSk 747 * @next_to_use: Next descriptor to use 748 * @next_to_clean: Next descriptor to clean 749 * @next_to_alloc: RX buffer to allocate at 750 * @pending: number of buffers to refill (Xsk) 751 * @hdr_truesize: truesize for buffer headers 752 * @truesize: truesize for data buffers 753 * @q_id: Queue id 754 * @size: Length of descriptor ring in bytes 755 * @dma: Physical address of ring 756 * @q_vector: Backreference to associated vector 757 * @rx_buffer_low_watermark: RX buffer low watermark 758 * @rx_hbuf_size: Header buffer size 759 * @rx_buf_size: Buffer size 760 */ 761 struct idpf_buf_queue { 762 __cacheline_group_begin_aligned(read_mostly); 763 struct virtchnl2_splitq_rx_buf_desc *split_buf; 764 union { 765 struct { 766 struct libeth_fqe *buf; 767 struct page_pool *pp; 768 }; 769 struct { 770 struct libeth_xdp_buff **xsk_buf; 771 struct xsk_buff_pool *pool; 772 }; 773 }; 774 struct libeth_fqe *hdr_buf; 775 struct page_pool *hdr_pp; 776 void __iomem *tail; 777 778 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); 779 u32 desc_count; 780 781 u32 thresh; 782 __cacheline_group_end_aligned(read_mostly); 783 784 __cacheline_group_begin_aligned(read_write); 785 u32 next_to_use; 786 u32 next_to_clean; 787 u32 next_to_alloc; 788 789 u32 pending; 790 u32 hdr_truesize; 791 u32 truesize; 792 __cacheline_group_end_aligned(read_write); 793 794 __cacheline_group_begin_aligned(cold); 795 u32 q_id; 796 u32 size; 797 dma_addr_t dma; 798 799 struct idpf_q_vector *q_vector; 800 801 u16 rx_buffer_low_watermark; 802 u16 rx_hbuf_size; 803 u16 rx_buf_size; 804 __cacheline_group_end_aligned(cold); 805 }; 806 libeth_cacheline_set_assert(struct idpf_buf_queue, 64, 24, 32); 807 808 /** 809 * struct idpf_compl_queue - software structure representing a completion queue 810 * @comp: 8-byte completion descriptor array 811 * @comp_4b: 4-byte completion descriptor array 812 * @desc_ring: virtual descriptor ring address 813 * @txq_grp: See struct idpf_txq_group 814 * @flags: See enum idpf_queue_flags_t 815 * @desc_count: Number of descriptors 816 * @clean_budget: queue cleaning budget 817 * @netdev: &net_device corresponding to this queue 818 * @next_to_use: Next descriptor to use. Relevant in both split & single txq 819 * and bufq. 820 * @next_to_clean: Next descriptor to clean 821 * @num_completions: Only relevant for TX completion queue. It tracks the 822 * number of completions received to compare against the 823 * number of completions pending, as accumulated by the 824 * TX queues. 825 * @q_id: Queue id 826 * @size: Length of descriptor ring in bytes 827 * @dma: Physical address of ring 828 * @q_vector: Backreference to associated vector 829 */ 830 struct idpf_compl_queue { 831 __cacheline_group_begin_aligned(read_mostly); 832 union { 833 struct idpf_splitq_tx_compl_desc *comp; 834 struct idpf_splitq_4b_tx_compl_desc *comp_4b; 835 836 void *desc_ring; 837 }; 838 struct idpf_txq_group *txq_grp; 839 840 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); 841 u32 desc_count; 842 843 u32 clean_budget; 844 struct net_device *netdev; 845 __cacheline_group_end_aligned(read_mostly); 846 847 __cacheline_group_begin_aligned(read_write); 848 u32 next_to_use; 849 u32 next_to_clean; 850 851 aligned_u64 num_completions; 852 __cacheline_group_end_aligned(read_write); 853 854 __cacheline_group_begin_aligned(cold); 855 u32 q_id; 856 u32 size; 857 dma_addr_t dma; 858 859 struct idpf_q_vector *q_vector; 860 __cacheline_group_end_aligned(cold); 861 }; 862 libeth_cacheline_set_assert(struct idpf_compl_queue, 40, 16, 24); 863 864 /** 865 * struct idpf_sw_queue 866 * @ring: Pointer to the ring 867 * @flags: See enum idpf_queue_flags_t 868 * @desc_count: Descriptor count 869 * @next_to_use: Buffer to allocate at 870 * @next_to_clean: Next descriptor to clean 871 * 872 * Software queues are used in splitq mode to manage buffers between rxq 873 * producer and the bufq consumer. These are required in order to maintain a 874 * lockless buffer management system and are strictly software only constructs. 875 */ 876 struct idpf_sw_queue { 877 __cacheline_group_begin_aligned(read_mostly); 878 u32 *ring; 879 880 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); 881 u32 desc_count; 882 __cacheline_group_end_aligned(read_mostly); 883 884 __cacheline_group_begin_aligned(read_write); 885 u32 next_to_use; 886 u32 next_to_clean; 887 __cacheline_group_end_aligned(read_write); 888 }; 889 libeth_cacheline_group_assert(struct idpf_sw_queue, read_mostly, 24); 890 libeth_cacheline_group_assert(struct idpf_sw_queue, read_write, 8); 891 libeth_cacheline_struct_assert(struct idpf_sw_queue, 24, 8); 892 893 /** 894 * struct idpf_rxq_set 895 * @rxq: RX queue 896 * @refillq: pointers to refill queues 897 * 898 * Splitq only. idpf_rxq_set associates an rxq with at an array of refillqs. 899 * Each rxq needs a refillq to return used buffers back to the respective bufq. 900 * Bufqs then clean these refillqs for buffers to give to hardware. 901 */ 902 struct idpf_rxq_set { 903 struct idpf_rx_queue rxq; 904 struct idpf_sw_queue *refillq[IDPF_MAX_BUFQS_PER_RXQ_GRP]; 905 }; 906 907 /** 908 * struct idpf_bufq_set 909 * @bufq: Buffer queue 910 * @num_refillqs: Number of refill queues. This is always equal to num_rxq_sets 911 * in idpf_rxq_group. 912 * @refillqs: Pointer to refill queues array. 913 * 914 * Splitq only. idpf_bufq_set associates a bufq to an array of refillqs. 915 * In this bufq_set, there will be one refillq for each rxq in this rxq_group. 916 * Used buffers received by rxqs will be put on refillqs which bufqs will 917 * clean to return new buffers back to hardware. 918 * 919 * Buffers needed by some number of rxqs associated in this rxq_group are 920 * managed by at most two bufqs (depending on performance configuration). 921 */ 922 struct idpf_bufq_set { 923 struct idpf_buf_queue bufq; 924 int num_refillqs; 925 struct idpf_sw_queue *refillqs; 926 }; 927 928 /** 929 * struct idpf_rxq_group 930 * @vport: Vport back pointer 931 * @singleq: Struct with single queue related members 932 * @singleq.num_rxq: Number of RX queues associated 933 * @singleq.rxqs: Array of RX queue pointers 934 * @splitq: Struct with split queue related members 935 * @splitq.num_rxq_sets: Number of RX queue sets 936 * @splitq.num_rxq_sets: Number of Buffer queue sets 937 * @splitq.rxq_sets: Array of RX queue sets 938 * @splitq.bufq_sets: Buffer queue set pointer 939 * 940 * In singleq mode, an rxq_group is simply an array of rxqs. In splitq, a 941 * rxq_group contains all the rxqs, bufqs and refillqs needed to 942 * manage buffers in splitq mode. 943 */ 944 struct idpf_rxq_group { 945 struct idpf_vport *vport; 946 947 union { 948 struct { 949 u16 num_rxq; 950 struct idpf_rx_queue *rxqs[IDPF_LARGE_MAX_Q]; 951 } singleq; 952 struct { 953 u16 num_rxq_sets; 954 u16 num_bufq_sets; 955 struct idpf_rxq_set *rxq_sets[IDPF_LARGE_MAX_Q]; 956 struct idpf_bufq_set *bufq_sets; 957 } splitq; 958 }; 959 }; 960 961 /** 962 * struct idpf_txq_group 963 * @vport: Vport back pointer 964 * @num_txq: Number of TX queues associated 965 * @txqs: Array of TX queue pointers 966 * @complq: Associated completion queue pointer, split queue only 967 * @num_completions_pending: Total number of completions pending for the 968 * completion queue, acculumated for all TX queues 969 * associated with that completion queue. 970 * 971 * Between singleq and splitq, a txq_group is largely the same except for the 972 * complq. In splitq a single complq is responsible for handling completions 973 * for some number of txqs associated in this txq_group. 974 */ 975 struct idpf_txq_group { 976 struct idpf_vport *vport; 977 978 u16 num_txq; 979 struct idpf_tx_queue *txqs[IDPF_LARGE_MAX_Q]; 980 981 struct idpf_compl_queue *complq; 982 983 aligned_u64 num_completions_pending; 984 }; 985 986 static inline int idpf_q_vector_to_mem(const struct idpf_q_vector *q_vector) 987 { 988 u32 cpu; 989 990 if (!q_vector) 991 return NUMA_NO_NODE; 992 993 cpu = cpumask_first(&q_vector->napi.config->affinity_mask); 994 995 return cpu < nr_cpu_ids ? cpu_to_mem(cpu) : NUMA_NO_NODE; 996 } 997 998 /** 999 * idpf_size_to_txd_count - Get number of descriptors needed for large Tx frag 1000 * @size: transmit request size in bytes 1001 * 1002 * In the case where a large frag (>= 16K) needs to be split across multiple 1003 * descriptors, we need to assume that we can have no more than 12K of data 1004 * per descriptor due to hardware alignment restrictions (4K alignment). 1005 */ 1006 static inline u32 idpf_size_to_txd_count(unsigned int size) 1007 { 1008 return DIV_ROUND_UP(size, IDPF_TX_MAX_DESC_DATA_ALIGNED); 1009 } 1010 1011 /** 1012 * idpf_tx_singleq_build_ctob - populate command tag offset and size 1013 * @td_cmd: Command to be filled in desc 1014 * @td_offset: Offset to be filled in desc 1015 * @size: Size of the buffer 1016 * @td_tag: td tag to be filled 1017 * 1018 * Returns the 64 bit value populated with the input parameters 1019 */ 1020 static inline __le64 idpf_tx_singleq_build_ctob(u64 td_cmd, u64 td_offset, 1021 unsigned int size, u64 td_tag) 1022 { 1023 return cpu_to_le64(IDPF_TX_DESC_DTYPE_DATA | 1024 (td_cmd << IDPF_TXD_QW1_CMD_S) | 1025 (td_offset << IDPF_TXD_QW1_OFFSET_S) | 1026 ((u64)size << IDPF_TXD_QW1_TX_BUF_SZ_S) | 1027 (td_tag << IDPF_TXD_QW1_L2TAG1_S)); 1028 } 1029 1030 void idpf_tx_splitq_build_ctb(union idpf_tx_flex_desc *desc, 1031 struct idpf_tx_splitq_params *params, 1032 u16 td_cmd, u16 size); 1033 void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc, 1034 struct idpf_tx_splitq_params *params, 1035 u16 td_cmd, u16 size); 1036 /** 1037 * idpf_tx_splitq_build_desc - determine which type of data descriptor to build 1038 * @desc: descriptor to populate 1039 * @params: pointer to tx params struct 1040 * @td_cmd: command to be filled in desc 1041 * @size: size of buffer 1042 */ 1043 static inline void idpf_tx_splitq_build_desc(union idpf_tx_flex_desc *desc, 1044 struct idpf_tx_splitq_params *params, 1045 u16 td_cmd, u16 size) 1046 { 1047 if (params->dtype == IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2) 1048 idpf_tx_splitq_build_ctb(desc, params, td_cmd, size); 1049 else 1050 idpf_tx_splitq_build_flow_desc(desc, params, td_cmd, size); 1051 } 1052 1053 /** 1054 * idpf_vport_intr_set_wb_on_itr - enable descriptor writeback on disabled interrupts 1055 * @q_vector: pointer to queue vector struct 1056 */ 1057 static inline void idpf_vport_intr_set_wb_on_itr(struct idpf_q_vector *q_vector) 1058 { 1059 struct idpf_intr_reg *reg; 1060 1061 if (q_vector->wb_on_itr) 1062 return; 1063 1064 q_vector->wb_on_itr = true; 1065 reg = &q_vector->intr_reg; 1066 1067 writel(reg->dyn_ctl_wb_on_itr_m | reg->dyn_ctl_intena_msk_m | 1068 (IDPF_NO_ITR_UPDATE_IDX << reg->dyn_ctl_itridx_s), 1069 reg->dyn_ctl); 1070 } 1071 1072 /** 1073 * idpf_tx_splitq_get_free_bufs - get number of free buf_ids in refillq 1074 * @refillq: pointer to refillq containing buf_ids 1075 */ 1076 static inline u32 idpf_tx_splitq_get_free_bufs(struct idpf_sw_queue *refillq) 1077 { 1078 return (refillq->next_to_use > refillq->next_to_clean ? 1079 0 : refillq->desc_count) + 1080 refillq->next_to_use - refillq->next_to_clean - 1; 1081 } 1082 1083 int idpf_vport_singleq_napi_poll(struct napi_struct *napi, int budget); 1084 void idpf_vport_init_num_qs(struct idpf_vport *vport, 1085 struct virtchnl2_create_vport *vport_msg, 1086 struct idpf_q_vec_rsrc *rsrc); 1087 void idpf_vport_calc_num_q_desc(struct idpf_vport *vport, 1088 struct idpf_q_vec_rsrc *rsrc); 1089 int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_index, 1090 struct virtchnl2_create_vport *vport_msg, 1091 struct idpf_vport_max_q *max_q); 1092 void idpf_vport_calc_num_q_groups(struct idpf_q_vec_rsrc *rsrc); 1093 int idpf_vport_queues_alloc(struct idpf_vport *vport, 1094 struct idpf_q_vec_rsrc *rsrc); 1095 void idpf_vport_queues_rel(struct idpf_vport *vport, 1096 struct idpf_q_vec_rsrc *rsrc); 1097 void idpf_vport_intr_rel(struct idpf_q_vec_rsrc *rsrc); 1098 int idpf_vport_intr_alloc(struct idpf_vport *vport, 1099 struct idpf_q_vec_rsrc *rsrc); 1100 void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector); 1101 void idpf_vport_intr_deinit(struct idpf_vport *vport, 1102 struct idpf_q_vec_rsrc *rsrc); 1103 int idpf_vport_intr_init(struct idpf_vport *vport, 1104 struct idpf_q_vec_rsrc *rsrc); 1105 void idpf_vport_intr_ena(struct idpf_vport *vport, 1106 struct idpf_q_vec_rsrc *rsrc); 1107 void idpf_fill_dflt_rss_lut(struct idpf_vport *vport, 1108 struct idpf_rss_data *rss_data); 1109 int idpf_config_rss(struct idpf_vport *vport, struct idpf_rss_data *rss_data); 1110 int idpf_init_rss_lut(struct idpf_vport *vport, struct idpf_rss_data *rss_data); 1111 void idpf_deinit_rss_lut(struct idpf_rss_data *rss_data); 1112 int idpf_rx_bufs_init_all(struct idpf_vport *vport, 1113 struct idpf_q_vec_rsrc *rsrc); 1114 1115 struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, 1116 u32 q_num); 1117 struct idpf_q_vector *idpf_find_txq_vec(const struct idpf_vport *vport, 1118 u32 q_num); 1119 int idpf_qp_switch(struct idpf_vport *vport, u32 qid, bool en); 1120 1121 void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val, 1122 bool xmit_more); 1123 unsigned int idpf_size_to_txd_count(unsigned int size); 1124 netdev_tx_t idpf_tx_drop_skb(struct idpf_tx_queue *tx_q, struct sk_buff *skb); 1125 unsigned int idpf_tx_res_count_required(struct idpf_tx_queue *txq, 1126 struct sk_buff *skb, u32 *buf_count); 1127 void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue); 1128 netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, 1129 struct idpf_tx_queue *tx_q); 1130 netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev); 1131 bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_rx_queue *rxq, 1132 u16 cleaned_count); 1133 bool idpf_rx_process_skb_fields(struct sk_buff *skb, 1134 const struct libeth_xdp_buff *xdp, 1135 struct libeth_rq_napi_stats *rs); 1136 int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off); 1137 1138 void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq); 1139 1140 #endif /* !_IDPF_TXRX_H_ */ 1141