1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (C) 2023 Intel Corporation */ 3 4 #ifndef _IDPF_TXRX_H_ 5 #define _IDPF_TXRX_H_ 6 7 #include <linux/dim.h> 8 9 #include <net/libeth/cache.h> 10 #include <net/tcp.h> 11 #include <net/netdev_queues.h> 12 13 #include "idpf_lan_txrx.h" 14 #include "virtchnl2_lan_desc.h" 15 16 #define IDPF_LARGE_MAX_Q 256 17 #define IDPF_MAX_Q 16 18 #define IDPF_MIN_Q 2 19 /* Mailbox Queue */ 20 #define IDPF_MAX_MBXQ 1 21 22 #define IDPF_MIN_TXQ_DESC 64 23 #define IDPF_MIN_RXQ_DESC 64 24 #define IDPF_MIN_TXQ_COMPLQ_DESC 256 25 #define IDPF_MAX_QIDS 256 26 27 /* Number of descriptors in a queue should be a multiple of 32. RX queue 28 * descriptors alone should be a multiple of IDPF_REQ_RXQ_DESC_MULTIPLE 29 * to achieve BufQ descriptors aligned to 32 30 */ 31 #define IDPF_REQ_DESC_MULTIPLE 32 32 #define IDPF_REQ_RXQ_DESC_MULTIPLE (IDPF_MAX_BUFQS_PER_RXQ_GRP * 32) 33 #define IDPF_MIN_TX_DESC_NEEDED (MAX_SKB_FRAGS + 6) 34 #define IDPF_TX_WAKE_THRESH ((u16)IDPF_MIN_TX_DESC_NEEDED * 2) 35 36 #define IDPF_MAX_DESCS 8160 37 #define IDPF_MAX_TXQ_DESC ALIGN_DOWN(IDPF_MAX_DESCS, IDPF_REQ_DESC_MULTIPLE) 38 #define IDPF_MAX_RXQ_DESC ALIGN_DOWN(IDPF_MAX_DESCS, IDPF_REQ_RXQ_DESC_MULTIPLE) 39 #define MIN_SUPPORT_TXDID (\ 40 VIRTCHNL2_TXDID_FLEX_FLOW_SCHED |\ 41 VIRTCHNL2_TXDID_FLEX_TSO_CTX) 42 43 #define IDPF_DFLT_SINGLEQ_TX_Q_GROUPS 1 44 #define IDPF_DFLT_SINGLEQ_RX_Q_GROUPS 1 45 #define IDPF_DFLT_SINGLEQ_TXQ_PER_GROUP 4 46 #define IDPF_DFLT_SINGLEQ_RXQ_PER_GROUP 4 47 48 #define IDPF_COMPLQ_PER_GROUP 1 49 #define IDPF_SINGLE_BUFQ_PER_RXQ_GRP 1 50 #define IDPF_MAX_BUFQS_PER_RXQ_GRP 2 51 #define IDPF_BUFQ2_ENA 1 52 #define IDPF_NUMQ_PER_CHUNK 1 53 54 #define IDPF_DFLT_SPLITQ_TXQ_PER_GROUP 1 55 #define IDPF_DFLT_SPLITQ_RXQ_PER_GROUP 1 56 57 /* Default vector sharing */ 58 #define IDPF_MBX_Q_VEC 1 59 #define IDPF_MIN_Q_VEC 1 60 61 #define IDPF_DFLT_TX_Q_DESC_COUNT 512 62 #define IDPF_DFLT_TX_COMPLQ_DESC_COUNT 512 63 #define IDPF_DFLT_RX_Q_DESC_COUNT 512 64 65 /* IMPORTANT: We absolutely _cannot_ have more buffers in the system than a 66 * given RX completion queue has descriptors. This includes _ALL_ buffer 67 * queues. E.g.: If you have two buffer queues of 512 descriptors and buffers, 68 * you have a total of 1024 buffers so your RX queue _must_ have at least that 69 * many descriptors. This macro divides a given number of RX descriptors by 70 * number of buffer queues to calculate how many descriptors each buffer queue 71 * can have without overrunning the RX queue. 72 * 73 * If you give hardware more buffers than completion descriptors what will 74 * happen is that if hardware gets a chance to post more than ring wrap of 75 * descriptors before SW gets an interrupt and overwrites SW head, the gen bit 76 * in the descriptor will be wrong. Any overwritten descriptors' buffers will 77 * be gone forever and SW has no reasonable way to tell that this has happened. 78 * From SW perspective, when we finally get an interrupt, it looks like we're 79 * still waiting for descriptor to be done, stalling forever. 80 */ 81 #define IDPF_RX_BUFQ_DESC_COUNT(RXD, NUM_BUFQ) ((RXD) / (NUM_BUFQ)) 82 83 #define IDPF_RX_BUFQ_WORKING_SET(rxq) ((rxq)->desc_count - 1) 84 85 #define IDPF_RX_BUMP_NTC(rxq, ntc) \ 86 do { \ 87 if (unlikely(++(ntc) == (rxq)->desc_count)) { \ 88 ntc = 0; \ 89 idpf_queue_change(GEN_CHK, rxq); \ 90 } \ 91 } while (0) 92 93 #define IDPF_SINGLEQ_BUMP_RING_IDX(q, idx) \ 94 do { \ 95 if (unlikely(++(idx) == (q)->desc_count)) \ 96 idx = 0; \ 97 } while (0) 98 99 #define IDPF_RX_BUF_STRIDE 32 100 #define IDPF_RX_BUF_POST_STRIDE 16 101 #define IDPF_LOW_WATERMARK 64 102 103 #define IDPF_TX_TSO_MIN_MSS 88 104 105 /* Minimum number of descriptors between 2 descriptors with the RE bit set; 106 * only relevant in flow scheduling mode 107 */ 108 #define IDPF_TX_SPLITQ_RE_MIN_GAP 64 109 110 #define IDPF_RX_BI_GEN_M BIT(16) 111 #define IDPF_RX_BI_BUFID_M GENMASK(15, 0) 112 113 #define IDPF_RXD_EOF_SPLITQ VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_EOF_M 114 #define IDPF_RXD_EOF_SINGLEQ VIRTCHNL2_RX_BASE_DESC_STATUS_EOF_M 115 116 #define IDPF_DESC_UNUSED(txq) \ 117 ((((txq)->next_to_clean > (txq)->next_to_use) ? 0 : (txq)->desc_count) + \ 118 (txq)->next_to_clean - (txq)->next_to_use - 1) 119 120 #define IDPF_TX_BUF_RSV_UNUSED(txq) ((txq)->stash->buf_stack.top) 121 #define IDPF_TX_BUF_RSV_LOW(txq) (IDPF_TX_BUF_RSV_UNUSED(txq) < \ 122 (txq)->desc_count >> 2) 123 124 #define IDPF_TX_COMPLQ_OVERFLOW_THRESH(txcq) ((txcq)->desc_count >> 1) 125 /* Determine the absolute number of completions pending, i.e. the number of 126 * completions that are expected to arrive on the TX completion queue. 127 */ 128 #define IDPF_TX_COMPLQ_PENDING(txq) \ 129 (((txq)->num_completions_pending >= (txq)->complq->num_completions ? \ 130 0 : U32_MAX) + \ 131 (txq)->num_completions_pending - (txq)->complq->num_completions) 132 133 #define IDPF_TX_SPLITQ_COMPL_TAG_WIDTH 16 134 /* Adjust the generation for the completion tag and wrap if necessary */ 135 #define IDPF_TX_ADJ_COMPL_TAG_GEN(txq) \ 136 ((++(txq)->compl_tag_cur_gen) >= (txq)->compl_tag_gen_max ? \ 137 0 : (txq)->compl_tag_cur_gen) 138 139 #define IDPF_TXD_LAST_DESC_CMD (IDPF_TX_DESC_CMD_EOP | IDPF_TX_DESC_CMD_RS) 140 141 #define IDPF_TX_FLAGS_TSO BIT(0) 142 #define IDPF_TX_FLAGS_IPV4 BIT(1) 143 #define IDPF_TX_FLAGS_IPV6 BIT(2) 144 #define IDPF_TX_FLAGS_TUNNEL BIT(3) 145 #define IDPF_TX_FLAGS_TSYN BIT(4) 146 147 union idpf_tx_flex_desc { 148 struct idpf_flex_tx_desc q; /* queue based scheduling */ 149 struct idpf_flex_tx_sched_desc flow; /* flow based scheduling */ 150 }; 151 152 #define idpf_tx_buf libeth_sqe 153 154 /** 155 * struct idpf_buf_lifo - LIFO for managing OOO completions 156 * @top: Used to know how many buffers are left 157 * @size: Total size of LIFO 158 * @bufs: Backing array 159 */ 160 struct idpf_buf_lifo { 161 u16 top; 162 u16 size; 163 struct idpf_tx_stash **bufs; 164 }; 165 166 /** 167 * struct idpf_tx_offload_params - Offload parameters for a given packet 168 * @tx_flags: Feature flags enabled for this packet 169 * @hdr_offsets: Offset parameter for single queue model 170 * @cd_tunneling: Type of tunneling enabled for single queue model 171 * @tso_len: Total length of payload to segment 172 * @mss: Segment size 173 * @tso_segs: Number of segments to be sent 174 * @tso_hdr_len: Length of headers to be duplicated 175 * @td_cmd: Command field to be inserted into descriptor 176 */ 177 struct idpf_tx_offload_params { 178 u32 tx_flags; 179 180 u32 hdr_offsets; 181 u32 cd_tunneling; 182 183 u32 tso_len; 184 u16 mss; 185 u16 tso_segs; 186 u16 tso_hdr_len; 187 188 u16 td_cmd; 189 }; 190 191 /** 192 * struct idpf_tx_splitq_params 193 * @dtype: General descriptor info 194 * @eop_cmd: Type of EOP 195 * @compl_tag: Associated tag for completion 196 * @td_tag: Descriptor tunneling tag 197 * @offload: Offload parameters 198 */ 199 struct idpf_tx_splitq_params { 200 enum idpf_tx_desc_dtype_value dtype; 201 u16 eop_cmd; 202 union { 203 u16 compl_tag; 204 u16 td_tag; 205 }; 206 207 struct idpf_tx_offload_params offload; 208 }; 209 210 enum idpf_tx_ctx_desc_eipt_offload { 211 IDPF_TX_CTX_EXT_IP_NONE = 0x0, 212 IDPF_TX_CTX_EXT_IP_IPV6 = 0x1, 213 IDPF_TX_CTX_EXT_IP_IPV4_NO_CSUM = 0x2, 214 IDPF_TX_CTX_EXT_IP_IPV4 = 0x3 215 }; 216 217 #define IDPF_TX_COMPLQ_CLEAN_BUDGET 256 218 #define IDPF_TX_MIN_PKT_LEN 17 219 #define IDPF_TX_DESCS_FOR_SKB_DATA_PTR 1 220 #define IDPF_TX_DESCS_PER_CACHE_LINE (L1_CACHE_BYTES / \ 221 sizeof(struct idpf_flex_tx_desc)) 222 #define IDPF_TX_DESCS_FOR_CTX 1 223 /* TX descriptors needed, worst case */ 224 #define IDPF_TX_DESC_NEEDED (MAX_SKB_FRAGS + IDPF_TX_DESCS_FOR_CTX + \ 225 IDPF_TX_DESCS_PER_CACHE_LINE + \ 226 IDPF_TX_DESCS_FOR_SKB_DATA_PTR) 227 228 /* The size limit for a transmit buffer in a descriptor is (16K - 1). 229 * In order to align with the read requests we will align the value to 230 * the nearest 4K which represents our maximum read request size. 231 */ 232 #define IDPF_TX_MAX_READ_REQ_SIZE SZ_4K 233 #define IDPF_TX_MAX_DESC_DATA (SZ_16K - 1) 234 #define IDPF_TX_MAX_DESC_DATA_ALIGNED \ 235 ALIGN_DOWN(IDPF_TX_MAX_DESC_DATA, IDPF_TX_MAX_READ_REQ_SIZE) 236 237 #define idpf_rx_buf libeth_fqe 238 239 #define IDPF_RX_MAX_PTYPE_PROTO_IDS 32 240 #define IDPF_RX_MAX_PTYPE_SZ (sizeof(struct virtchnl2_ptype) + \ 241 (sizeof(u16) * IDPF_RX_MAX_PTYPE_PROTO_IDS)) 242 #define IDPF_RX_PTYPE_HDR_SZ sizeof(struct virtchnl2_get_ptype_info) 243 #define IDPF_RX_MAX_PTYPES_PER_BUF \ 244 DIV_ROUND_DOWN_ULL((IDPF_CTLQ_MAX_BUF_LEN - IDPF_RX_PTYPE_HDR_SZ), \ 245 IDPF_RX_MAX_PTYPE_SZ) 246 247 #define IDPF_GET_PTYPE_SIZE(p) struct_size((p), proto_id, (p)->proto_id_count) 248 249 #define IDPF_TUN_IP_GRE (\ 250 IDPF_PTYPE_TUNNEL_IP |\ 251 IDPF_PTYPE_TUNNEL_IP_GRENAT) 252 253 #define IDPF_TUN_IP_GRE_MAC (\ 254 IDPF_TUN_IP_GRE |\ 255 IDPF_PTYPE_TUNNEL_IP_GRENAT_MAC) 256 257 #define IDPF_RX_MAX_PTYPE 1024 258 #define IDPF_RX_MAX_BASE_PTYPE 256 259 #define IDPF_INVALID_PTYPE_ID 0xFFFF 260 261 enum idpf_tunnel_state { 262 IDPF_PTYPE_TUNNEL_IP = BIT(0), 263 IDPF_PTYPE_TUNNEL_IP_GRENAT = BIT(1), 264 IDPF_PTYPE_TUNNEL_IP_GRENAT_MAC = BIT(2), 265 }; 266 267 struct idpf_ptype_state { 268 bool outer_ip:1; 269 bool outer_frag:1; 270 u8 tunnel_state:6; 271 }; 272 273 /** 274 * enum idpf_queue_flags_t 275 * @__IDPF_Q_GEN_CHK: Queues operating in splitq mode use a generation bit to 276 * identify new descriptor writebacks on the ring. HW sets 277 * the gen bit to 1 on the first writeback of any given 278 * descriptor. After the ring wraps, HW sets the gen bit of 279 * those descriptors to 0, and continues flipping 280 * 0->1 or 1->0 on each ring wrap. SW maintains its own 281 * gen bit to know what value will indicate writebacks on 282 * the next pass around the ring. E.g. it is initialized 283 * to 1 and knows that reading a gen bit of 1 in any 284 * descriptor on the initial pass of the ring indicates a 285 * writeback. It also flips on every ring wrap. 286 * @__IDPF_Q_RFL_GEN_CHK: Refill queues are SW only, so Q_GEN acts as the HW 287 * bit and Q_RFL_GEN is the SW bit. 288 * @__IDPF_Q_FLOW_SCH_EN: Enable flow scheduling 289 * @__IDPF_Q_SW_MARKER: Used to indicate TX queue marker completions 290 * @__IDPF_Q_POLL_MODE: Enable poll mode 291 * @__IDPF_Q_CRC_EN: enable CRC offload in singleq mode 292 * @__IDPF_Q_HSPLIT_EN: enable header split on Rx (splitq) 293 * @__IDPF_Q_PTP: indicates whether the Rx timestamping is enabled for the 294 * queue 295 * @__IDPF_Q_FLAGS_NBITS: Must be last 296 */ 297 enum idpf_queue_flags_t { 298 __IDPF_Q_GEN_CHK, 299 __IDPF_Q_RFL_GEN_CHK, 300 __IDPF_Q_FLOW_SCH_EN, 301 __IDPF_Q_SW_MARKER, 302 __IDPF_Q_POLL_MODE, 303 __IDPF_Q_CRC_EN, 304 __IDPF_Q_HSPLIT_EN, 305 __IDPF_Q_PTP, 306 307 __IDPF_Q_FLAGS_NBITS, 308 }; 309 310 #define idpf_queue_set(f, q) __set_bit(__IDPF_Q_##f, (q)->flags) 311 #define idpf_queue_clear(f, q) __clear_bit(__IDPF_Q_##f, (q)->flags) 312 #define idpf_queue_change(f, q) __change_bit(__IDPF_Q_##f, (q)->flags) 313 #define idpf_queue_has(f, q) test_bit(__IDPF_Q_##f, (q)->flags) 314 315 #define idpf_queue_has_clear(f, q) \ 316 __test_and_clear_bit(__IDPF_Q_##f, (q)->flags) 317 #define idpf_queue_assign(f, q, v) \ 318 __assign_bit(__IDPF_Q_##f, (q)->flags, v) 319 320 /** 321 * struct idpf_vec_regs 322 * @dyn_ctl_reg: Dynamic control interrupt register offset 323 * @itrn_reg: Interrupt Throttling Rate register offset 324 * @itrn_index_spacing: Register spacing between ITR registers of the same 325 * vector 326 */ 327 struct idpf_vec_regs { 328 u32 dyn_ctl_reg; 329 u32 itrn_reg; 330 u32 itrn_index_spacing; 331 }; 332 333 /** 334 * struct idpf_intr_reg 335 * @dyn_ctl: Dynamic control interrupt register 336 * @dyn_ctl_intena_m: Mask for dyn_ctl interrupt enable 337 * @dyn_ctl_intena_msk_m: Mask for dyn_ctl interrupt enable mask 338 * @dyn_ctl_itridx_s: Register bit offset for ITR index 339 * @dyn_ctl_itridx_m: Mask for ITR index 340 * @dyn_ctl_intrvl_s: Register bit offset for ITR interval 341 * @dyn_ctl_wb_on_itr_m: Mask for WB on ITR feature 342 * @dyn_ctl_sw_itridx_ena_m: Mask for SW ITR index 343 * @dyn_ctl_swint_trig_m: Mask for dyn_ctl SW triggered interrupt enable 344 * @rx_itr: RX ITR register 345 * @tx_itr: TX ITR register 346 * @icr_ena: Interrupt cause register offset 347 * @icr_ena_ctlq_m: Mask for ICR 348 */ 349 struct idpf_intr_reg { 350 void __iomem *dyn_ctl; 351 u32 dyn_ctl_intena_m; 352 u32 dyn_ctl_intena_msk_m; 353 u32 dyn_ctl_itridx_s; 354 u32 dyn_ctl_itridx_m; 355 u32 dyn_ctl_intrvl_s; 356 u32 dyn_ctl_wb_on_itr_m; 357 u32 dyn_ctl_sw_itridx_ena_m; 358 u32 dyn_ctl_swint_trig_m; 359 void __iomem *rx_itr; 360 void __iomem *tx_itr; 361 void __iomem *icr_ena; 362 u32 icr_ena_ctlq_m; 363 }; 364 365 /** 366 * struct idpf_q_vector 367 * @vport: Vport back pointer 368 * @num_rxq: Number of RX queues 369 * @num_txq: Number of TX queues 370 * @num_bufq: Number of buffer queues 371 * @num_complq: number of completion queues 372 * @rx: Array of RX queues to service 373 * @tx: Array of TX queues to service 374 * @bufq: Array of buffer queues to service 375 * @complq: array of completion queues 376 * @intr_reg: See struct idpf_intr_reg 377 * @napi: napi handler 378 * @total_events: Number of interrupts processed 379 * @wb_on_itr: whether WB on ITR is enabled 380 * @tx_dim: Data for TX net_dim algorithm 381 * @tx_itr_value: TX interrupt throttling rate 382 * @tx_intr_mode: Dynamic ITR or not 383 * @tx_itr_idx: TX ITR index 384 * @rx_dim: Data for RX net_dim algorithm 385 * @rx_itr_value: RX interrupt throttling rate 386 * @rx_intr_mode: Dynamic ITR or not 387 * @rx_itr_idx: RX ITR index 388 * @v_idx: Vector index 389 */ 390 struct idpf_q_vector { 391 __cacheline_group_begin_aligned(read_mostly); 392 struct idpf_vport *vport; 393 394 u16 num_rxq; 395 u16 num_txq; 396 u16 num_bufq; 397 u16 num_complq; 398 struct idpf_rx_queue **rx; 399 struct idpf_tx_queue **tx; 400 struct idpf_buf_queue **bufq; 401 struct idpf_compl_queue **complq; 402 403 struct idpf_intr_reg intr_reg; 404 __cacheline_group_end_aligned(read_mostly); 405 406 __cacheline_group_begin_aligned(read_write); 407 struct napi_struct napi; 408 u16 total_events; 409 bool wb_on_itr; 410 411 struct dim tx_dim; 412 u16 tx_itr_value; 413 bool tx_intr_mode; 414 u32 tx_itr_idx; 415 416 struct dim rx_dim; 417 u16 rx_itr_value; 418 bool rx_intr_mode; 419 u32 rx_itr_idx; 420 __cacheline_group_end_aligned(read_write); 421 422 __cacheline_group_begin_aligned(cold); 423 u16 v_idx; 424 425 __cacheline_group_end_aligned(cold); 426 }; 427 libeth_cacheline_set_assert(struct idpf_q_vector, 120, 428 24 + sizeof(struct napi_struct) + 429 2 * sizeof(struct dim), 430 8); 431 432 struct idpf_rx_queue_stats { 433 u64_stats_t packets; 434 u64_stats_t bytes; 435 u64_stats_t rsc_pkts; 436 u64_stats_t hw_csum_err; 437 u64_stats_t hsplit_pkts; 438 u64_stats_t hsplit_buf_ovf; 439 u64_stats_t bad_descs; 440 }; 441 442 struct idpf_tx_queue_stats { 443 u64_stats_t packets; 444 u64_stats_t bytes; 445 u64_stats_t lso_pkts; 446 u64_stats_t linearize; 447 u64_stats_t q_busy; 448 u64_stats_t skb_drops; 449 u64_stats_t dma_map_errs; 450 u64_stats_t tstamp_skipped; 451 }; 452 453 #define IDPF_ITR_DYNAMIC 1 454 #define IDPF_ITR_MAX 0x1FE0 455 #define IDPF_ITR_20K 0x0032 456 #define IDPF_ITR_GRAN_S 1 /* Assume ITR granularity is 2us */ 457 #define IDPF_ITR_MASK 0x1FFE /* ITR register value alignment mask */ 458 #define ITR_REG_ALIGN(setting) ((setting) & IDPF_ITR_MASK) 459 #define IDPF_ITR_IS_DYNAMIC(itr_mode) (itr_mode) 460 #define IDPF_ITR_TX_DEF IDPF_ITR_20K 461 #define IDPF_ITR_RX_DEF IDPF_ITR_20K 462 /* Index used for 'SW ITR' update in DYN_CTL register */ 463 #define IDPF_SW_ITR_UPDATE_IDX 2 464 /* Index used for 'No ITR' update in DYN_CTL register */ 465 #define IDPF_NO_ITR_UPDATE_IDX 3 466 #define IDPF_ITR_IDX_SPACING(spacing, dflt) (spacing ? spacing : dflt) 467 #define IDPF_DIM_DEFAULT_PROFILE_IX 1 468 469 /** 470 * struct idpf_txq_stash - Tx buffer stash for Flow-based scheduling mode 471 * @buf_stack: Stack of empty buffers to store buffer info for out of order 472 * buffer completions. See struct idpf_buf_lifo 473 * @sched_buf_hash: Hash table to store buffers 474 */ 475 struct idpf_txq_stash { 476 struct idpf_buf_lifo buf_stack; 477 DECLARE_HASHTABLE(sched_buf_hash, 12); 478 } ____cacheline_aligned; 479 480 /** 481 * struct idpf_rx_queue - software structure representing a receive queue 482 * @rx: universal receive descriptor array 483 * @single_buf: buffer descriptor array in singleq 484 * @desc_ring: virtual descriptor ring address 485 * @bufq_sets: Pointer to the array of buffer queues in splitq mode 486 * @napi: NAPI instance corresponding to this queue (splitq) 487 * @rx_buf: See struct &libeth_fqe 488 * @pp: Page pool pointer in singleq mode 489 * @netdev: &net_device corresponding to this queue 490 * @tail: Tail offset. Used for both queue models single and split. 491 * @flags: See enum idpf_queue_flags_t 492 * @idx: For RX queue, it is used to index to total RX queue across groups and 493 * used for skb reporting. 494 * @desc_count: Number of descriptors 495 * @rxdids: Supported RX descriptor ids 496 * @rx_ptype_lkup: LUT of Rx ptypes 497 * @next_to_use: Next descriptor to use 498 * @next_to_clean: Next descriptor to clean 499 * @next_to_alloc: RX buffer to allocate at 500 * @skb: Pointer to the skb 501 * @truesize: data buffer truesize in singleq 502 * @cached_phc_time: Cached PHC time for the Rx queue 503 * @stats_sync: See struct u64_stats_sync 504 * @q_stats: See union idpf_rx_queue_stats 505 * @q_id: Queue id 506 * @size: Length of descriptor ring in bytes 507 * @dma: Physical address of ring 508 * @q_vector: Backreference to associated vector 509 * @rx_buffer_low_watermark: RX buffer low watermark 510 * @rx_hbuf_size: Header buffer size 511 * @rx_buf_size: Buffer size 512 * @rx_max_pkt_size: RX max packet size 513 */ 514 struct idpf_rx_queue { 515 __cacheline_group_begin_aligned(read_mostly); 516 union { 517 union virtchnl2_rx_desc *rx; 518 struct virtchnl2_singleq_rx_buf_desc *single_buf; 519 520 void *desc_ring; 521 }; 522 union { 523 struct { 524 struct idpf_bufq_set *bufq_sets; 525 struct napi_struct *napi; 526 }; 527 struct { 528 struct libeth_fqe *rx_buf; 529 struct page_pool *pp; 530 }; 531 }; 532 struct net_device *netdev; 533 void __iomem *tail; 534 535 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); 536 u16 idx; 537 u16 desc_count; 538 539 u32 rxdids; 540 const struct libeth_rx_pt *rx_ptype_lkup; 541 __cacheline_group_end_aligned(read_mostly); 542 543 __cacheline_group_begin_aligned(read_write); 544 u16 next_to_use; 545 u16 next_to_clean; 546 u16 next_to_alloc; 547 548 struct sk_buff *skb; 549 u32 truesize; 550 u64 cached_phc_time; 551 552 struct u64_stats_sync stats_sync; 553 struct idpf_rx_queue_stats q_stats; 554 __cacheline_group_end_aligned(read_write); 555 556 __cacheline_group_begin_aligned(cold); 557 u32 q_id; 558 u32 size; 559 dma_addr_t dma; 560 561 struct idpf_q_vector *q_vector; 562 563 u16 rx_buffer_low_watermark; 564 u16 rx_hbuf_size; 565 u16 rx_buf_size; 566 u16 rx_max_pkt_size; 567 __cacheline_group_end_aligned(cold); 568 }; 569 libeth_cacheline_set_assert(struct idpf_rx_queue, 64, 570 88 + sizeof(struct u64_stats_sync), 571 32); 572 573 /** 574 * struct idpf_tx_queue - software structure representing a transmit queue 575 * @base_tx: base Tx descriptor array 576 * @base_ctx: base Tx context descriptor array 577 * @flex_tx: flex Tx descriptor array 578 * @flex_ctx: flex Tx context descriptor array 579 * @desc_ring: virtual descriptor ring address 580 * @tx_buf: See struct idpf_tx_buf 581 * @txq_grp: See struct idpf_txq_group 582 * @dev: Device back pointer for DMA mapping 583 * @tail: Tail offset. Used for both queue models single and split 584 * @flags: See enum idpf_queue_flags_t 585 * @idx: For TX queue, it is used as index to map between TX queue group and 586 * hot path TX pointers stored in vport. Used in both singleq/splitq. 587 * @desc_count: Number of descriptors 588 * @tx_min_pkt_len: Min supported packet length 589 * @compl_tag_gen_s: Completion tag generation bit 590 * The format of the completion tag will change based on the TXQ 591 * descriptor ring size so that we can maintain roughly the same level 592 * of "uniqueness" across all descriptor sizes. For example, if the 593 * TXQ descriptor ring size is 64 (the minimum size supported), the 594 * completion tag will be formatted as below: 595 * 15 6 5 0 596 * -------------------------------- 597 * | GEN=0-1023 |IDX = 0-63| 598 * -------------------------------- 599 * 600 * This gives us 64*1024 = 65536 possible unique values. Similarly, if 601 * the TXQ descriptor ring size is 8160 (the maximum size supported), 602 * the completion tag will be formatted as below: 603 * 15 13 12 0 604 * -------------------------------- 605 * |GEN | IDX = 0-8159 | 606 * -------------------------------- 607 * 608 * This gives us 8*8160 = 65280 possible unique values. 609 * @netdev: &net_device corresponding to this queue 610 * @next_to_use: Next descriptor to use 611 * @next_to_clean: Next descriptor to clean 612 * @cleaned_bytes: Splitq only, TXQ only: When a TX completion is received on 613 * the TX completion queue, it can be for any TXQ associated 614 * with that completion queue. This means we can clean up to 615 * N TXQs during a single call to clean the completion queue. 616 * cleaned_bytes|pkts tracks the clean stats per TXQ during 617 * that single call to clean the completion queue. By doing so, 618 * we can update BQL with aggregate cleaned stats for each TXQ 619 * only once at the end of the cleaning routine. 620 * @clean_budget: singleq only, queue cleaning budget 621 * @cleaned_pkts: Number of packets cleaned for the above said case 622 * @tx_max_bufs: Max buffers that can be transmitted with scatter-gather 623 * @stash: Tx buffer stash for Flow-based scheduling mode 624 * @compl_tag_bufid_m: Completion tag buffer id mask 625 * @compl_tag_cur_gen: Used to keep track of current completion tag generation 626 * @compl_tag_gen_max: To determine when compl_tag_cur_gen should be reset 627 * @cached_tstamp_caps: Tx timestamp capabilities negotiated with the CP 628 * @tstamp_task: Work that handles Tx timestamp read 629 * @stats_sync: See struct u64_stats_sync 630 * @q_stats: See union idpf_tx_queue_stats 631 * @q_id: Queue id 632 * @size: Length of descriptor ring in bytes 633 * @dma: Physical address of ring 634 * @q_vector: Backreference to associated vector 635 */ 636 struct idpf_tx_queue { 637 __cacheline_group_begin_aligned(read_mostly); 638 union { 639 struct idpf_base_tx_desc *base_tx; 640 struct idpf_base_tx_ctx_desc *base_ctx; 641 union idpf_tx_flex_desc *flex_tx; 642 union idpf_flex_tx_ctx_desc *flex_ctx; 643 644 void *desc_ring; 645 }; 646 struct libeth_sqe *tx_buf; 647 struct idpf_txq_group *txq_grp; 648 struct device *dev; 649 void __iomem *tail; 650 651 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); 652 u16 idx; 653 u16 desc_count; 654 655 u16 tx_min_pkt_len; 656 u16 compl_tag_gen_s; 657 658 struct net_device *netdev; 659 __cacheline_group_end_aligned(read_mostly); 660 661 __cacheline_group_begin_aligned(read_write); 662 u16 next_to_use; 663 u16 next_to_clean; 664 665 union { 666 u32 cleaned_bytes; 667 u32 clean_budget; 668 }; 669 u16 cleaned_pkts; 670 671 u16 tx_max_bufs; 672 struct idpf_txq_stash *stash; 673 674 u16 compl_tag_bufid_m; 675 u16 compl_tag_cur_gen; 676 u16 compl_tag_gen_max; 677 678 struct idpf_ptp_vport_tx_tstamp_caps *cached_tstamp_caps; 679 struct work_struct *tstamp_task; 680 681 struct u64_stats_sync stats_sync; 682 struct idpf_tx_queue_stats q_stats; 683 __cacheline_group_end_aligned(read_write); 684 685 __cacheline_group_begin_aligned(cold); 686 u32 q_id; 687 u32 size; 688 dma_addr_t dma; 689 690 struct idpf_q_vector *q_vector; 691 __cacheline_group_end_aligned(cold); 692 }; 693 libeth_cacheline_set_assert(struct idpf_tx_queue, 64, 694 112 + sizeof(struct u64_stats_sync), 695 24); 696 697 /** 698 * struct idpf_buf_queue - software structure representing a buffer queue 699 * @split_buf: buffer descriptor array 700 * @hdr_buf: &libeth_fqe for header buffers 701 * @hdr_pp: &page_pool for header buffers 702 * @buf: &libeth_fqe for data buffers 703 * @pp: &page_pool for data buffers 704 * @tail: Tail offset 705 * @flags: See enum idpf_queue_flags_t 706 * @desc_count: Number of descriptors 707 * @next_to_use: Next descriptor to use 708 * @next_to_clean: Next descriptor to clean 709 * @next_to_alloc: RX buffer to allocate at 710 * @hdr_truesize: truesize for buffer headers 711 * @truesize: truesize for data buffers 712 * @q_id: Queue id 713 * @size: Length of descriptor ring in bytes 714 * @dma: Physical address of ring 715 * @q_vector: Backreference to associated vector 716 * @rx_buffer_low_watermark: RX buffer low watermark 717 * @rx_hbuf_size: Header buffer size 718 * @rx_buf_size: Buffer size 719 */ 720 struct idpf_buf_queue { 721 __cacheline_group_begin_aligned(read_mostly); 722 struct virtchnl2_splitq_rx_buf_desc *split_buf; 723 struct libeth_fqe *hdr_buf; 724 struct page_pool *hdr_pp; 725 struct libeth_fqe *buf; 726 struct page_pool *pp; 727 void __iomem *tail; 728 729 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); 730 u32 desc_count; 731 __cacheline_group_end_aligned(read_mostly); 732 733 __cacheline_group_begin_aligned(read_write); 734 u32 next_to_use; 735 u32 next_to_clean; 736 u32 next_to_alloc; 737 738 u32 hdr_truesize; 739 u32 truesize; 740 __cacheline_group_end_aligned(read_write); 741 742 __cacheline_group_begin_aligned(cold); 743 u32 q_id; 744 u32 size; 745 dma_addr_t dma; 746 747 struct idpf_q_vector *q_vector; 748 749 u16 rx_buffer_low_watermark; 750 u16 rx_hbuf_size; 751 u16 rx_buf_size; 752 __cacheline_group_end_aligned(cold); 753 }; 754 libeth_cacheline_set_assert(struct idpf_buf_queue, 64, 24, 32); 755 756 /** 757 * struct idpf_compl_queue - software structure representing a completion queue 758 * @comp: completion descriptor array 759 * @txq_grp: See struct idpf_txq_group 760 * @flags: See enum idpf_queue_flags_t 761 * @desc_count: Number of descriptors 762 * @clean_budget: queue cleaning budget 763 * @netdev: &net_device corresponding to this queue 764 * @next_to_use: Next descriptor to use. Relevant in both split & single txq 765 * and bufq. 766 * @next_to_clean: Next descriptor to clean 767 * @num_completions: Only relevant for TX completion queue. It tracks the 768 * number of completions received to compare against the 769 * number of completions pending, as accumulated by the 770 * TX queues. 771 * @q_id: Queue id 772 * @size: Length of descriptor ring in bytes 773 * @dma: Physical address of ring 774 * @q_vector: Backreference to associated vector 775 */ 776 struct idpf_compl_queue { 777 __cacheline_group_begin_aligned(read_mostly); 778 struct idpf_splitq_tx_compl_desc *comp; 779 struct idpf_txq_group *txq_grp; 780 781 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); 782 u32 desc_count; 783 784 u32 clean_budget; 785 struct net_device *netdev; 786 __cacheline_group_end_aligned(read_mostly); 787 788 __cacheline_group_begin_aligned(read_write); 789 u32 next_to_use; 790 u32 next_to_clean; 791 792 aligned_u64 num_completions; 793 __cacheline_group_end_aligned(read_write); 794 795 __cacheline_group_begin_aligned(cold); 796 u32 q_id; 797 u32 size; 798 dma_addr_t dma; 799 800 struct idpf_q_vector *q_vector; 801 __cacheline_group_end_aligned(cold); 802 }; 803 libeth_cacheline_set_assert(struct idpf_compl_queue, 40, 16, 24); 804 805 /** 806 * struct idpf_sw_queue 807 * @ring: Pointer to the ring 808 * @flags: See enum idpf_queue_flags_t 809 * @desc_count: Descriptor count 810 * @next_to_use: Buffer to allocate at 811 * @next_to_clean: Next descriptor to clean 812 * 813 * Software queues are used in splitq mode to manage buffers between rxq 814 * producer and the bufq consumer. These are required in order to maintain a 815 * lockless buffer management system and are strictly software only constructs. 816 */ 817 struct idpf_sw_queue { 818 __cacheline_group_begin_aligned(read_mostly); 819 u32 *ring; 820 821 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); 822 u32 desc_count; 823 __cacheline_group_end_aligned(read_mostly); 824 825 __cacheline_group_begin_aligned(read_write); 826 u32 next_to_use; 827 u32 next_to_clean; 828 __cacheline_group_end_aligned(read_write); 829 }; 830 libeth_cacheline_group_assert(struct idpf_sw_queue, read_mostly, 24); 831 libeth_cacheline_group_assert(struct idpf_sw_queue, read_write, 8); 832 libeth_cacheline_struct_assert(struct idpf_sw_queue, 24, 8); 833 834 /** 835 * struct idpf_rxq_set 836 * @rxq: RX queue 837 * @refillq: pointers to refill queues 838 * 839 * Splitq only. idpf_rxq_set associates an rxq with at an array of refillqs. 840 * Each rxq needs a refillq to return used buffers back to the respective bufq. 841 * Bufqs then clean these refillqs for buffers to give to hardware. 842 */ 843 struct idpf_rxq_set { 844 struct idpf_rx_queue rxq; 845 struct idpf_sw_queue *refillq[IDPF_MAX_BUFQS_PER_RXQ_GRP]; 846 }; 847 848 /** 849 * struct idpf_bufq_set 850 * @bufq: Buffer queue 851 * @num_refillqs: Number of refill queues. This is always equal to num_rxq_sets 852 * in idpf_rxq_group. 853 * @refillqs: Pointer to refill queues array. 854 * 855 * Splitq only. idpf_bufq_set associates a bufq to an array of refillqs. 856 * In this bufq_set, there will be one refillq for each rxq in this rxq_group. 857 * Used buffers received by rxqs will be put on refillqs which bufqs will 858 * clean to return new buffers back to hardware. 859 * 860 * Buffers needed by some number of rxqs associated in this rxq_group are 861 * managed by at most two bufqs (depending on performance configuration). 862 */ 863 struct idpf_bufq_set { 864 struct idpf_buf_queue bufq; 865 int num_refillqs; 866 struct idpf_sw_queue *refillqs; 867 }; 868 869 /** 870 * struct idpf_rxq_group 871 * @vport: Vport back pointer 872 * @singleq: Struct with single queue related members 873 * @singleq.num_rxq: Number of RX queues associated 874 * @singleq.rxqs: Array of RX queue pointers 875 * @splitq: Struct with split queue related members 876 * @splitq.num_rxq_sets: Number of RX queue sets 877 * @splitq.rxq_sets: Array of RX queue sets 878 * @splitq.bufq_sets: Buffer queue set pointer 879 * 880 * In singleq mode, an rxq_group is simply an array of rxqs. In splitq, a 881 * rxq_group contains all the rxqs, bufqs and refillqs needed to 882 * manage buffers in splitq mode. 883 */ 884 struct idpf_rxq_group { 885 struct idpf_vport *vport; 886 887 union { 888 struct { 889 u16 num_rxq; 890 struct idpf_rx_queue *rxqs[IDPF_LARGE_MAX_Q]; 891 } singleq; 892 struct { 893 u16 num_rxq_sets; 894 struct idpf_rxq_set *rxq_sets[IDPF_LARGE_MAX_Q]; 895 struct idpf_bufq_set *bufq_sets; 896 } splitq; 897 }; 898 }; 899 900 /** 901 * struct idpf_txq_group 902 * @vport: Vport back pointer 903 * @num_txq: Number of TX queues associated 904 * @txqs: Array of TX queue pointers 905 * @stashes: array of OOO stashes for the queues 906 * @complq: Associated completion queue pointer, split queue only 907 * @num_completions_pending: Total number of completions pending for the 908 * completion queue, acculumated for all TX queues 909 * associated with that completion queue. 910 * 911 * Between singleq and splitq, a txq_group is largely the same except for the 912 * complq. In splitq a single complq is responsible for handling completions 913 * for some number of txqs associated in this txq_group. 914 */ 915 struct idpf_txq_group { 916 struct idpf_vport *vport; 917 918 u16 num_txq; 919 struct idpf_tx_queue *txqs[IDPF_LARGE_MAX_Q]; 920 struct idpf_txq_stash *stashes; 921 922 struct idpf_compl_queue *complq; 923 924 aligned_u64 num_completions_pending; 925 }; 926 927 static inline int idpf_q_vector_to_mem(const struct idpf_q_vector *q_vector) 928 { 929 u32 cpu; 930 931 if (!q_vector) 932 return NUMA_NO_NODE; 933 934 cpu = cpumask_first(&q_vector->napi.config->affinity_mask); 935 936 return cpu < nr_cpu_ids ? cpu_to_mem(cpu) : NUMA_NO_NODE; 937 } 938 939 /** 940 * idpf_size_to_txd_count - Get number of descriptors needed for large Tx frag 941 * @size: transmit request size in bytes 942 * 943 * In the case where a large frag (>= 16K) needs to be split across multiple 944 * descriptors, we need to assume that we can have no more than 12K of data 945 * per descriptor due to hardware alignment restrictions (4K alignment). 946 */ 947 static inline u32 idpf_size_to_txd_count(unsigned int size) 948 { 949 return DIV_ROUND_UP(size, IDPF_TX_MAX_DESC_DATA_ALIGNED); 950 } 951 952 /** 953 * idpf_tx_singleq_build_ctob - populate command tag offset and size 954 * @td_cmd: Command to be filled in desc 955 * @td_offset: Offset to be filled in desc 956 * @size: Size of the buffer 957 * @td_tag: td tag to be filled 958 * 959 * Returns the 64 bit value populated with the input parameters 960 */ 961 static inline __le64 idpf_tx_singleq_build_ctob(u64 td_cmd, u64 td_offset, 962 unsigned int size, u64 td_tag) 963 { 964 return cpu_to_le64(IDPF_TX_DESC_DTYPE_DATA | 965 (td_cmd << IDPF_TXD_QW1_CMD_S) | 966 (td_offset << IDPF_TXD_QW1_OFFSET_S) | 967 ((u64)size << IDPF_TXD_QW1_TX_BUF_SZ_S) | 968 (td_tag << IDPF_TXD_QW1_L2TAG1_S)); 969 } 970 971 void idpf_tx_splitq_build_ctb(union idpf_tx_flex_desc *desc, 972 struct idpf_tx_splitq_params *params, 973 u16 td_cmd, u16 size); 974 void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc, 975 struct idpf_tx_splitq_params *params, 976 u16 td_cmd, u16 size); 977 /** 978 * idpf_tx_splitq_build_desc - determine which type of data descriptor to build 979 * @desc: descriptor to populate 980 * @params: pointer to tx params struct 981 * @td_cmd: command to be filled in desc 982 * @size: size of buffer 983 */ 984 static inline void idpf_tx_splitq_build_desc(union idpf_tx_flex_desc *desc, 985 struct idpf_tx_splitq_params *params, 986 u16 td_cmd, u16 size) 987 { 988 if (params->dtype == IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2) 989 idpf_tx_splitq_build_ctb(desc, params, td_cmd, size); 990 else 991 idpf_tx_splitq_build_flow_desc(desc, params, td_cmd, size); 992 } 993 994 /** 995 * idpf_vport_intr_set_wb_on_itr - enable descriptor writeback on disabled interrupts 996 * @q_vector: pointer to queue vector struct 997 */ 998 static inline void idpf_vport_intr_set_wb_on_itr(struct idpf_q_vector *q_vector) 999 { 1000 struct idpf_intr_reg *reg; 1001 1002 if (q_vector->wb_on_itr) 1003 return; 1004 1005 q_vector->wb_on_itr = true; 1006 reg = &q_vector->intr_reg; 1007 1008 writel(reg->dyn_ctl_wb_on_itr_m | reg->dyn_ctl_intena_msk_m | 1009 (IDPF_NO_ITR_UPDATE_IDX << reg->dyn_ctl_itridx_s), 1010 reg->dyn_ctl); 1011 } 1012 1013 int idpf_vport_singleq_napi_poll(struct napi_struct *napi, int budget); 1014 void idpf_vport_init_num_qs(struct idpf_vport *vport, 1015 struct virtchnl2_create_vport *vport_msg); 1016 void idpf_vport_calc_num_q_desc(struct idpf_vport *vport); 1017 int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_index, 1018 struct virtchnl2_create_vport *vport_msg, 1019 struct idpf_vport_max_q *max_q); 1020 void idpf_vport_calc_num_q_groups(struct idpf_vport *vport); 1021 int idpf_vport_queues_alloc(struct idpf_vport *vport); 1022 void idpf_vport_queues_rel(struct idpf_vport *vport); 1023 void idpf_vport_intr_rel(struct idpf_vport *vport); 1024 int idpf_vport_intr_alloc(struct idpf_vport *vport); 1025 void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector); 1026 void idpf_vport_intr_deinit(struct idpf_vport *vport); 1027 int idpf_vport_intr_init(struct idpf_vport *vport); 1028 void idpf_vport_intr_ena(struct idpf_vport *vport); 1029 int idpf_config_rss(struct idpf_vport *vport); 1030 int idpf_init_rss(struct idpf_vport *vport); 1031 void idpf_deinit_rss(struct idpf_vport *vport); 1032 int idpf_rx_bufs_init_all(struct idpf_vport *vport); 1033 void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb, 1034 unsigned int size); 1035 struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size); 1036 void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val, 1037 bool xmit_more); 1038 unsigned int idpf_size_to_txd_count(unsigned int size); 1039 netdev_tx_t idpf_tx_drop_skb(struct idpf_tx_queue *tx_q, struct sk_buff *skb); 1040 void idpf_tx_dma_map_error(struct idpf_tx_queue *txq, struct sk_buff *skb, 1041 struct idpf_tx_buf *first, u16 ring_idx); 1042 unsigned int idpf_tx_desc_count_required(struct idpf_tx_queue *txq, 1043 struct sk_buff *skb); 1044 void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue); 1045 netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, 1046 struct idpf_tx_queue *tx_q); 1047 netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev); 1048 bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_rx_queue *rxq, 1049 u16 cleaned_count); 1050 int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off); 1051 1052 #endif /* !_IDPF_TXRX_H_ */ 1053