1 /*
2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #ifndef __MLX5_EN_H__
33 #define __MLX5_EN_H__
34
35 #include <linux/if_vlan.h>
36 #include <linux/etherdevice.h>
37 #include <linux/timecounter.h>
38 #include <linux/net_tstamp.h>
39 #include <linux/crash_dump.h>
40 #include <linux/mlx5/driver.h>
41 #include <linux/mlx5/qp.h>
42 #include <linux/mlx5/cq.h>
43 #include <linux/mlx5/port.h>
44 #include <linux/mlx5/vport.h>
45 #include <linux/mlx5/transobj.h>
46 #include <linux/mlx5/fs.h>
47 #include <linux/rhashtable.h>
48 #include <net/udp_tunnel.h>
49 #include <net/switchdev.h>
50 #include <net/psp/types.h>
51 #include <net/xdp.h>
52 #include <linux/dim.h>
53 #include <linux/bits.h>
54 #include "wq.h"
55 #include "mlx5_core.h"
56 #include "en_stats.h"
57 #include "en/dcbnl.h"
58 #include "en/fs.h"
59 #include "en/qos.h"
60 #include "lib/hv_vhca.h"
61 #include "lib/clock.h"
62 #include "en/rx_res.h"
63 #include "en/selq.h"
64 #include "lib/sd.h"
65
66 extern const struct net_device_ops mlx5e_netdev_ops;
67 struct page_pool;
68
69 #define MLX5E_METADATA_ETHER_TYPE (0x8CE4)
70 #define MLX5E_METADATA_ETHER_LEN 8
71
72 #define MLX5E_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN)
73
74 #define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu))
75 #define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu))
76
77 #define MLX5E_MAX_NUM_MQPRIO_CH_TC TC_QOPT_MAX_QUEUE
78
79 #define MLX5_RX_HEADROOM NET_SKB_PAD
80 #define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \
81 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
82
83 #define MLX5E_PAGECNT_BIAS_MAX U16_MAX
84 #define MLX5E_RX_MAX_HEAD (256)
85 #define MLX5E_XDP_LOG_MAX_LINEAR_SZ \
86 order_base_2(MLX5_SKB_FRAG_SZ(XDP_PACKET_HEADROOM + MLX5E_RX_MAX_HEAD))
87
88 #define MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE (8)
89 #define MLX5E_SHAMPO_WQ_HEADER_PER_PAGE \
90 (PAGE_SIZE >> MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE)
91 #define MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE \
92 (PAGE_SHIFT - MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE)
93 #define MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE_SHIFT (6)
94 #define MLX5E_SHAMPO_WQ_RESRV_SIZE_BASE_SHIFT (12)
95 #define MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE (16)
96 #define MLX5E_SHAMPO_WQ_RESRV_SIZE BIT(MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE)
97
98 #define MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev) \
99 (6 + MLX5_CAP_GEN(mdev, cache_line_128byte)) /* HW restriction */
100 #define MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, req) \
101 max_t(u32, MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev), req)
102 #define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) \
103 MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, order_base_2(MLX5E_RX_MAX_HEAD))
104
105 /* Keep in sync with mlx5e_mpwrq_log_wqe_sz.
106 * These are theoretical maximums, which can be further restricted by
107 * capabilities. These values are used for static resource allocations and
108 * sanity checks.
109 * MLX5_SEND_WQE_MAX_SIZE is a bit bigger than the maximum cacheline-aligned WQE
110 * size actually used at runtime, but it's not a problem when calculating static
111 * array sizes.
112 */
113 #define MLX5_UMR_MAX_FLEX_SPACE \
114 (ALIGN_DOWN(MLX5_SEND_WQE_MAX_SIZE - sizeof(struct mlx5e_umr_wqe), \
115 MLX5_UMR_FLEX_ALIGNMENT))
116 #define MLX5_MPWRQ_MAX_PAGES_PER_WQE \
117 rounddown_pow_of_two(MLX5_UMR_MAX_FLEX_SPACE / sizeof(struct mlx5_mtt))
118
119 #define MLX5E_MAX_RQ_NUM_MTTS \
120 (ALIGN_DOWN(U16_MAX, 4) * 2) /* Fits into u16 and aligned by WQEBB. */
121 #define MLX5E_MAX_RQ_NUM_KSMS (U16_MAX - 1) /* So that num_ksms fits into u16. */
122 #define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024))
123
124 #define MLX5E_MIN_SKB_FRAG_SZ (MLX5_SKB_FRAG_SZ(MLX5_RX_HEADROOM))
125 #define MLX5E_LOG_MAX_RX_WQE_BULK \
126 (ilog2(PAGE_SIZE / roundup_pow_of_two(MLX5E_MIN_SKB_FRAG_SZ)))
127
128 #define MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE 0x6
129 #define MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE 0xa
130 #define MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE 0xd
131
132 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE (1 + MLX5E_LOG_MAX_RX_WQE_BULK)
133 #define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE 0xa
134 #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE 0xd
135
136 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW 0x2
137
138 #define MLX5E_DEFAULT_LRO_TIMEOUT 32
139 #define MLX5E_DEFAULT_SHAMPO_TIMEOUT 1024
140
141 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC 0x10
142 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE 0x3
143 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS 0x20
144 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC 0x10
145 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE 0x10
146 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS 0x20
147 #define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES 0x80
148 #define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES_MPW 0x2
149
150 #define MLX5E_MIN_NUM_CHANNELS 0x1
151 #define MLX5E_MAX_NUM_CHANNELS 256
152 #define MLX5E_TX_CQ_POLL_BUDGET 128
153 #define MLX5E_TX_XSK_POLL_BUDGET 64
154 #define MLX5E_SQ_RECOVER_MIN_INTERVAL 500 /* msecs */
155
156 #define mlx5e_state_dereference(priv, p) \
157 rcu_dereference_protected((p), lockdep_is_held(&(priv)->state_lock))
158
159 enum mlx5e_devcom_events {
160 MPV_DEVCOM_MASTER_UP,
161 MPV_DEVCOM_MASTER_DOWN,
162 MPV_DEVCOM_IPSEC_MASTER_UP,
163 MPV_DEVCOM_IPSEC_MASTER_DOWN,
164 };
165
mlx5e_get_num_lag_ports(struct mlx5_core_dev * mdev)166 static inline u8 mlx5e_get_num_lag_ports(struct mlx5_core_dev *mdev)
167 {
168 if (mlx5_lag_is_lacp_owner(mdev))
169 return 1;
170
171 return clamp_t(u8, MLX5_CAP_GEN(mdev, num_lag_ports), 1, MLX5_MAX_PORTS);
172 }
173
mlx5_min_rx_wqes(int wq_type,u32 wq_size)174 static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size)
175 {
176 switch (wq_type) {
177 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
178 return min_t(u16, MLX5E_PARAMS_DEFAULT_MIN_RX_WQES_MPW,
179 wq_size / 2);
180 default:
181 return min_t(u16, MLX5E_PARAMS_DEFAULT_MIN_RX_WQES,
182 wq_size / 2);
183 }
184 }
185
186 /* Use this function to get max num channels (rxqs/txqs) only to create netdev */
187 static inline unsigned int
mlx5e_get_max_num_channels(struct mlx5_core_dev * mdev)188 mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
189 {
190 return is_kdump_kernel() ?
191 MLX5E_MIN_NUM_CHANNELS :
192 min3(mlx5_comp_vectors_max(mdev), (u32)MLX5E_MAX_NUM_CHANNELS,
193 (u32)(1 << MLX5_CAP_GEN(mdev, log_max_rqt_size)));
194 }
195
196 /* The maximum WQE size can be retrieved by max_wqe_sz_sq in
197 * bytes units. Driver hardens the limitation to 1KB (16
198 * WQEBBs), unless firmware capability is stricter.
199 */
mlx5e_get_max_sq_wqebbs(struct mlx5_core_dev * mdev)200 static inline u8 mlx5e_get_max_sq_wqebbs(struct mlx5_core_dev *mdev)
201 {
202 BUILD_BUG_ON(MLX5_SEND_WQE_MAX_WQEBBS > U8_MAX);
203
204 return (u8)min_t(u16, MLX5_SEND_WQE_MAX_WQEBBS,
205 MLX5_CAP_GEN(mdev, max_wqe_sz_sq) / MLX5_SEND_WQE_BB);
206 }
207
mlx5e_get_max_sq_aligned_wqebbs(struct mlx5_core_dev * mdev)208 static inline u8 mlx5e_get_max_sq_aligned_wqebbs(struct mlx5_core_dev *mdev)
209 {
210 /* The return value will be multiplied by MLX5_SEND_WQEBB_NUM_DS.
211 * Since max_sq_wqebbs may be up to MLX5_SEND_WQE_MAX_WQEBBS == 16,
212 * see mlx5e_get_max_sq_wqebbs(), the multiplication (16 * 4 == 64)
213 * overflows the 6-bit DS field of Ctrl Segment. Use a bound lower
214 * than MLX5_SEND_WQE_MAX_WQEBBS to let a full-session WQE be
215 * cache-aligned.
216 */
217 u8 wqebbs = mlx5e_get_max_sq_wqebbs(mdev);
218
219 wqebbs = min_t(u8, wqebbs, MLX5_SEND_WQE_MAX_WQEBBS - 1);
220 #if L1_CACHE_BYTES >= 128
221 wqebbs = ALIGN_DOWN(wqebbs, 2);
222 #endif
223 return wqebbs;
224 }
225
226 struct mlx5e_tx_wqe {
227 struct mlx5_wqe_ctrl_seg ctrl;
228 struct mlx5_wqe_eth_seg eth;
229 struct mlx5_wqe_data_seg data[];
230 };
231
232 struct mlx5e_rx_wqe_ll {
233 struct mlx5_wqe_srq_next_seg next;
234 struct mlx5_wqe_data_seg data[];
235 };
236
237 struct mlx5e_rx_wqe_cyc {
238 DECLARE_FLEX_ARRAY(struct mlx5_wqe_data_seg, data);
239 };
240
241 struct mlx5e_umr_wqe_hdr {
242 struct mlx5_wqe_ctrl_seg ctrl;
243 struct mlx5_wqe_umr_ctrl_seg uctrl;
244 struct mlx5_mkey_seg mkc;
245 };
246
247 struct mlx5e_umr_wqe {
248 struct mlx5e_umr_wqe_hdr hdr;
249 union {
250 DECLARE_FLEX_ARRAY(struct mlx5_mtt, inline_mtts);
251 DECLARE_FLEX_ARRAY(struct mlx5_klm, inline_klms);
252 DECLARE_FLEX_ARRAY(struct mlx5_ksm, inline_ksms);
253 };
254 };
255 static_assert(offsetof(struct mlx5e_umr_wqe, inline_mtts) == sizeof(struct mlx5e_umr_wqe_hdr),
256 "struct members should be included in struct mlx5e_umr_wqe_hdr, not in struct mlx5e_umr_wqe");
257
258 enum mlx5e_priv_flag {
259 MLX5E_PFLAG_RX_CQE_BASED_MODER,
260 MLX5E_PFLAG_TX_CQE_BASED_MODER,
261 MLX5E_PFLAG_RX_CQE_COMPRESS,
262 MLX5E_PFLAG_RX_STRIDING_RQ,
263 MLX5E_PFLAG_RX_NO_CSUM_COMPLETE,
264 MLX5E_PFLAG_XDP_TX_MPWQE,
265 MLX5E_PFLAG_SKB_TX_MPWQE,
266 MLX5E_PFLAG_TX_PORT_TS,
267 MLX5E_NUM_PFLAGS, /* Keep last */
268 };
269
270 #define MLX5E_SET_PFLAG(params, pflag, enable) \
271 do { \
272 if (enable) \
273 (params)->pflags |= BIT(pflag); \
274 else \
275 (params)->pflags &= ~(BIT(pflag)); \
276 } while (0)
277
278 #define MLX5E_GET_PFLAG(params, pflag) (!!((params)->pflags & (BIT(pflag))))
279
280 enum packet_merge {
281 MLX5E_PACKET_MERGE_NONE,
282 MLX5E_PACKET_MERGE_LRO,
283 MLX5E_PACKET_MERGE_SHAMPO,
284 };
285
286 struct mlx5e_packet_merge_param {
287 enum packet_merge type;
288 u32 timeout;
289 };
290
291 struct mlx5e_params {
292 u8 log_sq_size;
293 u8 rq_wq_type;
294 u8 log_rq_mtu_frames;
295 u16 num_channels;
296 struct {
297 u16 mode;
298 u8 num_tc;
299 struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
300 struct {
301 u64 max_rate[TC_MAX_QUEUE];
302 u32 hw_id[TC_MAX_QUEUE];
303 } channel;
304 } mqprio;
305 bool rx_cqe_compress_def;
306 struct dim_cq_moder rx_cq_moderation;
307 struct dim_cq_moder tx_cq_moderation;
308 struct mlx5e_packet_merge_param packet_merge;
309 u8 tx_min_inline_mode;
310 bool vlan_strip_disable;
311 bool scatter_fcs_en;
312 bool rx_dim_enabled;
313 bool tx_dim_enabled;
314 bool rx_moder_use_cqe_mode;
315 bool tx_moder_use_cqe_mode;
316 u32 pflags;
317 struct bpf_prog *xdp_prog;
318 struct mlx5e_xsk *xsk;
319 unsigned int sw_mtu;
320 int hard_mtu;
321 bool ptp_rx;
322 __be32 terminate_lkey_be;
323 };
324
mlx5e_get_dcb_num_tc(struct mlx5e_params * params)325 static inline u8 mlx5e_get_dcb_num_tc(struct mlx5e_params *params)
326 {
327 return params->mqprio.mode == TC_MQPRIO_MODE_DCB ?
328 params->mqprio.num_tc : 1;
329 }
330
331 /* Keep this enum consistent with the corresponding strings array
332 * declared in en/reporter_rx.c
333 */
334 enum {
335 MLX5E_RQ_STATE_ENABLED = 0,
336 MLX5E_RQ_STATE_RECOVERING,
337 MLX5E_RQ_STATE_DIM,
338 MLX5E_RQ_STATE_NO_CSUM_COMPLETE,
339 MLX5E_RQ_STATE_CSUM_FULL, /* cqe_csum_full hw bit is set */
340 MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, /* set when mini_cqe_resp_stride_index cap is used */
341 MLX5E_RQ_STATE_SHAMPO, /* set when SHAMPO cap is used */
342 MLX5E_RQ_STATE_MINI_CQE_ENHANCED, /* set when enhanced mini_cqe_cap is used */
343 MLX5E_RQ_STATE_XSK, /* set to indicate an xsk rq */
344 MLX5E_NUM_RQ_STATES, /* Must be kept last */
345 };
346
347 struct mlx5e_cq {
348 /* data path - accessed per cqe */
349 struct mlx5_cqwq wq;
350
351 /* data path - accessed per napi poll */
352 u16 event_ctr;
353 struct napi_struct *napi;
354 struct mlx5_uars_page *uar;
355 struct mlx5_core_cq mcq;
356 struct mlx5e_ch_stats *ch_stats;
357
358 /* control */
359 struct net_device *netdev;
360 struct mlx5_core_dev *mdev;
361 struct workqueue_struct *workqueue;
362 struct mlx5_wq_ctrl wq_ctrl;
363 } ____cacheline_aligned_in_smp;
364
365 struct mlx5e_cq_decomp {
366 /* cqe decompression */
367 struct mlx5_cqe64 title;
368 struct mlx5_mini_cqe8 mini_arr[MLX5_MINI_CQE_ARRAY_SIZE];
369 u8 mini_arr_idx;
370 u16 left;
371 u16 wqe_counter;
372 bool last_cqe_title;
373 } ____cacheline_aligned_in_smp;
374
375 enum mlx5e_dma_map_type {
376 MLX5E_DMA_MAP_SINGLE,
377 MLX5E_DMA_MAP_PAGE
378 };
379
380 struct mlx5e_sq_dma {
381 dma_addr_t addr;
382 u32 size;
383 enum mlx5e_dma_map_type type;
384 };
385
386 /* Keep this enum consistent with the corresponding strings array
387 * declared in en/reporter_tx.c
388 */
389 enum {
390 MLX5E_SQ_STATE_ENABLED = 0,
391 MLX5E_SQ_STATE_MPWQE,
392 MLX5E_SQ_STATE_RECOVERING,
393 MLX5E_SQ_STATE_IPSEC,
394 MLX5E_SQ_STATE_DIM,
395 MLX5E_SQ_STATE_PENDING_XSK_TX,
396 MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC,
397 MLX5E_SQ_STATE_LOCK_NEEDED,
398 MLX5E_NUM_SQ_STATES, /* Must be kept last */
399 };
400
401 struct mlx5e_tx_mpwqe {
402 /* Current MPWQE session */
403 struct mlx5e_tx_wqe *wqe;
404 u32 bytes_count;
405 u8 ds_count;
406 u8 ds_count_max;
407 u8 pkt_count;
408 u8 inline_on;
409 };
410
411 struct mlx5e_skb_fifo {
412 struct sk_buff **fifo;
413 u16 *pc;
414 u16 *cc;
415 u16 mask;
416 };
417
418 struct mlx5e_ptpsq;
419
420 struct mlx5e_txqsq {
421 /* data path */
422
423 /* dirtied @completion */
424 u16 cc;
425 u16 skb_fifo_cc;
426 u32 dma_fifo_cc;
427 struct dim *dim; /* Adaptive Moderation */
428
429 /* dirtied @xmit */
430 u16 pc ____cacheline_aligned_in_smp;
431 u16 skb_fifo_pc;
432 u32 dma_fifo_pc;
433 struct mlx5e_tx_mpwqe mpwqe;
434
435 struct mlx5e_cq cq;
436
437 /* read only */
438 struct mlx5_wq_cyc wq;
439 u32 dma_fifo_mask;
440 struct mlx5e_sq_stats *stats;
441 struct {
442 struct mlx5e_sq_dma *dma_fifo;
443 struct mlx5e_skb_fifo skb_fifo;
444 struct mlx5e_tx_wqe_info *wqe_info;
445 } db;
446 void __iomem *uar_map;
447 struct netdev_queue *txq;
448 u32 sqn;
449 u16 stop_room;
450 u8 max_sq_mpw_wqebbs;
451 u8 min_inline_mode;
452 struct device *pdev;
453 __be32 mkey_be;
454 unsigned long state;
455 unsigned int hw_mtu;
456 struct mlx5_clock *clock;
457 struct net_device *netdev;
458 struct mlx5_core_dev *mdev;
459 struct mlx5e_channel *channel;
460 struct mlx5e_priv *priv;
461
462 /* control path */
463 struct mlx5_wq_ctrl wq_ctrl;
464 int ch_ix;
465 int txq_ix;
466 u32 rate_limit;
467 struct work_struct recover_work;
468 struct mlx5e_ptpsq *ptpsq;
469 cqe_ts_to_ns ptp_cyc2time;
470 } ____cacheline_aligned_in_smp;
471
472 struct mlx5e_xdp_info_fifo {
473 union mlx5e_xdp_info *xi;
474 u32 *cc;
475 u32 *pc;
476 u32 mask;
477 };
478
479 struct mlx5e_xdpsq;
480 struct mlx5e_xmit_data;
481 struct xsk_tx_metadata;
482 typedef int (*mlx5e_fp_xmit_xdp_frame_check)(struct mlx5e_xdpsq *);
483 typedef bool (*mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq *,
484 struct mlx5e_xmit_data *,
485 int,
486 struct xsk_tx_metadata **);
487
488 struct mlx5e_xdpsq {
489 /* data path */
490
491 /* dirtied @completion */
492 u32 xdpi_fifo_cc;
493 u16 cc;
494
495 /* dirtied @xmit */
496 u32 xdpi_fifo_pc ____cacheline_aligned_in_smp;
497 u16 pc;
498 struct mlx5_wqe_ctrl_seg *doorbell_cseg;
499 struct mlx5e_tx_mpwqe mpwqe;
500
501 struct mlx5e_cq cq;
502
503 /* read only */
504 struct xsk_buff_pool *xsk_pool;
505 struct mlx5_wq_cyc wq;
506 struct mlx5e_xdpsq_stats *stats;
507 mlx5e_fp_xmit_xdp_frame_check xmit_xdp_frame_check;
508 mlx5e_fp_xmit_xdp_frame xmit_xdp_frame;
509 struct {
510 struct mlx5e_xdp_wqe_info *wqe_info;
511 struct mlx5e_xdp_info_fifo xdpi_fifo;
512 } db;
513 void __iomem *uar_map;
514 u32 sqn;
515 struct device *pdev;
516 __be32 mkey_be;
517 u16 stop_room;
518 u8 max_sq_mpw_wqebbs;
519 u8 min_inline_mode;
520 unsigned long state;
521 unsigned int hw_mtu;
522
523 /* control path */
524 struct mlx5_wq_ctrl wq_ctrl;
525 struct mlx5e_channel *channel;
526 } ____cacheline_aligned_in_smp;
527
528 struct mlx5e_xdp_buff {
529 struct xdp_buff xdp;
530 struct mlx5_cqe64 *cqe;
531 struct mlx5e_rq *rq;
532 };
533
534 struct mlx5e_ktls_resync_resp;
535
536 struct mlx5e_icosq {
537 /* data path */
538 u16 cc;
539 u16 pc;
540
541 struct mlx5_wqe_ctrl_seg *doorbell_cseg;
542 struct mlx5e_cq cq;
543
544 /* write@xmit, read@completion */
545 struct {
546 struct mlx5e_icosq_wqe_info *wqe_info;
547 } db;
548
549 /* read only */
550 struct mlx5_wq_cyc wq;
551 void __iomem *uar_map;
552 u32 sqn;
553 u16 reserved_room;
554 unsigned long state;
555 /* icosq can be accessed from any CPU and from different contexts
556 * (NAPI softirq or process/workqueue). Always use spin_lock_bh for
557 * simplicity and correctness across all contexts.
558 */
559 spinlock_t lock;
560 struct mlx5e_ktls_resync_resp *ktls_resync;
561
562 /* control path */
563 struct mlx5_wq_ctrl wq_ctrl;
564 struct mlx5e_channel *channel;
565
566 struct work_struct recover_work;
567 } ____cacheline_aligned_in_smp;
568
569 struct mlx5e_frag_page {
570 netmem_ref netmem;
571 u16 frags;
572 };
573
574 enum mlx5e_wqe_frag_flag {
575 MLX5E_WQE_FRAG_LAST_IN_PAGE,
576 MLX5E_WQE_FRAG_SKIP_RELEASE,
577 };
578
579 struct mlx5e_wqe_frag_info {
580 union {
581 struct mlx5e_frag_page *frag_page;
582 struct xdp_buff **xskp;
583 };
584 u32 offset;
585 u8 flags;
586 };
587
588 union mlx5e_alloc_units {
589 DECLARE_FLEX_ARRAY(struct mlx5e_frag_page, frag_pages);
590 DECLARE_FLEX_ARRAY(struct page *, pages);
591 DECLARE_FLEX_ARRAY(struct xdp_buff *, xsk_buffs);
592 };
593
594 struct mlx5e_mpw_info {
595 u16 consumed_strides;
596 DECLARE_BITMAP(skip_release_bitmap, MLX5_MPWRQ_MAX_PAGES_PER_WQE);
597 union mlx5e_alloc_units alloc_units;
598 };
599
600 struct mlx5e_mpw_linear_info {
601 struct mlx5e_frag_page frag_page;
602 u16 max_frags;
603 };
604
605 #define MLX5E_MAX_RX_FRAGS 4
606
607 struct mlx5e_rq;
608 typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*);
609 typedef struct sk_buff *
610 (*mlx5e_fp_skb_from_cqe_mpwrq)(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
611 struct mlx5_cqe64 *cqe, u16 cqe_bcnt,
612 u32 head_offset, u32 page_idx);
613 typedef struct sk_buff *
614 (*mlx5e_fp_skb_from_cqe)(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
615 struct mlx5_cqe64 *cqe, u32 cqe_bcnt);
616 typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq);
617 typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);
618 typedef void (*mlx5e_fp_shampo_dealloc_hd)(struct mlx5e_rq*, u16, u16, bool);
619
620 int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk);
621 void mlx5e_rq_set_trap_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params);
622
623 enum mlx5e_rq_flag {
624 MLX5E_RQ_FLAG_XDP_XMIT,
625 MLX5E_RQ_FLAG_XDP_REDIRECT,
626 };
627
628 struct mlx5e_rq_frag_info {
629 int frag_size;
630 int frag_stride;
631 };
632
633 struct mlx5e_rq_frags_info {
634 struct mlx5e_rq_frag_info arr[MLX5E_MAX_RX_FRAGS];
635 u8 num_frags;
636 u8 log_num_frags;
637 u16 wqe_bulk;
638 u16 refill_unit;
639 u8 wqe_index_mask;
640 };
641
642 struct mlx5e_dma_info {
643 dma_addr_t addr;
644 union {
645 struct mlx5e_frag_page *frag_page;
646 struct page *page;
647 };
648 };
649
650 struct mlx5e_shampo_hd {
651 u32 hd_per_wq;
652 u32 hd_buf_size;
653 u32 mkey;
654 u32 nentries;
655 DECLARE_FLEX_ARRAY(struct mlx5e_dma_info, hd_buf_pages);
656 };
657
658 struct mlx5e_hw_gro_data {
659 struct sk_buff *skb;
660 struct flow_keys fk;
661 int second_ip_id;
662 };
663
664 enum mlx5e_mpwrq_umr_mode {
665 MLX5E_MPWRQ_UMR_MODE_ALIGNED,
666 MLX5E_MPWRQ_UMR_MODE_UNALIGNED,
667 MLX5E_MPWRQ_UMR_MODE_OVERSIZED,
668 MLX5E_MPWRQ_UMR_MODE_TRIPLE,
669 };
670
671 struct mlx5e_rq {
672 /* data path */
673 union {
674 struct {
675 struct mlx5_wq_cyc wq;
676 struct mlx5e_wqe_frag_info *frags;
677 union mlx5e_alloc_units *alloc_units;
678 struct mlx5e_rq_frags_info info;
679 mlx5e_fp_skb_from_cqe skb_from_cqe;
680 } wqe;
681 struct {
682 struct mlx5_wq_ll wq;
683 struct mlx5e_umr_wqe_hdr umr_wqe;
684 struct mlx5e_mpw_info *info;
685 mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq;
686 __be32 umr_mkey_be;
687 u16 num_strides;
688 u16 actual_wq_head;
689 u8 log_stride_sz;
690 u8 umr_in_progress;
691 u8 umr_last_bulk;
692 u8 umr_completed;
693 u8 min_wqe_bulk;
694 u8 page_shift;
695 u8 pages_per_wqe;
696 u8 umr_wqebbs;
697 u8 mtts_per_wqe;
698 u8 umr_mode;
699 struct mlx5e_mpw_linear_info *linear_info;
700 struct mlx5e_shampo_hd *shampo;
701 } mpwqe;
702 };
703 struct {
704 u16 headroom;
705 u32 frame0_sz;
706 u8 map_dir; /* dma map direction */
707 } buff;
708
709 struct device *pdev;
710 struct net_device *netdev;
711 struct mlx5e_rq_stats *stats;
712 struct mlx5e_cq cq;
713 struct mlx5e_cq_decomp cqd;
714 struct kernel_hwtstamp_config *hwtstamp_config;
715 struct mlx5_clock *clock;
716 struct mlx5e_icosq *icosq;
717 struct mlx5e_priv *priv;
718
719 struct mlx5e_hw_gro_data *hw_gro_data;
720
721 mlx5e_fp_handle_rx_cqe handle_rx_cqe;
722 mlx5e_fp_post_rx_wqes post_wqes;
723 mlx5e_fp_dealloc_wqe dealloc_wqe;
724
725 unsigned long state;
726 int ix;
727 unsigned int hw_mtu;
728
729 struct dim *dim; /* Dynamic Interrupt Moderation */
730
731 /* XDP */
732 struct bpf_prog __rcu *xdp_prog;
733 struct mlx5e_xdpsq *xdpsq;
734 DECLARE_BITMAP(flags, 8);
735
736 /* page pools */
737 struct page_pool *page_pool;
738 struct page_pool *hd_page_pool;
739
740 struct mlx5e_xdp_buff mxbuf;
741
742 /* AF_XDP zero-copy */
743 struct xsk_buff_pool *xsk_pool;
744
745 struct work_struct recover_work;
746 struct work_struct rx_timeout_work;
747
748 /* control */
749 struct mlx5_wq_ctrl wq_ctrl;
750 __be32 mkey_be;
751 u8 wq_type;
752 u32 rqn;
753 struct mlx5_core_dev *mdev;
754 struct mlx5e_channel *channel;
755 struct mlx5e_dma_info wqe_overflow;
756
757 /* XDP read-mostly */
758 struct xdp_rxq_info xdp_rxq;
759 cqe_ts_to_ns ptp_cyc2time;
760 } ____cacheline_aligned_in_smp;
761
762 enum mlx5e_channel_state {
763 MLX5E_CHANNEL_STATE_XSK,
764 MLX5E_CHANNEL_NUM_STATES
765 };
766
767 struct mlx5e_channel {
768 /* data path */
769 struct mlx5e_rq rq;
770 struct mlx5e_xdpsq rq_xdpsq;
771 struct mlx5e_txqsq sq[MLX5_MAX_NUM_TC];
772 struct mlx5e_icosq icosq; /* internal control operations */
773 struct mlx5e_txqsq __rcu * __rcu *qos_sqs;
774 bool xdp;
775 struct napi_struct napi;
776 struct device *pdev;
777 struct net_device *netdev;
778 __be32 mkey_be;
779 u16 qos_sqs_size;
780 u8 num_tc;
781 u8 lag_port;
782
783 /* XDP_REDIRECT */
784 struct mlx5e_xdpsq *xdpsq;
785
786 /* AF_XDP zero-copy */
787 struct mlx5e_rq xskrq;
788 struct mlx5e_xdpsq xsksq;
789
790 /* Async ICOSQ */
791 struct mlx5e_icosq *async_icosq;
792
793 /* data path - accessed per napi poll */
794 const struct cpumask *aff_mask;
795 struct mlx5e_ch_stats *stats;
796
797 /* control */
798 struct mlx5e_priv *priv;
799 struct mlx5_core_dev *mdev;
800 DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES);
801 int ix;
802 int vec_ix;
803 int sd_ix;
804 int cpu;
805 struct mlx5_sq_bfreg *bfreg;
806 /* Sync between icosq recovery and XSK enable/disable. */
807 struct mutex icosq_recovery_lock;
808
809 /* coalescing configuration */
810 struct dim_cq_moder rx_cq_moder;
811 struct dim_cq_moder tx_cq_moder;
812 };
813
mlx5e_icosq_sync_lock(struct mlx5e_icosq * sq)814 static inline bool mlx5e_icosq_sync_lock(struct mlx5e_icosq *sq)
815 {
816 if (likely(!test_bit(MLX5E_SQ_STATE_LOCK_NEEDED, &sq->state)))
817 return false;
818
819 spin_lock_bh(&sq->lock);
820 return true;
821 }
822
mlx5e_icosq_sync_unlock(struct mlx5e_icosq * sq,bool locked)823 static inline void mlx5e_icosq_sync_unlock(struct mlx5e_icosq *sq, bool locked)
824 {
825 if (unlikely(locked))
826 spin_unlock_bh(&sq->lock);
827 }
828
829 struct mlx5e_ptp;
830
831 struct mlx5e_channels {
832 struct mlx5e_channel **c;
833 struct mlx5e_ptp *ptp;
834 unsigned int num;
835 struct mlx5e_params params;
836 };
837
838 struct mlx5e_channel_stats {
839 struct mlx5e_ch_stats ch;
840 struct mlx5e_sq_stats sq[MLX5_MAX_NUM_TC];
841 struct mlx5e_rq_stats rq;
842 struct mlx5e_rq_stats xskrq;
843 struct mlx5e_xdpsq_stats rq_xdpsq;
844 struct mlx5e_xdpsq_stats xdpsq;
845 struct mlx5e_xdpsq_stats xsksq;
846 } ____cacheline_aligned_in_smp;
847
848 struct mlx5e_ptp_stats {
849 struct mlx5e_ch_stats ch;
850 struct mlx5e_sq_stats sq[MLX5_MAX_NUM_TC];
851 struct mlx5e_ptp_cq_stats cq[MLX5_MAX_NUM_TC];
852 struct mlx5e_rq_stats rq;
853 } ____cacheline_aligned_in_smp;
854
855 enum {
856 MLX5E_STATE_OPENED,
857 MLX5E_STATE_DESTROYING,
858 MLX5E_STATE_XDP_TX_ENABLED,
859 MLX5E_STATE_XDP_ACTIVE,
860 MLX5E_STATE_CHANNELS_ACTIVE,
861 };
862
863 struct mlx5e_modify_sq_param {
864 int curr_state;
865 int next_state;
866 int rl_update;
867 int rl_index;
868 bool qos_update;
869 u16 qos_queue_group_id;
870 };
871
872 #if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE)
873 struct mlx5e_hv_vhca_stats_agent {
874 struct mlx5_hv_vhca_agent *agent;
875 struct delayed_work work;
876 u16 delay;
877 void *buf;
878 };
879 #endif
880
881 struct mlx5e_xsk {
882 /* XSK buffer pools are stored separately from channels,
883 * because we don't want to lose them when channels are
884 * recreated. The kernel also stores buffer pool, but it doesn't
885 * distinguish between zero-copy and non-zero-copy UMEMs, so
886 * rely on our mechanism.
887 */
888 struct xsk_buff_pool **pools;
889 u16 refcnt;
890 bool ever_used;
891 };
892
893 /* Temporary storage for variables that are allocated when struct mlx5e_priv is
894 * initialized, and used where we can't allocate them because that functions
895 * must not fail. Use with care and make sure the same variable is not used
896 * simultaneously by multiple users.
897 */
898 struct mlx5e_scratchpad {
899 cpumask_var_t cpumask;
900 };
901
902 struct mlx5e_trap;
903 struct mlx5e_htb;
904
905 struct mlx5e_priv {
906 /* priv data path fields - start */
907 struct mlx5e_selq selq;
908 struct mlx5e_txqsq **txq2sq;
909 struct mlx5e_sq_stats **txq2sq_stats;
910
911 #ifdef CONFIG_MLX5_CORE_EN_DCB
912 struct mlx5e_dcbx_dp dcbx_dp;
913 #endif
914 /* priv data path fields - end */
915
916 unsigned long state;
917 struct mutex state_lock; /* Protects Interface state */
918 struct mlx5e_rq drop_rq;
919
920 struct mlx5e_channels channels;
921 struct mlx5e_rx_res *rx_res;
922 u32 *tx_rates;
923
924 struct mlx5e_flow_steering *fs;
925
926 struct workqueue_struct *wq;
927 struct work_struct update_carrier_work;
928 struct work_struct set_rx_mode_work;
929 struct work_struct tx_timeout_work;
930 struct work_struct update_stats_work;
931 struct work_struct monitor_counters_work;
932 struct mlx5_nb monitor_counters_nb;
933
934 struct mlx5_core_dev *mdev;
935 struct net_device *netdev;
936 struct mlx5e_trap *en_trap;
937 struct mlx5e_stats stats;
938 struct mlx5e_channel_stats **channel_stats;
939 struct mlx5e_channel_stats trap_stats;
940 struct mlx5e_ptp_stats ptp_stats;
941 struct mlx5e_sq_stats **htb_qos_sq_stats;
942 u16 htb_max_qos_sqs;
943 u16 stats_nch;
944 u16 max_nch;
945 u8 max_opened_tc;
946 bool tx_ptp_opened;
947 bool rx_ptp_opened;
948 bool ktls_rx_was_enabled;
949 struct kernel_hwtstamp_config hwtstamp_config;
950 u16 q_counter[MLX5_SD_MAX_GROUP_SZ];
951 u16 drop_rq_q_counter;
952 struct notifier_block events_nb;
953 struct notifier_block blocking_events_nb;
954
955 struct mlx5e_pcie_cong_event *cong_event;
956
957 struct udp_tunnel_nic_info nic_info;
958 #ifdef CONFIG_MLX5_CORE_EN_DCB
959 struct mlx5e_dcbx dcbx;
960 #endif
961
962 const struct mlx5e_profile *profile;
963 void *ppriv;
964 #ifdef CONFIG_MLX5_MACSEC
965 struct mlx5e_macsec *macsec;
966 #endif
967 #ifdef CONFIG_MLX5_EN_IPSEC
968 struct mlx5e_ipsec *ipsec;
969 #endif
970 #ifdef CONFIG_MLX5_EN_PSP
971 struct mlx5e_psp *psp;
972 #endif
973 #ifdef CONFIG_MLX5_EN_TLS
974 struct mlx5e_tls *tls;
975 #endif
976 struct devlink_health_reporter *tx_reporter;
977 struct devlink_health_reporter *rx_reporter;
978 struct mlx5e_xsk xsk;
979 #if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE)
980 struct mlx5e_hv_vhca_stats_agent stats_agent;
981 #endif
982 struct mlx5e_scratchpad scratchpad;
983 struct mlx5e_htb *htb;
984 struct mlx5e_mqprio_rl *mqprio_rl;
985 struct dentry *dfs_root;
986 struct mlx5_devcom_comp_dev *devcom;
987 };
988
mlx5e_stats_nch_read(const struct mlx5e_priv * priv)989 static inline u16 mlx5e_stats_nch_read(const struct mlx5e_priv *priv)
990 {
991 /* Pairs with smp_store_release in mlx5e_stats_nch_write(). */
992 return smp_load_acquire(&priv->stats_nch);
993 }
994
mlx5e_stats_nch_write(struct mlx5e_priv * priv,u16 n)995 static inline void mlx5e_stats_nch_write(struct mlx5e_priv *priv, u16 n)
996 {
997 /* Pairs with smp_load_acquire in mlx5e_stats_nch_read(). */
998 smp_store_release(&priv->stats_nch, n);
999 }
1000
1001 struct mlx5e_dev {
1002 struct net_device *netdev;
1003 struct devlink_port dl_port;
1004 };
1005
1006 struct mlx5e_rx_handlers {
1007 mlx5e_fp_handle_rx_cqe handle_rx_cqe;
1008 mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe;
1009 mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe_shampo;
1010 };
1011
1012 extern const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic;
1013
1014 enum mlx5e_profile_feature {
1015 MLX5E_PROFILE_FEATURE_PTP_RX,
1016 MLX5E_PROFILE_FEATURE_PTP_TX,
1017 MLX5E_PROFILE_FEATURE_QOS_HTB,
1018 MLX5E_PROFILE_FEATURE_FS_VLAN,
1019 MLX5E_PROFILE_FEATURE_FS_TC,
1020 };
1021
1022 struct mlx5e_profile {
1023 int (*init)(struct mlx5_core_dev *mdev,
1024 struct net_device *netdev);
1025 void (*cleanup)(struct mlx5e_priv *priv);
1026 int (*init_rx)(struct mlx5e_priv *priv);
1027 void (*cleanup_rx)(struct mlx5e_priv *priv);
1028 int (*init_tx)(struct mlx5e_priv *priv);
1029 void (*cleanup_tx)(struct mlx5e_priv *priv);
1030 int (*enable)(struct mlx5e_priv *priv);
1031 void (*disable)(struct mlx5e_priv *priv);
1032 int (*update_rx)(struct mlx5e_priv *priv);
1033 void (*update_stats)(struct mlx5e_priv *priv);
1034 void (*update_carrier)(struct mlx5e_priv *priv);
1035 int (*max_nch_limit)(struct mlx5_core_dev *mdev);
1036 u32 (*get_tisn)(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv,
1037 u8 lag_port, u8 tc);
1038 unsigned int (*stats_grps_num)(struct mlx5e_priv *priv);
1039 mlx5e_stats_grp_t *stats_grps;
1040 const struct mlx5e_rx_handlers *rx_handlers;
1041 int max_tc;
1042 u32 features;
1043 };
1044
1045 u32 mlx5e_profile_get_tisn(struct mlx5_core_dev *mdev,
1046 struct mlx5e_priv *priv,
1047 const struct mlx5e_profile *profile,
1048 u8 lag_port, u8 tc);
1049
1050 #define mlx5e_profile_feature_cap(profile, feature) \
1051 ((profile)->features & BIT(MLX5E_PROFILE_FEATURE_##feature))
1052
1053 void mlx5e_build_ptys2ethtool_map(void);
1054
1055 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift,
1056 enum mlx5e_mpwrq_umr_mode umr_mode);
1057
1058 void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats);
1059 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s);
1060
1061 int mlx5e_self_test_num(struct mlx5e_priv *priv);
1062 int mlx5e_self_test_fill_strings(struct mlx5e_priv *priv, u8 *data);
1063 void mlx5e_self_test(struct net_device *ndev, struct ethtool_test *etest,
1064 u64 *buf);
1065 void mlx5e_set_rx_mode_work(struct work_struct *work);
1066
1067 int mlx5e_hwtstamp_set(struct mlx5e_priv *priv,
1068 struct kernel_hwtstamp_config *config,
1069 struct netlink_ext_ack *extack);
1070 int mlx5e_hwtstamp_get(struct mlx5e_priv *priv,
1071 struct kernel_hwtstamp_config *config);
1072 int mlx5e_modify_rx_cqe_compression_locked(struct mlx5e_priv *priv, bool val, bool rx_filter);
1073
1074 int mlx5e_vlan_rx_add_vid(struct net_device *dev, __always_unused __be16 proto,
1075 u16 vid);
1076 int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __always_unused __be16 proto,
1077 u16 vid);
1078 void mlx5e_timestamp_init(struct mlx5e_priv *priv);
1079
1080 struct mlx5e_xsk_param;
1081
1082 struct mlx5e_rq_param;
1083 struct mlx5e_rq_opt_param;
1084 int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param,
1085 struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter,
1086 struct mlx5e_rq *rq);
1087 #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
1088 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time);
1089 void mlx5e_close_rq(struct mlx5e_rq *rq);
1090 int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param,
1091 u16 q_counter);
1092 void mlx5e_destroy_rq(struct mlx5e_rq *rq);
1093
1094 bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,
1095 bool dim_enabled);
1096 bool mlx5e_reset_rx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode,
1097 bool dim_enabled, bool keep_dim_state);
1098
1099 void mlx5e_mpwqe_dealloc_linear_page(struct mlx5e_rq *rq);
1100
1101 struct mlx5e_sq_param;
1102 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
1103 struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
1104 struct mlx5e_xdpsq *sq, bool is_redirect);
1105 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq);
1106
1107 struct mlx5e_create_cq_param {
1108 struct net_device *netdev;
1109 struct workqueue_struct *wq;
1110 struct napi_struct *napi;
1111 struct mlx5e_ch_stats *ch_stats;
1112 int node;
1113 int ix;
1114 struct mlx5_uars_page *uar;
1115 };
1116
1117 struct mlx5e_cq_param;
1118 int mlx5e_open_cq(struct mlx5_core_dev *mdev, struct dim_cq_moder moder,
1119 struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp,
1120 struct mlx5e_cq *cq);
1121 void mlx5e_close_cq(struct mlx5e_cq *cq);
1122 int mlx5e_modify_cq_period_mode(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
1123 u8 cq_period_mode);
1124 int mlx5e_modify_cq_moderation(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
1125 u16 cq_period, u16 cq_max_count, u8 cq_period_mode);
1126
1127 int mlx5e_open_locked(struct net_device *netdev);
1128 int mlx5e_close_locked(struct net_device *netdev);
1129
1130 void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c);
1131 void mlx5e_trigger_napi_async_icosq(struct mlx5e_channel *c);
1132 void mlx5e_trigger_napi_sched(struct napi_struct *napi);
1133
1134 int mlx5e_open_channels(struct mlx5e_priv *priv,
1135 struct mlx5e_channels *chs);
1136 void mlx5e_close_channels(struct mlx5e_channels *chs);
1137
1138 /* Function pointer to be used to modify HW or kernel settings while
1139 * switching channels
1140 */
1141 typedef int (*mlx5e_fp_preactivate)(struct mlx5e_priv *priv, void *context);
1142 #define MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(fn) \
1143 int fn##_ctx(struct mlx5e_priv *priv, void *context) \
1144 { \
1145 return fn(priv); \
1146 }
1147 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv);
1148 int mlx5e_safe_switch_params(struct mlx5e_priv *priv,
1149 struct mlx5e_params *new_params,
1150 mlx5e_fp_preactivate preactivate,
1151 void *context, bool reset);
1152 int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv);
1153 int mlx5e_num_channels_changed_ctx(struct mlx5e_priv *priv, void *context);
1154 int mlx5e_update_tc_and_tx_queues_ctx(struct mlx5e_priv *priv, void *context);
1155 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv);
1156 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv);
1157 int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx);
1158
1159 int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state);
1160 void mlx5e_activate_rq(struct mlx5e_rq *rq);
1161 void mlx5e_deactivate_rq(struct mlx5e_rq *rq);
1162 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq);
1163 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq);
1164
1165 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
1166 struct mlx5e_modify_sq_param *p);
1167 int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix,
1168 struct mlx5e_params *params, struct mlx5e_sq_param *param,
1169 struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id,
1170 struct mlx5e_sq_stats *sq_stats);
1171 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq);
1172 void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq);
1173 void mlx5e_free_txqsq(struct mlx5e_txqsq *sq);
1174 void mlx5e_tx_disable_queue(struct netdev_queue *txq);
1175 int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa);
1176 void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq);
1177 struct mlx5e_create_sq_param;
1178 int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
1179 struct mlx5e_sq_param *param,
1180 struct mlx5e_create_sq_param *csp,
1181 u16 qos_queue_group_id,
1182 u32 *sqn);
1183 void mlx5e_tx_err_cqe_work(struct work_struct *recover_work);
1184 void mlx5e_close_txqsq(struct mlx5e_txqsq *sq);
1185
1186 bool mlx5e_reset_tx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,
1187 bool dim_enabled);
1188 bool mlx5e_reset_tx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode,
1189 bool dim_enabled, bool keep_dim_state);
1190
mlx5_tx_swp_supported(struct mlx5_core_dev * mdev)1191 static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev)
1192 {
1193 return MLX5_CAP_ETH(mdev, swp) &&
1194 MLX5_CAP_ETH(mdev, swp_csum) && MLX5_CAP_ETH(mdev, swp_lso);
1195 }
1196
1197 extern const struct ethtool_ops mlx5e_ethtool_ops;
1198
1199 int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey);
1200 int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises);
1201 void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev);
1202 int mlx5e_modify_tirs_lb(struct mlx5_core_dev *mdev, bool enable_uc_lb,
1203 bool enable_mc_lb);
1204 int mlx5e_refresh_tirs(struct mlx5_core_dev *mdev, bool enable_uc_lb,
1205 bool enable_mc_lb);
1206 void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc);
1207
1208 /* common netdev helpers */
1209 void mlx5e_create_q_counters(struct mlx5e_priv *priv);
1210 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv);
1211 int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
1212 struct mlx5e_rq *drop_rq);
1213 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq);
1214
1215 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn);
1216 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn);
1217
1218 void mlx5e_update_carrier(struct mlx5e_priv *priv);
1219 int mlx5e_close(struct net_device *netdev);
1220 int mlx5e_open(struct net_device *netdev);
1221
1222 void mlx5e_queue_update_stats(struct mlx5e_priv *priv);
1223
1224 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv);
1225 int mlx5e_set_dev_port_mtu_ctx(struct mlx5e_priv *priv, void *context);
1226 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
1227 mlx5e_fp_preactivate preactivate);
1228 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv);
1229
1230 /* ethtool helpers */
1231 void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv,
1232 struct ethtool_drvinfo *drvinfo);
1233 void mlx5e_ethtool_get_strings(struct mlx5e_priv *priv,
1234 u32 stringset, u8 *data);
1235 int mlx5e_ethtool_get_sset_count(struct mlx5e_priv *priv, int sset);
1236 void mlx5e_ethtool_get_ethtool_stats(struct mlx5e_priv *priv,
1237 struct ethtool_stats *stats, u64 *data);
1238 void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv,
1239 struct ethtool_ringparam *param,
1240 struct kernel_ethtool_ringparam *kernel_param);
1241 int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv,
1242 struct ethtool_ringparam *param,
1243 struct netlink_ext_ack *extack);
1244 void mlx5e_ethtool_get_channels(struct mlx5e_priv *priv,
1245 struct ethtool_channels *ch);
1246 int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv,
1247 struct ethtool_channels *ch);
1248 int mlx5e_ethtool_get_coalesce(struct mlx5e_priv *priv,
1249 struct ethtool_coalesce *coal,
1250 struct kernel_ethtool_coalesce *kernel_coal,
1251 struct netlink_ext_ack *extack);
1252 int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv,
1253 struct ethtool_coalesce *coal,
1254 struct kernel_ethtool_coalesce *kernel_coal,
1255 struct netlink_ext_ack *extack);
1256 int mlx5e_get_per_queue_coalesce(struct net_device *dev, u32 queue,
1257 struct ethtool_coalesce *coal);
1258 int mlx5e_set_per_queue_coalesce(struct net_device *dev, u32 queue,
1259 struct ethtool_coalesce *coal);
1260 u32 mlx5e_ethtool_get_rxfh_key_size(struct mlx5e_priv *priv);
1261 u32 mlx5e_ethtool_get_rxfh_indir_size(struct mlx5e_priv *priv);
1262 int mlx5e_ethtool_get_ts_info(struct mlx5e_priv *priv,
1263 struct kernel_ethtool_ts_info *info);
1264 int mlx5e_ethtool_flash_device(struct mlx5e_priv *priv,
1265 struct ethtool_flash *flash);
1266
1267 /* mlx5e generic netdev management API */
1268 static inline bool
mlx5e_tx_mpwqe_supported(struct mlx5_core_dev * mdev)1269 mlx5e_tx_mpwqe_supported(struct mlx5_core_dev *mdev)
1270 {
1271 return !is_kdump_kernel() &&
1272 MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe);
1273 }
1274
1275 int mlx5e_get_pf_num_tirs(struct mlx5_core_dev *mdev);
1276 int mlx5e_priv_init(struct mlx5e_priv *priv,
1277 const struct mlx5e_profile *profile,
1278 struct net_device *netdev,
1279 struct mlx5_core_dev *mdev);
1280 void mlx5e_priv_cleanup(struct mlx5e_priv *priv);
1281 struct net_device *
1282 mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile);
1283 int mlx5e_attach_netdev(struct mlx5e_priv *priv);
1284 void mlx5e_detach_netdev(struct mlx5e_priv *priv);
1285 void mlx5e_destroy_netdev(struct net_device *netdev);
1286 int mlx5e_netdev_change_profile(struct net_device *netdev,
1287 struct mlx5_core_dev *mdev,
1288 const struct mlx5e_profile *new_profile,
1289 void *new_ppriv);
1290 void mlx5e_netdev_attach_nic_profile(struct net_device *netdev,
1291 struct mlx5_core_dev *mdev);
1292 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv);
1293 void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu);
1294
1295 void mlx5e_set_xdp_feature(struct mlx5e_priv *priv);
1296 netdev_features_t mlx5e_features_check(struct sk_buff *skb,
1297 struct net_device *netdev,
1298 netdev_features_t features);
1299 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features);
1300 #ifdef CONFIG_MLX5_ESWITCH
1301 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac);
1302 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, int max_tx_rate);
1303 int mlx5e_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi);
1304 int mlx5e_get_vf_stats(struct net_device *dev, int vf, struct ifla_vf_stats *vf_stats);
1305 #endif
1306 int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey);
1307 #endif /* __MLX5_EN_H__ */
1308