1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 /* Copyright (c) 2021, Microsoft Corporation. */
3
4 #ifndef _MANA_H
5 #define _MANA_H
6
7 #include <linux/dim.h>
8 #include <net/xdp.h>
9 #include <net/net_shaper.h>
10
11 #include "gdma.h"
12 #include "hw_channel.h"
13
14 /* Microsoft Azure Network Adapter (MANA)'s definitions
15 *
16 * Structures labeled with "HW DATA" are exchanged with the hardware. All of
17 * them are naturally aligned and hence don't need __packed.
18 */
19
20 /* MANA protocol version */
21 #define MANA_MAJOR_VERSION 0
22 #define MANA_MINOR_VERSION 1
23 #define MANA_MICRO_VERSION 1
24
25 typedef u64 mana_handle_t;
26 #define INVALID_MANA_HANDLE ((mana_handle_t)-1)
27
28 enum TRI_STATE {
29 TRI_STATE_UNKNOWN = -1,
30 TRI_STATE_FALSE = 0,
31 TRI_STATE_TRUE = 1
32 };
33
34 /* MANA ethtool private flag bit positions */
35 enum mana_priv_flag_bits {
36 MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF = 0,
37 MANA_PRIV_FLAG_MAX,
38 };
39
40 /* Number of entries for hardware indirection table must be in power of 2 */
41 #define MANA_INDIRECT_TABLE_MAX_SIZE 512
42 #define MANA_INDIRECT_TABLE_DEF_SIZE 64
43
44 /* The Toeplitz hash key's length in bytes: should be multiple of 8 */
45 #define MANA_HASH_KEY_SIZE 40
46
47 #define COMP_ENTRY_SIZE 64
48
49 /* This Max value for RX buffers is derived from __alloc_page()'s max page
50 * allocation calculation. It allows maximum 2^(MAX_ORDER -1) pages. RX buffer
51 * size beyond this value gets rejected by __alloc_page() call.
52 */
53 #define MAX_RX_BUFFERS_PER_QUEUE 8192
54 #define DEF_RX_BUFFERS_PER_QUEUE 1024
55 #define MIN_RX_BUFFERS_PER_QUEUE 128
56
57 /* This max value for TX buffers is derived as the maximum allocatable
58 * pages supported on host per guest through testing. TX buffer size beyond
59 * this value is rejected by the hardware.
60 */
61 #define MAX_TX_BUFFERS_PER_QUEUE 16384
62 #define DEF_TX_BUFFERS_PER_QUEUE 256
63 #define MIN_TX_BUFFERS_PER_QUEUE 128
64
65 #define EQ_SIZE (8 * MANA_PAGE_SIZE)
66
67 #define LOG2_EQ_THROTTLE 3
68
69 #define MAX_PORTS_IN_MANA_DEV 256
70
71 /* Maximum number of PPIs per coalesced CQE */
72 #define MANA_RXCOMP_OOB_NUM_PPI 4
73
74 /* 8-pkt mode packs up to 2 packets per PPI entry */
75 #define MANA_CQE_COAL_PKTS_8 8
76
77 /* Default/max interrupt moderation settings */
78 #define MANA_INTR_MODR_USEC_DEF 0
79 #define MANA_INTR_MODR_COMP_DEF 0
80
81 #define MANA_ADAPTIVE_RX_DEF true
82 #define MANA_ADAPTIVE_TX_DEF true
83
84 /* DIM doorbell value field layout */
85 #define MANA_INTR_MODR_USEC_MAX GENMASK(9, 0)
86 #define MANA_INTR_MODR_USEC_VLD BIT(15)
87 #define MANA_INTR_MODR_COMP_MAX GENMASK(7, 0)
88 #define MANA_INTR_MODR_COMP_MASK GENMASK(23, 16)
89
90 /* Update this count whenever the respective structures are changed */
91 #define MANA_STATS_RX_COUNT (6 + MANA_CQE_COAL_PKTS_8 - 1)
92 #define MANA_STATS_TX_COUNT 11
93
94 #define MANA_RX_FRAG_ALIGNMENT 64
95
96 struct mana_stats_rx {
97 u64 packets;
98 u64 bytes;
99 u64 xdp_drop;
100 u64 xdp_tx;
101 u64 xdp_redirect;
102 u64 pkt_len0_err;
103 u64 coalesced_cqe[MANA_CQE_COAL_PKTS_8 - 1];
104 struct u64_stats_sync syncp;
105 };
106
107 struct mana_stats_tx {
108 u64 packets;
109 u64 bytes;
110 u64 xdp_xmit;
111 u64 tso_packets;
112 u64 tso_bytes;
113 u64 tso_inner_packets;
114 u64 tso_inner_bytes;
115 u64 short_pkt_fmt;
116 u64 long_pkt_fmt;
117 u64 csum_partial;
118 u64 mana_map_err;
119 struct u64_stats_sync syncp;
120 };
121
122 struct mana_txq {
123 struct gdma_queue *gdma_sq;
124
125 union {
126 u32 gdma_txq_id;
127 struct {
128 u32 reserved1 : 10;
129 u32 vsq_frame : 14;
130 u32 reserved2 : 8;
131 };
132 };
133
134 u16 vp_offset;
135
136 struct net_device *ndev;
137
138 /* The SKBs are sent to the HW and we are waiting for the CQEs. */
139 struct sk_buff_head pending_skbs;
140 struct netdev_queue *net_txq;
141
142 atomic_t pending_sends;
143
144 bool napi_initialized;
145
146 struct mana_stats_tx stats;
147 };
148
149 /* skb data and frags dma mappings */
150 struct mana_skb_head {
151 /* GSO pkts may have 2 SGEs for the linear part*/
152 dma_addr_t dma_handle[MAX_SKB_FRAGS + 2];
153
154 u32 size[MAX_SKB_FRAGS + 2];
155 };
156
157 #define MANA_HEADROOM sizeof(struct mana_skb_head)
158
159 enum mana_tx_pkt_format {
160 MANA_SHORT_PKT_FMT = 0,
161 MANA_LONG_PKT_FMT = 1,
162 };
163
164 struct mana_tx_short_oob {
165 u32 pkt_fmt : 2;
166 u32 is_outer_ipv4 : 1;
167 u32 is_outer_ipv6 : 1;
168 u32 comp_iphdr_csum : 1;
169 u32 comp_tcp_csum : 1;
170 u32 comp_udp_csum : 1;
171 u32 supress_txcqe_gen : 1;
172 u32 vcq_num : 24;
173
174 u32 trans_off : 10; /* Transport header offset */
175 u32 vsq_frame : 14;
176 u32 short_vp_offset : 8;
177 }; /* HW DATA */
178
179 struct mana_tx_long_oob {
180 u32 is_encap : 1;
181 u32 inner_is_ipv6 : 1;
182 u32 inner_tcp_opt : 1;
183 u32 inject_vlan_pri_tag : 1;
184 u32 reserved1 : 12;
185 u32 pcp : 3; /* 802.1Q */
186 u32 dei : 1; /* 802.1Q */
187 u32 vlan_id : 12; /* 802.1Q */
188
189 u32 inner_frame_offset : 10;
190 u32 inner_ip_rel_offset : 6;
191 u32 long_vp_offset : 12;
192 u32 reserved2 : 4;
193
194 u32 reserved3;
195 u32 reserved4;
196 }; /* HW DATA */
197
198 struct mana_tx_oob {
199 struct mana_tx_short_oob s_oob;
200 struct mana_tx_long_oob l_oob;
201 }; /* HW DATA */
202
203 enum mana_cq_type {
204 MANA_CQ_TYPE_RX,
205 MANA_CQ_TYPE_TX,
206 };
207
208 enum mana_cqe_type {
209 CQE_INVALID = 0,
210 CQE_RX_OKAY = 1,
211 CQE_RX_COALESCED_4 = 2,
212 CQE_RX_OBJECT_FENCE = 3,
213 CQE_RX_TRUNCATED = 4,
214 CQE_RX_COALESCED_8 = 7,
215
216 CQE_TX_OKAY = 32,
217 CQE_TX_SA_DROP = 33,
218 CQE_TX_MTU_DROP = 34,
219 CQE_TX_INVALID_OOB = 35,
220 CQE_TX_INVALID_ETH_TYPE = 36,
221 CQE_TX_HDR_PROCESSING_ERROR = 37,
222 CQE_TX_VF_DISABLED = 38,
223 CQE_TX_VPORT_IDX_OUT_OF_RANGE = 39,
224 CQE_TX_VPORT_DISABLED = 40,
225 CQE_TX_VLAN_TAGGING_VIOLATION = 41,
226 };
227
228 #define MANA_CQE_COMPLETION 1
229
230 struct mana_cqe_header {
231 u32 cqe_type : 6;
232 u32 client_type : 2;
233 u32 vendor_err : 24;
234 }; /* HW DATA */
235
236 /* NDIS HASH Types */
237 #define NDIS_HASH_IPV4 BIT(0)
238 #define NDIS_HASH_TCP_IPV4 BIT(1)
239 #define NDIS_HASH_UDP_IPV4 BIT(2)
240 #define NDIS_HASH_IPV6 BIT(3)
241 #define NDIS_HASH_TCP_IPV6 BIT(4)
242 #define NDIS_HASH_UDP_IPV6 BIT(5)
243 #define NDIS_HASH_IPV6_EX BIT(6)
244 #define NDIS_HASH_TCP_IPV6_EX BIT(7)
245 #define NDIS_HASH_UDP_IPV6_EX BIT(8)
246
247 #define MANA_HASH_L3 (NDIS_HASH_IPV4 | NDIS_HASH_IPV6 | NDIS_HASH_IPV6_EX)
248 #define MANA_HASH_L4 \
249 (NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | NDIS_HASH_TCP_IPV6 | \
250 NDIS_HASH_UDP_IPV6 | NDIS_HASH_TCP_IPV6_EX | NDIS_HASH_UDP_IPV6_EX)
251 #define MANA_HASH_ENABLE_SUPPORTED \
252 (NDIS_HASH_IPV4 | NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | \
253 NDIS_HASH_IPV6 | NDIS_HASH_TCP_IPV6 | NDIS_HASH_UDP_IPV6)
254
255 /* Read PPI in two different layouts based on cqe_type */
256 union mana_rxcomp_perpkt_info {
257 struct {
258 u32 pkt_len : 16;
259 u32 reserved1 : 16;
260 u32 reserved2;
261 u32 pkt_hash;
262 };
263
264 /* Up to two pkts per PPI entry */
265 struct {
266 u32 pkt_hash0;
267 u16 pkt_len0;
268 u16 pkt_len1;
269 u32 pkt_hash1;
270 };
271 }; /* HW DATA */
272
273 /* Receive completion OOB */
274 struct mana_rxcomp_oob {
275 struct mana_cqe_header cqe_hdr;
276
277 u32 rx_vlan_id : 12;
278 u32 rx_vlantag_present : 1;
279 u32 rx_outer_iphdr_csum_succeed : 1;
280 u32 rx_outer_iphdr_csum_fail : 1;
281 u32 reserved1 : 1;
282 u32 rx_hashtype : 9;
283 u32 rx_iphdr_csum_succeed : 1;
284 u32 rx_iphdr_csum_fail : 1;
285 u32 rx_tcp_csum_succeed : 1;
286 u32 rx_tcp_csum_fail : 1;
287 u32 rx_udp_csum_succeed : 1;
288 u32 rx_udp_csum_fail : 1;
289 u32 reserved2 : 1;
290
291 union mana_rxcomp_perpkt_info ppi[MANA_RXCOMP_OOB_NUM_PPI];
292
293 u32 rx_wqe_offset;
294 }; /* HW DATA */
295
296 struct mana_tx_comp_oob {
297 struct mana_cqe_header cqe_hdr;
298
299 u32 tx_data_offset;
300
301 u32 tx_sgl_offset : 5;
302 u32 tx_wqe_offset : 27;
303
304 u32 reserved[12];
305 }; /* HW DATA */
306
307 struct mana_rxq;
308
309 #define CQE_POLLING_BUFFER 512
310
311 struct mana_cq {
312 struct gdma_queue *gdma_cq;
313
314 /* Cache the CQ id (used to verify if each CQE comes to the right CQ. */
315 u32 gdma_id;
316
317 /* Type of the CQ: TX or RX */
318 enum mana_cq_type type;
319
320 /* Pointer to the mana_rxq that is pushing RX CQEs to the queue.
321 * Only and must be non-NULL if type is MANA_CQ_TYPE_RX.
322 */
323 struct mana_rxq *rxq;
324
325 /* Pointer to the mana_txq that is pushing TX CQEs to the queue.
326 * Only and must be non-NULL if type is MANA_CQ_TYPE_TX.
327 */
328 struct mana_txq *txq;
329
330 /* Buffer which the CQ handler can copy the CQE's into. */
331 struct gdma_comp gdma_comp_buf[CQE_POLLING_BUFFER];
332
333 /* NAPI data */
334 struct napi_struct napi;
335 int work_done;
336 int work_done_since_doorbell;
337 int budget;
338
339 /* DIM - Dynamic Interrupt Moderation */
340 struct dim dim;
341 u16 dim_event_ctr;
342
343 /* Cumulative TX completions fed to DIM. Updated and read only in
344 * NAPI context (mana_poll_tx_cq() / mana_update_tx_dim()), so they
345 * measure the hardware completion rate and need no u64_stats_sync.
346 */
347 u64 tx_dim_pkts;
348 u64 tx_dim_bytes;
349 };
350
351 struct mana_recv_buf_oob {
352 /* A valid GDMA work request representing the data buffer. */
353 struct gdma_wqe_request wqe_req;
354
355 void *buf_va;
356 bool from_pool; /* allocated from a page pool */
357 /* head page of the page_pool fragment; valid only when
358 * from_pool && frag_count > 1.
359 */
360 struct page *pp_page;
361 /* Fragment offset plus rxq->headroom, passed to
362 * page_pool_dma_sync_for_cpu().
363 */
364 u32 dma_sync_offset;
365
366 /* SGL of the buffer going to be sent as part of the work request. */
367 u32 num_sge;
368 struct gdma_sge sgl[MAX_RX_WQE_SGL_ENTRIES];
369
370 /* Required to store the result of mana_gd_post_work_request.
371 * gdma_posted_wqe_info.wqe_size_in_bu is required for progressing the
372 * work queue when the WQE is consumed.
373 */
374 struct gdma_posted_wqe_info wqe_inf;
375 };
376
377 #define MANA_RXBUF_PAD (SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) \
378 + ETH_HLEN)
379
380 /* Headroom an RX buffer has to reserve while XDP is attached: the XDP program
381 * needs XDP_PACKET_HEADROOM, and the TX path needs LL_RESERVED_SPACE() (see
382 * mana_get_rxbuf_headroom()). LL_RESERVED_SPACE() grows with MAX_SKB_FRAGS and
383 * exceeds XDP_PACKET_HEADROOM once CONFIG_MAX_SKB_FRAGS is 19 or more.
384 */
mana_xdp_headroom(struct net_device * ndev)385 static inline u32 mana_xdp_headroom(struct net_device *ndev)
386 {
387 return max_t(u32, LL_RESERVED_SPACE(ndev), XDP_PACKET_HEADROOM);
388 }
389
390 #define MANA_XDP_MTU_MAX(ndev) \
391 (PAGE_SIZE - MANA_RXBUF_PAD - mana_xdp_headroom(ndev))
392
393 struct mana_rxq {
394 struct gdma_queue *gdma_rq;
395 /* Cache the gdma receive queue id */
396 u32 gdma_id;
397
398 /* Index of RQ in the vPort, not gdma receive queue id */
399 u32 rxq_idx;
400
401 u32 datasize;
402 u32 alloc_size;
403 u32 headroom;
404 u32 frag_count;
405
406 mana_handle_t rxobj;
407
408 struct mana_cq rx_cq;
409
410 struct completion fence_event;
411
412 struct net_device *ndev;
413
414 /* Total number of receive buffers to be allocated */
415 u32 num_rx_buf;
416
417 u32 buf_index;
418
419 struct mana_stats_rx stats;
420
421 struct bpf_prog __rcu *bpf_prog;
422 struct xdp_rxq_info xdp_rxq;
423 void *xdp_save_va; /* for reusing */
424 bool xdp_flush;
425 int xdp_rc; /* XDP redirect return code */
426
427 struct page_pool *page_pool;
428 struct dentry *mana_rx_debugfs;
429
430 /* MUST BE THE LAST MEMBER:
431 * Each receive buffer has an associated mana_recv_buf_oob.
432 */
433 struct mana_recv_buf_oob rx_oobs[] __counted_by(num_rx_buf);
434 };
435
436 struct mana_tx_qp {
437 struct mana_txq txq;
438
439 struct mana_cq tx_cq;
440
441 mana_handle_t tx_object;
442
443 struct dentry *mana_tx_debugfs;
444 };
445
446 struct mana_ethtool_stats {
447 u64 stop_queue;
448 u64 wake_queue;
449 u64 tx_cqe_err;
450 u64 tx_cqe_unknown_type;
451 u64 tx_linear_pkt_cnt;
452 u64 rx_cqe_unknown_type;
453 };
454
455 struct mana_ethtool_hc_stats {
456 u64 hc_rx_discards_no_wqe;
457 u64 hc_rx_err_vport_disabled;
458 u64 hc_rx_bytes;
459 u64 hc_rx_ucast_pkts;
460 u64 hc_rx_ucast_bytes;
461 u64 hc_rx_bcast_pkts;
462 u64 hc_rx_bcast_bytes;
463 u64 hc_rx_mcast_pkts;
464 u64 hc_rx_mcast_bytes;
465 u64 hc_tx_err_gf_disabled;
466 u64 hc_tx_err_vport_disabled;
467 u64 hc_tx_err_inval_vportoffset_pkt;
468 u64 hc_tx_err_vlan_enforcement;
469 u64 hc_tx_err_eth_type_enforcement;
470 u64 hc_tx_err_sa_enforcement;
471 u64 hc_tx_err_sqpdid_enforcement;
472 u64 hc_tx_err_cqpdid_enforcement;
473 u64 hc_tx_err_mtu_violation;
474 u64 hc_tx_err_inval_oob;
475 u64 hc_tx_bytes;
476 u64 hc_tx_ucast_pkts;
477 u64 hc_tx_ucast_bytes;
478 u64 hc_tx_bcast_pkts;
479 u64 hc_tx_bcast_bytes;
480 u64 hc_tx_mcast_pkts;
481 u64 hc_tx_mcast_bytes;
482 u64 hc_tx_err_gdma;
483 };
484
485 struct mana_ethtool_phy_stats {
486 /* Drop Counters */
487 u64 rx_pkt_drop_phy;
488 u64 tx_pkt_drop_phy;
489
490 /* Per TC traffic Counters */
491 u64 rx_pkt_tc0_phy;
492 u64 tx_pkt_tc0_phy;
493 u64 rx_pkt_tc1_phy;
494 u64 tx_pkt_tc1_phy;
495 u64 rx_pkt_tc2_phy;
496 u64 tx_pkt_tc2_phy;
497 u64 rx_pkt_tc3_phy;
498 u64 tx_pkt_tc3_phy;
499 u64 rx_pkt_tc4_phy;
500 u64 tx_pkt_tc4_phy;
501 u64 rx_pkt_tc5_phy;
502 u64 tx_pkt_tc5_phy;
503 u64 rx_pkt_tc6_phy;
504 u64 tx_pkt_tc6_phy;
505 u64 rx_pkt_tc7_phy;
506 u64 tx_pkt_tc7_phy;
507
508 u64 rx_byte_tc0_phy;
509 u64 tx_byte_tc0_phy;
510 u64 rx_byte_tc1_phy;
511 u64 tx_byte_tc1_phy;
512 u64 rx_byte_tc2_phy;
513 u64 tx_byte_tc2_phy;
514 u64 rx_byte_tc3_phy;
515 u64 tx_byte_tc3_phy;
516 u64 rx_byte_tc4_phy;
517 u64 tx_byte_tc4_phy;
518 u64 rx_byte_tc5_phy;
519 u64 tx_byte_tc5_phy;
520 u64 rx_byte_tc6_phy;
521 u64 tx_byte_tc6_phy;
522 u64 rx_byte_tc7_phy;
523 u64 tx_byte_tc7_phy;
524
525 /* Per TC pause Counters */
526 u64 rx_pause_tc0_phy;
527 u64 tx_pause_tc0_phy;
528 u64 rx_pause_tc1_phy;
529 u64 tx_pause_tc1_phy;
530 u64 rx_pause_tc2_phy;
531 u64 tx_pause_tc2_phy;
532 u64 rx_pause_tc3_phy;
533 u64 tx_pause_tc3_phy;
534 u64 rx_pause_tc4_phy;
535 u64 tx_pause_tc4_phy;
536 u64 rx_pause_tc5_phy;
537 u64 tx_pause_tc5_phy;
538 u64 rx_pause_tc6_phy;
539 u64 tx_pause_tc6_phy;
540 u64 rx_pause_tc7_phy;
541 u64 tx_pause_tc7_phy;
542 };
543
544 struct mana_context {
545 struct gdma_dev *gdma_dev;
546
547 u16 num_ports;
548 u8 bm_hostmode;
549
550 struct mana_ethtool_hc_stats hc_stats;
551 struct workqueue_struct *per_port_queue_reset_wq;
552 /* Workqueue for querying hardware stats */
553 struct delayed_work gf_stats_work;
554 bool hwc_timeout_occurred;
555
556 struct net_device *ports[MAX_PORTS_IN_MANA_DEV];
557
558 /* Link state change work */
559 struct work_struct link_change_work;
560 u32 link_event;
561 };
562
563 struct mana_port_context {
564 struct mana_context *ac;
565 struct net_device *ndev;
566 struct work_struct queue_reset_work;
567
568 /* Debug knob to log TX timeout but skip recovery reset */
569 bool tx_timeout_skip_reset;
570
571 u8 mac_addr[ETH_ALEN];
572
573 struct mana_eq *eqs;
574 struct dentry *mana_eqs_debugfs;
575
576 enum TRI_STATE rss_state;
577
578 mana_handle_t default_rxobj;
579 bool tx_shortform_allowed;
580 u16 tx_vp_offset;
581
582 struct mana_tx_qp **tx_qp;
583
584 /* Indirection Table for RX & TX. The values are queue indexes */
585 u32 *indir_table;
586 u32 indir_table_sz;
587
588 /* Indirection table containing RxObject Handles */
589 mana_handle_t *rxobj_table;
590
591 /* Hash key used by the NIC */
592 u8 hashkey[MANA_HASH_KEY_SIZE];
593
594 /* This points to an array of num_queues of RQ pointers. */
595 struct mana_rxq **rxqs;
596
597 /* pre-allocated rx buffer array */
598 void **rxbufs_pre;
599 dma_addr_t *das_pre;
600 int rxbpre_total;
601 u32 rxbpre_datasize;
602 u32 rxbpre_alloc_size;
603 u32 rxbpre_headroom;
604 u32 rxbpre_frag_count;
605
606 u32 priv_flags;
607
608 struct bpf_prog *bpf_prog;
609
610 /* Create num_queues EQs, SQs, SQ-CQs, RQs and RQ-CQs, respectively. */
611 unsigned int max_queues;
612 unsigned int num_queues;
613
614 unsigned int rx_queue_size;
615 unsigned int tx_queue_size;
616
617 mana_handle_t port_handle;
618 mana_handle_t pf_filter_handle;
619
620 /* Mutex for sharing access to vport_use_count */
621 struct mutex vport_mutex;
622 int vport_use_count;
623
624 /* Set by mana_set_channels() under vport_mutex to block RDMA
625 * from grabbing the vport during the detach/attach window.
626 * Checked by mana_cfg_vport() when called from the RDMA path.
627 */
628 bool channel_changing;
629
630 /* Net shaper handle*/
631 struct net_shaper_handle handle;
632
633 u16 port_idx;
634 /* Currently configured speed (mbps) */
635 u32 speed;
636 /* Maximum speed supported by the SKU (mbps) */
637 u32 max_speed;
638 /* 1 = not queried, 0 = cached success, negative = permanent error.
639 * Protected by the netdev instance lock.
640 */
641 int link_cfg_error;
642
643 bool port_is_up;
644 bool port_st_save; /* Saved port state */
645
646 u8 cqe_coalescing_enable;
647 u8 cqe8_coalescing_enable;
648 u32 cqe_coalescing_timeout_ns;
649
650 /* Interrupt moderation settings */
651 u16 intr_modr_rx_usec;
652 u16 intr_modr_rx_comp;
653 u16 intr_modr_tx_usec;
654 u16 intr_modr_tx_comp;
655
656 bool rx_dim_enabled;
657 bool tx_dim_enabled;
658
659 struct mana_ethtool_stats eth_stats;
660
661 struct mana_ethtool_phy_stats phy_stats;
662
663 /* Debugfs */
664 struct dentry *mana_port_debugfs;
665
666 /* Cached vport/steering config for debugfs */
667 u32 vport_max_sq;
668 u32 vport_max_rq;
669 u32 steer_rx;
670 u32 steer_rss;
671 bool steer_update_tab;
672 u32 steer_cqe_coalescing;
673 };
674
675 netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev);
676 int mana_config_rss(struct mana_port_context *ac, enum TRI_STATE rx,
677 bool update_hash, bool update_tab);
678 int mana_disable_vport_rx(struct mana_port_context *apc);
679
680 int mana_alloc_queues(struct net_device *ndev);
681 int mana_attach(struct net_device *ndev);
682 int mana_detach(struct net_device *ndev, bool from_close);
683
684 void mana_dim_change(struct mana_cq *cq, bool enable);
685
686 int mana_probe(struct gdma_dev *gd, bool resuming);
687 void mana_remove(struct gdma_dev *gd, bool suspending);
688
689 int mana_rdma_probe(struct gdma_dev *gd);
690 void mana_rdma_remove(struct gdma_dev *gd);
691
692 void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev);
693 int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames,
694 u32 flags);
695 u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq,
696 struct xdp_buff *xdp, void *buf_va, uint pkt_len);
697 struct bpf_prog *mana_xdp_get(struct mana_port_context *apc);
698 void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog);
699 int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf);
700 int mana_query_gf_stats(struct mana_context *ac);
701 int mana_query_link_cfg(struct mana_port_context *apc);
702 int mana_set_bw_clamp(struct mana_port_context *apc, u32 speed,
703 int enable_clamping);
704 void mana_query_phy_stats(struct mana_port_context *apc);
705 bool mana_single_rxbuf_per_page_forced(struct mana_port_context *apc, u32 mtu);
706 int mana_pre_alloc_rxbufs(struct mana_port_context *apc, int mtu, int num_queues);
707 void mana_pre_dealloc_rxbufs(struct mana_port_context *apc);
708 void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc);
709
710 extern const struct ethtool_ops mana_ethtool_ops;
711 extern struct dentry *mana_debugfs_root;
712
713 /* A CQ can be created not associated with any EQ */
714 #define GDMA_CQ_NO_EQ 0xffff
715
716 struct mana_obj_spec {
717 u32 queue_index;
718 u64 gdma_region;
719 u32 queue_size;
720 u32 attached_eq;
721 u32 modr_ctx_id;
722 u8 req_cq_moderation;
723 u16 cq_moderation_comp;
724 u16 cq_moderation_usec;
725 };
726
727 enum mana_command_code {
728 MANA_QUERY_DEV_CONFIG = 0x20001,
729 MANA_QUERY_GF_STAT = 0x20002,
730 MANA_CONFIG_VPORT_TX = 0x20003,
731 MANA_CREATE_WQ_OBJ = 0x20004,
732 MANA_DESTROY_WQ_OBJ = 0x20005,
733 MANA_FENCE_RQ = 0x20006,
734 MANA_CONFIG_VPORT_RX = 0x20007,
735 MANA_QUERY_VPORT_CONFIG = 0x20008,
736 MANA_QUERY_LINK_CONFIG = 0x2000A,
737 MANA_SET_BW_CLAMP = 0x2000B,
738 MANA_QUERY_PHY_STAT = 0x2000c,
739
740 /* Privileged commands for the PF mode */
741 MANA_REGISTER_FILTER = 0x28000,
742 MANA_DEREGISTER_FILTER = 0x28001,
743 MANA_REGISTER_HW_PORT = 0x28003,
744 MANA_DEREGISTER_HW_PORT = 0x28004,
745 };
746
747 /* Query Link Configuration*/
748 struct mana_query_link_config_req {
749 struct gdma_req_hdr hdr;
750 mana_handle_t vport;
751 }; /* HW DATA */
752
753 struct mana_query_link_config_resp {
754 struct gdma_resp_hdr hdr;
755 u32 qos_speed_mbps;
756 u8 qos_unconfigured;
757 u8 reserved1[3];
758 u32 link_speed_mbps;
759 u8 reserved2[4];
760 }; /* HW DATA */
761
762 /* Set Bandwidth Clamp*/
763 struct mana_set_bw_clamp_req {
764 struct gdma_req_hdr hdr;
765 mana_handle_t vport;
766 enum TRI_STATE enable_clamping;
767 u32 link_speed_mbps;
768 }; /* HW DATA */
769
770 struct mana_set_bw_clamp_resp {
771 struct gdma_resp_hdr hdr;
772 u8 qos_unconfigured;
773 u8 reserved[7];
774 }; /* HW DATA */
775
776 /* Query Device Configuration */
777 struct mana_query_device_cfg_req {
778 struct gdma_req_hdr hdr;
779
780 /* MANA Nic Driver Capability flags */
781 u64 mn_drv_cap_flags1;
782 u64 mn_drv_cap_flags2;
783 u64 mn_drv_cap_flags3;
784 u64 mn_drv_cap_flags4;
785
786 u32 proto_major_ver;
787 u32 proto_minor_ver;
788 u32 proto_micro_ver;
789
790 u32 reserved;
791 }; /* HW DATA */
792
793 #define MANA_PF_FLAG_1_CQE_8_COALESCING_SUPPORTED BIT(5)
794
795 struct mana_query_device_cfg_resp {
796 struct gdma_resp_hdr hdr;
797
798 u64 pf_cap_flags1;
799 u64 pf_cap_flags2;
800 u64 pf_cap_flags3;
801 u64 pf_cap_flags4;
802
803 u16 max_num_vports;
804 u8 bm_hostmode; /* response v3: Bare Metal Host Mode */
805 u8 reserved;
806 u32 max_num_eqs;
807
808 /* response v2: */
809 u16 adapter_mtu;
810 u16 reserved2;
811 u32 reserved3;
812 }; /* HW DATA */
813
814 /* Query vPort Configuration */
815 struct mana_query_vport_cfg_req {
816 struct gdma_req_hdr hdr;
817 u32 vport_index;
818 }; /* HW DATA */
819
820 struct mana_query_vport_cfg_resp {
821 struct gdma_resp_hdr hdr;
822 u32 max_num_sq;
823 u32 max_num_rq;
824 u32 num_indirection_ent;
825 u32 reserved1;
826 u8 mac_addr[6];
827 u8 reserved2[2];
828 mana_handle_t vport;
829 }; /* HW DATA */
830
831 /* Configure vPort */
832 struct mana_config_vport_req {
833 struct gdma_req_hdr hdr;
834 mana_handle_t vport;
835 u32 pdid;
836 u32 doorbell_pageid;
837 }; /* HW DATA */
838
839 struct mana_config_vport_resp {
840 struct gdma_resp_hdr hdr;
841 u16 tx_vport_offset;
842 u8 short_form_allowed;
843 u8 reserved;
844 }; /* HW DATA */
845
846 /* Create WQ Object */
847 struct mana_create_wqobj_req {
848 struct gdma_req_hdr hdr;
849 mana_handle_t vport;
850 u32 wq_type;
851 u32 reserved;
852 u64 wq_gdma_region;
853 u64 cq_gdma_region;
854 u32 wq_size;
855 u32 cq_size;
856 u32 cq_moderation_ctx_id;
857 u32 cq_parent_qid;
858
859 /* V2 */
860 u8 allow_rqwqe_chain;
861
862 /* V3 */
863 u8 req_cq_moderation;
864 u16 cq_moderation_comp;
865 u16 cq_moderation_usec;
866 u8 reserved2[2];
867 }; /* HW DATA */
868
869 struct mana_create_wqobj_resp {
870 struct gdma_resp_hdr hdr;
871 u32 wq_id;
872 u32 cq_id;
873 mana_handle_t wq_obj;
874
875 /* V2 */
876 u16 cq_moderation_comp;
877 u16 cq_moderation_usec;
878 u8 cq_moderation_enabled;
879 u8 reserved1[3];
880 }; /* HW DATA */
881
882 /* Destroy WQ Object */
883 struct mana_destroy_wqobj_req {
884 struct gdma_req_hdr hdr;
885 u32 wq_type;
886 u32 reserved;
887 mana_handle_t wq_obj_handle;
888 }; /* HW DATA */
889
890 struct mana_destroy_wqobj_resp {
891 struct gdma_resp_hdr hdr;
892 }; /* HW DATA */
893
894 /* Fence RQ */
895 struct mana_fence_rq_req {
896 struct gdma_req_hdr hdr;
897 mana_handle_t wq_obj_handle;
898 }; /* HW DATA */
899
900 struct mana_fence_rq_resp {
901 struct gdma_resp_hdr hdr;
902 }; /* HW DATA */
903
904 /* Query stats RQ */
905 struct mana_query_gf_stat_req {
906 struct gdma_req_hdr hdr;
907 u64 req_stats;
908 }; /* HW DATA */
909
910 struct mana_query_gf_stat_resp {
911 struct gdma_resp_hdr hdr;
912 u64 reported_stats;
913 /* rx errors/discards */
914 u64 rx_discards_nowqe;
915 u64 rx_err_vport_disabled;
916 /* rx bytes/packets */
917 u64 hc_rx_bytes;
918 u64 hc_rx_ucast_pkts;
919 u64 hc_rx_ucast_bytes;
920 u64 hc_rx_bcast_pkts;
921 u64 hc_rx_bcast_bytes;
922 u64 hc_rx_mcast_pkts;
923 u64 hc_rx_mcast_bytes;
924 /* tx errors */
925 u64 tx_err_gf_disabled;
926 u64 tx_err_vport_disabled;
927 u64 tx_err_inval_vport_offset_pkt;
928 u64 tx_err_vlan_enforcement;
929 u64 tx_err_ethtype_enforcement;
930 u64 tx_err_SA_enforcement;
931 u64 tx_err_SQPDID_enforcement;
932 u64 tx_err_CQPDID_enforcement;
933 u64 tx_err_mtu_violation;
934 u64 tx_err_inval_oob;
935 /* tx bytes/packets */
936 u64 hc_tx_bytes;
937 u64 hc_tx_ucast_pkts;
938 u64 hc_tx_ucast_bytes;
939 u64 hc_tx_bcast_pkts;
940 u64 hc_tx_bcast_bytes;
941 u64 hc_tx_mcast_pkts;
942 u64 hc_tx_mcast_bytes;
943 /* tx error */
944 u64 tx_err_gdma;
945 }; /* HW DATA */
946
947 /* Query phy stats */
948 struct mana_query_phy_stat_req {
949 struct gdma_req_hdr hdr;
950 u64 req_stats;
951 }; /* HW DATA */
952
953 struct mana_query_phy_stat_resp {
954 struct gdma_resp_hdr hdr;
955 u64 reported_stats;
956
957 /* Aggregate Drop Counters */
958 u64 rx_pkt_drop_phy;
959 u64 tx_pkt_drop_phy;
960
961 /* Per TC(Traffic class) traffic Counters */
962 u64 rx_pkt_tc0_phy;
963 u64 tx_pkt_tc0_phy;
964 u64 rx_pkt_tc1_phy;
965 u64 tx_pkt_tc1_phy;
966 u64 rx_pkt_tc2_phy;
967 u64 tx_pkt_tc2_phy;
968 u64 rx_pkt_tc3_phy;
969 u64 tx_pkt_tc3_phy;
970 u64 rx_pkt_tc4_phy;
971 u64 tx_pkt_tc4_phy;
972 u64 rx_pkt_tc5_phy;
973 u64 tx_pkt_tc5_phy;
974 u64 rx_pkt_tc6_phy;
975 u64 tx_pkt_tc6_phy;
976 u64 rx_pkt_tc7_phy;
977 u64 tx_pkt_tc7_phy;
978
979 u64 rx_byte_tc0_phy;
980 u64 tx_byte_tc0_phy;
981 u64 rx_byte_tc1_phy;
982 u64 tx_byte_tc1_phy;
983 u64 rx_byte_tc2_phy;
984 u64 tx_byte_tc2_phy;
985 u64 rx_byte_tc3_phy;
986 u64 tx_byte_tc3_phy;
987 u64 rx_byte_tc4_phy;
988 u64 tx_byte_tc4_phy;
989 u64 rx_byte_tc5_phy;
990 u64 tx_byte_tc5_phy;
991 u64 rx_byte_tc6_phy;
992 u64 tx_byte_tc6_phy;
993 u64 rx_byte_tc7_phy;
994 u64 tx_byte_tc7_phy;
995
996 /* Per TC(Traffic Class) pause Counters */
997 u64 rx_pause_tc0_phy;
998 u64 tx_pause_tc0_phy;
999 u64 rx_pause_tc1_phy;
1000 u64 tx_pause_tc1_phy;
1001 u64 rx_pause_tc2_phy;
1002 u64 tx_pause_tc2_phy;
1003 u64 rx_pause_tc3_phy;
1004 u64 tx_pause_tc3_phy;
1005 u64 rx_pause_tc4_phy;
1006 u64 tx_pause_tc4_phy;
1007 u64 rx_pause_tc5_phy;
1008 u64 tx_pause_tc5_phy;
1009 u64 rx_pause_tc6_phy;
1010 u64 tx_pause_tc6_phy;
1011 u64 rx_pause_tc7_phy;
1012 u64 tx_pause_tc7_phy;
1013 }; /* HW DATA */
1014
1015 /* Configure vPort Rx Steering */
1016 struct mana_cfg_rx_steer_req_v2 {
1017 struct gdma_req_hdr hdr;
1018 mana_handle_t vport;
1019 u16 num_indir_entries;
1020 u16 indir_tab_offset;
1021 u32 rx_enable;
1022 u32 rss_enable;
1023 u8 update_default_rxobj;
1024 u8 update_hashkey;
1025 u8 update_indir_tab;
1026 u8 reserved;
1027 mana_handle_t default_rxobj;
1028 u8 hashkey[MANA_HASH_KEY_SIZE];
1029 u8 cqe_coalescing_enable;
1030 u8 reserved2[3];
1031 u16 rss_hash_types;
1032 u8 cqe8_coalescing_enable; /* v5 message */
1033 u8 reserved3;
1034 mana_handle_t indir_tab[] __counted_by(num_indir_entries);
1035 }; /* HW DATA */
1036
1037 struct mana_cfg_rx_steer_resp {
1038 struct gdma_resp_hdr hdr;
1039
1040 /* V2 */
1041 u32 cqe_coalescing_timeout_ns;
1042 u32 reserved1;
1043 }; /* HW DATA */
1044
1045 /* Register HW vPort */
1046 struct mana_register_hw_vport_req {
1047 struct gdma_req_hdr hdr;
1048 u16 attached_gfid;
1049 u8 is_pf_default_vport;
1050 u8 reserved1;
1051 u8 allow_all_ether_types;
1052 u8 reserved2;
1053 u8 reserved3;
1054 u8 reserved4;
1055 }; /* HW DATA */
1056
1057 struct mana_register_hw_vport_resp {
1058 struct gdma_resp_hdr hdr;
1059 mana_handle_t hw_vport_handle;
1060 }; /* HW DATA */
1061
1062 /* Deregister HW vPort */
1063 struct mana_deregister_hw_vport_req {
1064 struct gdma_req_hdr hdr;
1065 mana_handle_t hw_vport_handle;
1066 }; /* HW DATA */
1067
1068 struct mana_deregister_hw_vport_resp {
1069 struct gdma_resp_hdr hdr;
1070 }; /* HW DATA */
1071
1072 /* Register filter */
1073 struct mana_register_filter_req {
1074 struct gdma_req_hdr hdr;
1075 mana_handle_t vport;
1076 u8 mac_addr[6];
1077 u8 reserved1;
1078 u8 reserved2;
1079 u8 reserved3;
1080 u8 reserved4;
1081 u16 reserved5;
1082 u32 reserved6;
1083 u32 reserved7;
1084 u32 reserved8;
1085 }; /* HW DATA */
1086
1087 struct mana_register_filter_resp {
1088 struct gdma_resp_hdr hdr;
1089 mana_handle_t filter_handle;
1090 }; /* HW DATA */
1091
1092 /* Deregister filter */
1093 struct mana_deregister_filter_req {
1094 struct gdma_req_hdr hdr;
1095 mana_handle_t filter_handle;
1096 }; /* HW DATA */
1097
1098 struct mana_deregister_filter_resp {
1099 struct gdma_resp_hdr hdr;
1100 }; /* HW DATA */
1101
1102 /* Requested GF stats Flags */
1103 /* Rx discards/Errors */
1104 #define STATISTICS_FLAGS_RX_DISCARDS_NO_WQE 0x0000000000000001
1105 #define STATISTICS_FLAGS_RX_ERRORS_VPORT_DISABLED 0x0000000000000002
1106 /* Rx bytes/pkts */
1107 #define STATISTICS_FLAGS_HC_RX_BYTES 0x0000000000000004
1108 #define STATISTICS_FLAGS_HC_RX_UCAST_PACKETS 0x0000000000000008
1109 #define STATISTICS_FLAGS_HC_RX_UCAST_BYTES 0x0000000000000010
1110 #define STATISTICS_FLAGS_HC_RX_MCAST_PACKETS 0x0000000000000020
1111 #define STATISTICS_FLAGS_HC_RX_MCAST_BYTES 0x0000000000000040
1112 #define STATISTICS_FLAGS_HC_RX_BCAST_PACKETS 0x0000000000000080
1113 #define STATISTICS_FLAGS_HC_RX_BCAST_BYTES 0x0000000000000100
1114 /* Tx errors */
1115 #define STATISTICS_FLAGS_TX_ERRORS_GF_DISABLED 0x0000000000000200
1116 #define STATISTICS_FLAGS_TX_ERRORS_VPORT_DISABLED 0x0000000000000400
1117 #define STATISTICS_FLAGS_TX_ERRORS_INVAL_VPORT_OFFSET_PACKETS \
1118 0x0000000000000800
1119 #define STATISTICS_FLAGS_TX_ERRORS_VLAN_ENFORCEMENT 0x0000000000001000
1120 #define STATISTICS_FLAGS_TX_ERRORS_ETH_TYPE_ENFORCEMENT \
1121 0x0000000000002000
1122 #define STATISTICS_FLAGS_TX_ERRORS_SA_ENFORCEMENT 0x0000000000004000
1123 #define STATISTICS_FLAGS_TX_ERRORS_SQPDID_ENFORCEMENT 0x0000000000008000
1124 #define STATISTICS_FLAGS_TX_ERRORS_CQPDID_ENFORCEMENT 0x0000000000010000
1125 #define STATISTICS_FLAGS_TX_ERRORS_MTU_VIOLATION 0x0000000000020000
1126 #define STATISTICS_FLAGS_TX_ERRORS_INVALID_OOB 0x0000000000040000
1127 /* Tx bytes/pkts */
1128 #define STATISTICS_FLAGS_HC_TX_BYTES 0x0000000000080000
1129 #define STATISTICS_FLAGS_HC_TX_UCAST_PACKETS 0x0000000000100000
1130 #define STATISTICS_FLAGS_HC_TX_UCAST_BYTES 0x0000000000200000
1131 #define STATISTICS_FLAGS_HC_TX_MCAST_PACKETS 0x0000000000400000
1132 #define STATISTICS_FLAGS_HC_TX_MCAST_BYTES 0x0000000000800000
1133 #define STATISTICS_FLAGS_HC_TX_BCAST_PACKETS 0x0000000001000000
1134 #define STATISTICS_FLAGS_HC_TX_BCAST_BYTES 0x0000000002000000
1135 /* Tx error */
1136 #define STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR 0x0000000004000000
1137
1138 #define MANA_MAX_NUM_QUEUES 64
1139 #define MANA_DEF_NUM_QUEUES 16
1140
1141 #define MANA_SHORT_VPORT_OFFSET_MAX ((1U << 8) - 1)
1142
1143 struct mana_tx_package {
1144 struct gdma_wqe_request wqe_req;
1145 struct gdma_sge sgl_array[5];
1146 struct gdma_sge *sgl_ptr;
1147
1148 struct mana_tx_oob tx_oob;
1149
1150 struct gdma_posted_wqe_info wqe_info;
1151 };
1152
1153 int mana_create_wq_obj(struct mana_port_context *apc,
1154 mana_handle_t vport,
1155 u32 wq_type, struct mana_obj_spec *wq_spec,
1156 struct mana_obj_spec *cq_spec,
1157 mana_handle_t *wq_obj);
1158
1159 void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type,
1160 mana_handle_t wq_obj);
1161
1162 int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id,
1163 u32 doorbell_pg_id, bool check_channel_changing);
1164 void mana_uncfg_vport(struct mana_port_context *apc);
1165 int mana_create_eq(struct mana_port_context *apc);
1166 void mana_destroy_eq(struct mana_port_context *apc);
1167
1168 struct net_device *mana_get_primary_netdev(struct mana_context *ac,
1169 u32 port_index,
1170 netdevice_tracker *tracker);
1171 #endif /* _MANA_H */
1172