1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */
3
4 #ifndef _FBNIC_TXRX_H_
5 #define _FBNIC_TXRX_H_
6
7 #include <linux/netdevice.h>
8 #include <linux/skbuff.h>
9 #include <linux/types.h>
10 #include <linux/u64_stats_sync.h>
11 #include <net/xdp.h>
12
13 struct fbnic_net;
14
15 /* Guarantee we have space needed for storing the buffer
16 * To store the buffer we need:
17 * 1 descriptor per page
18 * + 1 descriptor for skb head
19 * + 2 descriptors for metadata and optional metadata
20 * + 7 descriptors to keep tail out of the same cacheline as head
21 * If we cannot guarantee that then we should return TX_BUSY
22 */
23 #define FBNIC_MAX_SKB_DESC (MAX_SKB_FRAGS + 10)
24 #define FBNIC_TX_DESC_WAKEUP (FBNIC_MAX_SKB_DESC * 2)
25 #define FBNIC_TX_DESC_MIN roundup_pow_of_two(FBNIC_TX_DESC_WAKEUP)
26
27 /* To receive the worst case packet we need:
28 * 1 descriptor for primary metadata
29 * + 1 descriptor for optional metadata
30 * + 1 descriptor for headers
31 * + 4 descriptors for payload
32 */
33 #define FBNIC_MAX_RX_PKT_DESC 7
34 #define FBNIC_RX_DESC_MIN roundup_pow_of_two(FBNIC_MAX_RX_PKT_DESC * 2)
35
36 #define FBNIC_MAX_TXQS 128u
37 #define FBNIC_MAX_RXQS 128u
38 #define FBNIC_MAX_XDPQS 128u
39
40 /* These apply to TWQs, TCQ, RCQ */
41 #define FBNIC_QUEUE_SIZE_MIN 16u
42 #define FBNIC_QUEUE_SIZE_MAX SZ_64K
43
44 #define FBNIC_TXQ_SIZE_DEFAULT 1024
45 #define FBNIC_HPQ_SIZE_DEFAULT 256
46 #define FBNIC_PPQ_SIZE_DEFAULT 256
47 #define FBNIC_RCQ_SIZE_DEFAULT 1024
48 #define FBNIC_TX_USECS_DEFAULT 35
49 #define FBNIC_RX_USECS_DEFAULT 30
50 #define FBNIC_RX_FRAMES_DEFAULT 0
51
52 #define FBNIC_RX_TROOM \
53 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
54 #define FBNIC_RX_HROOM_PAD 128
55 #define FBNIC_RX_HROOM \
56 (ALIGN(FBNIC_RX_TROOM + FBNIC_RX_HROOM_PAD, 128) - FBNIC_RX_TROOM)
57 #define FBNIC_RX_PAD 0
58 #define FBNIC_RX_PAYLD_OFFSET 0
59 #define FBNIC_RX_PAYLD_PG_CL 0
60
61 #define FBNIC_RING_F_DISABLED BIT(0)
62 #define FBNIC_RING_F_CTX BIT(1)
63 #define FBNIC_RING_F_STATS BIT(2) /* Ring's stats may be used */
64
65 #define FBNIC_HDS_THRESH_MAX \
66 (4096 - FBNIC_RX_HROOM - FBNIC_RX_TROOM - FBNIC_RX_PAD)
67 #define FBNIC_HDS_THRESH_DEFAULT \
68 (1536 - FBNIC_RX_PAD)
69 #define FBNIC_HDR_BYTES_MIN 128
70
71 struct fbnic_pkt_buff {
72 struct xdp_buff buff;
73 ktime_t hwtstamp;
74 bool add_frag_failed;
75 };
76
77 struct fbnic_queue_stats {
78 u64 packets;
79 u64 bytes;
80 union {
81 struct {
82 u64 csum_partial;
83 u64 lso;
84 u64 ts_packets;
85 u64 ts_lost;
86 u64 stop;
87 u64 wake;
88 } twq;
89 struct {
90 u64 alloc_failed;
91 u64 csum_complete;
92 u64 csum_none;
93 u64 length_errors;
94 } rx;
95 struct {
96 u64 alloc_failed;
97 } bdq;
98 };
99 u64 dropped;
100 struct u64_stats_sync syncp;
101 };
102
103 #define FBNIC_PAGECNT_BIAS_MAX PAGE_SIZE
104
105 struct fbnic_rx_buf {
106 netmem_ref netmem;
107 long pagecnt_bias;
108 };
109
110 struct fbnic_ring {
111 /* Pointer to buffer specific info */
112 union {
113 struct fbnic_pkt_buff *pkt; /* RCQ */
114 struct fbnic_rx_buf *rx_buf; /* BDQ */
115 void **tx_buf; /* TWQ */
116 void *buffer; /* Generic pointer */
117 };
118
119 u32 __iomem *doorbell; /* Pointer to CSR space for ring */
120 __le64 *desc; /* Descriptor ring memory */
121 u16 size_mask; /* Size of ring in descriptors - 1 */
122 u8 q_idx; /* Logical netdev ring index */
123 u8 flags; /* Ring flags (FBNIC_RING_F_*) */
124
125 u32 head, tail; /* Head/Tail of ring */
126
127 union {
128 /* Rx BDQs only */
129 struct page_pool *page_pool;
130
131 /* Deferred_head is used to cache the head for TWQ1 if
132 * an attempt is made to clean TWQ1 with zero napi_budget.
133 * We do not use it for any other ring.
134 */
135 s32 deferred_head;
136 };
137
138 struct fbnic_queue_stats stats;
139
140 /* Slow path fields follow */
141 dma_addr_t dma; /* Phys addr of descriptor memory */
142 size_t size; /* Size of descriptor ring in memory */
143 };
144
145 struct fbnic_q_triad {
146 struct fbnic_ring sub0, sub1, cmpl;
147 struct xdp_rxq_info xdp_rxq;
148 };
149
150 struct fbnic_napi_vector {
151 struct napi_struct napi;
152 struct device *dev; /* Device for DMA unmapping */
153 struct fbnic_dev *fbd;
154
155 u16 v_idx;
156 u8 txt_count;
157 u8 rxt_count;
158
159 struct fbnic_q_triad qt[];
160 };
161
162 extern const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops;
163
164 netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev);
165 netdev_features_t
166 fbnic_features_check(struct sk_buff *skb, struct net_device *dev,
167 netdev_features_t features);
168
169 void fbnic_aggregate_ring_rx_counters(struct fbnic_net *fbn,
170 struct fbnic_ring *rxr);
171 void fbnic_aggregate_ring_bdq_counters(struct fbnic_net *fbn,
172 struct fbnic_ring *rxr);
173 void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn,
174 struct fbnic_ring *txr);
175 void fbnic_aggregate_ring_xdp_counters(struct fbnic_net *fbn,
176 struct fbnic_ring *xdpr);
177
178 int fbnic_alloc_napi_vectors(struct fbnic_net *fbn);
179 void fbnic_free_napi_vectors(struct fbnic_net *fbn);
180 int fbnic_alloc_resources(struct fbnic_net *fbn);
181 void fbnic_free_resources(struct fbnic_net *fbn);
182 int fbnic_set_netif_queues(struct fbnic_net *fbn);
183 void fbnic_reset_netif_queues(struct fbnic_net *fbn);
184 irqreturn_t fbnic_msix_clean_rings(int irq, void *data);
185 void fbnic_napi_enable(struct fbnic_net *fbn);
186 void fbnic_napi_disable(struct fbnic_net *fbn);
187 void fbnic_enable(struct fbnic_net *fbn);
188 void fbnic_disable(struct fbnic_net *fbn);
189 void fbnic_flush(struct fbnic_net *fbn);
190 void fbnic_fill(struct fbnic_net *fbn);
191
192 void fbnic_napi_depletion_check(struct net_device *netdev);
193 int fbnic_wait_all_queues_idle(struct fbnic_dev *fbd, bool may_fail);
194
fbnic_napi_idx(const struct fbnic_napi_vector * nv)195 static inline int fbnic_napi_idx(const struct fbnic_napi_vector *nv)
196 {
197 return nv->v_idx - FBNIC_NON_NAPI_VECTORS;
198 }
199
200 #endif /* _FBNIC_TXRX_H_ */
201