1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef TEST_XSK_H_
3 #define TEST_XSK_H_
4
5 #include <linux/ethtool.h>
6 #include <linux/if_xdp.h>
7
8 #include "../kselftest.h"
9 #include "xsk.h"
10
11 #ifndef SO_PREFER_BUSY_POLL
12 #define SO_PREFER_BUSY_POLL 69
13 #endif
14
15 #ifndef SO_BUSY_POLL_BUDGET
16 #define SO_BUSY_POLL_BUDGET 70
17 #endif
18
19 #define TEST_PASS 0
20 #define TEST_FAILURE -1
21 #define TEST_CONTINUE 1
22 #define TEST_SKIP 2
23
24 #define DEFAULT_PKT_CNT (4 * 1024)
25 #define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4)
26 #define HUGEPAGE_SIZE (2 * 1024 * 1024)
27 #define MIN_PKT_SIZE 64
28 #define MAX_ETH_PKT_SIZE 1518
29 #define MAX_INTERFACE_NAME_CHARS 16
30 #define MAX_TEST_NAME_SIZE 48
31 #define SOCK_RECONF_CTR 10
32 #define USLEEP_MAX 10000
33
34 #define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags"
35 #define SMP_CACHE_BYTES_PATH "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size"
36
37 extern bool opt_verbose;
38 #define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0)
39
40
ceil_u32(u32 a,u32 b)41 static inline u32 ceil_u32(u32 a, u32 b)
42 {
43 return (a + b - 1) / b;
44 }
45
ceil_u64(u64 a,u64 b)46 static inline u64 ceil_u64(u64 a, u64 b)
47 {
48 return (a + b - 1) / b;
49 }
50
read_procfs_val(const char * path)51 static inline unsigned int read_procfs_val(const char *path)
52 {
53 unsigned int read_val = 0;
54 FILE *file;
55
56 file = fopen(path, "r");
57 if (!file) {
58 ksft_print_msg("Error opening %s\n", path);
59 return 0;
60 }
61
62 if (fscanf(file, "%u", &read_val) != 1)
63 ksft_print_msg("Error reading %s\n", path);
64
65 fclose(file);
66 return read_val;
67 }
68
69 /* Simple test */
70 enum test_mode {
71 TEST_MODE_SKB,
72 TEST_MODE_DRV,
73 TEST_MODE_ZC,
74 TEST_MODE_ALL
75 };
76
77 struct ifobject;
78 struct test_spec;
79 typedef int (*validation_func_t)(struct ifobject *ifobj);
80 typedef void *(*thread_func_t)(void *arg);
81 typedef int (*test_func_t)(struct test_spec *test);
82
83 struct xsk_socket_info {
84 struct xsk_ring_cons rx;
85 struct xsk_ring_prod tx;
86 struct xsk_umem_info *umem_real;
87 struct xsk_umem_info *umem;
88 struct xsk_socket *xsk;
89 struct pkt_stream *pkt_stream;
90 u32 outstanding_tx;
91 u32 rxqsize;
92 u32 batch_size;
93 u8 dst_mac[ETH_ALEN];
94 u8 src_mac[ETH_ALEN];
95 bool check_consumer;
96 };
97
98 int kick_rx(struct xsk_socket_info *xsk);
99 int kick_tx(struct xsk_socket_info *xsk);
100
101 struct xsk_umem_info {
102 struct xsk_ring_prod fq;
103 struct xsk_ring_cons cq;
104 struct xsk_umem *umem;
105 u64 next_buffer;
106 u64 mmap_size;
107 u32 num_frames;
108 u32 frame_headroom;
109 void *buffer;
110 u32 frame_size;
111 u32 base_addr;
112 u32 fill_size;
113 u32 comp_size;
114 bool unaligned_mode;
115 };
116
117 struct set_hw_ring {
118 u32 default_tx;
119 u32 default_rx;
120 };
121
122 int hw_ring_size_reset(struct ifobject *ifobj);
123
124 struct ifobject {
125 char ifname[MAX_INTERFACE_NAME_CHARS];
126 struct xsk_socket_info *xsk;
127 struct xsk_socket_info *xsk_arr;
128 thread_func_t func_ptr;
129 validation_func_t validation_func;
130 struct xsk_xdp_progs *xdp_progs;
131 struct bpf_map *xskmap;
132 struct bpf_program *xdp_prog;
133 struct ethtool_ringparam ring;
134 struct set_hw_ring set_ring;
135 enum test_mode mode;
136 int ifindex;
137 int mtu;
138 u32 bind_flags;
139 u32 xdp_zc_max_segs;
140 u32 umem_tailroom;
141 u32 max_skb_frags;
142 bool tx_on;
143 bool rx_on;
144 bool use_poll;
145 bool busy_poll;
146 bool use_fill_ring;
147 bool release_rx;
148 bool shared_umem;
149 bool use_metadata;
150 bool unaligned_supp;
151 bool multi_buff_supp;
152 bool multi_buff_zc_supp;
153 bool hw_ring_size_supp;
154 };
155 struct ifobject *ifobject_create(void);
156 void ifobject_delete(struct ifobject *ifobj);
157 int init_iface(struct ifobject *ifobj, thread_func_t func_ptr);
158
159 int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, u64 size);
160 int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
161 struct ifobject *ifobject, bool shared);
162
163
164 struct pkt {
165 int offset;
166 u32 len;
167 u32 pkt_nb;
168 bool valid;
169 u16 options;
170 };
171
172 struct pkt_stream {
173 u32 nb_pkts;
174 u32 current_pkt_nb;
175 struct pkt *pkts;
176 u32 max_pkt_len;
177 u32 nb_rx_pkts;
178 u32 nb_valid_entries;
179 bool verbatim;
180 };
181
pkt_continues(u32 options)182 static inline bool pkt_continues(u32 options)
183 {
184 return options & XDP_PKT_CONTD;
185 }
186
187 struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len);
188 void pkt_stream_delete(struct pkt_stream *pkt_stream);
189 void pkt_stream_reset(struct pkt_stream *pkt_stream);
190 void pkt_stream_restore_default(struct test_spec *test);
191
192 struct test_spec {
193 struct ifobject *ifobj_tx;
194 struct ifobject *ifobj_rx;
195 struct pkt_stream *tx_pkt_stream_default;
196 struct pkt_stream *rx_pkt_stream_default;
197 struct bpf_program *xdp_prog_rx;
198 struct bpf_program *xdp_prog_tx;
199 struct bpf_map *xskmap_rx;
200 struct bpf_map *xskmap_tx;
201 test_func_t test_func;
202 int mtu;
203 u16 total_steps;
204 u16 current_step;
205 u16 nb_sockets;
206 bool fail;
207 bool set_ring;
208 bool adjust_tail;
209 bool adjust_tail_support;
210 bool poll_tmout;
211 bool use_barrier;
212 enum test_mode mode;
213 char name[MAX_TEST_NAME_SIZE];
214 };
215
216 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
mode_string(struct test_spec * test)217 static inline char *mode_string(struct test_spec *test)
218 {
219 switch (test->mode) {
220 case TEST_MODE_SKB:
221 return "SKB";
222 case TEST_MODE_DRV:
223 return "DRV";
224 case TEST_MODE_ZC:
225 return "ZC";
226 default:
227 return "BOGUS";
228 }
229 }
230
231 void test_init(struct test_spec *test, struct ifobject *ifobj_tx,
232 struct ifobject *ifobj_rx, enum test_mode mode,
233 const struct test_spec *test_to_run);
234
235 int testapp_adjust_tail_grow(struct test_spec *test);
236 int testapp_adjust_tail_grow_mb(struct test_spec *test);
237 int testapp_adjust_tail_shrink(struct test_spec *test);
238 int testapp_adjust_tail_shrink_mb(struct test_spec *test);
239 int testapp_aligned_inv_desc(struct test_spec *test);
240 int testapp_aligned_inv_desc_2k_frame(struct test_spec *test);
241 int testapp_aligned_inv_desc_mb(struct test_spec *test);
242 int testapp_bidirectional(struct test_spec *test);
243 int testapp_headroom(struct test_spec *test);
244 int testapp_hw_sw_max_ring_size(struct test_spec *test);
245 int testapp_hw_sw_min_ring_size(struct test_spec *test);
246 int testapp_poll_rx(struct test_spec *test);
247 int testapp_poll_rxq_tmout(struct test_spec *test);
248 int testapp_poll_tx(struct test_spec *test);
249 int testapp_poll_txq_tmout(struct test_spec *test);
250 int testapp_send_receive(struct test_spec *test);
251 int testapp_send_receive_2k_frame(struct test_spec *test);
252 int testapp_send_receive_mb(struct test_spec *test);
253 int testapp_send_receive_unaligned(struct test_spec *test);
254 int testapp_send_receive_unaligned_mb(struct test_spec *test);
255 int testapp_single_pkt(struct test_spec *test);
256 int testapp_stats_fill_empty(struct test_spec *test);
257 int testapp_stats_rx_dropped(struct test_spec *test);
258 int testapp_stats_tx_invalid_descs(struct test_spec *test);
259 int testapp_stats_rx_full(struct test_spec *test);
260 int testapp_teardown(struct test_spec *test);
261 int testapp_too_many_frags(struct test_spec *test);
262 int testapp_tx_queue_consumer(struct test_spec *test);
263 int testapp_unaligned_inv_desc(struct test_spec *test);
264 int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test);
265 int testapp_unaligned_inv_desc_mb(struct test_spec *test);
266 int testapp_xdp_drop(struct test_spec *test);
267 int testapp_xdp_metadata(struct test_spec *test);
268 int testapp_xdp_metadata_mb(struct test_spec *test);
269 int testapp_xdp_prog_cleanup(struct test_spec *test);
270 int testapp_xdp_shared_umem(struct test_spec *test);
271
272 void *worker_testapp_validate_rx(void *arg);
273 void *worker_testapp_validate_tx(void *arg);
274
275 static const struct test_spec tests[] = {
276 {.name = "SEND_RECEIVE", .test_func = testapp_send_receive},
277 {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame},
278 {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt},
279 {.name = "POLL_RX", .test_func = testapp_poll_rx},
280 {.name = "POLL_TX", .test_func = testapp_poll_tx},
281 {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout},
282 {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout},
283 {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc},
284 {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame},
285 {.name = "UMEM_HEADROOM", .test_func = testapp_headroom},
286 {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional},
287 {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped},
288 {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs},
289 {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full},
290 {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty},
291 {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup},
292 {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop},
293 {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem},
294 {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata},
295 {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb},
296 {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb},
297 {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags},
298 {.name = "XDP_ADJUST_TAIL_SHRINK", .test_func = testapp_adjust_tail_shrink},
299 {.name = "TX_QUEUE_CONSUMER", .test_func = testapp_tx_queue_consumer},
300 };
301
302 static const struct test_spec ci_skip_tests[] = {
303 /* Flaky tests */
304 {.name = "XDP_ADJUST_TAIL_SHRINK_MULTI_BUFF", .test_func = testapp_adjust_tail_shrink_mb},
305 {.name = "XDP_ADJUST_TAIL_GROW", .test_func = testapp_adjust_tail_grow},
306 {.name = "XDP_ADJUST_TAIL_GROW_MULTI_BUFF", .test_func = testapp_adjust_tail_grow_mb},
307 {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb},
308 /* Tests with huge page dependency */
309 {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned},
310 {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc},
311 {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE",
312 .test_func = testapp_unaligned_inv_desc_4001_frame},
313 {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS",
314 .test_func = testapp_send_receive_unaligned_mb},
315 {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb},
316 /* Test with HW ring size dependency */
317 {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size},
318 {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size},
319 /* Too long test */
320 {.name = "TEARDOWN", .test_func = testapp_teardown},
321 };
322
323
324 #endif /* TEST_XSK_H_ */
325