1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2020 Intel Corporation. */
3
4 /*
5 * Some functions in this program are taken from
6 * Linux kernel samples/bpf/xdpsock* and modified
7 * for use.
8 *
9 * See test_xsk.sh for detailed information on test topology
10 * and prerequisite network setup.
11 *
12 * This test program contains two threads, each thread is single socket with
13 * a unique UMEM. It validates in-order packet delivery and packet content
14 * by sending packets to each other.
15 *
16 * Tests Information:
17 * ------------------
18 * These selftests test AF_XDP SKB and Native/DRV modes using veth
19 * Virtual Ethernet interfaces.
20 *
21 * For each mode, the following tests are run:
22 * a. nopoll - soft-irq processing in run-to-completion mode
23 * b. poll - using poll() syscall
24 * c. Socket Teardown
25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
26 * both sockets, then repeat multiple times. Only nopoll mode is used
27 * d. Bi-directional sockets
28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and
29 * completion rings on each socket, tx/rx in both directions. Only nopoll
30 * mode is used
31 * e. Statistics
32 * Trigger some error conditions and ensure that the appropriate statistics
33 * are incremented. Within this test, the following statistics are tested:
34 * i. rx dropped
35 * Increase the UMEM frame headroom to a value which results in
36 * insufficient space in the rx buffer for both the packet and the headroom.
37 * ii. tx invalid
38 * Set the 'len' field of tx descriptors to an invalid value (umem frame
39 * size + 1).
40 * iii. rx ring full
41 * Reduce the size of the RX ring to a fraction of the fill ring size.
42 * iv. fill queue empty
43 * Do not populate the fill queue and then try to receive pkts.
44 * f. bpf_link resource persistence
45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
46 * then remove xsk sockets from queue 0 on both veth interfaces and
47 * finally run a traffic on queues ids 1
48 * g. unaligned mode
49 * h. tests for invalid and corner case Tx descriptors so that the correct ones
50 * are discarded and let through, respectively.
51 * i. 2K frame size tests
52 * j. If multi-buffer is supported, send 9k packets divided into 3 frames
53 * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame
54 * using unaligned mode
55 * l. If multi-buffer is supported, try various nasty combinations of descriptors to
56 * check if they pass the validation or not
57 *
58 * Flow:
59 * -----
60 * - Single process spawns two threads: Tx and Rx
61 * - Each of these two threads attach to a veth interface
62 * - Each thread creates one AF_XDP socket connected to a unique umem for each
63 * veth interface
64 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy>
65 * - Rx thread verifies if all packets were received and delivered in-order,
66 * and have the right content
67 *
68 * Enable/disable packet dump mode:
69 * --------------------------
70 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
71 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
72 */
73
74 #define _GNU_SOURCE
75 #include <assert.h>
76 #include <fcntl.h>
77 #include <getopt.h>
78 #include <linux/if_link.h>
79 #include <linux/if_ether.h>
80 #include <linux/mman.h>
81 #include <linux/netdev.h>
82 #include <linux/ethtool.h>
83 #include <arpa/inet.h>
84 #include <net/if.h>
85 #include <locale.h>
86 #include <stdio.h>
87 #include <stdlib.h>
88 #include <libgen.h>
89 #include <stddef.h>
90 #include <sys/mman.h>
91 #include <sys/types.h>
92
93 #include "prog_tests/test_xsk.h"
94 #include "xsk_xdp_progs.skel.h"
95 #include "xsk.h"
96 #include "xskxceiver.h"
97 #include <bpf/bpf.h>
98 #include <linux/filter.h>
99 #include "kselftest.h"
100 #include "xsk_xdp_common.h"
101
102 #include <network_helpers.h>
103
104 static bool opt_print_tests;
105 static enum test_mode opt_mode = TEST_MODE_ALL;
106 static u32 opt_run_test = RUN_ALL_TESTS;
107
test__fail(void)108 void test__fail(void) { /* for network_helpers.c */ }
109
__exit_with_error(int error,const char * file,const char * func,int line)110 static void __exit_with_error(int error, const char *file, const char *func, int line)
111 {
112 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line,
113 error, strerror(error));
114 ksft_exit_xfail();
115 }
116
117 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
118
ifobj_zc_avail(struct ifobject * ifobject)119 static bool ifobj_zc_avail(struct ifobject *ifobject)
120 {
121 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
122 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
123 struct xsk_socket_info *xsk;
124 struct xsk_umem_info *umem;
125 bool zc_avail = false;
126 void *bufs;
127 int ret;
128
129 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
130 if (bufs == MAP_FAILED)
131 exit_with_error(errno);
132
133 umem = calloc(1, sizeof(struct xsk_umem_info));
134 if (!umem) {
135 munmap(bufs, umem_sz);
136 exit_with_error(ENOMEM);
137 }
138 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
139 ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz);
140 if (ret)
141 exit_with_error(-ret);
142
143 xsk = calloc(1, sizeof(struct xsk_socket_info));
144 if (!xsk)
145 goto out;
146 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;
147 ifobject->rx_on = true;
148 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
149 ret = xsk_configure_socket(xsk, umem, ifobject, false);
150 if (!ret)
151 zc_avail = true;
152
153 xsk_socket__delete(xsk->xsk);
154 free(xsk);
155 out:
156 munmap(umem->buffer, umem_sz);
157 xsk_umem__delete(umem->umem);
158 free(umem);
159 return zc_avail;
160 }
161
162 static struct option long_options[] = {
163 {"interface", required_argument, 0, 'i'},
164 {"busy-poll", no_argument, 0, 'b'},
165 {"verbose", no_argument, 0, 'v'},
166 {"mode", required_argument, 0, 'm'},
167 {"list", no_argument, 0, 'l'},
168 {"test", required_argument, 0, 't'},
169 {"help", no_argument, 0, 'h'},
170 {0, 0, 0, 0}
171 };
172
print_usage(char ** argv)173 static void print_usage(char **argv)
174 {
175 const char *str =
176 " Usage: xskxceiver [OPTIONS]\n"
177 " Options:\n"
178 " -i, --interface Use interface\n"
179 " -v, --verbose Verbose output\n"
180 " -b, --busy-poll Enable busy poll\n"
181 " -m, --mode Run only mode skb, drv, or zc\n"
182 " -l, --list List all available tests\n"
183 " -t, --test Run a specific test. Enter number from -l option.\n"
184 " -h, --help Display this help and exit\n";
185
186 ksft_print_msg(str, basename(argv[0]));
187 ksft_exit_xfail();
188 }
189
validate_interface(struct ifobject * ifobj)190 static bool validate_interface(struct ifobject *ifobj)
191 {
192 if (!strcmp(ifobj->ifname, ""))
193 return false;
194 return true;
195 }
196
parse_command_line(struct ifobject * ifobj_tx,struct ifobject * ifobj_rx,int argc,char ** argv)197 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
198 char **argv)
199 {
200 struct ifobject *ifobj;
201 u32 interface_nb = 0;
202 int option_index, c;
203
204 opterr = 0;
205
206 for (;;) {
207 c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index);
208 if (c == -1)
209 break;
210
211 switch (c) {
212 case 'i':
213 if (interface_nb == 0)
214 ifobj = ifobj_tx;
215 else if (interface_nb == 1)
216 ifobj = ifobj_rx;
217 else
218 break;
219
220 memcpy(ifobj->ifname, optarg,
221 min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg)));
222
223 ifobj->ifindex = if_nametoindex(ifobj->ifname);
224 if (!ifobj->ifindex)
225 exit_with_error(errno);
226
227 interface_nb++;
228 break;
229 case 'v':
230 opt_verbose = true;
231 break;
232 case 'b':
233 ifobj_tx->busy_poll = true;
234 ifobj_rx->busy_poll = true;
235 break;
236 case 'm':
237 if (!strncmp("skb", optarg, strlen(optarg)))
238 opt_mode = TEST_MODE_SKB;
239 else if (!strncmp("drv", optarg, strlen(optarg)))
240 opt_mode = TEST_MODE_DRV;
241 else if (!strncmp("zc", optarg, strlen(optarg)))
242 opt_mode = TEST_MODE_ZC;
243 else
244 print_usage(argv);
245 break;
246 case 'l':
247 opt_print_tests = true;
248 break;
249 case 't':
250 errno = 0;
251 opt_run_test = strtol(optarg, NULL, 0);
252 if (errno)
253 print_usage(argv);
254 break;
255 case 'h':
256 default:
257 print_usage(argv);
258 }
259 }
260 }
261
xsk_unload_xdp_programs(struct ifobject * ifobj)262 static void xsk_unload_xdp_programs(struct ifobject *ifobj)
263 {
264 xsk_xdp_progs__destroy(ifobj->xdp_progs);
265 }
266
run_pkt_test(struct test_spec * test)267 static void run_pkt_test(struct test_spec *test)
268 {
269 int ret;
270
271 ret = test->test_func(test);
272
273 switch (ret) {
274 case TEST_PASS:
275 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
276 test->name);
277 break;
278 case TEST_SKIP:
279 ksft_test_result_skip("SKIP: %s %s%s\n", mode_string(test), busy_poll_string(test),
280 test->name);
281 break;
282 case TEST_FAILURE:
283 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
284 test->name);
285 break;
286 default:
287 ksft_test_result_fail("FAIL: %s %s%s -- Unexpected returned value (%d)\n",
288 mode_string(test), busy_poll_string(test), test->name, ret);
289 }
290
291 pkt_stream_restore_default(test);
292 }
293
is_xdp_supported(int ifindex)294 static bool is_xdp_supported(int ifindex)
295 {
296 int flags = XDP_FLAGS_DRV_MODE;
297
298 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
299 struct bpf_insn insns[2] = {
300 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
301 BPF_EXIT_INSN()
302 };
303 int prog_fd, insn_cnt = ARRAY_SIZE(insns);
304 int err;
305
306 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
307 if (prog_fd < 0)
308 return false;
309
310 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
311 if (err) {
312 close(prog_fd);
313 return false;
314 }
315
316 bpf_xdp_detach(ifindex, flags, NULL);
317 close(prog_fd);
318
319 return true;
320 }
321
print_tests(void)322 static void print_tests(void)
323 {
324 u32 i;
325
326 printf("Tests:\n");
327 for (i = 0; i < ARRAY_SIZE(tests); i++)
328 printf("%u: %s\n", i, tests[i].name);
329 for (i = ARRAY_SIZE(tests); i < ARRAY_SIZE(tests) + ARRAY_SIZE(ci_skip_tests); i++)
330 printf("%u: %s\n", i, ci_skip_tests[i - ARRAY_SIZE(tests)].name);
331 }
332
main(int argc,char ** argv)333 int main(int argc, char **argv)
334 {
335 const size_t total_tests = ARRAY_SIZE(tests) + ARRAY_SIZE(ci_skip_tests);
336 struct pkt_stream *rx_pkt_stream_default;
337 struct pkt_stream *tx_pkt_stream_default;
338 struct ifobject *ifobj_tx, *ifobj_rx;
339 u32 i, j, failed_tests = 0, nb_tests;
340 int modes = TEST_MODE_SKB + 1;
341 struct test_spec test;
342 bool shared_netdev;
343 int ret;
344
345 /* Use libbpf 1.0 API mode */
346 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
347
348 ifobj_tx = ifobject_create();
349 if (!ifobj_tx)
350 exit_with_error(ENOMEM);
351 ifobj_rx = ifobject_create();
352 if (!ifobj_rx)
353 exit_with_error(ENOMEM);
354
355 setlocale(LC_ALL, "");
356
357 parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
358
359 if (opt_print_tests) {
360 print_tests();
361 ksft_exit_xpass();
362 }
363 if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= total_tests) {
364 ksft_print_msg("Error: test %u does not exist.\n", opt_run_test);
365 ksft_exit_xfail();
366 }
367
368 shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex);
369 ifobj_tx->shared_umem = shared_netdev;
370 ifobj_rx->shared_umem = shared_netdev;
371
372 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx))
373 print_usage(argv);
374
375 if (is_xdp_supported(ifobj_tx->ifindex)) {
376 modes++;
377 if (ifobj_zc_avail(ifobj_tx))
378 modes++;
379 }
380
381 ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring);
382 if (!ret) {
383 ifobj_tx->hw_ring_size_supp = true;
384 ifobj_tx->set_ring.default_tx = ifobj_tx->ring.tx_pending;
385 ifobj_tx->set_ring.default_rx = ifobj_tx->ring.rx_pending;
386 }
387
388 if (init_iface(ifobj_rx, worker_testapp_validate_rx) ||
389 init_iface(ifobj_tx, worker_testapp_validate_tx)) {
390 ksft_print_msg("Error : can't initialize interfaces\n");
391 ksft_exit_xfail();
392 }
393
394 test_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]);
395 tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
396 rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
397 if (!tx_pkt_stream_default || !rx_pkt_stream_default)
398 exit_with_error(ENOMEM);
399 test.tx_pkt_stream_default = tx_pkt_stream_default;
400 test.rx_pkt_stream_default = rx_pkt_stream_default;
401
402 if (opt_run_test == RUN_ALL_TESTS)
403 nb_tests = total_tests;
404 else
405 nb_tests = 1;
406 if (opt_mode == TEST_MODE_ALL) {
407 ksft_set_plan(modes * nb_tests);
408 } else {
409 if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) {
410 ksft_print_msg("Error: XDP_DRV mode not supported.\n");
411 ksft_exit_xfail();
412 }
413 if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) {
414 ksft_print_msg("Error: zero-copy mode not supported.\n");
415 ksft_exit_xfail();
416 }
417
418 ksft_set_plan(nb_tests);
419 }
420
421 for (i = 0; i < modes; i++) {
422 if (opt_mode != TEST_MODE_ALL && i != opt_mode)
423 continue;
424
425 for (j = 0; j < total_tests; j++) {
426 if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test)
427 continue;
428
429 if (j < ARRAY_SIZE(tests))
430 test_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]);
431 else
432 test_init(&test, ifobj_tx, ifobj_rx, i,
433 &ci_skip_tests[j - ARRAY_SIZE(tests)]);
434 run_pkt_test(&test);
435 usleep(USLEEP_MAX);
436
437 if (test.fail)
438 failed_tests++;
439 }
440 }
441
442 if (ifobj_tx->hw_ring_size_supp)
443 hw_ring_size_reset(ifobj_tx);
444
445 pkt_stream_delete(tx_pkt_stream_default);
446 pkt_stream_delete(rx_pkt_stream_default);
447 xsk_unload_xdp_programs(ifobj_tx);
448 xsk_unload_xdp_programs(ifobj_rx);
449 ifobject_delete(ifobj_tx);
450 ifobject_delete(ifobj_rx);
451
452 if (failed_tests)
453 ksft_exit_fail();
454 else
455 ksft_exit_pass();
456 }
457