1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2020 Intel Corporation. */
3
4 /*
5 * Some functions in this program are taken from
6 * Linux kernel samples/bpf/xdpsock* and modified
7 * for use.
8 *
9 * See test_xsk.sh for detailed information on test topology
10 * and prerequisite network setup.
11 *
12 * This test program contains two threads, each thread is single socket with
13 * a unique UMEM. It validates in-order packet delivery and packet content
14 * by sending packets to each other.
15 *
16 * Tests Information:
17 * ------------------
18 * These selftests test AF_XDP SKB and Native/DRV modes using veth
19 * Virtual Ethernet interfaces.
20 *
21 * For each mode, the following tests are run:
22 * a. nopoll - soft-irq processing in run-to-completion mode
23 * b. poll - using poll() syscall
24 * c. Socket Teardown
25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
26 * both sockets, then repeat multiple times. Only nopoll mode is used
27 * d. Bi-directional sockets
28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and
29 * completion rings on each socket, tx/rx in both directions. Only nopoll
30 * mode is used
31 * e. Statistics
32 * Trigger some error conditions and ensure that the appropriate statistics
33 * are incremented. Within this test, the following statistics are tested:
34 * i. rx dropped
35 * Increase the UMEM frame headroom to a value which results in
36 * insufficient space in the rx buffer for both the packet and the headroom.
37 * ii. tx invalid
38 * Set the 'len' field of tx descriptors to an invalid value (umem frame
39 * size + 1).
40 * iii. rx ring full
41 * Reduce the size of the RX ring to a fraction of the fill ring size.
42 * iv. fill queue empty
43 * Do not populate the fill queue and then try to receive pkts.
44 * f. bpf_link resource persistence
45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
46 * then remove xsk sockets from queue 0 on both veth interfaces and
47 * finally run a traffic on queues ids 1
48 * g. unaligned mode
49 * h. tests for invalid and corner case Tx descriptors so that the correct ones
50 * are discarded and let through, respectively.
51 * i. 2K frame size tests
52 * j. If multi-buffer is supported, send 9k packets divided into 3 frames
53 * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame
54 * using unaligned mode
55 * l. If multi-buffer is supported, try various nasty combinations of descriptors to
56 * check if they pass the validation or not
57 *
58 * Flow:
59 * -----
60 * - Single process spawns two threads: Tx and Rx
61 * - Each of these two threads attach to a veth interface
62 * - Each thread creates one AF_XDP socket connected to a unique umem for each
63 * veth interface
64 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy>
65 * - Rx thread verifies if all packets were received and delivered in-order,
66 * and have the right content
67 *
68 * Enable/disable packet dump mode:
69 * --------------------------
70 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
71 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
72 */
73
74 #define _GNU_SOURCE
75 #include <assert.h>
76 #include <fcntl.h>
77 #include <getopt.h>
78 #include <linux/if_link.h>
79 #include <linux/if_ether.h>
80 #include <linux/mman.h>
81 #include <linux/netdev.h>
82 #include <linux/ethtool.h>
83 #include <linux/align.h>
84 #include <arpa/inet.h>
85 #include <net/if.h>
86 #include <locale.h>
87 #include <stdio.h>
88 #include <stdlib.h>
89 #include <libgen.h>
90 #include <stddef.h>
91 #include <sys/mman.h>
92 #include <sys/types.h>
93
94 #include "prog_tests/test_xsk.h"
95 #include "xsk_xdp_progs.skel.h"
96 #include "xsk.h"
97 #include "xskxceiver.h"
98 #include <bpf/bpf.h>
99 #include <linux/filter.h>
100 #include "kselftest.h"
101 #include "xsk_xdp_common.h"
102
103 #include <network_helpers.h>
104
105 static bool opt_print_tests;
106 static enum test_mode opt_mode = TEST_MODE_ALL;
107 static u32 opt_run_test = RUN_ALL_TESTS;
108
test__fail(void)109 void test__fail(void) { /* for network_helpers.c */ }
110
__exit_with_error(int error,const char * file,const char * func,int line)111 static void __exit_with_error(int error, const char *file, const char *func, int line)
112 {
113 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line,
114 error, strerror(error));
115 ksft_exit_xfail();
116 }
117
118 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
119
ifobj_zc_avail(struct ifobject * ifobject)120 static bool ifobj_zc_avail(struct ifobject *ifobject)
121 {
122 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
123 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
124 struct xsk_socket_info *xsk;
125 struct xsk_umem_info *umem;
126 bool zc_avail = false;
127 void *bufs;
128 int ret;
129
130 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
131 if (bufs == MAP_FAILED)
132 exit_with_error(errno);
133
134 umem = calloc(1, sizeof(struct xsk_umem_info));
135 if (!umem) {
136 munmap(bufs, umem_sz);
137 exit_with_error(ENOMEM);
138 }
139 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
140 ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz);
141 if (ret)
142 exit_with_error(-ret);
143
144 xsk = calloc(1, sizeof(struct xsk_socket_info));
145 if (!xsk)
146 goto out;
147 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;
148 ifobject->rx_on = true;
149 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
150 ret = xsk_configure_socket(xsk, umem, ifobject, false);
151 if (!ret)
152 zc_avail = true;
153
154 xsk_socket__delete(xsk->xsk);
155 free(xsk);
156 out:
157 munmap(umem->buffer, umem_sz);
158 xsk_umem__delete(umem->umem);
159 free(umem);
160 return zc_avail;
161 }
162
163 static struct option long_options[] = {
164 {"interface", required_argument, 0, 'i'},
165 {"busy-poll", no_argument, 0, 'b'},
166 {"verbose", no_argument, 0, 'v'},
167 {"mode", required_argument, 0, 'm'},
168 {"list", no_argument, 0, 'l'},
169 {"test", required_argument, 0, 't'},
170 {"help", no_argument, 0, 'h'},
171 {0, 0, 0, 0}
172 };
173
print_usage(char ** argv)174 static void print_usage(char **argv)
175 {
176 const char *str =
177 " Usage: xskxceiver [OPTIONS]\n"
178 " Options:\n"
179 " -i, --interface Use interface\n"
180 " -v, --verbose Verbose output\n"
181 " -b, --busy-poll Enable busy poll\n"
182 " -m, --mode Run only mode skb, drv, or zc\n"
183 " -l, --list List all available tests\n"
184 " -t, --test Run a specific test. Enter number from -l option.\n"
185 " -h, --help Display this help and exit\n";
186
187 ksft_print_msg(str, basename(argv[0]));
188 ksft_exit_xfail();
189 }
190
validate_interface(struct ifobject * ifobj)191 static bool validate_interface(struct ifobject *ifobj)
192 {
193 if (!strcmp(ifobj->ifname, ""))
194 return false;
195 return true;
196 }
197
parse_command_line(struct ifobject * ifobj_tx,struct ifobject * ifobj_rx,int argc,char ** argv)198 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
199 char **argv)
200 {
201 struct ifobject *ifobj;
202 u32 interface_nb = 0;
203 int option_index, c;
204
205 opterr = 0;
206
207 for (;;) {
208 c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index);
209 if (c == -1)
210 break;
211
212 switch (c) {
213 case 'i':
214 if (interface_nb == 0)
215 ifobj = ifobj_tx;
216 else if (interface_nb == 1)
217 ifobj = ifobj_rx;
218 else
219 break;
220
221 memcpy(ifobj->ifname, optarg,
222 min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg)));
223
224 ifobj->ifindex = if_nametoindex(ifobj->ifname);
225 if (!ifobj->ifindex)
226 exit_with_error(errno);
227
228 interface_nb++;
229 break;
230 case 'v':
231 opt_verbose = true;
232 break;
233 case 'b':
234 ifobj_tx->busy_poll = true;
235 ifobj_rx->busy_poll = true;
236 break;
237 case 'm':
238 if (!strncmp("skb", optarg, strlen(optarg)))
239 opt_mode = TEST_MODE_SKB;
240 else if (!strncmp("drv", optarg, strlen(optarg)))
241 opt_mode = TEST_MODE_DRV;
242 else if (!strncmp("zc", optarg, strlen(optarg)))
243 opt_mode = TEST_MODE_ZC;
244 else
245 print_usage(argv);
246 break;
247 case 'l':
248 opt_print_tests = true;
249 break;
250 case 't':
251 errno = 0;
252 opt_run_test = strtol(optarg, NULL, 0);
253 if (errno)
254 print_usage(argv);
255 break;
256 case 'h':
257 default:
258 print_usage(argv);
259 }
260 }
261 }
262
xsk_unload_xdp_programs(struct ifobject * ifobj)263 static void xsk_unload_xdp_programs(struct ifobject *ifobj)
264 {
265 xsk_xdp_progs__destroy(ifobj->xdp_progs);
266 }
267
run_pkt_test(struct test_spec * test)268 static void run_pkt_test(struct test_spec *test)
269 {
270 int ret;
271
272 ret = test->test_func(test);
273
274 switch (ret) {
275 case TEST_PASS:
276 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
277 test->name);
278 break;
279 case TEST_SKIP:
280 ksft_test_result_skip("SKIP: %s %s%s\n", mode_string(test), busy_poll_string(test),
281 test->name);
282 break;
283 case TEST_FAILURE:
284 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
285 test->name);
286 break;
287 default:
288 ksft_test_result_fail("FAIL: %s %s%s -- Unexpected returned value (%d)\n",
289 mode_string(test), busy_poll_string(test), test->name, ret);
290 }
291
292 pkt_stream_restore_default(test);
293 }
294
is_xdp_supported(int ifindex)295 static bool is_xdp_supported(int ifindex)
296 {
297 int flags = XDP_FLAGS_DRV_MODE;
298
299 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
300 struct bpf_insn insns[2] = {
301 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
302 BPF_EXIT_INSN()
303 };
304 int prog_fd, insn_cnt = ARRAY_SIZE(insns);
305 int err;
306
307 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
308 if (prog_fd < 0)
309 return false;
310
311 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
312 if (err) {
313 close(prog_fd);
314 return false;
315 }
316
317 bpf_xdp_detach(ifindex, flags, NULL);
318 close(prog_fd);
319
320 return true;
321 }
322
print_tests(void)323 static void print_tests(void)
324 {
325 u32 i;
326
327 printf("Tests:\n");
328 for (i = 0; i < ARRAY_SIZE(tests); i++)
329 printf("%u: %s\n", i, tests[i].name);
330 for (i = ARRAY_SIZE(tests); i < ARRAY_SIZE(tests) + ARRAY_SIZE(ci_skip_tests); i++)
331 printf("%u: %s\n", i, ci_skip_tests[i - ARRAY_SIZE(tests)].name);
332 }
333
main(int argc,char ** argv)334 int main(int argc, char **argv)
335 {
336 const size_t total_tests = ARRAY_SIZE(tests) + ARRAY_SIZE(ci_skip_tests);
337 u32 cache_line_size, max_frags, umem_tailroom;
338 struct pkt_stream *rx_pkt_stream_default;
339 struct pkt_stream *tx_pkt_stream_default;
340 struct ifobject *ifobj_tx, *ifobj_rx;
341 u32 i, j, failed_tests = 0, nb_tests;
342 int modes = TEST_MODE_SKB + 1;
343 struct test_spec test;
344 bool shared_netdev;
345 int ret;
346
347 /* Use libbpf 1.0 API mode */
348 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
349
350 ifobj_tx = ifobject_create();
351 if (!ifobj_tx)
352 exit_with_error(ENOMEM);
353 ifobj_rx = ifobject_create();
354 if (!ifobj_rx)
355 exit_with_error(ENOMEM);
356
357 setlocale(LC_ALL, "");
358
359 cache_line_size = read_procfs_val(SMP_CACHE_BYTES_PATH);
360 if (!cache_line_size) {
361 ksft_print_msg("Can't get SMP_CACHE_BYTES from system, using default (64)\n");
362 cache_line_size = 64;
363 }
364
365 max_frags = read_procfs_val(MAX_SKB_FRAGS_PATH);
366 if (!max_frags) {
367 ksft_print_msg("Can't get MAX_SKB_FRAGS from system, using default (17)\n");
368 max_frags = 17;
369 }
370 ifobj_tx->max_skb_frags = max_frags;
371 ifobj_rx->max_skb_frags = max_frags;
372
373 /* 48 bytes is a part of skb_shared_info w/o frags array;
374 * 16 bytes is sizeof(skb_frag_t)
375 */
376 umem_tailroom = ALIGN(48 + (max_frags * 16), cache_line_size);
377 ifobj_tx->umem_tailroom = umem_tailroom;
378 ifobj_rx->umem_tailroom = umem_tailroom;
379
380 parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
381
382 if (opt_print_tests) {
383 print_tests();
384 ksft_exit_xpass();
385 }
386 if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= total_tests) {
387 ksft_print_msg("Error: test %u does not exist.\n", opt_run_test);
388 ksft_exit_xfail();
389 }
390
391 shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex);
392 ifobj_tx->shared_umem = shared_netdev;
393 ifobj_rx->shared_umem = shared_netdev;
394
395 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx))
396 print_usage(argv);
397
398 if (is_xdp_supported(ifobj_tx->ifindex)) {
399 modes++;
400 if (ifobj_zc_avail(ifobj_tx))
401 modes++;
402 }
403
404 ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring);
405 if (!ret) {
406 ifobj_tx->hw_ring_size_supp = true;
407 ifobj_tx->set_ring.default_tx = ifobj_tx->ring.tx_pending;
408 ifobj_tx->set_ring.default_rx = ifobj_tx->ring.rx_pending;
409 }
410
411 if (init_iface(ifobj_rx, worker_testapp_validate_rx) ||
412 init_iface(ifobj_tx, worker_testapp_validate_tx)) {
413 ksft_print_msg("Error : can't initialize interfaces\n");
414 ksft_exit_xfail();
415 }
416
417 test_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]);
418 tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
419 rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
420 if (!tx_pkt_stream_default || !rx_pkt_stream_default)
421 exit_with_error(ENOMEM);
422 test.tx_pkt_stream_default = tx_pkt_stream_default;
423 test.rx_pkt_stream_default = rx_pkt_stream_default;
424
425 if (opt_run_test == RUN_ALL_TESTS)
426 nb_tests = total_tests;
427 else
428 nb_tests = 1;
429 if (opt_mode == TEST_MODE_ALL) {
430 ksft_set_plan(modes * nb_tests);
431 } else {
432 if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) {
433 ksft_print_msg("Error: XDP_DRV mode not supported.\n");
434 ksft_exit_xfail();
435 }
436 if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) {
437 ksft_print_msg("Error: zero-copy mode not supported.\n");
438 ksft_exit_xfail();
439 }
440
441 ksft_set_plan(nb_tests);
442 }
443
444 for (i = 0; i < modes; i++) {
445 if (opt_mode != TEST_MODE_ALL && i != opt_mode)
446 continue;
447
448 for (j = 0; j < total_tests; j++) {
449 if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test)
450 continue;
451
452 if (j < ARRAY_SIZE(tests))
453 test_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]);
454 else
455 test_init(&test, ifobj_tx, ifobj_rx, i,
456 &ci_skip_tests[j - ARRAY_SIZE(tests)]);
457 run_pkt_test(&test);
458 usleep(USLEEP_MAX);
459
460 if (test.fail)
461 failed_tests++;
462 }
463 }
464
465 if (ifobj_tx->hw_ring_size_supp)
466 hw_ring_size_reset(ifobj_tx);
467
468 pkt_stream_delete(tx_pkt_stream_default);
469 pkt_stream_delete(rx_pkt_stream_default);
470 xsk_unload_xdp_programs(ifobj_tx);
471 xsk_unload_xdp_programs(ifobj_rx);
472 ifobject_delete(ifobj_tx);
473 ifobject_delete(ifobj_rx);
474
475 if (failed_tests)
476 ksft_exit_fail();
477 else
478 ksft_exit_pass();
479 }
480