1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2020 Intel Corporation. */
3
4 /*
5 * Some functions in this program are taken from
6 * Linux kernel samples/bpf/xdpsock* and modified
7 * for use.
8 *
9 * See test_xsk.sh for detailed information on test topology
10 * and prerequisite network setup.
11 *
12 * This test program contains two threads, each thread is single socket with
13 * a unique UMEM. It validates in-order packet delivery and packet content
14 * by sending packets to each other.
15 *
16 * Tests Information:
17 * ------------------
18 * These selftests test AF_XDP SKB and Native/DRV modes using veth
19 * Virtual Ethernet interfaces.
20 *
21 * For each mode, the following tests are run:
22 * a. nopoll - soft-irq processing in run-to-completion mode
23 * b. poll - using poll() syscall
24 * c. Socket Teardown
25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
26 * both sockets, then repeat multiple times. Only nopoll mode is used
27 * d. Bi-directional sockets
28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and
29 * completion rings on each socket, tx/rx in both directions. Only nopoll
30 * mode is used
31 * e. Statistics
32 * Trigger some error conditions and ensure that the appropriate statistics
33 * are incremented. Within this test, the following statistics are tested:
34 * i. rx dropped
35 * Increase the UMEM frame headroom to a value which results in
36 * insufficient space in the rx buffer for both the packet and the headroom.
37 * ii. tx invalid
38 * Set the 'len' field of tx descriptors to an invalid value (umem frame
39 * size + 1).
40 * iii. rx ring full
41 * Reduce the size of the RX ring to a fraction of the fill ring size.
42 * iv. fill queue empty
43 * Do not populate the fill queue and then try to receive pkts.
44 * f. bpf_link resource persistence
45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
46 * then remove xsk sockets from queue 0 on both veth interfaces and
47 * finally run a traffic on queues ids 1
48 * g. unaligned mode
49 * h. tests for invalid and corner case Tx descriptors so that the correct ones
50 * are discarded and let through, respectively.
51 * i. 2K frame size tests
52 * j. If multi-buffer is supported, send 9k packets divided into 3 frames
53 * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame
54 * using unaligned mode
55 * l. If multi-buffer is supported, try various nasty combinations of descriptors to
56 * check if they pass the validation or not
57 *
58 * Flow:
59 * -----
60 * - Single process spawns two threads: Tx and Rx
61 * - Each of these two threads attach to a veth interface
62 * - Each thread creates one AF_XDP socket connected to a unique umem for each
63 * veth interface
64 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy>
65 * - Rx thread verifies if all packets were received and delivered in-order,
66 * and have the right content
67 *
68 * Enable/disable packet dump mode:
69 * --------------------------
70 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
71 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
72 */
73
74 #define _GNU_SOURCE
75 #include <assert.h>
76 #include <fcntl.h>
77 #include <errno.h>
78 #include <getopt.h>
79 #include <linux/if_link.h>
80 #include <linux/if_ether.h>
81 #include <linux/mman.h>
82 #include <linux/netdev.h>
83 #include <linux/bitmap.h>
84 #include <linux/ethtool.h>
85 #include <arpa/inet.h>
86 #include <net/if.h>
87 #include <locale.h>
88 #include <poll.h>
89 #include <pthread.h>
90 #include <signal.h>
91 #include <stdio.h>
92 #include <stdlib.h>
93 #include <libgen.h>
94 #include <string.h>
95 #include <stddef.h>
96 #include <sys/mman.h>
97 #include <sys/socket.h>
98 #include <sys/time.h>
99 #include <sys/types.h>
100 #include <unistd.h>
101
102 #include "xsk_xdp_progs.skel.h"
103 #include "xsk.h"
104 #include "xskxceiver.h"
105 #include <bpf/bpf.h>
106 #include <linux/filter.h>
107 #include "../kselftest.h"
108 #include "xsk_xdp_common.h"
109
110 #include <network_helpers.h>
111
112 #define MAX_TX_BUDGET_DEFAULT 32
113
114 static bool opt_verbose;
115 static bool opt_print_tests;
116 static enum test_mode opt_mode = TEST_MODE_ALL;
117 static u32 opt_run_test = RUN_ALL_TESTS;
118
test__fail(void)119 void test__fail(void) { /* for network_helpers.c */ }
120
__exit_with_error(int error,const char * file,const char * func,int line)121 static void __exit_with_error(int error, const char *file, const char *func, int line)
122 {
123 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error,
124 strerror(error));
125 ksft_exit_xfail();
126 }
127
128 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
129 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
mode_string(struct test_spec * test)130 static char *mode_string(struct test_spec *test)
131 {
132 switch (test->mode) {
133 case TEST_MODE_SKB:
134 return "SKB";
135 case TEST_MODE_DRV:
136 return "DRV";
137 case TEST_MODE_ZC:
138 return "ZC";
139 default:
140 return "BOGUS";
141 }
142 }
143
report_failure(struct test_spec * test)144 static void report_failure(struct test_spec *test)
145 {
146 if (test->fail)
147 return;
148
149 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
150 test->name);
151 test->fail = true;
152 }
153
154 /* The payload is a word consisting of a packet sequence number in the upper
155 * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's
156 * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0.
157 */
write_payload(void * dest,u32 pkt_nb,u32 start,u32 size)158 static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size)
159 {
160 u32 *ptr = (u32 *)dest, i;
161
162 start /= sizeof(*ptr);
163 size /= sizeof(*ptr);
164 for (i = 0; i < size; i++)
165 ptr[i] = htonl(pkt_nb << 16 | (i + start));
166 }
167
gen_eth_hdr(struct xsk_socket_info * xsk,struct ethhdr * eth_hdr)168 static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr)
169 {
170 memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN);
171 memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN);
172 eth_hdr->h_proto = htons(ETH_P_LOOPBACK);
173 }
174
is_umem_valid(struct ifobject * ifobj)175 static bool is_umem_valid(struct ifobject *ifobj)
176 {
177 return !!ifobj->umem->umem;
178 }
179
mode_to_xdp_flags(enum test_mode mode)180 static u32 mode_to_xdp_flags(enum test_mode mode)
181 {
182 return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;
183 }
184
umem_size(struct xsk_umem_info * umem)185 static u64 umem_size(struct xsk_umem_info *umem)
186 {
187 return umem->num_frames * umem->frame_size;
188 }
189
xsk_configure_umem(struct ifobject * ifobj,struct xsk_umem_info * umem,void * buffer,u64 size)190 static int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer,
191 u64 size)
192 {
193 struct xsk_umem_config cfg = {
194 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
195 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
196 .frame_size = umem->frame_size,
197 .frame_headroom = umem->frame_headroom,
198 .flags = XSK_UMEM__DEFAULT_FLAGS
199 };
200 int ret;
201
202 if (umem->fill_size)
203 cfg.fill_size = umem->fill_size;
204
205 if (umem->comp_size)
206 cfg.comp_size = umem->comp_size;
207
208 if (umem->unaligned_mode)
209 cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
210
211 ret = xsk_umem__create(&umem->umem, buffer, size,
212 &umem->fq, &umem->cq, &cfg);
213 if (ret)
214 return ret;
215
216 umem->buffer = buffer;
217 if (ifobj->shared_umem && ifobj->rx_on) {
218 umem->base_addr = umem_size(umem);
219 umem->next_buffer = umem_size(umem);
220 }
221
222 return 0;
223 }
224
umem_alloc_buffer(struct xsk_umem_info * umem)225 static u64 umem_alloc_buffer(struct xsk_umem_info *umem)
226 {
227 u64 addr;
228
229 addr = umem->next_buffer;
230 umem->next_buffer += umem->frame_size;
231 if (umem->next_buffer >= umem->base_addr + umem_size(umem))
232 umem->next_buffer = umem->base_addr;
233
234 return addr;
235 }
236
umem_reset_alloc(struct xsk_umem_info * umem)237 static void umem_reset_alloc(struct xsk_umem_info *umem)
238 {
239 umem->next_buffer = 0;
240 }
241
enable_busy_poll(struct xsk_socket_info * xsk)242 static void enable_busy_poll(struct xsk_socket_info *xsk)
243 {
244 int sock_opt;
245
246 sock_opt = 1;
247 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL,
248 (void *)&sock_opt, sizeof(sock_opt)) < 0)
249 exit_with_error(errno);
250
251 sock_opt = 20;
252 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL,
253 (void *)&sock_opt, sizeof(sock_opt)) < 0)
254 exit_with_error(errno);
255
256 sock_opt = xsk->batch_size;
257 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET,
258 (void *)&sock_opt, sizeof(sock_opt)) < 0)
259 exit_with_error(errno);
260 }
261
__xsk_configure_socket(struct xsk_socket_info * xsk,struct xsk_umem_info * umem,struct ifobject * ifobject,bool shared)262 static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
263 struct ifobject *ifobject, bool shared)
264 {
265 struct xsk_socket_config cfg = {};
266 struct xsk_ring_cons *rxr;
267 struct xsk_ring_prod *txr;
268
269 xsk->umem = umem;
270 cfg.rx_size = xsk->rxqsize;
271 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
272 cfg.bind_flags = ifobject->bind_flags;
273 if (shared)
274 cfg.bind_flags |= XDP_SHARED_UMEM;
275 if (ifobject->mtu > MAX_ETH_PKT_SIZE)
276 cfg.bind_flags |= XDP_USE_SG;
277 if (umem->comp_size)
278 cfg.tx_size = umem->comp_size;
279 if (umem->fill_size)
280 cfg.rx_size = umem->fill_size;
281
282 txr = ifobject->tx_on ? &xsk->tx : NULL;
283 rxr = ifobject->rx_on ? &xsk->rx : NULL;
284 return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg);
285 }
286
ifobj_zc_avail(struct ifobject * ifobject)287 static bool ifobj_zc_avail(struct ifobject *ifobject)
288 {
289 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
290 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
291 struct xsk_socket_info *xsk;
292 struct xsk_umem_info *umem;
293 bool zc_avail = false;
294 void *bufs;
295 int ret;
296
297 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
298 if (bufs == MAP_FAILED)
299 exit_with_error(errno);
300
301 umem = calloc(1, sizeof(struct xsk_umem_info));
302 if (!umem) {
303 munmap(bufs, umem_sz);
304 exit_with_error(ENOMEM);
305 }
306 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
307 ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz);
308 if (ret)
309 exit_with_error(-ret);
310
311 xsk = calloc(1, sizeof(struct xsk_socket_info));
312 if (!xsk)
313 goto out;
314 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;
315 ifobject->rx_on = true;
316 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
317 ret = __xsk_configure_socket(xsk, umem, ifobject, false);
318 if (!ret)
319 zc_avail = true;
320
321 xsk_socket__delete(xsk->xsk);
322 free(xsk);
323 out:
324 munmap(umem->buffer, umem_sz);
325 xsk_umem__delete(umem->umem);
326 free(umem);
327 return zc_avail;
328 }
329
330 #define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags"
get_max_skb_frags(void)331 static unsigned int get_max_skb_frags(void)
332 {
333 unsigned int max_skb_frags = 0;
334 FILE *file;
335
336 file = fopen(MAX_SKB_FRAGS_PATH, "r");
337 if (!file) {
338 ksft_print_msg("Error opening %s\n", MAX_SKB_FRAGS_PATH);
339 return 0;
340 }
341
342 if (fscanf(file, "%u", &max_skb_frags) != 1)
343 ksft_print_msg("Error reading %s\n", MAX_SKB_FRAGS_PATH);
344
345 fclose(file);
346 return max_skb_frags;
347 }
348
349 static struct option long_options[] = {
350 {"interface", required_argument, 0, 'i'},
351 {"busy-poll", no_argument, 0, 'b'},
352 {"verbose", no_argument, 0, 'v'},
353 {"mode", required_argument, 0, 'm'},
354 {"list", no_argument, 0, 'l'},
355 {"test", required_argument, 0, 't'},
356 {"help", no_argument, 0, 'h'},
357 {0, 0, 0, 0}
358 };
359
print_usage(char ** argv)360 static void print_usage(char **argv)
361 {
362 const char *str =
363 " Usage: xskxceiver [OPTIONS]\n"
364 " Options:\n"
365 " -i, --interface Use interface\n"
366 " -v, --verbose Verbose output\n"
367 " -b, --busy-poll Enable busy poll\n"
368 " -m, --mode Run only mode skb, drv, or zc\n"
369 " -l, --list List all available tests\n"
370 " -t, --test Run a specific test. Enter number from -l option.\n"
371 " -h, --help Display this help and exit\n";
372
373 ksft_print_msg(str, basename(argv[0]));
374 ksft_exit_xfail();
375 }
376
validate_interface(struct ifobject * ifobj)377 static bool validate_interface(struct ifobject *ifobj)
378 {
379 if (!strcmp(ifobj->ifname, ""))
380 return false;
381 return true;
382 }
383
parse_command_line(struct ifobject * ifobj_tx,struct ifobject * ifobj_rx,int argc,char ** argv)384 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
385 char **argv)
386 {
387 struct ifobject *ifobj;
388 u32 interface_nb = 0;
389 int option_index, c;
390
391 opterr = 0;
392
393 for (;;) {
394 c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index);
395 if (c == -1)
396 break;
397
398 switch (c) {
399 case 'i':
400 if (interface_nb == 0)
401 ifobj = ifobj_tx;
402 else if (interface_nb == 1)
403 ifobj = ifobj_rx;
404 else
405 break;
406
407 memcpy(ifobj->ifname, optarg,
408 min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg)));
409
410 ifobj->ifindex = if_nametoindex(ifobj->ifname);
411 if (!ifobj->ifindex)
412 exit_with_error(errno);
413
414 interface_nb++;
415 break;
416 case 'v':
417 opt_verbose = true;
418 break;
419 case 'b':
420 ifobj_tx->busy_poll = true;
421 ifobj_rx->busy_poll = true;
422 break;
423 case 'm':
424 if (!strncmp("skb", optarg, strlen(optarg)))
425 opt_mode = TEST_MODE_SKB;
426 else if (!strncmp("drv", optarg, strlen(optarg)))
427 opt_mode = TEST_MODE_DRV;
428 else if (!strncmp("zc", optarg, strlen(optarg)))
429 opt_mode = TEST_MODE_ZC;
430 else
431 print_usage(argv);
432 break;
433 case 'l':
434 opt_print_tests = true;
435 break;
436 case 't':
437 errno = 0;
438 opt_run_test = strtol(optarg, NULL, 0);
439 if (errno)
440 print_usage(argv);
441 break;
442 case 'h':
443 default:
444 print_usage(argv);
445 }
446 }
447 }
448
set_ring_size(struct ifobject * ifobj)449 static int set_ring_size(struct ifobject *ifobj)
450 {
451 int ret;
452 u32 ctr = 0;
453
454 while (ctr++ < SOCK_RECONF_CTR) {
455 ret = set_hw_ring_size(ifobj->ifname, &ifobj->ring);
456 if (!ret)
457 break;
458
459 /* Retry if it fails */
460 if (ctr >= SOCK_RECONF_CTR || errno != EBUSY)
461 return -errno;
462
463 usleep(USLEEP_MAX);
464 }
465
466 return ret;
467 }
468
hw_ring_size_reset(struct ifobject * ifobj)469 static int hw_ring_size_reset(struct ifobject *ifobj)
470 {
471 ifobj->ring.tx_pending = ifobj->set_ring.default_tx;
472 ifobj->ring.rx_pending = ifobj->set_ring.default_rx;
473 return set_ring_size(ifobj);
474 }
475
__test_spec_init(struct test_spec * test,struct ifobject * ifobj_tx,struct ifobject * ifobj_rx)476 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
477 struct ifobject *ifobj_rx)
478 {
479 u32 i, j;
480
481 for (i = 0; i < MAX_INTERFACES; i++) {
482 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
483
484 ifobj->xsk = &ifobj->xsk_arr[0];
485 ifobj->use_poll = false;
486 ifobj->use_fill_ring = true;
487 ifobj->release_rx = true;
488 ifobj->validation_func = NULL;
489 ifobj->use_metadata = false;
490
491 if (i == 0) {
492 ifobj->rx_on = false;
493 ifobj->tx_on = true;
494 } else {
495 ifobj->rx_on = true;
496 ifobj->tx_on = false;
497 }
498
499 memset(ifobj->umem, 0, sizeof(*ifobj->umem));
500 ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;
501 ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
502
503 for (j = 0; j < MAX_SOCKETS; j++) {
504 memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
505 ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
506 ifobj->xsk_arr[j].batch_size = DEFAULT_BATCH_SIZE;
507 if (i == 0)
508 ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default;
509 else
510 ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default;
511
512 memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN);
513 memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN);
514 ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0);
515 ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1);
516 }
517 }
518
519 if (ifobj_tx->hw_ring_size_supp)
520 hw_ring_size_reset(ifobj_tx);
521
522 test->ifobj_tx = ifobj_tx;
523 test->ifobj_rx = ifobj_rx;
524 test->current_step = 0;
525 test->total_steps = 1;
526 test->nb_sockets = 1;
527 test->fail = false;
528 test->set_ring = false;
529 test->adjust_tail = false;
530 test->adjust_tail_support = false;
531 test->mtu = MAX_ETH_PKT_SIZE;
532 test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog;
533 test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk;
534 test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog;
535 test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk;
536 }
537
test_spec_init(struct test_spec * test,struct ifobject * ifobj_tx,struct ifobject * ifobj_rx,enum test_mode mode,const struct test_spec * test_to_run)538 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
539 struct ifobject *ifobj_rx, enum test_mode mode,
540 const struct test_spec *test_to_run)
541 {
542 struct pkt_stream *tx_pkt_stream;
543 struct pkt_stream *rx_pkt_stream;
544 u32 i;
545
546 tx_pkt_stream = test->tx_pkt_stream_default;
547 rx_pkt_stream = test->rx_pkt_stream_default;
548 memset(test, 0, sizeof(*test));
549 test->tx_pkt_stream_default = tx_pkt_stream;
550 test->rx_pkt_stream_default = rx_pkt_stream;
551
552 for (i = 0; i < MAX_INTERFACES; i++) {
553 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
554
555 ifobj->bind_flags = XDP_USE_NEED_WAKEUP;
556 if (mode == TEST_MODE_ZC)
557 ifobj->bind_flags |= XDP_ZEROCOPY;
558 else
559 ifobj->bind_flags |= XDP_COPY;
560 }
561
562 strncpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE);
563 test->test_func = test_to_run->test_func;
564 test->mode = mode;
565 __test_spec_init(test, ifobj_tx, ifobj_rx);
566 }
567
test_spec_reset(struct test_spec * test)568 static void test_spec_reset(struct test_spec *test)
569 {
570 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx);
571 }
572
test_spec_set_xdp_prog(struct test_spec * test,struct bpf_program * xdp_prog_rx,struct bpf_program * xdp_prog_tx,struct bpf_map * xskmap_rx,struct bpf_map * xskmap_tx)573 static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx,
574 struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx,
575 struct bpf_map *xskmap_tx)
576 {
577 test->xdp_prog_rx = xdp_prog_rx;
578 test->xdp_prog_tx = xdp_prog_tx;
579 test->xskmap_rx = xskmap_rx;
580 test->xskmap_tx = xskmap_tx;
581 }
582
test_spec_set_mtu(struct test_spec * test,int mtu)583 static int test_spec_set_mtu(struct test_spec *test, int mtu)
584 {
585 int err;
586
587 if (test->ifobj_rx->mtu != mtu) {
588 err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu);
589 if (err)
590 return err;
591 test->ifobj_rx->mtu = mtu;
592 }
593 if (test->ifobj_tx->mtu != mtu) {
594 err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu);
595 if (err)
596 return err;
597 test->ifobj_tx->mtu = mtu;
598 }
599
600 return 0;
601 }
602
pkt_stream_reset(struct pkt_stream * pkt_stream)603 static void pkt_stream_reset(struct pkt_stream *pkt_stream)
604 {
605 if (pkt_stream) {
606 pkt_stream->current_pkt_nb = 0;
607 pkt_stream->nb_rx_pkts = 0;
608 }
609 }
610
pkt_stream_get_next_tx_pkt(struct pkt_stream * pkt_stream)611 static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream)
612 {
613 if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts)
614 return NULL;
615
616 return &pkt_stream->pkts[pkt_stream->current_pkt_nb++];
617 }
618
pkt_stream_get_next_rx_pkt(struct pkt_stream * pkt_stream,u32 * pkts_sent)619 static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent)
620 {
621 while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) {
622 (*pkts_sent)++;
623 if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid)
624 return &pkt_stream->pkts[pkt_stream->current_pkt_nb++];
625 pkt_stream->current_pkt_nb++;
626 }
627 return NULL;
628 }
629
pkt_stream_delete(struct pkt_stream * pkt_stream)630 static void pkt_stream_delete(struct pkt_stream *pkt_stream)
631 {
632 free(pkt_stream->pkts);
633 free(pkt_stream);
634 }
635
pkt_stream_restore_default(struct test_spec * test)636 static void pkt_stream_restore_default(struct test_spec *test)
637 {
638 struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream;
639 struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream;
640
641 if (tx_pkt_stream != test->tx_pkt_stream_default) {
642 pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream);
643 test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default;
644 }
645
646 if (rx_pkt_stream != test->rx_pkt_stream_default) {
647 pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream);
648 test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default;
649 }
650 }
651
__pkt_stream_alloc(u32 nb_pkts)652 static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
653 {
654 struct pkt_stream *pkt_stream;
655
656 pkt_stream = calloc(1, sizeof(*pkt_stream));
657 if (!pkt_stream)
658 return NULL;
659
660 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));
661 if (!pkt_stream->pkts) {
662 free(pkt_stream);
663 return NULL;
664 }
665
666 pkt_stream->nb_pkts = nb_pkts;
667 return pkt_stream;
668 }
669
pkt_continues(u32 options)670 static bool pkt_continues(u32 options)
671 {
672 return options & XDP_PKT_CONTD;
673 }
674
ceil_u32(u32 a,u32 b)675 static u32 ceil_u32(u32 a, u32 b)
676 {
677 return (a + b - 1) / b;
678 }
679
pkt_nb_frags(u32 frame_size,struct pkt_stream * pkt_stream,struct pkt * pkt)680 static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt)
681 {
682 u32 nb_frags = 1, next_frag;
683
684 if (!pkt)
685 return 1;
686
687 if (!pkt_stream->verbatim) {
688 if (!pkt->valid || !pkt->len)
689 return 1;
690 return ceil_u32(pkt->len, frame_size);
691 }
692
693 /* Search for the end of the packet in verbatim mode */
694 if (!pkt_continues(pkt->options))
695 return nb_frags;
696
697 next_frag = pkt_stream->current_pkt_nb;
698 pkt++;
699 while (next_frag++ < pkt_stream->nb_pkts) {
700 nb_frags++;
701 if (!pkt_continues(pkt->options) || !pkt->valid)
702 break;
703 pkt++;
704 }
705 return nb_frags;
706 }
707
set_pkt_valid(int offset,u32 len)708 static bool set_pkt_valid(int offset, u32 len)
709 {
710 return len <= MAX_ETH_JUMBO_SIZE;
711 }
712
pkt_set(struct pkt_stream * pkt_stream,struct pkt * pkt,int offset,u32 len)713 static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len)
714 {
715 pkt->offset = offset;
716 pkt->len = len;
717 pkt->valid = set_pkt_valid(offset, len);
718 }
719
pkt_stream_pkt_set(struct pkt_stream * pkt_stream,struct pkt * pkt,int offset,u32 len)720 static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len)
721 {
722 bool prev_pkt_valid = pkt->valid;
723
724 pkt_set(pkt_stream, pkt, offset, len);
725 pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid;
726 }
727
pkt_get_buffer_len(struct xsk_umem_info * umem,u32 len)728 static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len)
729 {
730 return ceil_u32(len, umem->frame_size) * umem->frame_size;
731 }
732
__pkt_stream_generate(u32 nb_pkts,u32 pkt_len,u32 nb_start,u32 nb_off)733 static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off)
734 {
735 struct pkt_stream *pkt_stream;
736 u32 i;
737
738 pkt_stream = __pkt_stream_alloc(nb_pkts);
739 if (!pkt_stream)
740 exit_with_error(ENOMEM);
741
742 pkt_stream->nb_pkts = nb_pkts;
743 pkt_stream->max_pkt_len = pkt_len;
744 for (i = 0; i < nb_pkts; i++) {
745 struct pkt *pkt = &pkt_stream->pkts[i];
746
747 pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len);
748 pkt->pkt_nb = nb_start + i * nb_off;
749 }
750
751 return pkt_stream;
752 }
753
pkt_stream_generate(u32 nb_pkts,u32 pkt_len)754 static struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len)
755 {
756 return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1);
757 }
758
pkt_stream_clone(struct pkt_stream * pkt_stream)759 static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream)
760 {
761 return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
762 }
763
pkt_stream_replace_ifobject(struct ifobject * ifobj,u32 nb_pkts,u32 pkt_len)764 static void pkt_stream_replace_ifobject(struct ifobject *ifobj, u32 nb_pkts, u32 pkt_len)
765 {
766 ifobj->xsk->pkt_stream = pkt_stream_generate(nb_pkts, pkt_len);
767 }
768
pkt_stream_replace(struct test_spec * test,u32 nb_pkts,u32 pkt_len)769 static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
770 {
771 pkt_stream_replace_ifobject(test->ifobj_tx, nb_pkts, pkt_len);
772 pkt_stream_replace_ifobject(test->ifobj_rx, nb_pkts, pkt_len);
773 }
774
__pkt_stream_replace_half(struct ifobject * ifobj,u32 pkt_len,int offset)775 static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,
776 int offset)
777 {
778 struct pkt_stream *pkt_stream;
779 u32 i;
780
781 pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream);
782 for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2)
783 pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len);
784
785 ifobj->xsk->pkt_stream = pkt_stream;
786 }
787
pkt_stream_replace_half(struct test_spec * test,u32 pkt_len,int offset)788 static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
789 {
790 __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);
791 __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset);
792 }
793
pkt_stream_receive_half(struct test_spec * test)794 static void pkt_stream_receive_half(struct test_spec *test)
795 {
796 struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream;
797 u32 i;
798
799 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts,
800 pkt_stream->pkts[0].len);
801 pkt_stream = test->ifobj_rx->xsk->pkt_stream;
802 for (i = 1; i < pkt_stream->nb_pkts; i += 2)
803 pkt_stream->pkts[i].valid = false;
804
805 pkt_stream->nb_valid_entries /= 2;
806 }
807
pkt_stream_even_odd_sequence(struct test_spec * test)808 static void pkt_stream_even_odd_sequence(struct test_spec *test)
809 {
810 struct pkt_stream *pkt_stream;
811 u32 i;
812
813 for (i = 0; i < test->nb_sockets; i++) {
814 pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream;
815 pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2,
816 pkt_stream->pkts[0].len, i, 2);
817 test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream;
818
819 pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream;
820 pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2,
821 pkt_stream->pkts[0].len, i, 2);
822 test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream;
823 }
824 }
825
pkt_get_addr(struct pkt * pkt,struct xsk_umem_info * umem)826 static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem)
827 {
828 if (!pkt->valid)
829 return pkt->offset;
830 return pkt->offset + umem_alloc_buffer(umem);
831 }
832
pkt_stream_cancel(struct pkt_stream * pkt_stream)833 static void pkt_stream_cancel(struct pkt_stream *pkt_stream)
834 {
835 pkt_stream->current_pkt_nb--;
836 }
837
pkt_generate(struct xsk_socket_info * xsk,struct xsk_umem_info * umem,u64 addr,u32 len,u32 pkt_nb,u32 bytes_written)838 static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len,
839 u32 pkt_nb, u32 bytes_written)
840 {
841 void *data = xsk_umem__get_data(umem->buffer, addr);
842
843 if (len < MIN_PKT_SIZE)
844 return;
845
846 if (!bytes_written) {
847 gen_eth_hdr(xsk, data);
848
849 len -= PKT_HDR_SIZE;
850 data += PKT_HDR_SIZE;
851 } else {
852 bytes_written -= PKT_HDR_SIZE;
853 }
854
855 write_payload(data, pkt_nb, bytes_written, len);
856 }
857
__pkt_stream_generate_custom(struct ifobject * ifobj,struct pkt * frames,u32 nb_frames,bool verbatim)858 static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames,
859 u32 nb_frames, bool verbatim)
860 {
861 u32 i, len = 0, pkt_nb = 0, payload = 0;
862 struct pkt_stream *pkt_stream;
863
864 pkt_stream = __pkt_stream_alloc(nb_frames);
865 if (!pkt_stream)
866 exit_with_error(ENOMEM);
867
868 for (i = 0; i < nb_frames; i++) {
869 struct pkt *pkt = &pkt_stream->pkts[pkt_nb];
870 struct pkt *frame = &frames[i];
871
872 pkt->offset = frame->offset;
873 if (verbatim) {
874 *pkt = *frame;
875 pkt->pkt_nb = payload;
876 if (!frame->valid || !pkt_continues(frame->options))
877 payload++;
878 } else {
879 if (frame->valid)
880 len += frame->len;
881 if (frame->valid && pkt_continues(frame->options))
882 continue;
883
884 pkt->pkt_nb = pkt_nb;
885 pkt->len = len;
886 pkt->valid = frame->valid;
887 pkt->options = 0;
888
889 len = 0;
890 }
891
892 print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n",
893 pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb);
894
895 if (pkt->valid && pkt->len > pkt_stream->max_pkt_len)
896 pkt_stream->max_pkt_len = pkt->len;
897
898 if (pkt->valid)
899 pkt_stream->nb_valid_entries++;
900
901 pkt_nb++;
902 }
903
904 pkt_stream->nb_pkts = pkt_nb;
905 pkt_stream->verbatim = verbatim;
906 return pkt_stream;
907 }
908
pkt_stream_generate_custom(struct test_spec * test,struct pkt * pkts,u32 nb_pkts)909 static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
910 {
911 struct pkt_stream *pkt_stream;
912
913 pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true);
914 test->ifobj_tx->xsk->pkt_stream = pkt_stream;
915
916 pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false);
917 test->ifobj_rx->xsk->pkt_stream = pkt_stream;
918 }
919
pkt_print_data(u32 * data,u32 cnt)920 static void pkt_print_data(u32 *data, u32 cnt)
921 {
922 u32 i;
923
924 for (i = 0; i < cnt; i++) {
925 u32 seqnum, pkt_nb;
926
927 seqnum = ntohl(*data) & 0xffff;
928 pkt_nb = ntohl(*data) >> 16;
929 ksft_print_msg("%u:%u ", pkt_nb, seqnum);
930 data++;
931 }
932 }
933
pkt_dump(void * pkt,u32 len,bool eth_header)934 static void pkt_dump(void *pkt, u32 len, bool eth_header)
935 {
936 struct ethhdr *ethhdr = pkt;
937 u32 i, *data;
938
939 if (eth_header) {
940 /*extract L2 frame */
941 ksft_print_msg("DEBUG>> L2: dst mac: ");
942 for (i = 0; i < ETH_ALEN; i++)
943 ksft_print_msg("%02X", ethhdr->h_dest[i]);
944
945 ksft_print_msg("\nDEBUG>> L2: src mac: ");
946 for (i = 0; i < ETH_ALEN; i++)
947 ksft_print_msg("%02X", ethhdr->h_source[i]);
948
949 data = pkt + PKT_HDR_SIZE;
950 } else {
951 data = pkt;
952 }
953
954 /*extract L5 frame */
955 ksft_print_msg("\nDEBUG>> L5: seqnum: ");
956 pkt_print_data(data, PKT_DUMP_NB_TO_PRINT);
957 ksft_print_msg("....");
958 if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) {
959 ksft_print_msg("\n.... ");
960 pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT,
961 PKT_DUMP_NB_TO_PRINT);
962 }
963 ksft_print_msg("\n---------------------------------------\n");
964 }
965
is_offset_correct(struct xsk_umem_info * umem,struct pkt * pkt,u64 addr)966 static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr)
967 {
968 u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom;
969 u32 offset = addr % umem->frame_size, expected_offset;
970 int pkt_offset = pkt->valid ? pkt->offset : 0;
971
972 if (!umem->unaligned_mode)
973 pkt_offset = 0;
974
975 expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size;
976
977 if (offset == expected_offset)
978 return true;
979
980 ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset);
981 return false;
982 }
983
is_metadata_correct(struct pkt * pkt,void * buffer,u64 addr)984 static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr)
985 {
986 void *data = xsk_umem__get_data(buffer, addr);
987 struct xdp_info *meta = data - sizeof(struct xdp_info);
988
989 if (meta->count != pkt->pkt_nb) {
990 ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%llu]\n",
991 __func__, pkt->pkt_nb,
992 (unsigned long long)meta->count);
993 return false;
994 }
995
996 return true;
997 }
998
is_adjust_tail_supported(struct xsk_xdp_progs * skel_rx)999 static bool is_adjust_tail_supported(struct xsk_xdp_progs *skel_rx)
1000 {
1001 struct bpf_map *data_map;
1002 int adjust_value = 0;
1003 int key = 0;
1004 int ret;
1005
1006 data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss");
1007 if (!data_map || !bpf_map__is_internal(data_map)) {
1008 ksft_print_msg("Error: could not find bss section of XDP program\n");
1009 exit_with_error(errno);
1010 }
1011
1012 ret = bpf_map_lookup_elem(bpf_map__fd(data_map), &key, &adjust_value);
1013 if (ret) {
1014 ksft_print_msg("Error: bpf_map_lookup_elem failed with error %d\n", ret);
1015 exit_with_error(errno);
1016 }
1017
1018 /* Set the 'adjust_value' variable to -EOPNOTSUPP in the XDP program if the adjust_tail
1019 * helper is not supported. Skip the adjust_tail test case in this scenario.
1020 */
1021 return adjust_value != -EOPNOTSUPP;
1022 }
1023
is_frag_valid(struct xsk_umem_info * umem,u64 addr,u32 len,u32 expected_pkt_nb,u32 bytes_processed)1024 static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb,
1025 u32 bytes_processed)
1026 {
1027 u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum;
1028 void *data = xsk_umem__get_data(umem->buffer, addr);
1029
1030 addr -= umem->base_addr;
1031
1032 if (addr >= umem->num_frames * umem->frame_size ||
1033 addr + len > umem->num_frames * umem->frame_size) {
1034 ksft_print_msg("Frag invalid addr: %llx len: %u\n",
1035 (unsigned long long)addr, len);
1036 return false;
1037 }
1038 if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) {
1039 ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n",
1040 (unsigned long long)addr, len);
1041 return false;
1042 }
1043
1044 pkt_data = data;
1045 if (!bytes_processed) {
1046 pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data);
1047 len -= PKT_HDR_SIZE;
1048 } else {
1049 bytes_processed -= PKT_HDR_SIZE;
1050 }
1051
1052 expected_seqnum = bytes_processed / sizeof(*pkt_data);
1053 seqnum = ntohl(*pkt_data) & 0xffff;
1054 pkt_nb = ntohl(*pkt_data) >> 16;
1055
1056 if (expected_pkt_nb != pkt_nb) {
1057 ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n",
1058 __func__, expected_pkt_nb, pkt_nb);
1059 goto error;
1060 }
1061 if (expected_seqnum != seqnum) {
1062 ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n",
1063 __func__, expected_seqnum, seqnum);
1064 goto error;
1065 }
1066
1067 words_to_end = len / sizeof(*pkt_data) - 1;
1068 pkt_data += words_to_end;
1069 seqnum = ntohl(*pkt_data) & 0xffff;
1070 expected_seqnum += words_to_end;
1071 if (expected_seqnum != seqnum) {
1072 ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n",
1073 __func__, expected_seqnum, seqnum);
1074 goto error;
1075 }
1076
1077 return true;
1078
1079 error:
1080 pkt_dump(data, len, !bytes_processed);
1081 return false;
1082 }
1083
is_pkt_valid(struct pkt * pkt,void * buffer,u64 addr,u32 len)1084 static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len)
1085 {
1086 if (pkt->len != len) {
1087 ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n",
1088 __func__, pkt->len, len);
1089 pkt_dump(xsk_umem__get_data(buffer, addr), len, true);
1090 return false;
1091 }
1092
1093 return true;
1094 }
1095
load_value(u32 * counter)1096 static u32 load_value(u32 *counter)
1097 {
1098 return __atomic_load_n(counter, __ATOMIC_ACQUIRE);
1099 }
1100
kick_tx_with_check(struct xsk_socket_info * xsk,int * ret)1101 static bool kick_tx_with_check(struct xsk_socket_info *xsk, int *ret)
1102 {
1103 u32 max_budget = MAX_TX_BUDGET_DEFAULT;
1104 u32 cons, ready_to_send;
1105 int delta;
1106
1107 cons = load_value(xsk->tx.consumer);
1108 ready_to_send = load_value(xsk->tx.producer) - cons;
1109 *ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
1110
1111 delta = load_value(xsk->tx.consumer) - cons;
1112 /* By default, xsk should consume exact @max_budget descs at one
1113 * send in this case where hitting the max budget limit in while
1114 * loop is triggered in __xsk_generic_xmit(). Please make sure that
1115 * the number of descs to be sent is larger than @max_budget, or
1116 * else the tx.consumer will be updated in xskq_cons_peek_desc()
1117 * in time which hides the issue we try to verify.
1118 */
1119 if (ready_to_send > max_budget && delta != max_budget)
1120 return false;
1121
1122 return true;
1123 }
1124
kick_tx(struct xsk_socket_info * xsk)1125 static int kick_tx(struct xsk_socket_info *xsk)
1126 {
1127 int ret;
1128
1129 if (xsk->check_consumer) {
1130 if (!kick_tx_with_check(xsk, &ret))
1131 return TEST_FAILURE;
1132 } else {
1133 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
1134 }
1135 if (ret >= 0)
1136 return TEST_PASS;
1137 if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) {
1138 usleep(100);
1139 return TEST_PASS;
1140 }
1141 return TEST_FAILURE;
1142 }
1143
kick_rx(struct xsk_socket_info * xsk)1144 static int kick_rx(struct xsk_socket_info *xsk)
1145 {
1146 int ret;
1147
1148 ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL);
1149 if (ret < 0)
1150 return TEST_FAILURE;
1151
1152 return TEST_PASS;
1153 }
1154
complete_pkts(struct xsk_socket_info * xsk,int batch_size)1155 static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)
1156 {
1157 unsigned int rcvd;
1158 u32 idx;
1159 int ret;
1160
1161 if (xsk_ring_prod__needs_wakeup(&xsk->tx)) {
1162 ret = kick_tx(xsk);
1163 if (ret)
1164 return TEST_FAILURE;
1165 }
1166
1167 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
1168 if (rcvd) {
1169 if (rcvd > xsk->outstanding_tx) {
1170 u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1);
1171
1172 ksft_print_msg("[%s] Too many packets completed\n", __func__);
1173 ksft_print_msg("Last completion address: %llx\n",
1174 (unsigned long long)addr);
1175 return TEST_FAILURE;
1176 }
1177
1178 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
1179 xsk->outstanding_tx -= rcvd;
1180 }
1181
1182 return TEST_PASS;
1183 }
1184
__receive_pkts(struct test_spec * test,struct xsk_socket_info * xsk)1185 static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk)
1186 {
1187 u32 frags_processed = 0, nb_frags = 0, pkt_len = 0;
1188 u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0;
1189 struct pkt_stream *pkt_stream = xsk->pkt_stream;
1190 struct ifobject *ifobj = test->ifobj_rx;
1191 struct xsk_umem_info *umem = xsk->umem;
1192 struct pollfd fds = { };
1193 struct pkt *pkt;
1194 u64 first_addr = 0;
1195 int ret;
1196
1197 fds.fd = xsk_socket__fd(xsk->xsk);
1198 fds.events = POLLIN;
1199
1200 ret = kick_rx(xsk);
1201 if (ret)
1202 return TEST_FAILURE;
1203
1204 if (ifobj->use_poll) {
1205 ret = poll(&fds, 1, POLL_TMOUT);
1206 if (ret < 0)
1207 return TEST_FAILURE;
1208
1209 if (!ret) {
1210 if (!is_umem_valid(test->ifobj_tx))
1211 return TEST_PASS;
1212
1213 ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);
1214 return TEST_CONTINUE;
1215 }
1216
1217 if (!(fds.revents & POLLIN))
1218 return TEST_CONTINUE;
1219 }
1220
1221 rcvd = xsk_ring_cons__peek(&xsk->rx, xsk->batch_size, &idx_rx);
1222 if (!rcvd)
1223 return TEST_CONTINUE;
1224
1225 if (ifobj->use_fill_ring) {
1226 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
1227 while (ret != rcvd) {
1228 if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
1229 ret = poll(&fds, 1, POLL_TMOUT);
1230 if (ret < 0)
1231 return TEST_FAILURE;
1232 }
1233 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
1234 }
1235 }
1236
1237 while (frags_processed < rcvd) {
1238 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
1239 u64 addr = desc->addr, orig;
1240
1241 orig = xsk_umem__extract_addr(addr);
1242 addr = xsk_umem__add_offset_to_addr(addr);
1243
1244 if (!nb_frags) {
1245 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
1246 if (!pkt) {
1247 ksft_print_msg("[%s] received too many packets addr: %lx len %u\n",
1248 __func__, addr, desc->len);
1249 return TEST_FAILURE;
1250 }
1251 }
1252
1253 print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n",
1254 addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid);
1255
1256 if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) ||
1257 !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata &&
1258 !is_metadata_correct(pkt, umem->buffer, addr)))
1259 return TEST_FAILURE;
1260
1261 if (!nb_frags++)
1262 first_addr = addr;
1263 frags_processed++;
1264 pkt_len += desc->len;
1265 if (ifobj->use_fill_ring)
1266 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;
1267
1268 if (pkt_continues(desc->options))
1269 continue;
1270
1271 /* The complete packet has been received */
1272 if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) ||
1273 !is_offset_correct(umem, pkt, addr))
1274 return TEST_FAILURE;
1275
1276 pkt_stream->nb_rx_pkts++;
1277 nb_frags = 0;
1278 pkt_len = 0;
1279 }
1280
1281 if (nb_frags) {
1282 /* In the middle of a packet. Start over from beginning of packet. */
1283 idx_rx -= nb_frags;
1284 xsk_ring_cons__cancel(&xsk->rx, nb_frags);
1285 if (ifobj->use_fill_ring) {
1286 idx_fq -= nb_frags;
1287 xsk_ring_prod__cancel(&umem->fq, nb_frags);
1288 }
1289 frags_processed -= nb_frags;
1290 }
1291
1292 if (ifobj->use_fill_ring)
1293 xsk_ring_prod__submit(&umem->fq, frags_processed);
1294 if (ifobj->release_rx)
1295 xsk_ring_cons__release(&xsk->rx, frags_processed);
1296
1297 pthread_mutex_lock(&pacing_mutex);
1298 pkts_in_flight -= pkts_sent;
1299 pthread_mutex_unlock(&pacing_mutex);
1300 pkts_sent = 0;
1301
1302 return TEST_CONTINUE;
1303 }
1304
all_packets_received(struct test_spec * test,struct xsk_socket_info * xsk,u32 sock_num,unsigned long * bitmap)1305 bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num,
1306 unsigned long *bitmap)
1307 {
1308 struct pkt_stream *pkt_stream = xsk->pkt_stream;
1309
1310 if (!pkt_stream) {
1311 __set_bit(sock_num, bitmap);
1312 return false;
1313 }
1314
1315 if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) {
1316 __set_bit(sock_num, bitmap);
1317 if (bitmap_full(bitmap, test->nb_sockets))
1318 return true;
1319 }
1320
1321 return false;
1322 }
1323
receive_pkts(struct test_spec * test)1324 static int receive_pkts(struct test_spec *test)
1325 {
1326 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
1327 DECLARE_BITMAP(bitmap, test->nb_sockets);
1328 struct xsk_socket_info *xsk;
1329 u32 sock_num = 0;
1330 int res, ret;
1331
1332 ret = gettimeofday(&tv_now, NULL);
1333 if (ret)
1334 exit_with_error(errno);
1335
1336 timeradd(&tv_now, &tv_timeout, &tv_end);
1337
1338 while (1) {
1339 xsk = &test->ifobj_rx->xsk_arr[sock_num];
1340
1341 if ((all_packets_received(test, xsk, sock_num, bitmap)))
1342 break;
1343
1344 res = __receive_pkts(test, xsk);
1345 if (!(res == TEST_PASS || res == TEST_CONTINUE))
1346 return res;
1347
1348 ret = gettimeofday(&tv_now, NULL);
1349 if (ret)
1350 exit_with_error(errno);
1351
1352 if (timercmp(&tv_now, &tv_end, >)) {
1353 ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);
1354 return TEST_FAILURE;
1355 }
1356 sock_num = (sock_num + 1) % test->nb_sockets;
1357 }
1358
1359 return TEST_PASS;
1360 }
1361
__send_pkts(struct ifobject * ifobject,struct xsk_socket_info * xsk,bool timeout)1362 static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout)
1363 {
1364 u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len;
1365 struct pkt_stream *pkt_stream = xsk->pkt_stream;
1366 struct xsk_umem_info *umem = ifobject->umem;
1367 bool use_poll = ifobject->use_poll;
1368 struct pollfd fds = { };
1369 int ret;
1370
1371 buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len);
1372 /* pkts_in_flight might be negative if many invalid packets are sent */
1373 if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) /
1374 buffer_len)) {
1375 ret = kick_tx(xsk);
1376 if (ret)
1377 return TEST_FAILURE;
1378 return TEST_CONTINUE;
1379 }
1380
1381 fds.fd = xsk_socket__fd(xsk->xsk);
1382 fds.events = POLLOUT;
1383
1384 while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) {
1385 if (use_poll) {
1386 ret = poll(&fds, 1, POLL_TMOUT);
1387 if (timeout) {
1388 if (ret < 0) {
1389 ksft_print_msg("ERROR: [%s] Poll error %d\n",
1390 __func__, errno);
1391 return TEST_FAILURE;
1392 }
1393 if (ret == 0)
1394 return TEST_PASS;
1395 break;
1396 }
1397 if (ret <= 0) {
1398 ksft_print_msg("ERROR: [%s] Poll error %d\n",
1399 __func__, errno);
1400 return TEST_FAILURE;
1401 }
1402 }
1403
1404 complete_pkts(xsk, xsk->batch_size);
1405 }
1406
1407 for (i = 0; i < xsk->batch_size; i++) {
1408 struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream);
1409 u32 nb_frags_left, nb_frags, bytes_written = 0;
1410
1411 if (!pkt)
1412 break;
1413
1414 nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt);
1415 if (nb_frags > xsk->batch_size - i) {
1416 pkt_stream_cancel(pkt_stream);
1417 xsk_ring_prod__cancel(&xsk->tx, xsk->batch_size - i);
1418 break;
1419 }
1420 nb_frags_left = nb_frags;
1421
1422 while (nb_frags_left--) {
1423 struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);
1424
1425 tx_desc->addr = pkt_get_addr(pkt, ifobject->umem);
1426 if (pkt_stream->verbatim) {
1427 tx_desc->len = pkt->len;
1428 tx_desc->options = pkt->options;
1429 } else if (nb_frags_left) {
1430 tx_desc->len = umem->frame_size;
1431 tx_desc->options = XDP_PKT_CONTD;
1432 } else {
1433 tx_desc->len = pkt->len - bytes_written;
1434 tx_desc->options = 0;
1435 }
1436 if (pkt->valid)
1437 pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb,
1438 bytes_written);
1439 bytes_written += tx_desc->len;
1440
1441 print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n",
1442 tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb);
1443
1444 if (nb_frags_left) {
1445 i++;
1446 if (pkt_stream->verbatim)
1447 pkt = pkt_stream_get_next_tx_pkt(pkt_stream);
1448 }
1449 }
1450
1451 if (pkt && pkt->valid) {
1452 valid_pkts++;
1453 valid_frags += nb_frags;
1454 }
1455 }
1456
1457 pthread_mutex_lock(&pacing_mutex);
1458 pkts_in_flight += valid_pkts;
1459 pthread_mutex_unlock(&pacing_mutex);
1460
1461 xsk_ring_prod__submit(&xsk->tx, i);
1462 xsk->outstanding_tx += valid_frags;
1463
1464 if (use_poll) {
1465 ret = poll(&fds, 1, POLL_TMOUT);
1466 if (ret <= 0) {
1467 if (ret == 0 && timeout)
1468 return TEST_PASS;
1469
1470 ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret);
1471 return TEST_FAILURE;
1472 }
1473 }
1474
1475 if (!timeout) {
1476 if (complete_pkts(xsk, i))
1477 return TEST_FAILURE;
1478
1479 usleep(10);
1480 return TEST_PASS;
1481 }
1482
1483 return TEST_CONTINUE;
1484 }
1485
wait_for_tx_completion(struct xsk_socket_info * xsk)1486 static int wait_for_tx_completion(struct xsk_socket_info *xsk)
1487 {
1488 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
1489 int ret;
1490
1491 ret = gettimeofday(&tv_now, NULL);
1492 if (ret)
1493 exit_with_error(errno);
1494 timeradd(&tv_now, &tv_timeout, &tv_end);
1495
1496 while (xsk->outstanding_tx) {
1497 ret = gettimeofday(&tv_now, NULL);
1498 if (ret)
1499 exit_with_error(errno);
1500 if (timercmp(&tv_now, &tv_end, >)) {
1501 ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__);
1502 return TEST_FAILURE;
1503 }
1504
1505 complete_pkts(xsk, xsk->batch_size);
1506 }
1507
1508 return TEST_PASS;
1509 }
1510
all_packets_sent(struct test_spec * test,unsigned long * bitmap)1511 bool all_packets_sent(struct test_spec *test, unsigned long *bitmap)
1512 {
1513 return bitmap_full(bitmap, test->nb_sockets);
1514 }
1515
send_pkts(struct test_spec * test,struct ifobject * ifobject)1516 static int send_pkts(struct test_spec *test, struct ifobject *ifobject)
1517 {
1518 bool timeout = !is_umem_valid(test->ifobj_rx);
1519 DECLARE_BITMAP(bitmap, test->nb_sockets);
1520 u32 i, ret;
1521
1522 while (!(all_packets_sent(test, bitmap))) {
1523 for (i = 0; i < test->nb_sockets; i++) {
1524 struct pkt_stream *pkt_stream;
1525
1526 pkt_stream = ifobject->xsk_arr[i].pkt_stream;
1527 if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) {
1528 __set_bit(i, bitmap);
1529 continue;
1530 }
1531 ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout);
1532 if (ret == TEST_CONTINUE && !test->fail)
1533 continue;
1534
1535 if ((ret || test->fail) && !timeout)
1536 return TEST_FAILURE;
1537
1538 if (ret == TEST_PASS && timeout)
1539 return ret;
1540
1541 ret = wait_for_tx_completion(&ifobject->xsk_arr[i]);
1542 if (ret)
1543 return TEST_FAILURE;
1544 }
1545 }
1546
1547 return TEST_PASS;
1548 }
1549
get_xsk_stats(struct xsk_socket * xsk,struct xdp_statistics * stats)1550 static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats)
1551 {
1552 int fd = xsk_socket__fd(xsk), err;
1553 socklen_t optlen, expected_len;
1554
1555 optlen = sizeof(*stats);
1556 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen);
1557 if (err) {
1558 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1559 __func__, -err, strerror(-err));
1560 return TEST_FAILURE;
1561 }
1562
1563 expected_len = sizeof(struct xdp_statistics);
1564 if (optlen != expected_len) {
1565 ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n",
1566 __func__, expected_len, optlen);
1567 return TEST_FAILURE;
1568 }
1569
1570 return TEST_PASS;
1571 }
1572
validate_rx_dropped(struct ifobject * ifobject)1573 static int validate_rx_dropped(struct ifobject *ifobject)
1574 {
1575 struct xsk_socket *xsk = ifobject->xsk->xsk;
1576 struct xdp_statistics stats;
1577 int err;
1578
1579 err = kick_rx(ifobject->xsk);
1580 if (err)
1581 return TEST_FAILURE;
1582
1583 err = get_xsk_stats(xsk, &stats);
1584 if (err)
1585 return TEST_FAILURE;
1586
1587 /* The receiver calls getsockopt after receiving the last (valid)
1588 * packet which is not the final packet sent in this test (valid and
1589 * invalid packets are sent in alternating fashion with the final
1590 * packet being invalid). Since the last packet may or may not have
1591 * been dropped already, both outcomes must be allowed.
1592 */
1593 if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 ||
1594 stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1)
1595 return TEST_PASS;
1596
1597 return TEST_FAILURE;
1598 }
1599
validate_rx_full(struct ifobject * ifobject)1600 static int validate_rx_full(struct ifobject *ifobject)
1601 {
1602 struct xsk_socket *xsk = ifobject->xsk->xsk;
1603 struct xdp_statistics stats;
1604 int err;
1605
1606 usleep(1000);
1607 err = kick_rx(ifobject->xsk);
1608 if (err)
1609 return TEST_FAILURE;
1610
1611 err = get_xsk_stats(xsk, &stats);
1612 if (err)
1613 return TEST_FAILURE;
1614
1615 if (stats.rx_ring_full)
1616 return TEST_PASS;
1617
1618 return TEST_FAILURE;
1619 }
1620
validate_fill_empty(struct ifobject * ifobject)1621 static int validate_fill_empty(struct ifobject *ifobject)
1622 {
1623 struct xsk_socket *xsk = ifobject->xsk->xsk;
1624 struct xdp_statistics stats;
1625 int err;
1626
1627 usleep(1000);
1628 err = kick_rx(ifobject->xsk);
1629 if (err)
1630 return TEST_FAILURE;
1631
1632 err = get_xsk_stats(xsk, &stats);
1633 if (err)
1634 return TEST_FAILURE;
1635
1636 if (stats.rx_fill_ring_empty_descs)
1637 return TEST_PASS;
1638
1639 return TEST_FAILURE;
1640 }
1641
validate_tx_invalid_descs(struct ifobject * ifobject)1642 static int validate_tx_invalid_descs(struct ifobject *ifobject)
1643 {
1644 struct xsk_socket *xsk = ifobject->xsk->xsk;
1645 int fd = xsk_socket__fd(xsk);
1646 struct xdp_statistics stats;
1647 socklen_t optlen;
1648 int err;
1649
1650 optlen = sizeof(stats);
1651 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
1652 if (err) {
1653 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1654 __func__, -err, strerror(-err));
1655 return TEST_FAILURE;
1656 }
1657
1658 if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) {
1659 ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n",
1660 __func__,
1661 (unsigned long long)stats.tx_invalid_descs,
1662 ifobject->xsk->pkt_stream->nb_pkts);
1663 return TEST_FAILURE;
1664 }
1665
1666 return TEST_PASS;
1667 }
1668
xsk_configure_socket(struct test_spec * test,struct ifobject * ifobject,struct xsk_umem_info * umem,bool tx)1669 static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject,
1670 struct xsk_umem_info *umem, bool tx)
1671 {
1672 int i, ret;
1673
1674 for (i = 0; i < test->nb_sockets; i++) {
1675 bool shared = (ifobject->shared_umem && tx) ? true : !!i;
1676 u32 ctr = 0;
1677
1678 while (ctr++ < SOCK_RECONF_CTR) {
1679 ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,
1680 ifobject, shared);
1681 if (!ret)
1682 break;
1683
1684 /* Retry if it fails as xsk_socket__create() is asynchronous */
1685 if (ctr >= SOCK_RECONF_CTR)
1686 exit_with_error(-ret);
1687 usleep(USLEEP_MAX);
1688 }
1689 if (ifobject->busy_poll)
1690 enable_busy_poll(&ifobject->xsk_arr[i]);
1691 }
1692 }
1693
thread_common_ops_tx(struct test_spec * test,struct ifobject * ifobject)1694 static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)
1695 {
1696 xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);
1697 ifobject->xsk = &ifobject->xsk_arr[0];
1698 ifobject->xskmap = test->ifobj_rx->xskmap;
1699 memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));
1700 ifobject->umem->base_addr = 0;
1701 }
1702
xsk_populate_fill_ring(struct xsk_umem_info * umem,struct pkt_stream * pkt_stream,bool fill_up)1703 static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream,
1704 bool fill_up)
1705 {
1706 u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM;
1707 u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts;
1708 int ret;
1709
1710 if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
1711 buffers_to_fill = umem->num_frames;
1712 else
1713 buffers_to_fill = umem->fill_size;
1714
1715 ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
1716 if (ret != buffers_to_fill)
1717 exit_with_error(ENOSPC);
1718
1719 while (filled < buffers_to_fill) {
1720 struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts);
1721 u64 addr;
1722 u32 i;
1723
1724 for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) {
1725 if (!pkt) {
1726 if (!fill_up)
1727 break;
1728 addr = filled * umem->frame_size + umem->base_addr;
1729 } else if (pkt->offset >= 0) {
1730 addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem);
1731 } else {
1732 addr = pkt->offset + umem_alloc_buffer(umem);
1733 }
1734
1735 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
1736 if (++filled >= buffers_to_fill)
1737 break;
1738 }
1739 }
1740 xsk_ring_prod__submit(&umem->fq, filled);
1741 xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled);
1742
1743 pkt_stream_reset(pkt_stream);
1744 umem_reset_alloc(umem);
1745 }
1746
thread_common_ops(struct test_spec * test,struct ifobject * ifobject)1747 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
1748 {
1749 u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
1750 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
1751 LIBBPF_OPTS(bpf_xdp_query_opts, opts);
1752 void *bufs;
1753 int ret;
1754 u32 i;
1755
1756 if (ifobject->umem->unaligned_mode)
1757 mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB;
1758
1759 if (ifobject->shared_umem)
1760 umem_sz *= 2;
1761
1762 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
1763 if (bufs == MAP_FAILED)
1764 exit_with_error(errno);
1765
1766 ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz);
1767 if (ret)
1768 exit_with_error(-ret);
1769
1770 xsk_configure_socket(test, ifobject, ifobject->umem, false);
1771
1772 ifobject->xsk = &ifobject->xsk_arr[0];
1773
1774 if (!ifobject->rx_on)
1775 return;
1776
1777 xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring);
1778
1779 for (i = 0; i < test->nb_sockets; i++) {
1780 ifobject->xsk = &ifobject->xsk_arr[i];
1781 ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i);
1782 if (ret)
1783 exit_with_error(errno);
1784 }
1785 }
1786
worker_testapp_validate_tx(void * arg)1787 static void *worker_testapp_validate_tx(void *arg)
1788 {
1789 struct test_spec *test = (struct test_spec *)arg;
1790 struct ifobject *ifobject = test->ifobj_tx;
1791 int err;
1792
1793 if (test->current_step == 1) {
1794 if (!ifobject->shared_umem)
1795 thread_common_ops(test, ifobject);
1796 else
1797 thread_common_ops_tx(test, ifobject);
1798 }
1799
1800 err = send_pkts(test, ifobject);
1801
1802 if (!err && ifobject->validation_func)
1803 err = ifobject->validation_func(ifobject);
1804 if (err)
1805 report_failure(test);
1806
1807 pthread_exit(NULL);
1808 }
1809
worker_testapp_validate_rx(void * arg)1810 static void *worker_testapp_validate_rx(void *arg)
1811 {
1812 struct test_spec *test = (struct test_spec *)arg;
1813 struct ifobject *ifobject = test->ifobj_rx;
1814 int err;
1815
1816 if (test->current_step == 1) {
1817 thread_common_ops(test, ifobject);
1818 } else {
1819 xsk_clear_xskmap(ifobject->xskmap);
1820 err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0);
1821 if (err) {
1822 ksft_print_msg("Error: Failed to update xskmap, error %s\n",
1823 strerror(-err));
1824 exit_with_error(-err);
1825 }
1826 }
1827
1828 pthread_barrier_wait(&barr);
1829
1830 err = receive_pkts(test);
1831
1832 if (!err && ifobject->validation_func)
1833 err = ifobject->validation_func(ifobject);
1834
1835 if (err) {
1836 if (test->adjust_tail && !is_adjust_tail_supported(ifobject->xdp_progs))
1837 test->adjust_tail_support = false;
1838 else
1839 report_failure(test);
1840 }
1841
1842 pthread_exit(NULL);
1843 }
1844
ceil_u64(u64 a,u64 b)1845 static u64 ceil_u64(u64 a, u64 b)
1846 {
1847 return (a + b - 1) / b;
1848 }
1849
testapp_clean_xsk_umem(struct ifobject * ifobj)1850 static void testapp_clean_xsk_umem(struct ifobject *ifobj)
1851 {
1852 u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size;
1853
1854 if (ifobj->shared_umem)
1855 umem_sz *= 2;
1856
1857 umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE;
1858 xsk_umem__delete(ifobj->umem->umem);
1859 munmap(ifobj->umem->buffer, umem_sz);
1860 }
1861
handler(int signum)1862 static void handler(int signum)
1863 {
1864 pthread_exit(NULL);
1865 }
1866
xdp_prog_changed_rx(struct test_spec * test)1867 static bool xdp_prog_changed_rx(struct test_spec *test)
1868 {
1869 struct ifobject *ifobj = test->ifobj_rx;
1870
1871 return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode;
1872 }
1873
xdp_prog_changed_tx(struct test_spec * test)1874 static bool xdp_prog_changed_tx(struct test_spec *test)
1875 {
1876 struct ifobject *ifobj = test->ifobj_tx;
1877
1878 return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode;
1879 }
1880
xsk_reattach_xdp(struct ifobject * ifobj,struct bpf_program * xdp_prog,struct bpf_map * xskmap,enum test_mode mode)1881 static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog,
1882 struct bpf_map *xskmap, enum test_mode mode)
1883 {
1884 int err;
1885
1886 xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode));
1887 err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode));
1888 if (err) {
1889 ksft_print_msg("Error attaching XDP program\n");
1890 exit_with_error(-err);
1891 }
1892
1893 if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC))
1894 if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) {
1895 ksft_print_msg("ERROR: XDP prog not in DRV mode\n");
1896 exit_with_error(EINVAL);
1897 }
1898
1899 ifobj->xdp_prog = xdp_prog;
1900 ifobj->xskmap = xskmap;
1901 ifobj->mode = mode;
1902 }
1903
xsk_attach_xdp_progs(struct test_spec * test,struct ifobject * ifobj_rx,struct ifobject * ifobj_tx)1904 static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx,
1905 struct ifobject *ifobj_tx)
1906 {
1907 if (xdp_prog_changed_rx(test))
1908 xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode);
1909
1910 if (!ifobj_tx || ifobj_tx->shared_umem)
1911 return;
1912
1913 if (xdp_prog_changed_tx(test))
1914 xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode);
1915 }
1916
__testapp_validate_traffic(struct test_spec * test,struct ifobject * ifobj1,struct ifobject * ifobj2)1917 static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1,
1918 struct ifobject *ifobj2)
1919 {
1920 pthread_t t0, t1;
1921 int err;
1922
1923 if (test->mtu > MAX_ETH_PKT_SIZE) {
1924 if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp ||
1925 (ifobj2 && !ifobj2->multi_buff_zc_supp))) {
1926 ksft_test_result_skip("Multi buffer for zero-copy not supported.\n");
1927 return TEST_SKIP;
1928 }
1929 if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp ||
1930 (ifobj2 && !ifobj2->multi_buff_supp))) {
1931 ksft_test_result_skip("Multi buffer not supported.\n");
1932 return TEST_SKIP;
1933 }
1934 }
1935 err = test_spec_set_mtu(test, test->mtu);
1936 if (err) {
1937 ksft_print_msg("Error, could not set mtu.\n");
1938 exit_with_error(err);
1939 }
1940
1941 if (ifobj2) {
1942 if (pthread_barrier_init(&barr, NULL, 2))
1943 exit_with_error(errno);
1944 pkt_stream_reset(ifobj2->xsk->pkt_stream);
1945 }
1946
1947 test->current_step++;
1948 pkt_stream_reset(ifobj1->xsk->pkt_stream);
1949 pkts_in_flight = 0;
1950
1951 signal(SIGUSR1, handler);
1952 /*Spawn RX thread */
1953 pthread_create(&t0, NULL, ifobj1->func_ptr, test);
1954
1955 if (ifobj2) {
1956 pthread_barrier_wait(&barr);
1957 if (pthread_barrier_destroy(&barr))
1958 exit_with_error(errno);
1959
1960 /*Spawn TX thread */
1961 pthread_create(&t1, NULL, ifobj2->func_ptr, test);
1962
1963 pthread_join(t1, NULL);
1964 }
1965
1966 if (!ifobj2)
1967 pthread_kill(t0, SIGUSR1);
1968 else
1969 pthread_join(t0, NULL);
1970
1971 if (test->total_steps == test->current_step || test->fail) {
1972 u32 i;
1973
1974 if (ifobj2)
1975 for (i = 0; i < test->nb_sockets; i++)
1976 xsk_socket__delete(ifobj2->xsk_arr[i].xsk);
1977
1978 for (i = 0; i < test->nb_sockets; i++)
1979 xsk_socket__delete(ifobj1->xsk_arr[i].xsk);
1980
1981 testapp_clean_xsk_umem(ifobj1);
1982 if (ifobj2 && !ifobj2->shared_umem)
1983 testapp_clean_xsk_umem(ifobj2);
1984 }
1985
1986 return !!test->fail;
1987 }
1988
testapp_validate_traffic(struct test_spec * test)1989 static int testapp_validate_traffic(struct test_spec *test)
1990 {
1991 struct ifobject *ifobj_rx = test->ifobj_rx;
1992 struct ifobject *ifobj_tx = test->ifobj_tx;
1993
1994 if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) ||
1995 (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) {
1996 ksft_test_result_skip("No huge pages present.\n");
1997 return TEST_SKIP;
1998 }
1999
2000 if (test->set_ring) {
2001 if (ifobj_tx->hw_ring_size_supp) {
2002 if (set_ring_size(ifobj_tx)) {
2003 ksft_test_result_skip("Failed to change HW ring size.\n");
2004 return TEST_FAILURE;
2005 }
2006 } else {
2007 ksft_test_result_skip("Changing HW ring size not supported.\n");
2008 return TEST_SKIP;
2009 }
2010 }
2011
2012 xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx);
2013 return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx);
2014 }
2015
testapp_validate_traffic_single_thread(struct test_spec * test,struct ifobject * ifobj)2016 static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj)
2017 {
2018 return __testapp_validate_traffic(test, ifobj, NULL);
2019 }
2020
testapp_teardown(struct test_spec * test)2021 static int testapp_teardown(struct test_spec *test)
2022 {
2023 int i;
2024
2025 for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
2026 if (testapp_validate_traffic(test))
2027 return TEST_FAILURE;
2028 test_spec_reset(test);
2029 }
2030
2031 return TEST_PASS;
2032 }
2033
swap_directions(struct ifobject ** ifobj1,struct ifobject ** ifobj2)2034 static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)
2035 {
2036 thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr;
2037 struct ifobject *tmp_ifobj = (*ifobj1);
2038
2039 (*ifobj1)->func_ptr = (*ifobj2)->func_ptr;
2040 (*ifobj2)->func_ptr = tmp_func_ptr;
2041
2042 *ifobj1 = *ifobj2;
2043 *ifobj2 = tmp_ifobj;
2044 }
2045
testapp_bidirectional(struct test_spec * test)2046 static int testapp_bidirectional(struct test_spec *test)
2047 {
2048 int res;
2049
2050 test->ifobj_tx->rx_on = true;
2051 test->ifobj_rx->tx_on = true;
2052 test->total_steps = 2;
2053 if (testapp_validate_traffic(test))
2054 return TEST_FAILURE;
2055
2056 print_verbose("Switching Tx/Rx direction\n");
2057 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
2058 res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx);
2059
2060 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
2061 return res;
2062 }
2063
swap_xsk_resources(struct test_spec * test)2064 static int swap_xsk_resources(struct test_spec *test)
2065 {
2066 int ret;
2067
2068 test->ifobj_tx->xsk_arr[0].pkt_stream = NULL;
2069 test->ifobj_rx->xsk_arr[0].pkt_stream = NULL;
2070 test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default;
2071 test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default;
2072 test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1];
2073 test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1];
2074
2075 ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0);
2076 if (ret)
2077 return TEST_FAILURE;
2078
2079 return TEST_PASS;
2080 }
2081
testapp_xdp_prog_cleanup(struct test_spec * test)2082 static int testapp_xdp_prog_cleanup(struct test_spec *test)
2083 {
2084 test->total_steps = 2;
2085 test->nb_sockets = 2;
2086 if (testapp_validate_traffic(test))
2087 return TEST_FAILURE;
2088
2089 if (swap_xsk_resources(test))
2090 return TEST_FAILURE;
2091 return testapp_validate_traffic(test);
2092 }
2093
testapp_headroom(struct test_spec * test)2094 static int testapp_headroom(struct test_spec *test)
2095 {
2096 test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE;
2097 return testapp_validate_traffic(test);
2098 }
2099
testapp_stats_rx_dropped(struct test_spec * test)2100 static int testapp_stats_rx_dropped(struct test_spec *test)
2101 {
2102 if (test->mode == TEST_MODE_ZC) {
2103 ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n");
2104 return TEST_SKIP;
2105 }
2106
2107 pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
2108 test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
2109 XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;
2110 pkt_stream_receive_half(test);
2111 test->ifobj_rx->validation_func = validate_rx_dropped;
2112 return testapp_validate_traffic(test);
2113 }
2114
testapp_stats_tx_invalid_descs(struct test_spec * test)2115 static int testapp_stats_tx_invalid_descs(struct test_spec *test)
2116 {
2117 pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0);
2118 test->ifobj_tx->validation_func = validate_tx_invalid_descs;
2119 return testapp_validate_traffic(test);
2120 }
2121
testapp_stats_rx_full(struct test_spec * test)2122 static int testapp_stats_rx_full(struct test_spec *test)
2123 {
2124 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE);
2125 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
2126
2127 test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS;
2128 test->ifobj_rx->release_rx = false;
2129 test->ifobj_rx->validation_func = validate_rx_full;
2130 return testapp_validate_traffic(test);
2131 }
2132
testapp_stats_fill_empty(struct test_spec * test)2133 static int testapp_stats_fill_empty(struct test_spec *test)
2134 {
2135 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE);
2136 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
2137
2138 test->ifobj_rx->use_fill_ring = false;
2139 test->ifobj_rx->validation_func = validate_fill_empty;
2140 return testapp_validate_traffic(test);
2141 }
2142
testapp_send_receive_unaligned(struct test_spec * test)2143 static int testapp_send_receive_unaligned(struct test_spec *test)
2144 {
2145 test->ifobj_tx->umem->unaligned_mode = true;
2146 test->ifobj_rx->umem->unaligned_mode = true;
2147 /* Let half of the packets straddle a 4K buffer boundary */
2148 pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2);
2149
2150 return testapp_validate_traffic(test);
2151 }
2152
testapp_send_receive_unaligned_mb(struct test_spec * test)2153 static int testapp_send_receive_unaligned_mb(struct test_spec *test)
2154 {
2155 test->mtu = MAX_ETH_JUMBO_SIZE;
2156 test->ifobj_tx->umem->unaligned_mode = true;
2157 test->ifobj_rx->umem->unaligned_mode = true;
2158 pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE);
2159 return testapp_validate_traffic(test);
2160 }
2161
testapp_single_pkt(struct test_spec * test)2162 static int testapp_single_pkt(struct test_spec *test)
2163 {
2164 struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}};
2165
2166 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
2167 return testapp_validate_traffic(test);
2168 }
2169
testapp_send_receive_mb(struct test_spec * test)2170 static int testapp_send_receive_mb(struct test_spec *test)
2171 {
2172 test->mtu = MAX_ETH_JUMBO_SIZE;
2173 pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE);
2174
2175 return testapp_validate_traffic(test);
2176 }
2177
testapp_invalid_desc_mb(struct test_spec * test)2178 static int testapp_invalid_desc_mb(struct test_spec *test)
2179 {
2180 struct xsk_umem_info *umem = test->ifobj_tx->umem;
2181 u64 umem_size = umem->num_frames * umem->frame_size;
2182 struct pkt pkts[] = {
2183 /* Valid packet for synch to start with */
2184 {0, MIN_PKT_SIZE, 0, true, 0},
2185 /* Zero frame len is not legal */
2186 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2187 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2188 {0, 0, 0, false, 0},
2189 /* Invalid address in the second frame */
2190 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2191 {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2192 /* Invalid len in the middle */
2193 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2194 {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2195 /* Invalid options in the middle */
2196 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2197 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION},
2198 /* Transmit 2 frags, receive 3 */
2199 {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD},
2200 {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0},
2201 /* Middle frame crosses chunk boundary with small length */
2202 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2203 {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0},
2204 /* Valid packet for synch so that something is received */
2205 {0, MIN_PKT_SIZE, 0, true, 0}};
2206
2207 if (umem->unaligned_mode) {
2208 /* Crossing a chunk boundary allowed */
2209 pkts[12].valid = true;
2210 pkts[13].valid = true;
2211 }
2212
2213 test->mtu = MAX_ETH_JUMBO_SIZE;
2214 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
2215 return testapp_validate_traffic(test);
2216 }
2217
testapp_invalid_desc(struct test_spec * test)2218 static int testapp_invalid_desc(struct test_spec *test)
2219 {
2220 struct xsk_umem_info *umem = test->ifobj_tx->umem;
2221 u64 umem_size = umem->num_frames * umem->frame_size;
2222 struct pkt pkts[] = {
2223 /* Zero packet address allowed */
2224 {0, MIN_PKT_SIZE, 0, true},
2225 /* Allowed packet */
2226 {0, MIN_PKT_SIZE, 0, true},
2227 /* Straddling the start of umem */
2228 {-2, MIN_PKT_SIZE, 0, false},
2229 /* Packet too large */
2230 {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false},
2231 /* Up to end of umem allowed */
2232 {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true},
2233 /* After umem ends */
2234 {umem_size, MIN_PKT_SIZE, 0, false},
2235 /* Straddle the end of umem */
2236 {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false},
2237 /* Straddle a 4K boundary */
2238 {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false},
2239 /* Straddle a 2K boundary */
2240 {0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true},
2241 /* Valid packet for synch so that something is received */
2242 {0, MIN_PKT_SIZE, 0, true}};
2243
2244 if (umem->unaligned_mode) {
2245 /* Crossing a page boundary allowed */
2246 pkts[7].valid = true;
2247 }
2248 if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) {
2249 /* Crossing a 2K frame size boundary not allowed */
2250 pkts[8].valid = false;
2251 }
2252
2253 if (test->ifobj_tx->shared_umem) {
2254 pkts[4].offset += umem_size;
2255 pkts[5].offset += umem_size;
2256 pkts[6].offset += umem_size;
2257 }
2258
2259 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
2260 return testapp_validate_traffic(test);
2261 }
2262
testapp_xdp_drop(struct test_spec * test)2263 static int testapp_xdp_drop(struct test_spec *test)
2264 {
2265 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2266 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2267
2268 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop,
2269 skel_rx->maps.xsk, skel_tx->maps.xsk);
2270
2271 pkt_stream_receive_half(test);
2272 return testapp_validate_traffic(test);
2273 }
2274
testapp_xdp_metadata_copy(struct test_spec * test)2275 static int testapp_xdp_metadata_copy(struct test_spec *test)
2276 {
2277 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2278 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2279 struct bpf_map *data_map;
2280 int count = 0;
2281 int key = 0;
2282
2283 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata,
2284 skel_tx->progs.xsk_xdp_populate_metadata,
2285 skel_rx->maps.xsk, skel_tx->maps.xsk);
2286 test->ifobj_rx->use_metadata = true;
2287
2288 data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss");
2289 if (!data_map || !bpf_map__is_internal(data_map)) {
2290 ksft_print_msg("Error: could not find bss section of XDP program\n");
2291 return TEST_FAILURE;
2292 }
2293
2294 if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY)) {
2295 ksft_print_msg("Error: could not update count element\n");
2296 return TEST_FAILURE;
2297 }
2298
2299 return testapp_validate_traffic(test);
2300 }
2301
testapp_xdp_shared_umem(struct test_spec * test)2302 static int testapp_xdp_shared_umem(struct test_spec *test)
2303 {
2304 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2305 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2306
2307 test->total_steps = 1;
2308 test->nb_sockets = 2;
2309
2310 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem,
2311 skel_tx->progs.xsk_xdp_shared_umem,
2312 skel_rx->maps.xsk, skel_tx->maps.xsk);
2313
2314 pkt_stream_even_odd_sequence(test);
2315
2316 return testapp_validate_traffic(test);
2317 }
2318
testapp_poll_txq_tmout(struct test_spec * test)2319 static int testapp_poll_txq_tmout(struct test_spec *test)
2320 {
2321 test->ifobj_tx->use_poll = true;
2322 /* create invalid frame by set umem frame_size and pkt length equal to 2048 */
2323 test->ifobj_tx->umem->frame_size = 2048;
2324 pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048);
2325 return testapp_validate_traffic_single_thread(test, test->ifobj_tx);
2326 }
2327
testapp_poll_rxq_tmout(struct test_spec * test)2328 static int testapp_poll_rxq_tmout(struct test_spec *test)
2329 {
2330 test->ifobj_rx->use_poll = true;
2331 return testapp_validate_traffic_single_thread(test, test->ifobj_rx);
2332 }
2333
testapp_too_many_frags(struct test_spec * test)2334 static int testapp_too_many_frags(struct test_spec *test)
2335 {
2336 struct pkt *pkts;
2337 u32 max_frags, i;
2338 int ret;
2339
2340 if (test->mode == TEST_MODE_ZC) {
2341 max_frags = test->ifobj_tx->xdp_zc_max_segs;
2342 } else {
2343 max_frags = get_max_skb_frags();
2344 if (!max_frags) {
2345 ksft_print_msg("Couldn't retrieve MAX_SKB_FRAGS from system, using default (17) value\n");
2346 max_frags = 17;
2347 }
2348 max_frags += 1;
2349 }
2350
2351 pkts = calloc(2 * max_frags + 2, sizeof(struct pkt));
2352 if (!pkts)
2353 return TEST_FAILURE;
2354
2355 test->mtu = MAX_ETH_JUMBO_SIZE;
2356
2357 /* Valid packet for synch */
2358 pkts[0].len = MIN_PKT_SIZE;
2359 pkts[0].valid = true;
2360
2361 /* One valid packet with the max amount of frags */
2362 for (i = 1; i < max_frags + 1; i++) {
2363 pkts[i].len = MIN_PKT_SIZE;
2364 pkts[i].options = XDP_PKT_CONTD;
2365 pkts[i].valid = true;
2366 }
2367 pkts[max_frags].options = 0;
2368
2369 /* An invalid packet with the max amount of frags but signals packet
2370 * continues on the last frag
2371 */
2372 for (i = max_frags + 1; i < 2 * max_frags + 1; i++) {
2373 pkts[i].len = MIN_PKT_SIZE;
2374 pkts[i].options = XDP_PKT_CONTD;
2375 pkts[i].valid = false;
2376 }
2377
2378 /* Valid packet for synch */
2379 pkts[2 * max_frags + 1].len = MIN_PKT_SIZE;
2380 pkts[2 * max_frags + 1].valid = true;
2381
2382 pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2);
2383 ret = testapp_validate_traffic(test);
2384
2385 free(pkts);
2386 return ret;
2387 }
2388
xsk_load_xdp_programs(struct ifobject * ifobj)2389 static int xsk_load_xdp_programs(struct ifobject *ifobj)
2390 {
2391 ifobj->xdp_progs = xsk_xdp_progs__open_and_load();
2392 if (libbpf_get_error(ifobj->xdp_progs))
2393 return libbpf_get_error(ifobj->xdp_progs);
2394
2395 return 0;
2396 }
2397
xsk_unload_xdp_programs(struct ifobject * ifobj)2398 static void xsk_unload_xdp_programs(struct ifobject *ifobj)
2399 {
2400 xsk_xdp_progs__destroy(ifobj->xdp_progs);
2401 }
2402
2403 /* Simple test */
hugepages_present(void)2404 static bool hugepages_present(void)
2405 {
2406 size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
2407 void *bufs;
2408
2409 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
2410 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB);
2411 if (bufs == MAP_FAILED)
2412 return false;
2413
2414 mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE;
2415 munmap(bufs, mmap_sz);
2416 return true;
2417 }
2418
init_iface(struct ifobject * ifobj,thread_func_t func_ptr)2419 static void init_iface(struct ifobject *ifobj, thread_func_t func_ptr)
2420 {
2421 LIBBPF_OPTS(bpf_xdp_query_opts, query_opts);
2422 int err;
2423
2424 ifobj->func_ptr = func_ptr;
2425
2426 err = xsk_load_xdp_programs(ifobj);
2427 if (err) {
2428 ksft_print_msg("Error loading XDP program\n");
2429 exit_with_error(err);
2430 }
2431
2432 if (hugepages_present())
2433 ifobj->unaligned_supp = true;
2434
2435 err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts);
2436 if (err) {
2437 ksft_print_msg("Error querying XDP capabilities\n");
2438 exit_with_error(-err);
2439 }
2440 if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG)
2441 ifobj->multi_buff_supp = true;
2442 if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
2443 if (query_opts.xdp_zc_max_segs > 1) {
2444 ifobj->multi_buff_zc_supp = true;
2445 ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs;
2446 } else {
2447 ifobj->xdp_zc_max_segs = 0;
2448 }
2449 }
2450 }
2451
testapp_send_receive(struct test_spec * test)2452 static int testapp_send_receive(struct test_spec *test)
2453 {
2454 return testapp_validate_traffic(test);
2455 }
2456
testapp_send_receive_2k_frame(struct test_spec * test)2457 static int testapp_send_receive_2k_frame(struct test_spec *test)
2458 {
2459 test->ifobj_tx->umem->frame_size = 2048;
2460 test->ifobj_rx->umem->frame_size = 2048;
2461 pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE);
2462 return testapp_validate_traffic(test);
2463 }
2464
testapp_poll_rx(struct test_spec * test)2465 static int testapp_poll_rx(struct test_spec *test)
2466 {
2467 test->ifobj_rx->use_poll = true;
2468 return testapp_validate_traffic(test);
2469 }
2470
testapp_poll_tx(struct test_spec * test)2471 static int testapp_poll_tx(struct test_spec *test)
2472 {
2473 test->ifobj_tx->use_poll = true;
2474 return testapp_validate_traffic(test);
2475 }
2476
testapp_aligned_inv_desc(struct test_spec * test)2477 static int testapp_aligned_inv_desc(struct test_spec *test)
2478 {
2479 return testapp_invalid_desc(test);
2480 }
2481
testapp_aligned_inv_desc_2k_frame(struct test_spec * test)2482 static int testapp_aligned_inv_desc_2k_frame(struct test_spec *test)
2483 {
2484 test->ifobj_tx->umem->frame_size = 2048;
2485 test->ifobj_rx->umem->frame_size = 2048;
2486 return testapp_invalid_desc(test);
2487 }
2488
testapp_unaligned_inv_desc(struct test_spec * test)2489 static int testapp_unaligned_inv_desc(struct test_spec *test)
2490 {
2491 test->ifobj_tx->umem->unaligned_mode = true;
2492 test->ifobj_rx->umem->unaligned_mode = true;
2493 return testapp_invalid_desc(test);
2494 }
2495
testapp_unaligned_inv_desc_4001_frame(struct test_spec * test)2496 static int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test)
2497 {
2498 u64 page_size, umem_size;
2499
2500 /* Odd frame size so the UMEM doesn't end near a page boundary. */
2501 test->ifobj_tx->umem->frame_size = 4001;
2502 test->ifobj_rx->umem->frame_size = 4001;
2503 test->ifobj_tx->umem->unaligned_mode = true;
2504 test->ifobj_rx->umem->unaligned_mode = true;
2505 /* This test exists to test descriptors that staddle the end of
2506 * the UMEM but not a page.
2507 */
2508 page_size = sysconf(_SC_PAGESIZE);
2509 umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size;
2510 assert(umem_size % page_size > MIN_PKT_SIZE);
2511 assert(umem_size % page_size < page_size - MIN_PKT_SIZE);
2512
2513 return testapp_invalid_desc(test);
2514 }
2515
testapp_aligned_inv_desc_mb(struct test_spec * test)2516 static int testapp_aligned_inv_desc_mb(struct test_spec *test)
2517 {
2518 return testapp_invalid_desc_mb(test);
2519 }
2520
testapp_unaligned_inv_desc_mb(struct test_spec * test)2521 static int testapp_unaligned_inv_desc_mb(struct test_spec *test)
2522 {
2523 test->ifobj_tx->umem->unaligned_mode = true;
2524 test->ifobj_rx->umem->unaligned_mode = true;
2525 return testapp_invalid_desc_mb(test);
2526 }
2527
testapp_xdp_metadata(struct test_spec * test)2528 static int testapp_xdp_metadata(struct test_spec *test)
2529 {
2530 return testapp_xdp_metadata_copy(test);
2531 }
2532
testapp_xdp_metadata_mb(struct test_spec * test)2533 static int testapp_xdp_metadata_mb(struct test_spec *test)
2534 {
2535 test->mtu = MAX_ETH_JUMBO_SIZE;
2536 return testapp_xdp_metadata_copy(test);
2537 }
2538
testapp_hw_sw_min_ring_size(struct test_spec * test)2539 static int testapp_hw_sw_min_ring_size(struct test_spec *test)
2540 {
2541 int ret;
2542
2543 test->set_ring = true;
2544 test->total_steps = 2;
2545 test->ifobj_tx->ring.tx_pending = DEFAULT_BATCH_SIZE;
2546 test->ifobj_tx->ring.rx_pending = DEFAULT_BATCH_SIZE * 2;
2547 test->ifobj_tx->xsk->batch_size = 1;
2548 test->ifobj_rx->xsk->batch_size = 1;
2549 ret = testapp_validate_traffic(test);
2550 if (ret)
2551 return ret;
2552
2553 /* Set batch size to hw_ring_size - 1 */
2554 test->ifobj_tx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1;
2555 test->ifobj_rx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1;
2556 return testapp_validate_traffic(test);
2557 }
2558
testapp_hw_sw_max_ring_size(struct test_spec * test)2559 static int testapp_hw_sw_max_ring_size(struct test_spec *test)
2560 {
2561 u32 max_descs = XSK_RING_PROD__DEFAULT_NUM_DESCS * 4;
2562 int ret;
2563
2564 test->set_ring = true;
2565 test->total_steps = 2;
2566 test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending;
2567 test->ifobj_tx->ring.rx_pending = test->ifobj_tx->ring.rx_max_pending;
2568 test->ifobj_rx->umem->num_frames = max_descs;
2569 test->ifobj_rx->umem->fill_size = max_descs;
2570 test->ifobj_rx->umem->comp_size = max_descs;
2571 test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
2572 test->ifobj_rx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
2573
2574 ret = testapp_validate_traffic(test);
2575 if (ret)
2576 return ret;
2577
2578 /* Set batch_size to 8152 for testing, as the ice HW ignores the 3 lowest bits when
2579 * updating the Rx HW tail register.
2580 */
2581 test->ifobj_tx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8;
2582 test->ifobj_rx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8;
2583 pkt_stream_replace(test, max_descs, MIN_PKT_SIZE);
2584 return testapp_validate_traffic(test);
2585 }
2586
testapp_xdp_adjust_tail(struct test_spec * test,int adjust_value)2587 static int testapp_xdp_adjust_tail(struct test_spec *test, int adjust_value)
2588 {
2589 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2590 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2591
2592 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_adjust_tail,
2593 skel_tx->progs.xsk_xdp_adjust_tail,
2594 skel_rx->maps.xsk, skel_tx->maps.xsk);
2595
2596 skel_rx->bss->adjust_value = adjust_value;
2597
2598 return testapp_validate_traffic(test);
2599 }
2600
testapp_adjust_tail(struct test_spec * test,u32 value,u32 pkt_len)2601 static int testapp_adjust_tail(struct test_spec *test, u32 value, u32 pkt_len)
2602 {
2603 int ret;
2604
2605 test->adjust_tail_support = true;
2606 test->adjust_tail = true;
2607 test->total_steps = 1;
2608
2609 pkt_stream_replace_ifobject(test->ifobj_tx, DEFAULT_BATCH_SIZE, pkt_len);
2610 pkt_stream_replace_ifobject(test->ifobj_rx, DEFAULT_BATCH_SIZE, pkt_len + value);
2611
2612 ret = testapp_xdp_adjust_tail(test, value);
2613 if (ret)
2614 return ret;
2615
2616 if (!test->adjust_tail_support) {
2617 ksft_test_result_skip("%s %sResize pkt with bpf_xdp_adjust_tail() not supported\n",
2618 mode_string(test), busy_poll_string(test));
2619 return TEST_SKIP;
2620 }
2621
2622 return 0;
2623 }
2624
testapp_adjust_tail_shrink(struct test_spec * test)2625 static int testapp_adjust_tail_shrink(struct test_spec *test)
2626 {
2627 /* Shrink by 4 bytes for testing purpose */
2628 return testapp_adjust_tail(test, -4, MIN_PKT_SIZE * 2);
2629 }
2630
testapp_adjust_tail_shrink_mb(struct test_spec * test)2631 static int testapp_adjust_tail_shrink_mb(struct test_spec *test)
2632 {
2633 test->mtu = MAX_ETH_JUMBO_SIZE;
2634 /* Shrink by the frag size */
2635 return testapp_adjust_tail(test, -XSK_UMEM__MAX_FRAME_SIZE, XSK_UMEM__LARGE_FRAME_SIZE * 2);
2636 }
2637
testapp_adjust_tail_grow(struct test_spec * test)2638 static int testapp_adjust_tail_grow(struct test_spec *test)
2639 {
2640 /* Grow by 4 bytes for testing purpose */
2641 return testapp_adjust_tail(test, 4, MIN_PKT_SIZE * 2);
2642 }
2643
testapp_adjust_tail_grow_mb(struct test_spec * test)2644 static int testapp_adjust_tail_grow_mb(struct test_spec *test)
2645 {
2646 test->mtu = MAX_ETH_JUMBO_SIZE;
2647 /* Grow by (frag_size - last_frag_Size) - 1 to stay inside the last fragment */
2648 return testapp_adjust_tail(test, (XSK_UMEM__MAX_FRAME_SIZE / 2) - 1,
2649 XSK_UMEM__LARGE_FRAME_SIZE * 2);
2650 }
2651
testapp_tx_queue_consumer(struct test_spec * test)2652 static int testapp_tx_queue_consumer(struct test_spec *test)
2653 {
2654 int nr_packets;
2655
2656 if (test->mode == TEST_MODE_ZC) {
2657 ksft_test_result_skip("Can not run TX_QUEUE_CONSUMER test for ZC mode\n");
2658 return TEST_SKIP;
2659 }
2660
2661 nr_packets = MAX_TX_BUDGET_DEFAULT + 1;
2662 pkt_stream_replace(test, nr_packets, MIN_PKT_SIZE);
2663 test->ifobj_tx->xsk->batch_size = nr_packets;
2664 test->ifobj_tx->xsk->check_consumer = true;
2665
2666 return testapp_validate_traffic(test);
2667 }
2668
run_pkt_test(struct test_spec * test)2669 static void run_pkt_test(struct test_spec *test)
2670 {
2671 int ret;
2672
2673 ret = test->test_func(test);
2674
2675 if (ret == TEST_PASS)
2676 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
2677 test->name);
2678 pkt_stream_restore_default(test);
2679 }
2680
ifobject_create(void)2681 static struct ifobject *ifobject_create(void)
2682 {
2683 struct ifobject *ifobj;
2684
2685 ifobj = calloc(1, sizeof(struct ifobject));
2686 if (!ifobj)
2687 return NULL;
2688
2689 ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr));
2690 if (!ifobj->xsk_arr)
2691 goto out_xsk_arr;
2692
2693 ifobj->umem = calloc(1, sizeof(*ifobj->umem));
2694 if (!ifobj->umem)
2695 goto out_umem;
2696
2697 return ifobj;
2698
2699 out_umem:
2700 free(ifobj->xsk_arr);
2701 out_xsk_arr:
2702 free(ifobj);
2703 return NULL;
2704 }
2705
ifobject_delete(struct ifobject * ifobj)2706 static void ifobject_delete(struct ifobject *ifobj)
2707 {
2708 free(ifobj->umem);
2709 free(ifobj->xsk_arr);
2710 free(ifobj);
2711 }
2712
is_xdp_supported(int ifindex)2713 static bool is_xdp_supported(int ifindex)
2714 {
2715 int flags = XDP_FLAGS_DRV_MODE;
2716
2717 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
2718 struct bpf_insn insns[2] = {
2719 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
2720 BPF_EXIT_INSN()
2721 };
2722 int prog_fd, insn_cnt = ARRAY_SIZE(insns);
2723 int err;
2724
2725 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
2726 if (prog_fd < 0)
2727 return false;
2728
2729 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
2730 if (err) {
2731 close(prog_fd);
2732 return false;
2733 }
2734
2735 bpf_xdp_detach(ifindex, flags, NULL);
2736 close(prog_fd);
2737
2738 return true;
2739 }
2740
2741 static const struct test_spec tests[] = {
2742 {.name = "SEND_RECEIVE", .test_func = testapp_send_receive},
2743 {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame},
2744 {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt},
2745 {.name = "POLL_RX", .test_func = testapp_poll_rx},
2746 {.name = "POLL_TX", .test_func = testapp_poll_tx},
2747 {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout},
2748 {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout},
2749 {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned},
2750 {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc},
2751 {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame},
2752 {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc},
2753 {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE",
2754 .test_func = testapp_unaligned_inv_desc_4001_frame},
2755 {.name = "UMEM_HEADROOM", .test_func = testapp_headroom},
2756 {.name = "TEARDOWN", .test_func = testapp_teardown},
2757 {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional},
2758 {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped},
2759 {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs},
2760 {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full},
2761 {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty},
2762 {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup},
2763 {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop},
2764 {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem},
2765 {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata},
2766 {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb},
2767 {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb},
2768 {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS",
2769 .test_func = testapp_send_receive_unaligned_mb},
2770 {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb},
2771 {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb},
2772 {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags},
2773 {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size},
2774 {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size},
2775 {.name = "XDP_ADJUST_TAIL_SHRINK", .test_func = testapp_adjust_tail_shrink},
2776 {.name = "XDP_ADJUST_TAIL_SHRINK_MULTI_BUFF", .test_func = testapp_adjust_tail_shrink_mb},
2777 {.name = "XDP_ADJUST_TAIL_GROW", .test_func = testapp_adjust_tail_grow},
2778 {.name = "XDP_ADJUST_TAIL_GROW_MULTI_BUFF", .test_func = testapp_adjust_tail_grow_mb},
2779 {.name = "TX_QUEUE_CONSUMER", .test_func = testapp_tx_queue_consumer},
2780 };
2781
print_tests(void)2782 static void print_tests(void)
2783 {
2784 u32 i;
2785
2786 printf("Tests:\n");
2787 for (i = 0; i < ARRAY_SIZE(tests); i++)
2788 printf("%u: %s\n", i, tests[i].name);
2789 }
2790
main(int argc,char ** argv)2791 int main(int argc, char **argv)
2792 {
2793 struct pkt_stream *rx_pkt_stream_default;
2794 struct pkt_stream *tx_pkt_stream_default;
2795 struct ifobject *ifobj_tx, *ifobj_rx;
2796 u32 i, j, failed_tests = 0, nb_tests;
2797 int modes = TEST_MODE_SKB + 1;
2798 struct test_spec test;
2799 bool shared_netdev;
2800 int ret;
2801
2802 /* Use libbpf 1.0 API mode */
2803 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2804
2805 ifobj_tx = ifobject_create();
2806 if (!ifobj_tx)
2807 exit_with_error(ENOMEM);
2808 ifobj_rx = ifobject_create();
2809 if (!ifobj_rx)
2810 exit_with_error(ENOMEM);
2811
2812 setlocale(LC_ALL, "");
2813
2814 parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
2815
2816 if (opt_print_tests) {
2817 print_tests();
2818 ksft_exit_xpass();
2819 }
2820 if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= ARRAY_SIZE(tests)) {
2821 ksft_print_msg("Error: test %u does not exist.\n", opt_run_test);
2822 ksft_exit_xfail();
2823 }
2824
2825 shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex);
2826 ifobj_tx->shared_umem = shared_netdev;
2827 ifobj_rx->shared_umem = shared_netdev;
2828
2829 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx))
2830 print_usage(argv);
2831
2832 if (is_xdp_supported(ifobj_tx->ifindex)) {
2833 modes++;
2834 if (ifobj_zc_avail(ifobj_tx))
2835 modes++;
2836 }
2837
2838 ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring);
2839 if (!ret) {
2840 ifobj_tx->hw_ring_size_supp = true;
2841 ifobj_tx->set_ring.default_tx = ifobj_tx->ring.tx_pending;
2842 ifobj_tx->set_ring.default_rx = ifobj_tx->ring.rx_pending;
2843 }
2844
2845 init_iface(ifobj_rx, worker_testapp_validate_rx);
2846 init_iface(ifobj_tx, worker_testapp_validate_tx);
2847
2848 test_spec_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]);
2849 tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
2850 rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
2851 if (!tx_pkt_stream_default || !rx_pkt_stream_default)
2852 exit_with_error(ENOMEM);
2853 test.tx_pkt_stream_default = tx_pkt_stream_default;
2854 test.rx_pkt_stream_default = rx_pkt_stream_default;
2855
2856 if (opt_run_test == RUN_ALL_TESTS)
2857 nb_tests = ARRAY_SIZE(tests);
2858 else
2859 nb_tests = 1;
2860 if (opt_mode == TEST_MODE_ALL) {
2861 ksft_set_plan(modes * nb_tests);
2862 } else {
2863 if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) {
2864 ksft_print_msg("Error: XDP_DRV mode not supported.\n");
2865 ksft_exit_xfail();
2866 }
2867 if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) {
2868 ksft_print_msg("Error: zero-copy mode not supported.\n");
2869 ksft_exit_xfail();
2870 }
2871
2872 ksft_set_plan(nb_tests);
2873 }
2874
2875 for (i = 0; i < modes; i++) {
2876 if (opt_mode != TEST_MODE_ALL && i != opt_mode)
2877 continue;
2878
2879 for (j = 0; j < ARRAY_SIZE(tests); j++) {
2880 if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test)
2881 continue;
2882
2883 test_spec_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]);
2884 run_pkt_test(&test);
2885 usleep(USLEEP_MAX);
2886
2887 if (test.fail)
2888 failed_tests++;
2889 }
2890 }
2891
2892 if (ifobj_tx->hw_ring_size_supp)
2893 hw_ring_size_reset(ifobj_tx);
2894
2895 pkt_stream_delete(tx_pkt_stream_default);
2896 pkt_stream_delete(rx_pkt_stream_default);
2897 xsk_unload_xdp_programs(ifobj_tx);
2898 xsk_unload_xdp_programs(ifobj_rx);
2899 ifobject_delete(ifobj_tx);
2900 ifobject_delete(ifobj_rx);
2901
2902 if (failed_tests)
2903 ksft_exit_fail();
2904 else
2905 ksft_exit_pass();
2906 }
2907