1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 /* 5 * Some functions in this program are taken from 6 * Linux kernel samples/bpf/xdpsock* and modified 7 * for use. 8 * 9 * See test_xsk.sh for detailed information on test topology 10 * and prerequisite network setup. 11 * 12 * This test program contains two threads, each thread is single socket with 13 * a unique UMEM. It validates in-order packet delivery and packet content 14 * by sending packets to each other. 15 * 16 * Tests Information: 17 * ------------------ 18 * These selftests test AF_XDP SKB and Native/DRV modes using veth 19 * Virtual Ethernet interfaces. 20 * 21 * For each mode, the following tests are run: 22 * a. nopoll - soft-irq processing in run-to-completion mode 23 * b. poll - using poll() syscall 24 * c. Socket Teardown 25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy 26 * both sockets, then repeat multiple times. Only nopoll mode is used 27 * d. Bi-directional sockets 28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and 29 * completion rings on each socket, tx/rx in both directions. Only nopoll 30 * mode is used 31 * e. Statistics 32 * Trigger some error conditions and ensure that the appropriate statistics 33 * are incremented. Within this test, the following statistics are tested: 34 * i. rx dropped 35 * Increase the UMEM frame headroom to a value which results in 36 * insufficient space in the rx buffer for both the packet and the headroom. 37 * ii. tx invalid 38 * Set the 'len' field of tx descriptors to an invalid value (umem frame 39 * size + 1). 40 * iii. rx ring full 41 * Reduce the size of the RX ring to a fraction of the fill ring size. 42 * iv. fill queue empty 43 * Do not populate the fill queue and then try to receive pkts. 44 * f. bpf_link resource persistence 45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0, 46 * then remove xsk sockets from queue 0 on both veth interfaces and 47 * finally run a traffic on queues ids 1 48 * g. unaligned mode 49 * h. tests for invalid and corner case Tx descriptors so that the correct ones 50 * are discarded and let through, respectively. 51 * i. 2K frame size tests 52 * j. If multi-buffer is supported, send 9k packets divided into 3 frames 53 * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame 54 * using unaligned mode 55 * l. If multi-buffer is supported, try various nasty combinations of descriptors to 56 * check if they pass the validation or not 57 * 58 * Flow: 59 * ----- 60 * - Single process spawns two threads: Tx and Rx 61 * - Each of these two threads attach to a veth interface 62 * - Each thread creates one AF_XDP socket connected to a unique umem for each 63 * veth interface 64 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy> 65 * - Rx thread verifies if all packets were received and delivered in-order, 66 * and have the right content 67 * 68 * Enable/disable packet dump mode: 69 * -------------------------- 70 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add 71 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D") 72 */ 73 74 #define _GNU_SOURCE 75 #include <assert.h> 76 #include <fcntl.h> 77 #include <getopt.h> 78 #include <linux/if_link.h> 79 #include <linux/if_ether.h> 80 #include <linux/mman.h> 81 #include <linux/netdev.h> 82 #include <linux/ethtool.h> 83 #include <arpa/inet.h> 84 #include <net/if.h> 85 #include <locale.h> 86 #include <stdio.h> 87 #include <stdlib.h> 88 #include <libgen.h> 89 #include <stddef.h> 90 #include <sys/mman.h> 91 #include <sys/types.h> 92 93 #include "prog_tests/test_xsk.h" 94 #include "xsk_xdp_progs.skel.h" 95 #include "xsk.h" 96 #include "xskxceiver.h" 97 #include <bpf/bpf.h> 98 #include <linux/filter.h> 99 #include "../kselftest.h" 100 #include "xsk_xdp_common.h" 101 102 #include <network_helpers.h> 103 104 static bool opt_print_tests; 105 static enum test_mode opt_mode = TEST_MODE_ALL; 106 static u32 opt_run_test = RUN_ALL_TESTS; 107 108 void test__fail(void) { /* for network_helpers.c */ } 109 110 static void __exit_with_error(int error, const char *file, const char *func, int line) 111 { 112 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, 113 error, strerror(error)); 114 ksft_exit_xfail(); 115 } 116 117 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) 118 119 static bool ifobj_zc_avail(struct ifobject *ifobject) 120 { 121 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; 122 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; 123 struct xsk_socket_info *xsk; 124 struct xsk_umem_info *umem; 125 bool zc_avail = false; 126 void *bufs; 127 int ret; 128 129 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); 130 if (bufs == MAP_FAILED) 131 exit_with_error(errno); 132 133 umem = calloc(1, sizeof(struct xsk_umem_info)); 134 if (!umem) { 135 munmap(bufs, umem_sz); 136 exit_with_error(ENOMEM); 137 } 138 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; 139 ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz); 140 if (ret) 141 exit_with_error(-ret); 142 143 xsk = calloc(1, sizeof(struct xsk_socket_info)); 144 if (!xsk) 145 goto out; 146 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY; 147 ifobject->rx_on = true; 148 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 149 ret = xsk_configure_socket(xsk, umem, ifobject, false); 150 if (!ret) 151 zc_avail = true; 152 153 xsk_socket__delete(xsk->xsk); 154 free(xsk); 155 out: 156 munmap(umem->buffer, umem_sz); 157 xsk_umem__delete(umem->umem); 158 free(umem); 159 return zc_avail; 160 } 161 162 static struct option long_options[] = { 163 {"interface", required_argument, 0, 'i'}, 164 {"busy-poll", no_argument, 0, 'b'}, 165 {"verbose", no_argument, 0, 'v'}, 166 {"mode", required_argument, 0, 'm'}, 167 {"list", no_argument, 0, 'l'}, 168 {"test", required_argument, 0, 't'}, 169 {"help", no_argument, 0, 'h'}, 170 {0, 0, 0, 0} 171 }; 172 173 static void print_usage(char **argv) 174 { 175 const char *str = 176 " Usage: xskxceiver [OPTIONS]\n" 177 " Options:\n" 178 " -i, --interface Use interface\n" 179 " -v, --verbose Verbose output\n" 180 " -b, --busy-poll Enable busy poll\n" 181 " -m, --mode Run only mode skb, drv, or zc\n" 182 " -l, --list List all available tests\n" 183 " -t, --test Run a specific test. Enter number from -l option.\n" 184 " -h, --help Display this help and exit\n"; 185 186 ksft_print_msg(str, basename(argv[0])); 187 ksft_exit_xfail(); 188 } 189 190 static bool validate_interface(struct ifobject *ifobj) 191 { 192 if (!strcmp(ifobj->ifname, "")) 193 return false; 194 return true; 195 } 196 197 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc, 198 char **argv) 199 { 200 struct ifobject *ifobj; 201 u32 interface_nb = 0; 202 int option_index, c; 203 204 opterr = 0; 205 206 for (;;) { 207 c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index); 208 if (c == -1) 209 break; 210 211 switch (c) { 212 case 'i': 213 if (interface_nb == 0) 214 ifobj = ifobj_tx; 215 else if (interface_nb == 1) 216 ifobj = ifobj_rx; 217 else 218 break; 219 220 memcpy(ifobj->ifname, optarg, 221 min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg))); 222 223 ifobj->ifindex = if_nametoindex(ifobj->ifname); 224 if (!ifobj->ifindex) 225 exit_with_error(errno); 226 227 interface_nb++; 228 break; 229 case 'v': 230 opt_verbose = true; 231 break; 232 case 'b': 233 ifobj_tx->busy_poll = true; 234 ifobj_rx->busy_poll = true; 235 break; 236 case 'm': 237 if (!strncmp("skb", optarg, strlen(optarg))) 238 opt_mode = TEST_MODE_SKB; 239 else if (!strncmp("drv", optarg, strlen(optarg))) 240 opt_mode = TEST_MODE_DRV; 241 else if (!strncmp("zc", optarg, strlen(optarg))) 242 opt_mode = TEST_MODE_ZC; 243 else 244 print_usage(argv); 245 break; 246 case 'l': 247 opt_print_tests = true; 248 break; 249 case 't': 250 errno = 0; 251 opt_run_test = strtol(optarg, NULL, 0); 252 if (errno) 253 print_usage(argv); 254 break; 255 case 'h': 256 default: 257 print_usage(argv); 258 } 259 } 260 } 261 262 static void xsk_unload_xdp_programs(struct ifobject *ifobj) 263 { 264 xsk_xdp_progs__destroy(ifobj->xdp_progs); 265 } 266 267 static void run_pkt_test(struct test_spec *test) 268 { 269 int ret; 270 271 ret = test->test_func(test); 272 273 switch (ret) { 274 case TEST_PASS: 275 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), 276 test->name); 277 break; 278 case TEST_SKIP: 279 ksft_test_result_skip("SKIP: %s %s%s\n", mode_string(test), busy_poll_string(test), 280 test->name); 281 break; 282 case TEST_FAILURE: 283 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test), 284 test->name); 285 break; 286 default: 287 ksft_test_result_fail("FAIL: %s %s%s -- Unexpected returned value (%d)\n", 288 mode_string(test), busy_poll_string(test), test->name, ret); 289 } 290 291 pkt_stream_restore_default(test); 292 } 293 294 static bool is_xdp_supported(int ifindex) 295 { 296 int flags = XDP_FLAGS_DRV_MODE; 297 298 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); 299 struct bpf_insn insns[2] = { 300 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), 301 BPF_EXIT_INSN() 302 }; 303 int prog_fd, insn_cnt = ARRAY_SIZE(insns); 304 int err; 305 306 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); 307 if (prog_fd < 0) 308 return false; 309 310 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); 311 if (err) { 312 close(prog_fd); 313 return false; 314 } 315 316 bpf_xdp_detach(ifindex, flags, NULL); 317 close(prog_fd); 318 319 return true; 320 } 321 322 static void print_tests(void) 323 { 324 u32 i; 325 326 printf("Tests:\n"); 327 for (i = 0; i < ARRAY_SIZE(tests); i++) 328 printf("%u: %s\n", i, tests[i].name); 329 for (i = ARRAY_SIZE(tests); i < ARRAY_SIZE(tests) + ARRAY_SIZE(ci_skip_tests); i++) 330 printf("%u: %s\n", i, ci_skip_tests[i - ARRAY_SIZE(tests)].name); 331 } 332 333 int main(int argc, char **argv) 334 { 335 const size_t total_tests = ARRAY_SIZE(tests) + ARRAY_SIZE(ci_skip_tests); 336 struct pkt_stream *rx_pkt_stream_default; 337 struct pkt_stream *tx_pkt_stream_default; 338 struct ifobject *ifobj_tx, *ifobj_rx; 339 u32 i, j, failed_tests = 0, nb_tests; 340 int modes = TEST_MODE_SKB + 1; 341 struct test_spec test; 342 bool shared_netdev; 343 int ret; 344 345 /* Use libbpf 1.0 API mode */ 346 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 347 348 ifobj_tx = ifobject_create(); 349 if (!ifobj_tx) 350 exit_with_error(ENOMEM); 351 ifobj_rx = ifobject_create(); 352 if (!ifobj_rx) 353 exit_with_error(ENOMEM); 354 355 setlocale(LC_ALL, ""); 356 357 parse_command_line(ifobj_tx, ifobj_rx, argc, argv); 358 359 if (opt_print_tests) { 360 print_tests(); 361 ksft_exit_xpass(); 362 } 363 if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= total_tests) { 364 ksft_print_msg("Error: test %u does not exist.\n", opt_run_test); 365 ksft_exit_xfail(); 366 } 367 368 shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex); 369 ifobj_tx->shared_umem = shared_netdev; 370 ifobj_rx->shared_umem = shared_netdev; 371 372 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) 373 print_usage(argv); 374 375 if (is_xdp_supported(ifobj_tx->ifindex)) { 376 modes++; 377 if (ifobj_zc_avail(ifobj_tx)) 378 modes++; 379 } 380 381 ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring); 382 if (!ret) { 383 ifobj_tx->hw_ring_size_supp = true; 384 ifobj_tx->set_ring.default_tx = ifobj_tx->ring.tx_pending; 385 ifobj_tx->set_ring.default_rx = ifobj_tx->ring.rx_pending; 386 } 387 388 if (init_iface(ifobj_rx, worker_testapp_validate_rx) || 389 init_iface(ifobj_tx, worker_testapp_validate_tx)) { 390 ksft_print_msg("Error : can't initialize interfaces\n"); 391 ksft_exit_xfail(); 392 } 393 394 test_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]); 395 tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); 396 rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); 397 if (!tx_pkt_stream_default || !rx_pkt_stream_default) 398 exit_with_error(ENOMEM); 399 test.tx_pkt_stream_default = tx_pkt_stream_default; 400 test.rx_pkt_stream_default = rx_pkt_stream_default; 401 402 if (opt_run_test == RUN_ALL_TESTS) 403 nb_tests = total_tests; 404 else 405 nb_tests = 1; 406 if (opt_mode == TEST_MODE_ALL) { 407 ksft_set_plan(modes * nb_tests); 408 } else { 409 if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) { 410 ksft_print_msg("Error: XDP_DRV mode not supported.\n"); 411 ksft_exit_xfail(); 412 } 413 if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) { 414 ksft_print_msg("Error: zero-copy mode not supported.\n"); 415 ksft_exit_xfail(); 416 } 417 418 ksft_set_plan(nb_tests); 419 } 420 421 for (i = 0; i < modes; i++) { 422 if (opt_mode != TEST_MODE_ALL && i != opt_mode) 423 continue; 424 425 for (j = 0; j < total_tests; j++) { 426 if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test) 427 continue; 428 429 if (j < ARRAY_SIZE(tests)) 430 test_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]); 431 else 432 test_init(&test, ifobj_tx, ifobj_rx, i, 433 &ci_skip_tests[j - ARRAY_SIZE(tests)]); 434 run_pkt_test(&test); 435 usleep(USLEEP_MAX); 436 437 if (test.fail) 438 failed_tests++; 439 } 440 } 441 442 if (ifobj_tx->hw_ring_size_supp) 443 hw_ring_size_reset(ifobj_tx); 444 445 pkt_stream_delete(tx_pkt_stream_default); 446 pkt_stream_delete(rx_pkt_stream_default); 447 xsk_unload_xdp_programs(ifobj_tx); 448 xsk_unload_xdp_programs(ifobj_rx); 449 ifobject_delete(ifobj_tx); 450 ifobject_delete(ifobj_rx); 451 452 if (failed_tests) 453 ksft_exit_fail(); 454 else 455 ksft_exit_pass(); 456 } 457