1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 /* 5 * Some functions in this program are taken from 6 * Linux kernel samples/bpf/xdpsock* and modified 7 * for use. 8 * 9 * See test_xsk.sh for detailed information on test topology 10 * and prerequisite network setup. 11 * 12 * This test program contains two threads, each thread is single socket with 13 * a unique UMEM. It validates in-order packet delivery and packet content 14 * by sending packets to each other. 15 * 16 * Tests Information: 17 * ------------------ 18 * These selftests test AF_XDP SKB and Native/DRV modes using veth 19 * Virtual Ethernet interfaces. 20 * 21 * For each mode, the following tests are run: 22 * a. nopoll - soft-irq processing in run-to-completion mode 23 * b. poll - using poll() syscall 24 * c. Socket Teardown 25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy 26 * both sockets, then repeat multiple times. Only nopoll mode is used 27 * d. Bi-directional sockets 28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and 29 * completion rings on each socket, tx/rx in both directions. Only nopoll 30 * mode is used 31 * e. Statistics 32 * Trigger some error conditions and ensure that the appropriate statistics 33 * are incremented. Within this test, the following statistics are tested: 34 * i. rx dropped 35 * Increase the UMEM frame headroom to a value which results in 36 * insufficient space in the rx buffer for both the packet and the headroom. 37 * ii. tx invalid 38 * Set the 'len' field of tx descriptors to an invalid value (umem frame 39 * size + 1). 40 * iii. rx ring full 41 * Reduce the size of the RX ring to a fraction of the fill ring size. 42 * iv. fill queue empty 43 * Do not populate the fill queue and then try to receive pkts. 44 * f. bpf_link resource persistence 45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0, 46 * then remove xsk sockets from queue 0 on both veth interfaces and 47 * finally run a traffic on queues ids 1 48 * g. unaligned mode 49 * h. tests for invalid and corner case Tx descriptors so that the correct ones 50 * are discarded and let through, respectively. 51 * i. 2K frame size tests 52 * j. If multi-buffer is supported, send 9k packets divided into 3 frames 53 * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame 54 * using unaligned mode 55 * l. If multi-buffer is supported, try various nasty combinations of descriptors to 56 * check if they pass the validation or not 57 * 58 * Flow: 59 * ----- 60 * - Single process spawns two threads: Tx and Rx 61 * - Each of these two threads attach to a veth interface 62 * - Each thread creates one AF_XDP socket connected to a unique umem for each 63 * veth interface 64 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy> 65 * - Rx thread verifies if all packets were received and delivered in-order, 66 * and have the right content 67 * 68 * Enable/disable packet dump mode: 69 * -------------------------- 70 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add 71 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D") 72 */ 73 74 #define _GNU_SOURCE 75 #include <assert.h> 76 #include <fcntl.h> 77 #include <getopt.h> 78 #include <linux/if_link.h> 79 #include <linux/if_ether.h> 80 #include <linux/mman.h> 81 #include <linux/netdev.h> 82 #include <linux/ethtool.h> 83 #include <linux/align.h> 84 #include <arpa/inet.h> 85 #include <net/if.h> 86 #include <locale.h> 87 #include <stdio.h> 88 #include <stdlib.h> 89 #include <libgen.h> 90 #include <stddef.h> 91 #include <sys/mman.h> 92 #include <sys/types.h> 93 94 #include "prog_tests/test_xsk.h" 95 #include "xsk_xdp_progs.skel.h" 96 #include "xsk.h" 97 #include "xskxceiver.h" 98 #include <bpf/bpf.h> 99 #include <linux/filter.h> 100 #include "kselftest.h" 101 #include "xsk_xdp_common.h" 102 103 #include <network_helpers.h> 104 105 static bool opt_print_tests; 106 static enum test_mode opt_mode = TEST_MODE_ALL; 107 static u32 opt_run_test = RUN_ALL_TESTS; 108 109 void test__fail(void) { /* for network_helpers.c */ } 110 111 static void __exit_with_error(int error, const char *file, const char *func, int line) 112 { 113 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, 114 error, strerror(error)); 115 ksft_exit_xfail(); 116 } 117 118 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) 119 120 static bool ifobj_zc_avail(struct ifobject *ifobject) 121 { 122 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; 123 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; 124 struct xsk_socket_info *xsk; 125 struct xsk_umem_info *umem; 126 bool zc_avail = false; 127 void *bufs; 128 int ret; 129 130 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); 131 if (bufs == MAP_FAILED) 132 exit_with_error(errno); 133 134 umem = calloc(1, sizeof(struct xsk_umem_info)); 135 if (!umem) { 136 munmap(bufs, umem_sz); 137 exit_with_error(ENOMEM); 138 } 139 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; 140 ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz); 141 if (ret) 142 exit_with_error(-ret); 143 144 xsk = calloc(1, sizeof(struct xsk_socket_info)); 145 if (!xsk) 146 goto out; 147 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY; 148 ifobject->rx_on = true; 149 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 150 ret = xsk_configure_socket(xsk, umem, ifobject, false); 151 if (!ret) 152 zc_avail = true; 153 154 xsk_socket__delete(xsk->xsk); 155 free(xsk); 156 out: 157 munmap(umem->buffer, umem_sz); 158 xsk_umem__delete(umem->umem); 159 free(umem); 160 return zc_avail; 161 } 162 163 static struct option long_options[] = { 164 {"interface", required_argument, 0, 'i'}, 165 {"busy-poll", no_argument, 0, 'b'}, 166 {"verbose", no_argument, 0, 'v'}, 167 {"mode", required_argument, 0, 'm'}, 168 {"list", no_argument, 0, 'l'}, 169 {"test", required_argument, 0, 't'}, 170 {"help", no_argument, 0, 'h'}, 171 {0, 0, 0, 0} 172 }; 173 174 static void print_usage(char **argv) 175 { 176 const char *str = 177 " Usage: xskxceiver [OPTIONS]\n" 178 " Options:\n" 179 " -i, --interface Use interface\n" 180 " -v, --verbose Verbose output\n" 181 " -b, --busy-poll Enable busy poll\n" 182 " -m, --mode Run only mode skb, drv, or zc\n" 183 " -l, --list List all available tests\n" 184 " -t, --test Run a specific test. Enter number from -l option.\n" 185 " -h, --help Display this help and exit\n"; 186 187 ksft_print_msg(str, basename(argv[0])); 188 ksft_exit_xfail(); 189 } 190 191 static bool validate_interface(struct ifobject *ifobj) 192 { 193 if (!strcmp(ifobj->ifname, "")) 194 return false; 195 return true; 196 } 197 198 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc, 199 char **argv) 200 { 201 struct ifobject *ifobj; 202 u32 interface_nb = 0; 203 int option_index, c; 204 205 opterr = 0; 206 207 for (;;) { 208 c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index); 209 if (c == -1) 210 break; 211 212 switch (c) { 213 case 'i': 214 if (interface_nb == 0) 215 ifobj = ifobj_tx; 216 else if (interface_nb == 1) 217 ifobj = ifobj_rx; 218 else 219 break; 220 221 memcpy(ifobj->ifname, optarg, 222 min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg))); 223 224 ifobj->ifindex = if_nametoindex(ifobj->ifname); 225 if (!ifobj->ifindex) 226 exit_with_error(errno); 227 228 interface_nb++; 229 break; 230 case 'v': 231 opt_verbose = true; 232 break; 233 case 'b': 234 ifobj_tx->busy_poll = true; 235 ifobj_rx->busy_poll = true; 236 break; 237 case 'm': 238 if (!strncmp("skb", optarg, strlen(optarg))) 239 opt_mode = TEST_MODE_SKB; 240 else if (!strncmp("drv", optarg, strlen(optarg))) 241 opt_mode = TEST_MODE_DRV; 242 else if (!strncmp("zc", optarg, strlen(optarg))) 243 opt_mode = TEST_MODE_ZC; 244 else 245 print_usage(argv); 246 break; 247 case 'l': 248 opt_print_tests = true; 249 break; 250 case 't': 251 errno = 0; 252 opt_run_test = strtol(optarg, NULL, 0); 253 if (errno) 254 print_usage(argv); 255 break; 256 case 'h': 257 default: 258 print_usage(argv); 259 } 260 } 261 } 262 263 static void xsk_unload_xdp_programs(struct ifobject *ifobj) 264 { 265 xsk_xdp_progs__destroy(ifobj->xdp_progs); 266 } 267 268 static void run_pkt_test(struct test_spec *test) 269 { 270 int ret; 271 272 ret = test->test_func(test); 273 274 switch (ret) { 275 case TEST_PASS: 276 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), 277 test->name); 278 break; 279 case TEST_SKIP: 280 ksft_test_result_skip("SKIP: %s %s%s\n", mode_string(test), busy_poll_string(test), 281 test->name); 282 break; 283 case TEST_FAILURE: 284 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test), 285 test->name); 286 break; 287 default: 288 ksft_test_result_fail("FAIL: %s %s%s -- Unexpected returned value (%d)\n", 289 mode_string(test), busy_poll_string(test), test->name, ret); 290 } 291 292 pkt_stream_restore_default(test); 293 } 294 295 static bool is_xdp_supported(int ifindex) 296 { 297 int flags = XDP_FLAGS_DRV_MODE; 298 299 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); 300 struct bpf_insn insns[2] = { 301 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), 302 BPF_EXIT_INSN() 303 }; 304 int prog_fd, insn_cnt = ARRAY_SIZE(insns); 305 int err; 306 307 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); 308 if (prog_fd < 0) 309 return false; 310 311 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); 312 if (err) { 313 close(prog_fd); 314 return false; 315 } 316 317 bpf_xdp_detach(ifindex, flags, NULL); 318 close(prog_fd); 319 320 return true; 321 } 322 323 static void print_tests(void) 324 { 325 u32 i; 326 327 printf("Tests:\n"); 328 for (i = 0; i < ARRAY_SIZE(tests); i++) 329 printf("%u: %s\n", i, tests[i].name); 330 for (i = ARRAY_SIZE(tests); i < ARRAY_SIZE(tests) + ARRAY_SIZE(ci_skip_tests); i++) 331 printf("%u: %s\n", i, ci_skip_tests[i - ARRAY_SIZE(tests)].name); 332 } 333 334 int main(int argc, char **argv) 335 { 336 const size_t total_tests = ARRAY_SIZE(tests) + ARRAY_SIZE(ci_skip_tests); 337 u32 cache_line_size, max_frags, umem_tailroom; 338 struct pkt_stream *rx_pkt_stream_default; 339 struct pkt_stream *tx_pkt_stream_default; 340 struct ifobject *ifobj_tx, *ifobj_rx; 341 u32 i, j, failed_tests = 0, nb_tests; 342 int modes = TEST_MODE_SKB + 1; 343 struct test_spec test; 344 bool shared_netdev; 345 int ret; 346 347 /* Use libbpf 1.0 API mode */ 348 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 349 350 ifobj_tx = ifobject_create(); 351 if (!ifobj_tx) 352 exit_with_error(ENOMEM); 353 ifobj_rx = ifobject_create(); 354 if (!ifobj_rx) 355 exit_with_error(ENOMEM); 356 357 setlocale(LC_ALL, ""); 358 359 cache_line_size = read_procfs_val(SMP_CACHE_BYTES_PATH); 360 if (!cache_line_size) { 361 ksft_print_msg("Can't get SMP_CACHE_BYTES from system, using default (64)\n"); 362 cache_line_size = 64; 363 } 364 365 max_frags = read_procfs_val(MAX_SKB_FRAGS_PATH); 366 if (!max_frags) { 367 ksft_print_msg("Can't get MAX_SKB_FRAGS from system, using default (17)\n"); 368 max_frags = 17; 369 } 370 ifobj_tx->max_skb_frags = max_frags; 371 ifobj_rx->max_skb_frags = max_frags; 372 373 /* 48 bytes is a part of skb_shared_info w/o frags array; 374 * 16 bytes is sizeof(skb_frag_t) 375 */ 376 umem_tailroom = ALIGN(48 + (max_frags * 16), cache_line_size); 377 ifobj_tx->umem_tailroom = umem_tailroom; 378 ifobj_rx->umem_tailroom = umem_tailroom; 379 380 parse_command_line(ifobj_tx, ifobj_rx, argc, argv); 381 382 if (opt_print_tests) { 383 print_tests(); 384 ksft_exit_xpass(); 385 } 386 if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= total_tests) { 387 ksft_print_msg("Error: test %u does not exist.\n", opt_run_test); 388 ksft_exit_xfail(); 389 } 390 391 shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex); 392 ifobj_tx->shared_umem = shared_netdev; 393 ifobj_rx->shared_umem = shared_netdev; 394 395 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) 396 print_usage(argv); 397 398 if (is_xdp_supported(ifobj_tx->ifindex)) { 399 modes++; 400 if (ifobj_zc_avail(ifobj_tx)) 401 modes++; 402 } 403 404 ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring); 405 if (!ret) { 406 ifobj_tx->hw_ring_size_supp = true; 407 ifobj_tx->set_ring.default_tx = ifobj_tx->ring.tx_pending; 408 ifobj_tx->set_ring.default_rx = ifobj_tx->ring.rx_pending; 409 } 410 411 if (init_iface(ifobj_rx, worker_testapp_validate_rx) || 412 init_iface(ifobj_tx, worker_testapp_validate_tx)) { 413 ksft_print_msg("Error : can't initialize interfaces\n"); 414 ksft_exit_xfail(); 415 } 416 417 test_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]); 418 tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); 419 rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); 420 if (!tx_pkt_stream_default || !rx_pkt_stream_default) 421 exit_with_error(ENOMEM); 422 test.tx_pkt_stream_default = tx_pkt_stream_default; 423 test.rx_pkt_stream_default = rx_pkt_stream_default; 424 425 if (opt_run_test == RUN_ALL_TESTS) 426 nb_tests = total_tests; 427 else 428 nb_tests = 1; 429 if (opt_mode == TEST_MODE_ALL) { 430 ksft_set_plan(modes * nb_tests); 431 } else { 432 if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) { 433 ksft_print_msg("Error: XDP_DRV mode not supported.\n"); 434 ksft_exit_xfail(); 435 } 436 if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) { 437 ksft_print_msg("Error: zero-copy mode not supported.\n"); 438 ksft_exit_xfail(); 439 } 440 441 ksft_set_plan(nb_tests); 442 } 443 444 for (i = 0; i < modes; i++) { 445 if (opt_mode != TEST_MODE_ALL && i != opt_mode) 446 continue; 447 448 for (j = 0; j < total_tests; j++) { 449 if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test) 450 continue; 451 452 if (j < ARRAY_SIZE(tests)) 453 test_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]); 454 else 455 test_init(&test, ifobj_tx, ifobj_rx, i, 456 &ci_skip_tests[j - ARRAY_SIZE(tests)]); 457 run_pkt_test(&test); 458 usleep(USLEEP_MAX); 459 460 if (test.fail) 461 failed_tests++; 462 } 463 } 464 465 if (ifobj_tx->hw_ring_size_supp) 466 hw_ring_size_reset(ifobj_tx); 467 468 pkt_stream_delete(tx_pkt_stream_default); 469 pkt_stream_delete(rx_pkt_stream_default); 470 xsk_unload_xdp_programs(ifobj_tx); 471 xsk_unload_xdp_programs(ifobj_rx); 472 ifobject_delete(ifobj_tx); 473 ifobject_delete(ifobj_rx); 474 475 if (failed_tests) 476 ksft_exit_fail(); 477 else 478 ksft_exit_pass(); 479 } 480