1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 /* 5 * Some functions in this program are taken from 6 * Linux kernel samples/bpf/xdpsock* and modified 7 * for use. 8 * 9 * See test_xsk.sh for detailed information on test topology 10 * and prerequisite network setup. 11 * 12 * This test program contains two threads, each thread is single socket with 13 * a unique UMEM. It validates in-order packet delivery and packet content 14 * by sending packets to each other. 15 * 16 * Tests Information: 17 * ------------------ 18 * These selftests test AF_XDP SKB and Native/DRV modes using veth 19 * Virtual Ethernet interfaces. 20 * 21 * For each mode, the following tests are run: 22 * a. nopoll - soft-irq processing in run-to-completion mode 23 * b. poll - using poll() syscall 24 * c. Socket Teardown 25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy 26 * both sockets, then repeat multiple times. Only nopoll mode is used 27 * d. Bi-directional sockets 28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and 29 * completion rings on each socket, tx/rx in both directions. Only nopoll 30 * mode is used 31 * e. Statistics 32 * Trigger some error conditions and ensure that the appropriate statistics 33 * are incremented. Within this test, the following statistics are tested: 34 * i. rx dropped 35 * Increase the UMEM frame headroom to a value which results in 36 * insufficient space in the rx buffer for both the packet and the headroom. 37 * ii. tx invalid 38 * Set the 'len' field of tx descriptors to an invalid value (umem frame 39 * size + 1). 40 * iii. rx ring full 41 * Reduce the size of the RX ring to a fraction of the fill ring size. 42 * iv. fill queue empty 43 * Do not populate the fill queue and then try to receive pkts. 44 * f. bpf_link resource persistence 45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0, 46 * then remove xsk sockets from queue 0 on both veth interfaces and 47 * finally run a traffic on queues ids 1 48 * g. unaligned mode 49 * h. tests for invalid and corner case Tx descriptors so that the correct ones 50 * are discarded and let through, respectively. 51 * i. 2K frame size tests 52 * j. If multi-buffer is supported, send 9k packets divided into 3 frames 53 * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame 54 * using unaligned mode 55 * l. If multi-buffer is supported, try various nasty combinations of descriptors to 56 * check if they pass the validation or not 57 * 58 * Flow: 59 * ----- 60 * - Single process spawns two threads: Tx and Rx 61 * - Each of these two threads attach to a veth interface 62 * - Each thread creates one AF_XDP socket connected to a unique umem for each 63 * veth interface 64 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy> 65 * - Rx thread verifies if all packets were received and delivered in-order, 66 * and have the right content 67 * 68 * Enable/disable packet dump mode: 69 * -------------------------- 70 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add 71 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D") 72 */ 73 74 #define _GNU_SOURCE 75 #include <assert.h> 76 #include <fcntl.h> 77 #include <errno.h> 78 #include <getopt.h> 79 #include <linux/if_link.h> 80 #include <linux/if_ether.h> 81 #include <linux/mman.h> 82 #include <linux/netdev.h> 83 #include <linux/bitmap.h> 84 #include <arpa/inet.h> 85 #include <net/if.h> 86 #include <locale.h> 87 #include <poll.h> 88 #include <pthread.h> 89 #include <signal.h> 90 #include <stdio.h> 91 #include <stdlib.h> 92 #include <string.h> 93 #include <stddef.h> 94 #include <sys/mman.h> 95 #include <sys/socket.h> 96 #include <sys/time.h> 97 #include <sys/types.h> 98 #include <unistd.h> 99 100 #include "xsk_xdp_progs.skel.h" 101 #include "xsk.h" 102 #include "xskxceiver.h" 103 #include <bpf/bpf.h> 104 #include <linux/filter.h> 105 #include "../kselftest.h" 106 #include "xsk_xdp_common.h" 107 108 static bool opt_verbose; 109 static bool opt_print_tests; 110 static enum test_mode opt_mode = TEST_MODE_ALL; 111 static u32 opt_run_test = RUN_ALL_TESTS; 112 113 static void __exit_with_error(int error, const char *file, const char *func, int line) 114 { 115 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, 116 strerror(error)); 117 ksft_exit_xfail(); 118 } 119 120 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) 121 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" 122 static char *mode_string(struct test_spec *test) 123 { 124 switch (test->mode) { 125 case TEST_MODE_SKB: 126 return "SKB"; 127 case TEST_MODE_DRV: 128 return "DRV"; 129 case TEST_MODE_ZC: 130 return "ZC"; 131 default: 132 return "BOGUS"; 133 } 134 } 135 136 static void report_failure(struct test_spec *test) 137 { 138 if (test->fail) 139 return; 140 141 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test), 142 test->name); 143 test->fail = true; 144 } 145 146 /* The payload is a word consisting of a packet sequence number in the upper 147 * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's 148 * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0. 149 */ 150 static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size) 151 { 152 u32 *ptr = (u32 *)dest, i; 153 154 start /= sizeof(*ptr); 155 size /= sizeof(*ptr); 156 for (i = 0; i < size; i++) 157 ptr[i] = htonl(pkt_nb << 16 | (i + start)); 158 } 159 160 static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr) 161 { 162 memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN); 163 memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN); 164 eth_hdr->h_proto = htons(ETH_P_LOOPBACK); 165 } 166 167 static bool is_umem_valid(struct ifobject *ifobj) 168 { 169 return !!ifobj->umem->umem; 170 } 171 172 static u32 mode_to_xdp_flags(enum test_mode mode) 173 { 174 return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; 175 } 176 177 static u64 umem_size(struct xsk_umem_info *umem) 178 { 179 return umem->num_frames * umem->frame_size; 180 } 181 182 static int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, 183 u64 size) 184 { 185 struct xsk_umem_config cfg = { 186 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, 187 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, 188 .frame_size = umem->frame_size, 189 .frame_headroom = umem->frame_headroom, 190 .flags = XSK_UMEM__DEFAULT_FLAGS 191 }; 192 int ret; 193 194 if (umem->unaligned_mode) 195 cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; 196 197 ret = xsk_umem__create(&umem->umem, buffer, size, 198 &umem->fq, &umem->cq, &cfg); 199 if (ret) 200 return ret; 201 202 umem->buffer = buffer; 203 if (ifobj->shared_umem && ifobj->rx_on) { 204 umem->base_addr = umem_size(umem); 205 umem->next_buffer = umem_size(umem); 206 } 207 208 return 0; 209 } 210 211 static u64 umem_alloc_buffer(struct xsk_umem_info *umem) 212 { 213 u64 addr; 214 215 addr = umem->next_buffer; 216 umem->next_buffer += umem->frame_size; 217 if (umem->next_buffer >= umem->base_addr + umem_size(umem)) 218 umem->next_buffer = umem->base_addr; 219 220 return addr; 221 } 222 223 static void umem_reset_alloc(struct xsk_umem_info *umem) 224 { 225 umem->next_buffer = 0; 226 } 227 228 static void enable_busy_poll(struct xsk_socket_info *xsk) 229 { 230 int sock_opt; 231 232 sock_opt = 1; 233 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL, 234 (void *)&sock_opt, sizeof(sock_opt)) < 0) 235 exit_with_error(errno); 236 237 sock_opt = 20; 238 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL, 239 (void *)&sock_opt, sizeof(sock_opt)) < 0) 240 exit_with_error(errno); 241 242 sock_opt = BATCH_SIZE; 243 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET, 244 (void *)&sock_opt, sizeof(sock_opt)) < 0) 245 exit_with_error(errno); 246 } 247 248 static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, 249 struct ifobject *ifobject, bool shared) 250 { 251 struct xsk_socket_config cfg = {}; 252 struct xsk_ring_cons *rxr; 253 struct xsk_ring_prod *txr; 254 255 xsk->umem = umem; 256 cfg.rx_size = xsk->rxqsize; 257 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; 258 cfg.bind_flags = ifobject->bind_flags; 259 if (shared) 260 cfg.bind_flags |= XDP_SHARED_UMEM; 261 if (ifobject->mtu > MAX_ETH_PKT_SIZE) 262 cfg.bind_flags |= XDP_USE_SG; 263 264 txr = ifobject->tx_on ? &xsk->tx : NULL; 265 rxr = ifobject->rx_on ? &xsk->rx : NULL; 266 return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg); 267 } 268 269 static bool ifobj_zc_avail(struct ifobject *ifobject) 270 { 271 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; 272 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; 273 struct xsk_socket_info *xsk; 274 struct xsk_umem_info *umem; 275 bool zc_avail = false; 276 void *bufs; 277 int ret; 278 279 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); 280 if (bufs == MAP_FAILED) 281 exit_with_error(errno); 282 283 umem = calloc(1, sizeof(struct xsk_umem_info)); 284 if (!umem) { 285 munmap(bufs, umem_sz); 286 exit_with_error(ENOMEM); 287 } 288 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; 289 ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz); 290 if (ret) 291 exit_with_error(-ret); 292 293 xsk = calloc(1, sizeof(struct xsk_socket_info)); 294 if (!xsk) 295 goto out; 296 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY; 297 ifobject->rx_on = true; 298 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 299 ret = __xsk_configure_socket(xsk, umem, ifobject, false); 300 if (!ret) 301 zc_avail = true; 302 303 xsk_socket__delete(xsk->xsk); 304 free(xsk); 305 out: 306 munmap(umem->buffer, umem_sz); 307 xsk_umem__delete(umem->umem); 308 free(umem); 309 return zc_avail; 310 } 311 312 static struct option long_options[] = { 313 {"interface", required_argument, 0, 'i'}, 314 {"busy-poll", no_argument, 0, 'b'}, 315 {"verbose", no_argument, 0, 'v'}, 316 {"mode", required_argument, 0, 'm'}, 317 {"list", no_argument, 0, 'l'}, 318 {"test", required_argument, 0, 't'}, 319 {"help", no_argument, 0, 'h'}, 320 {0, 0, 0, 0} 321 }; 322 323 static void print_usage(char **argv) 324 { 325 const char *str = 326 " Usage: xskxceiver [OPTIONS]\n" 327 " Options:\n" 328 " -i, --interface Use interface\n" 329 " -v, --verbose Verbose output\n" 330 " -b, --busy-poll Enable busy poll\n" 331 " -m, --mode Run only mode skb, drv, or zc\n" 332 " -l, --list List all available tests\n" 333 " -t, --test Run a specific test. Enter number from -l option.\n" 334 " -h, --help Display this help and exit\n"; 335 336 ksft_print_msg(str, basename(argv[0])); 337 ksft_exit_xfail(); 338 } 339 340 static bool validate_interface(struct ifobject *ifobj) 341 { 342 if (!strcmp(ifobj->ifname, "")) 343 return false; 344 return true; 345 } 346 347 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc, 348 char **argv) 349 { 350 struct ifobject *ifobj; 351 u32 interface_nb = 0; 352 int option_index, c; 353 354 opterr = 0; 355 356 for (;;) { 357 c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index); 358 if (c == -1) 359 break; 360 361 switch (c) { 362 case 'i': 363 if (interface_nb == 0) 364 ifobj = ifobj_tx; 365 else if (interface_nb == 1) 366 ifobj = ifobj_rx; 367 else 368 break; 369 370 memcpy(ifobj->ifname, optarg, 371 min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg))); 372 373 ifobj->ifindex = if_nametoindex(ifobj->ifname); 374 if (!ifobj->ifindex) 375 exit_with_error(errno); 376 377 interface_nb++; 378 break; 379 case 'v': 380 opt_verbose = true; 381 break; 382 case 'b': 383 ifobj_tx->busy_poll = true; 384 ifobj_rx->busy_poll = true; 385 break; 386 case 'm': 387 if (!strncmp("skb", optarg, strlen(optarg))) 388 opt_mode = TEST_MODE_SKB; 389 else if (!strncmp("drv", optarg, strlen(optarg))) 390 opt_mode = TEST_MODE_DRV; 391 else if (!strncmp("zc", optarg, strlen(optarg))) 392 opt_mode = TEST_MODE_ZC; 393 else 394 print_usage(argv); 395 break; 396 case 'l': 397 opt_print_tests = true; 398 break; 399 case 't': 400 errno = 0; 401 opt_run_test = strtol(optarg, NULL, 0); 402 if (errno) 403 print_usage(argv); 404 break; 405 case 'h': 406 default: 407 print_usage(argv); 408 } 409 } 410 } 411 412 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, 413 struct ifobject *ifobj_rx) 414 { 415 u32 i, j; 416 417 for (i = 0; i < MAX_INTERFACES; i++) { 418 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; 419 420 ifobj->xsk = &ifobj->xsk_arr[0]; 421 ifobj->use_poll = false; 422 ifobj->use_fill_ring = true; 423 ifobj->release_rx = true; 424 ifobj->validation_func = NULL; 425 ifobj->use_metadata = false; 426 427 if (i == 0) { 428 ifobj->rx_on = false; 429 ifobj->tx_on = true; 430 } else { 431 ifobj->rx_on = true; 432 ifobj->tx_on = false; 433 } 434 435 memset(ifobj->umem, 0, sizeof(*ifobj->umem)); 436 ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; 437 ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; 438 439 for (j = 0; j < MAX_SOCKETS; j++) { 440 memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); 441 ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 442 if (i == 0) 443 ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default; 444 else 445 ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default; 446 447 memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN); 448 memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN); 449 ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0); 450 ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1); 451 } 452 } 453 454 test->ifobj_tx = ifobj_tx; 455 test->ifobj_rx = ifobj_rx; 456 test->current_step = 0; 457 test->total_steps = 1; 458 test->nb_sockets = 1; 459 test->fail = false; 460 test->mtu = MAX_ETH_PKT_SIZE; 461 test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog; 462 test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk; 463 test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog; 464 test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk; 465 } 466 467 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, 468 struct ifobject *ifobj_rx, enum test_mode mode, 469 const struct test_spec *test_to_run) 470 { 471 struct pkt_stream *tx_pkt_stream; 472 struct pkt_stream *rx_pkt_stream; 473 u32 i; 474 475 tx_pkt_stream = test->tx_pkt_stream_default; 476 rx_pkt_stream = test->rx_pkt_stream_default; 477 memset(test, 0, sizeof(*test)); 478 test->tx_pkt_stream_default = tx_pkt_stream; 479 test->rx_pkt_stream_default = rx_pkt_stream; 480 481 for (i = 0; i < MAX_INTERFACES; i++) { 482 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; 483 484 ifobj->bind_flags = XDP_USE_NEED_WAKEUP; 485 if (mode == TEST_MODE_ZC) 486 ifobj->bind_flags |= XDP_ZEROCOPY; 487 else 488 ifobj->bind_flags |= XDP_COPY; 489 } 490 491 strncpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE); 492 test->test_func = test_to_run->test_func; 493 test->mode = mode; 494 __test_spec_init(test, ifobj_tx, ifobj_rx); 495 } 496 497 static void test_spec_reset(struct test_spec *test) 498 { 499 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); 500 } 501 502 static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx, 503 struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx, 504 struct bpf_map *xskmap_tx) 505 { 506 test->xdp_prog_rx = xdp_prog_rx; 507 test->xdp_prog_tx = xdp_prog_tx; 508 test->xskmap_rx = xskmap_rx; 509 test->xskmap_tx = xskmap_tx; 510 } 511 512 static int test_spec_set_mtu(struct test_spec *test, int mtu) 513 { 514 int err; 515 516 if (test->ifobj_rx->mtu != mtu) { 517 err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu); 518 if (err) 519 return err; 520 test->ifobj_rx->mtu = mtu; 521 } 522 if (test->ifobj_tx->mtu != mtu) { 523 err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu); 524 if (err) 525 return err; 526 test->ifobj_tx->mtu = mtu; 527 } 528 529 return 0; 530 } 531 532 static void pkt_stream_reset(struct pkt_stream *pkt_stream) 533 { 534 if (pkt_stream) { 535 pkt_stream->current_pkt_nb = 0; 536 pkt_stream->nb_rx_pkts = 0; 537 } 538 } 539 540 static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream) 541 { 542 if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) 543 return NULL; 544 545 return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; 546 } 547 548 static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent) 549 { 550 while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) { 551 (*pkts_sent)++; 552 if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid) 553 return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; 554 pkt_stream->current_pkt_nb++; 555 } 556 return NULL; 557 } 558 559 static void pkt_stream_delete(struct pkt_stream *pkt_stream) 560 { 561 free(pkt_stream->pkts); 562 free(pkt_stream); 563 } 564 565 static void pkt_stream_restore_default(struct test_spec *test) 566 { 567 struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream; 568 struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream; 569 570 if (tx_pkt_stream != test->tx_pkt_stream_default) { 571 pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream); 572 test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default; 573 } 574 575 if (rx_pkt_stream != test->rx_pkt_stream_default) { 576 pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream); 577 test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default; 578 } 579 } 580 581 static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) 582 { 583 struct pkt_stream *pkt_stream; 584 585 pkt_stream = calloc(1, sizeof(*pkt_stream)); 586 if (!pkt_stream) 587 return NULL; 588 589 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts)); 590 if (!pkt_stream->pkts) { 591 free(pkt_stream); 592 return NULL; 593 } 594 595 pkt_stream->nb_pkts = nb_pkts; 596 return pkt_stream; 597 } 598 599 static bool pkt_continues(u32 options) 600 { 601 return options & XDP_PKT_CONTD; 602 } 603 604 static u32 ceil_u32(u32 a, u32 b) 605 { 606 return (a + b - 1) / b; 607 } 608 609 static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt) 610 { 611 u32 nb_frags = 1, next_frag; 612 613 if (!pkt) 614 return 1; 615 616 if (!pkt_stream->verbatim) { 617 if (!pkt->valid || !pkt->len) 618 return 1; 619 return ceil_u32(pkt->len, frame_size); 620 } 621 622 /* Search for the end of the packet in verbatim mode */ 623 if (!pkt_continues(pkt->options)) 624 return nb_frags; 625 626 next_frag = pkt_stream->current_pkt_nb; 627 pkt++; 628 while (next_frag++ < pkt_stream->nb_pkts) { 629 nb_frags++; 630 if (!pkt_continues(pkt->options) || !pkt->valid) 631 break; 632 pkt++; 633 } 634 return nb_frags; 635 } 636 637 static bool set_pkt_valid(int offset, u32 len) 638 { 639 return len <= MAX_ETH_JUMBO_SIZE; 640 } 641 642 static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) 643 { 644 pkt->offset = offset; 645 pkt->len = len; 646 pkt->valid = set_pkt_valid(offset, len); 647 } 648 649 static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) 650 { 651 bool prev_pkt_valid = pkt->valid; 652 653 pkt_set(pkt_stream, pkt, offset, len); 654 pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid; 655 } 656 657 static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len) 658 { 659 return ceil_u32(len, umem->frame_size) * umem->frame_size; 660 } 661 662 static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off) 663 { 664 struct pkt_stream *pkt_stream; 665 u32 i; 666 667 pkt_stream = __pkt_stream_alloc(nb_pkts); 668 if (!pkt_stream) 669 exit_with_error(ENOMEM); 670 671 pkt_stream->nb_pkts = nb_pkts; 672 pkt_stream->max_pkt_len = pkt_len; 673 for (i = 0; i < nb_pkts; i++) { 674 struct pkt *pkt = &pkt_stream->pkts[i]; 675 676 pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len); 677 pkt->pkt_nb = nb_start + i * nb_off; 678 } 679 680 return pkt_stream; 681 } 682 683 static struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len) 684 { 685 return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1); 686 } 687 688 static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream) 689 { 690 return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len); 691 } 692 693 static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) 694 { 695 struct pkt_stream *pkt_stream; 696 697 pkt_stream = pkt_stream_generate(nb_pkts, pkt_len); 698 test->ifobj_tx->xsk->pkt_stream = pkt_stream; 699 pkt_stream = pkt_stream_generate(nb_pkts, pkt_len); 700 test->ifobj_rx->xsk->pkt_stream = pkt_stream; 701 } 702 703 static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len, 704 int offset) 705 { 706 struct pkt_stream *pkt_stream; 707 u32 i; 708 709 pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream); 710 for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2) 711 pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len); 712 713 ifobj->xsk->pkt_stream = pkt_stream; 714 } 715 716 static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) 717 { 718 __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset); 719 __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset); 720 } 721 722 static void pkt_stream_receive_half(struct test_spec *test) 723 { 724 struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream; 725 u32 i; 726 727 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts, 728 pkt_stream->pkts[0].len); 729 pkt_stream = test->ifobj_rx->xsk->pkt_stream; 730 for (i = 1; i < pkt_stream->nb_pkts; i += 2) 731 pkt_stream->pkts[i].valid = false; 732 733 pkt_stream->nb_valid_entries /= 2; 734 } 735 736 static void pkt_stream_even_odd_sequence(struct test_spec *test) 737 { 738 struct pkt_stream *pkt_stream; 739 u32 i; 740 741 for (i = 0; i < test->nb_sockets; i++) { 742 pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream; 743 pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, 744 pkt_stream->pkts[0].len, i, 2); 745 test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream; 746 747 pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream; 748 pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, 749 pkt_stream->pkts[0].len, i, 2); 750 test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream; 751 } 752 } 753 754 static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem) 755 { 756 if (!pkt->valid) 757 return pkt->offset; 758 return pkt->offset + umem_alloc_buffer(umem); 759 } 760 761 static void pkt_stream_cancel(struct pkt_stream *pkt_stream) 762 { 763 pkt_stream->current_pkt_nb--; 764 } 765 766 static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len, 767 u32 pkt_nb, u32 bytes_written) 768 { 769 void *data = xsk_umem__get_data(umem->buffer, addr); 770 771 if (len < MIN_PKT_SIZE) 772 return; 773 774 if (!bytes_written) { 775 gen_eth_hdr(xsk, data); 776 777 len -= PKT_HDR_SIZE; 778 data += PKT_HDR_SIZE; 779 } else { 780 bytes_written -= PKT_HDR_SIZE; 781 } 782 783 write_payload(data, pkt_nb, bytes_written, len); 784 } 785 786 static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames, 787 u32 nb_frames, bool verbatim) 788 { 789 u32 i, len = 0, pkt_nb = 0, payload = 0; 790 struct pkt_stream *pkt_stream; 791 792 pkt_stream = __pkt_stream_alloc(nb_frames); 793 if (!pkt_stream) 794 exit_with_error(ENOMEM); 795 796 for (i = 0; i < nb_frames; i++) { 797 struct pkt *pkt = &pkt_stream->pkts[pkt_nb]; 798 struct pkt *frame = &frames[i]; 799 800 pkt->offset = frame->offset; 801 if (verbatim) { 802 *pkt = *frame; 803 pkt->pkt_nb = payload; 804 if (!frame->valid || !pkt_continues(frame->options)) 805 payload++; 806 } else { 807 if (frame->valid) 808 len += frame->len; 809 if (frame->valid && pkt_continues(frame->options)) 810 continue; 811 812 pkt->pkt_nb = pkt_nb; 813 pkt->len = len; 814 pkt->valid = frame->valid; 815 pkt->options = 0; 816 817 len = 0; 818 } 819 820 print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n", 821 pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb); 822 823 if (pkt->valid && pkt->len > pkt_stream->max_pkt_len) 824 pkt_stream->max_pkt_len = pkt->len; 825 826 if (pkt->valid) 827 pkt_stream->nb_valid_entries++; 828 829 pkt_nb++; 830 } 831 832 pkt_stream->nb_pkts = pkt_nb; 833 pkt_stream->verbatim = verbatim; 834 return pkt_stream; 835 } 836 837 static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) 838 { 839 struct pkt_stream *pkt_stream; 840 841 pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true); 842 test->ifobj_tx->xsk->pkt_stream = pkt_stream; 843 844 pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false); 845 test->ifobj_rx->xsk->pkt_stream = pkt_stream; 846 } 847 848 static void pkt_print_data(u32 *data, u32 cnt) 849 { 850 u32 i; 851 852 for (i = 0; i < cnt; i++) { 853 u32 seqnum, pkt_nb; 854 855 seqnum = ntohl(*data) & 0xffff; 856 pkt_nb = ntohl(*data) >> 16; 857 ksft_print_msg("%u:%u ", pkt_nb, seqnum); 858 data++; 859 } 860 } 861 862 static void pkt_dump(void *pkt, u32 len, bool eth_header) 863 { 864 struct ethhdr *ethhdr = pkt; 865 u32 i, *data; 866 867 if (eth_header) { 868 /*extract L2 frame */ 869 ksft_print_msg("DEBUG>> L2: dst mac: "); 870 for (i = 0; i < ETH_ALEN; i++) 871 ksft_print_msg("%02X", ethhdr->h_dest[i]); 872 873 ksft_print_msg("\nDEBUG>> L2: src mac: "); 874 for (i = 0; i < ETH_ALEN; i++) 875 ksft_print_msg("%02X", ethhdr->h_source[i]); 876 877 data = pkt + PKT_HDR_SIZE; 878 } else { 879 data = pkt; 880 } 881 882 /*extract L5 frame */ 883 ksft_print_msg("\nDEBUG>> L5: seqnum: "); 884 pkt_print_data(data, PKT_DUMP_NB_TO_PRINT); 885 ksft_print_msg("...."); 886 if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) { 887 ksft_print_msg("\n.... "); 888 pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT, 889 PKT_DUMP_NB_TO_PRINT); 890 } 891 ksft_print_msg("\n---------------------------------------\n"); 892 } 893 894 static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr) 895 { 896 u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom; 897 u32 offset = addr % umem->frame_size, expected_offset; 898 int pkt_offset = pkt->valid ? pkt->offset : 0; 899 900 if (!umem->unaligned_mode) 901 pkt_offset = 0; 902 903 expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size; 904 905 if (offset == expected_offset) 906 return true; 907 908 ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset); 909 return false; 910 } 911 912 static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr) 913 { 914 void *data = xsk_umem__get_data(buffer, addr); 915 struct xdp_info *meta = data - sizeof(struct xdp_info); 916 917 if (meta->count != pkt->pkt_nb) { 918 ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%llu]\n", 919 __func__, pkt->pkt_nb, 920 (unsigned long long)meta->count); 921 return false; 922 } 923 924 return true; 925 } 926 927 static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb, 928 u32 bytes_processed) 929 { 930 u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum; 931 void *data = xsk_umem__get_data(umem->buffer, addr); 932 933 addr -= umem->base_addr; 934 935 if (addr >= umem->num_frames * umem->frame_size || 936 addr + len > umem->num_frames * umem->frame_size) { 937 ksft_print_msg("Frag invalid addr: %llx len: %u\n", 938 (unsigned long long)addr, len); 939 return false; 940 } 941 if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) { 942 ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n", 943 (unsigned long long)addr, len); 944 return false; 945 } 946 947 pkt_data = data; 948 if (!bytes_processed) { 949 pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data); 950 len -= PKT_HDR_SIZE; 951 } else { 952 bytes_processed -= PKT_HDR_SIZE; 953 } 954 955 expected_seqnum = bytes_processed / sizeof(*pkt_data); 956 seqnum = ntohl(*pkt_data) & 0xffff; 957 pkt_nb = ntohl(*pkt_data) >> 16; 958 959 if (expected_pkt_nb != pkt_nb) { 960 ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n", 961 __func__, expected_pkt_nb, pkt_nb); 962 goto error; 963 } 964 if (expected_seqnum != seqnum) { 965 ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n", 966 __func__, expected_seqnum, seqnum); 967 goto error; 968 } 969 970 words_to_end = len / sizeof(*pkt_data) - 1; 971 pkt_data += words_to_end; 972 seqnum = ntohl(*pkt_data) & 0xffff; 973 expected_seqnum += words_to_end; 974 if (expected_seqnum != seqnum) { 975 ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n", 976 __func__, expected_seqnum, seqnum); 977 goto error; 978 } 979 980 return true; 981 982 error: 983 pkt_dump(data, len, !bytes_processed); 984 return false; 985 } 986 987 static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) 988 { 989 if (pkt->len != len) { 990 ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n", 991 __func__, pkt->len, len); 992 pkt_dump(xsk_umem__get_data(buffer, addr), len, true); 993 return false; 994 } 995 996 return true; 997 } 998 999 static int kick_tx(struct xsk_socket_info *xsk) 1000 { 1001 int ret; 1002 1003 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); 1004 if (ret >= 0) 1005 return TEST_PASS; 1006 if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) { 1007 usleep(100); 1008 return TEST_PASS; 1009 } 1010 return TEST_FAILURE; 1011 } 1012 1013 static int kick_rx(struct xsk_socket_info *xsk) 1014 { 1015 int ret; 1016 1017 ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); 1018 if (ret < 0) 1019 return TEST_FAILURE; 1020 1021 return TEST_PASS; 1022 } 1023 1024 static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) 1025 { 1026 unsigned int rcvd; 1027 u32 idx; 1028 int ret; 1029 1030 if (xsk_ring_prod__needs_wakeup(&xsk->tx)) { 1031 ret = kick_tx(xsk); 1032 if (ret) 1033 return TEST_FAILURE; 1034 } 1035 1036 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); 1037 if (rcvd) { 1038 if (rcvd > xsk->outstanding_tx) { 1039 u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1); 1040 1041 ksft_print_msg("[%s] Too many packets completed\n", __func__); 1042 ksft_print_msg("Last completion address: %llx\n", 1043 (unsigned long long)addr); 1044 return TEST_FAILURE; 1045 } 1046 1047 xsk_ring_cons__release(&xsk->umem->cq, rcvd); 1048 xsk->outstanding_tx -= rcvd; 1049 } 1050 1051 return TEST_PASS; 1052 } 1053 1054 static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) 1055 { 1056 u32 frags_processed = 0, nb_frags = 0, pkt_len = 0; 1057 u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0; 1058 struct pkt_stream *pkt_stream = xsk->pkt_stream; 1059 struct ifobject *ifobj = test->ifobj_rx; 1060 struct xsk_umem_info *umem = xsk->umem; 1061 struct pollfd fds = { }; 1062 struct pkt *pkt; 1063 u64 first_addr = 0; 1064 int ret; 1065 1066 fds.fd = xsk_socket__fd(xsk->xsk); 1067 fds.events = POLLIN; 1068 1069 ret = kick_rx(xsk); 1070 if (ret) 1071 return TEST_FAILURE; 1072 1073 if (ifobj->use_poll) { 1074 ret = poll(&fds, 1, POLL_TMOUT); 1075 if (ret < 0) 1076 return TEST_FAILURE; 1077 1078 if (!ret) { 1079 if (!is_umem_valid(test->ifobj_tx)) 1080 return TEST_PASS; 1081 1082 ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); 1083 return TEST_CONTINUE; 1084 } 1085 1086 if (!(fds.revents & POLLIN)) 1087 return TEST_CONTINUE; 1088 } 1089 1090 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); 1091 if (!rcvd) 1092 return TEST_CONTINUE; 1093 1094 if (ifobj->use_fill_ring) { 1095 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); 1096 while (ret != rcvd) { 1097 if (xsk_ring_prod__needs_wakeup(&umem->fq)) { 1098 ret = poll(&fds, 1, POLL_TMOUT); 1099 if (ret < 0) 1100 return TEST_FAILURE; 1101 } 1102 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); 1103 } 1104 } 1105 1106 while (frags_processed < rcvd) { 1107 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++); 1108 u64 addr = desc->addr, orig; 1109 1110 orig = xsk_umem__extract_addr(addr); 1111 addr = xsk_umem__add_offset_to_addr(addr); 1112 1113 if (!nb_frags) { 1114 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); 1115 if (!pkt) { 1116 ksft_print_msg("[%s] received too many packets addr: %lx len %u\n", 1117 __func__, addr, desc->len); 1118 return TEST_FAILURE; 1119 } 1120 } 1121 1122 print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n", 1123 addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid); 1124 1125 if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) || 1126 !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata && 1127 !is_metadata_correct(pkt, umem->buffer, addr))) 1128 return TEST_FAILURE; 1129 1130 if (!nb_frags++) 1131 first_addr = addr; 1132 frags_processed++; 1133 pkt_len += desc->len; 1134 if (ifobj->use_fill_ring) 1135 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig; 1136 1137 if (pkt_continues(desc->options)) 1138 continue; 1139 1140 /* The complete packet has been received */ 1141 if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) || 1142 !is_offset_correct(umem, pkt, addr)) 1143 return TEST_FAILURE; 1144 1145 pkt_stream->nb_rx_pkts++; 1146 nb_frags = 0; 1147 pkt_len = 0; 1148 } 1149 1150 if (nb_frags) { 1151 /* In the middle of a packet. Start over from beginning of packet. */ 1152 idx_rx -= nb_frags; 1153 xsk_ring_cons__cancel(&xsk->rx, nb_frags); 1154 if (ifobj->use_fill_ring) { 1155 idx_fq -= nb_frags; 1156 xsk_ring_prod__cancel(&umem->fq, nb_frags); 1157 } 1158 frags_processed -= nb_frags; 1159 } 1160 1161 if (ifobj->use_fill_ring) 1162 xsk_ring_prod__submit(&umem->fq, frags_processed); 1163 if (ifobj->release_rx) 1164 xsk_ring_cons__release(&xsk->rx, frags_processed); 1165 1166 pthread_mutex_lock(&pacing_mutex); 1167 pkts_in_flight -= pkts_sent; 1168 pthread_mutex_unlock(&pacing_mutex); 1169 pkts_sent = 0; 1170 1171 return TEST_CONTINUE; 1172 } 1173 1174 bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num, 1175 unsigned long *bitmap) 1176 { 1177 struct pkt_stream *pkt_stream = xsk->pkt_stream; 1178 1179 if (!pkt_stream) { 1180 __set_bit(sock_num, bitmap); 1181 return false; 1182 } 1183 1184 if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) { 1185 __set_bit(sock_num, bitmap); 1186 if (bitmap_full(bitmap, test->nb_sockets)) 1187 return true; 1188 } 1189 1190 return false; 1191 } 1192 1193 static int receive_pkts(struct test_spec *test) 1194 { 1195 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; 1196 DECLARE_BITMAP(bitmap, test->nb_sockets); 1197 struct xsk_socket_info *xsk; 1198 u32 sock_num = 0; 1199 int res, ret; 1200 1201 ret = gettimeofday(&tv_now, NULL); 1202 if (ret) 1203 exit_with_error(errno); 1204 1205 timeradd(&tv_now, &tv_timeout, &tv_end); 1206 1207 while (1) { 1208 xsk = &test->ifobj_rx->xsk_arr[sock_num]; 1209 1210 if ((all_packets_received(test, xsk, sock_num, bitmap))) 1211 break; 1212 1213 res = __receive_pkts(test, xsk); 1214 if (!(res == TEST_PASS || res == TEST_CONTINUE)) 1215 return res; 1216 1217 ret = gettimeofday(&tv_now, NULL); 1218 if (ret) 1219 exit_with_error(errno); 1220 1221 if (timercmp(&tv_now, &tv_end, >)) { 1222 ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__); 1223 return TEST_FAILURE; 1224 } 1225 sock_num = (sock_num + 1) % test->nb_sockets; 1226 } 1227 1228 return TEST_PASS; 1229 } 1230 1231 static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout) 1232 { 1233 u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len; 1234 struct pkt_stream *pkt_stream = xsk->pkt_stream; 1235 struct xsk_umem_info *umem = ifobject->umem; 1236 bool use_poll = ifobject->use_poll; 1237 struct pollfd fds = { }; 1238 int ret; 1239 1240 buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len); 1241 /* pkts_in_flight might be negative if many invalid packets are sent */ 1242 if (pkts_in_flight >= (int)((umem_size(umem) - BATCH_SIZE * buffer_len) / buffer_len)) { 1243 ret = kick_tx(xsk); 1244 if (ret) 1245 return TEST_FAILURE; 1246 return TEST_CONTINUE; 1247 } 1248 1249 fds.fd = xsk_socket__fd(xsk->xsk); 1250 fds.events = POLLOUT; 1251 1252 while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) { 1253 if (use_poll) { 1254 ret = poll(&fds, 1, POLL_TMOUT); 1255 if (timeout) { 1256 if (ret < 0) { 1257 ksft_print_msg("ERROR: [%s] Poll error %d\n", 1258 __func__, errno); 1259 return TEST_FAILURE; 1260 } 1261 if (ret == 0) 1262 return TEST_PASS; 1263 break; 1264 } 1265 if (ret <= 0) { 1266 ksft_print_msg("ERROR: [%s] Poll error %d\n", 1267 __func__, errno); 1268 return TEST_FAILURE; 1269 } 1270 } 1271 1272 complete_pkts(xsk, BATCH_SIZE); 1273 } 1274 1275 for (i = 0; i < BATCH_SIZE; i++) { 1276 struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream); 1277 u32 nb_frags_left, nb_frags, bytes_written = 0; 1278 1279 if (!pkt) 1280 break; 1281 1282 nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt); 1283 if (nb_frags > BATCH_SIZE - i) { 1284 pkt_stream_cancel(pkt_stream); 1285 xsk_ring_prod__cancel(&xsk->tx, BATCH_SIZE - i); 1286 break; 1287 } 1288 nb_frags_left = nb_frags; 1289 1290 while (nb_frags_left--) { 1291 struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); 1292 1293 tx_desc->addr = pkt_get_addr(pkt, ifobject->umem); 1294 if (pkt_stream->verbatim) { 1295 tx_desc->len = pkt->len; 1296 tx_desc->options = pkt->options; 1297 } else if (nb_frags_left) { 1298 tx_desc->len = umem->frame_size; 1299 tx_desc->options = XDP_PKT_CONTD; 1300 } else { 1301 tx_desc->len = pkt->len - bytes_written; 1302 tx_desc->options = 0; 1303 } 1304 if (pkt->valid) 1305 pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb, 1306 bytes_written); 1307 bytes_written += tx_desc->len; 1308 1309 print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n", 1310 tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb); 1311 1312 if (nb_frags_left) { 1313 i++; 1314 if (pkt_stream->verbatim) 1315 pkt = pkt_stream_get_next_tx_pkt(pkt_stream); 1316 } 1317 } 1318 1319 if (pkt && pkt->valid) { 1320 valid_pkts++; 1321 valid_frags += nb_frags; 1322 } 1323 } 1324 1325 pthread_mutex_lock(&pacing_mutex); 1326 pkts_in_flight += valid_pkts; 1327 pthread_mutex_unlock(&pacing_mutex); 1328 1329 xsk_ring_prod__submit(&xsk->tx, i); 1330 xsk->outstanding_tx += valid_frags; 1331 1332 if (use_poll) { 1333 ret = poll(&fds, 1, POLL_TMOUT); 1334 if (ret <= 0) { 1335 if (ret == 0 && timeout) 1336 return TEST_PASS; 1337 1338 ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); 1339 return TEST_FAILURE; 1340 } 1341 } 1342 1343 if (!timeout) { 1344 if (complete_pkts(xsk, i)) 1345 return TEST_FAILURE; 1346 1347 usleep(10); 1348 return TEST_PASS; 1349 } 1350 1351 return TEST_CONTINUE; 1352 } 1353 1354 static int wait_for_tx_completion(struct xsk_socket_info *xsk) 1355 { 1356 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; 1357 int ret; 1358 1359 ret = gettimeofday(&tv_now, NULL); 1360 if (ret) 1361 exit_with_error(errno); 1362 timeradd(&tv_now, &tv_timeout, &tv_end); 1363 1364 while (xsk->outstanding_tx) { 1365 ret = gettimeofday(&tv_now, NULL); 1366 if (ret) 1367 exit_with_error(errno); 1368 if (timercmp(&tv_now, &tv_end, >)) { 1369 ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__); 1370 return TEST_FAILURE; 1371 } 1372 1373 complete_pkts(xsk, BATCH_SIZE); 1374 } 1375 1376 return TEST_PASS; 1377 } 1378 1379 bool all_packets_sent(struct test_spec *test, unsigned long *bitmap) 1380 { 1381 return bitmap_full(bitmap, test->nb_sockets); 1382 } 1383 1384 static int send_pkts(struct test_spec *test, struct ifobject *ifobject) 1385 { 1386 bool timeout = !is_umem_valid(test->ifobj_rx); 1387 DECLARE_BITMAP(bitmap, test->nb_sockets); 1388 u32 i, ret; 1389 1390 while (!(all_packets_sent(test, bitmap))) { 1391 for (i = 0; i < test->nb_sockets; i++) { 1392 struct pkt_stream *pkt_stream; 1393 1394 pkt_stream = ifobject->xsk_arr[i].pkt_stream; 1395 if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) { 1396 __set_bit(i, bitmap); 1397 continue; 1398 } 1399 ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout); 1400 if (ret == TEST_CONTINUE && !test->fail) 1401 continue; 1402 1403 if ((ret || test->fail) && !timeout) 1404 return TEST_FAILURE; 1405 1406 if (ret == TEST_PASS && timeout) 1407 return ret; 1408 1409 ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); 1410 if (ret) 1411 return TEST_FAILURE; 1412 } 1413 } 1414 1415 return TEST_PASS; 1416 } 1417 1418 static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats) 1419 { 1420 int fd = xsk_socket__fd(xsk), err; 1421 socklen_t optlen, expected_len; 1422 1423 optlen = sizeof(*stats); 1424 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen); 1425 if (err) { 1426 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", 1427 __func__, -err, strerror(-err)); 1428 return TEST_FAILURE; 1429 } 1430 1431 expected_len = sizeof(struct xdp_statistics); 1432 if (optlen != expected_len) { 1433 ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n", 1434 __func__, expected_len, optlen); 1435 return TEST_FAILURE; 1436 } 1437 1438 return TEST_PASS; 1439 } 1440 1441 static int validate_rx_dropped(struct ifobject *ifobject) 1442 { 1443 struct xsk_socket *xsk = ifobject->xsk->xsk; 1444 struct xdp_statistics stats; 1445 int err; 1446 1447 err = kick_rx(ifobject->xsk); 1448 if (err) 1449 return TEST_FAILURE; 1450 1451 err = get_xsk_stats(xsk, &stats); 1452 if (err) 1453 return TEST_FAILURE; 1454 1455 /* The receiver calls getsockopt after receiving the last (valid) 1456 * packet which is not the final packet sent in this test (valid and 1457 * invalid packets are sent in alternating fashion with the final 1458 * packet being invalid). Since the last packet may or may not have 1459 * been dropped already, both outcomes must be allowed. 1460 */ 1461 if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 || 1462 stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1) 1463 return TEST_PASS; 1464 1465 return TEST_FAILURE; 1466 } 1467 1468 static int validate_rx_full(struct ifobject *ifobject) 1469 { 1470 struct xsk_socket *xsk = ifobject->xsk->xsk; 1471 struct xdp_statistics stats; 1472 int err; 1473 1474 usleep(1000); 1475 err = kick_rx(ifobject->xsk); 1476 if (err) 1477 return TEST_FAILURE; 1478 1479 err = get_xsk_stats(xsk, &stats); 1480 if (err) 1481 return TEST_FAILURE; 1482 1483 if (stats.rx_ring_full) 1484 return TEST_PASS; 1485 1486 return TEST_FAILURE; 1487 } 1488 1489 static int validate_fill_empty(struct ifobject *ifobject) 1490 { 1491 struct xsk_socket *xsk = ifobject->xsk->xsk; 1492 struct xdp_statistics stats; 1493 int err; 1494 1495 usleep(1000); 1496 err = kick_rx(ifobject->xsk); 1497 if (err) 1498 return TEST_FAILURE; 1499 1500 err = get_xsk_stats(xsk, &stats); 1501 if (err) 1502 return TEST_FAILURE; 1503 1504 if (stats.rx_fill_ring_empty_descs) 1505 return TEST_PASS; 1506 1507 return TEST_FAILURE; 1508 } 1509 1510 static int validate_tx_invalid_descs(struct ifobject *ifobject) 1511 { 1512 struct xsk_socket *xsk = ifobject->xsk->xsk; 1513 int fd = xsk_socket__fd(xsk); 1514 struct xdp_statistics stats; 1515 socklen_t optlen; 1516 int err; 1517 1518 optlen = sizeof(stats); 1519 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); 1520 if (err) { 1521 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", 1522 __func__, -err, strerror(-err)); 1523 return TEST_FAILURE; 1524 } 1525 1526 if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) { 1527 ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n", 1528 __func__, 1529 (unsigned long long)stats.tx_invalid_descs, 1530 ifobject->xsk->pkt_stream->nb_pkts); 1531 return TEST_FAILURE; 1532 } 1533 1534 return TEST_PASS; 1535 } 1536 1537 static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject, 1538 struct xsk_umem_info *umem, bool tx) 1539 { 1540 int i, ret; 1541 1542 for (i = 0; i < test->nb_sockets; i++) { 1543 bool shared = (ifobject->shared_umem && tx) ? true : !!i; 1544 u32 ctr = 0; 1545 1546 while (ctr++ < SOCK_RECONF_CTR) { 1547 ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem, 1548 ifobject, shared); 1549 if (!ret) 1550 break; 1551 1552 /* Retry if it fails as xsk_socket__create() is asynchronous */ 1553 if (ctr >= SOCK_RECONF_CTR) 1554 exit_with_error(-ret); 1555 usleep(USLEEP_MAX); 1556 } 1557 if (ifobject->busy_poll) 1558 enable_busy_poll(&ifobject->xsk_arr[i]); 1559 } 1560 } 1561 1562 static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) 1563 { 1564 xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true); 1565 ifobject->xsk = &ifobject->xsk_arr[0]; 1566 ifobject->xskmap = test->ifobj_rx->xskmap; 1567 memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); 1568 ifobject->umem->base_addr = 0; 1569 } 1570 1571 static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, 1572 bool fill_up) 1573 { 1574 u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM; 1575 u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts; 1576 int ret; 1577 1578 if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) 1579 buffers_to_fill = umem->num_frames; 1580 else 1581 buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS; 1582 1583 ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx); 1584 if (ret != buffers_to_fill) 1585 exit_with_error(ENOSPC); 1586 1587 while (filled < buffers_to_fill) { 1588 struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts); 1589 u64 addr; 1590 u32 i; 1591 1592 for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) { 1593 if (!pkt) { 1594 if (!fill_up) 1595 break; 1596 addr = filled * umem->frame_size + umem->base_addr; 1597 } else if (pkt->offset >= 0) { 1598 addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem); 1599 } else { 1600 addr = pkt->offset + umem_alloc_buffer(umem); 1601 } 1602 1603 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr; 1604 if (++filled >= buffers_to_fill) 1605 break; 1606 } 1607 } 1608 xsk_ring_prod__submit(&umem->fq, filled); 1609 xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled); 1610 1611 pkt_stream_reset(pkt_stream); 1612 umem_reset_alloc(umem); 1613 } 1614 1615 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) 1616 { 1617 u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; 1618 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; 1619 LIBBPF_OPTS(bpf_xdp_query_opts, opts); 1620 void *bufs; 1621 int ret; 1622 u32 i; 1623 1624 if (ifobject->umem->unaligned_mode) 1625 mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB; 1626 1627 if (ifobject->shared_umem) 1628 umem_sz *= 2; 1629 1630 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); 1631 if (bufs == MAP_FAILED) 1632 exit_with_error(errno); 1633 1634 ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz); 1635 if (ret) 1636 exit_with_error(-ret); 1637 1638 xsk_configure_socket(test, ifobject, ifobject->umem, false); 1639 1640 ifobject->xsk = &ifobject->xsk_arr[0]; 1641 1642 if (!ifobject->rx_on) 1643 return; 1644 1645 xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring); 1646 1647 for (i = 0; i < test->nb_sockets; i++) { 1648 ifobject->xsk = &ifobject->xsk_arr[i]; 1649 ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i); 1650 if (ret) 1651 exit_with_error(errno); 1652 } 1653 } 1654 1655 static void *worker_testapp_validate_tx(void *arg) 1656 { 1657 struct test_spec *test = (struct test_spec *)arg; 1658 struct ifobject *ifobject = test->ifobj_tx; 1659 int err; 1660 1661 if (test->current_step == 1) { 1662 if (!ifobject->shared_umem) 1663 thread_common_ops(test, ifobject); 1664 else 1665 thread_common_ops_tx(test, ifobject); 1666 } 1667 1668 err = send_pkts(test, ifobject); 1669 1670 if (!err && ifobject->validation_func) 1671 err = ifobject->validation_func(ifobject); 1672 if (err) 1673 report_failure(test); 1674 1675 pthread_exit(NULL); 1676 } 1677 1678 static void *worker_testapp_validate_rx(void *arg) 1679 { 1680 struct test_spec *test = (struct test_spec *)arg; 1681 struct ifobject *ifobject = test->ifobj_rx; 1682 int err; 1683 1684 if (test->current_step == 1) { 1685 thread_common_ops(test, ifobject); 1686 } else { 1687 xsk_clear_xskmap(ifobject->xskmap); 1688 err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0); 1689 if (err) { 1690 ksft_print_msg("Error: Failed to update xskmap, error %s\n", 1691 strerror(-err)); 1692 exit_with_error(-err); 1693 } 1694 } 1695 1696 pthread_barrier_wait(&barr); 1697 1698 err = receive_pkts(test); 1699 1700 if (!err && ifobject->validation_func) 1701 err = ifobject->validation_func(ifobject); 1702 if (err) 1703 report_failure(test); 1704 1705 pthread_exit(NULL); 1706 } 1707 1708 static u64 ceil_u64(u64 a, u64 b) 1709 { 1710 return (a + b - 1) / b; 1711 } 1712 1713 static void testapp_clean_xsk_umem(struct ifobject *ifobj) 1714 { 1715 u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; 1716 1717 if (ifobj->shared_umem) 1718 umem_sz *= 2; 1719 1720 umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; 1721 xsk_umem__delete(ifobj->umem->umem); 1722 munmap(ifobj->umem->buffer, umem_sz); 1723 } 1724 1725 static void handler(int signum) 1726 { 1727 pthread_exit(NULL); 1728 } 1729 1730 static bool xdp_prog_changed_rx(struct test_spec *test) 1731 { 1732 struct ifobject *ifobj = test->ifobj_rx; 1733 1734 return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode; 1735 } 1736 1737 static bool xdp_prog_changed_tx(struct test_spec *test) 1738 { 1739 struct ifobject *ifobj = test->ifobj_tx; 1740 1741 return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode; 1742 } 1743 1744 static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog, 1745 struct bpf_map *xskmap, enum test_mode mode) 1746 { 1747 int err; 1748 1749 xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode)); 1750 err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode)); 1751 if (err) { 1752 ksft_print_msg("Error attaching XDP program\n"); 1753 exit_with_error(-err); 1754 } 1755 1756 if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC)) 1757 if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) { 1758 ksft_print_msg("ERROR: XDP prog not in DRV mode\n"); 1759 exit_with_error(EINVAL); 1760 } 1761 1762 ifobj->xdp_prog = xdp_prog; 1763 ifobj->xskmap = xskmap; 1764 ifobj->mode = mode; 1765 } 1766 1767 static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx, 1768 struct ifobject *ifobj_tx) 1769 { 1770 if (xdp_prog_changed_rx(test)) 1771 xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode); 1772 1773 if (!ifobj_tx || ifobj_tx->shared_umem) 1774 return; 1775 1776 if (xdp_prog_changed_tx(test)) 1777 xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode); 1778 } 1779 1780 static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1, 1781 struct ifobject *ifobj2) 1782 { 1783 pthread_t t0, t1; 1784 int err; 1785 1786 if (test->mtu > MAX_ETH_PKT_SIZE) { 1787 if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp || 1788 (ifobj2 && !ifobj2->multi_buff_zc_supp))) { 1789 ksft_test_result_skip("Multi buffer for zero-copy not supported.\n"); 1790 return TEST_SKIP; 1791 } 1792 if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp || 1793 (ifobj2 && !ifobj2->multi_buff_supp))) { 1794 ksft_test_result_skip("Multi buffer not supported.\n"); 1795 return TEST_SKIP; 1796 } 1797 } 1798 err = test_spec_set_mtu(test, test->mtu); 1799 if (err) { 1800 ksft_print_msg("Error, could not set mtu.\n"); 1801 exit_with_error(err); 1802 } 1803 1804 if (ifobj2) { 1805 if (pthread_barrier_init(&barr, NULL, 2)) 1806 exit_with_error(errno); 1807 pkt_stream_reset(ifobj2->xsk->pkt_stream); 1808 } 1809 1810 test->current_step++; 1811 pkt_stream_reset(ifobj1->xsk->pkt_stream); 1812 pkts_in_flight = 0; 1813 1814 signal(SIGUSR1, handler); 1815 /*Spawn RX thread */ 1816 pthread_create(&t0, NULL, ifobj1->func_ptr, test); 1817 1818 if (ifobj2) { 1819 pthread_barrier_wait(&barr); 1820 if (pthread_barrier_destroy(&barr)) 1821 exit_with_error(errno); 1822 1823 /*Spawn TX thread */ 1824 pthread_create(&t1, NULL, ifobj2->func_ptr, test); 1825 1826 pthread_join(t1, NULL); 1827 } 1828 1829 if (!ifobj2) 1830 pthread_kill(t0, SIGUSR1); 1831 else 1832 pthread_join(t0, NULL); 1833 1834 if (test->total_steps == test->current_step || test->fail) { 1835 u32 i; 1836 1837 if (ifobj2) 1838 for (i = 0; i < test->nb_sockets; i++) 1839 xsk_socket__delete(ifobj2->xsk_arr[i].xsk); 1840 1841 for (i = 0; i < test->nb_sockets; i++) 1842 xsk_socket__delete(ifobj1->xsk_arr[i].xsk); 1843 1844 testapp_clean_xsk_umem(ifobj1); 1845 if (ifobj2 && !ifobj2->shared_umem) 1846 testapp_clean_xsk_umem(ifobj2); 1847 } 1848 1849 return !!test->fail; 1850 } 1851 1852 static int testapp_validate_traffic(struct test_spec *test) 1853 { 1854 struct ifobject *ifobj_rx = test->ifobj_rx; 1855 struct ifobject *ifobj_tx = test->ifobj_tx; 1856 1857 if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) || 1858 (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) { 1859 ksft_test_result_skip("No huge pages present.\n"); 1860 return TEST_SKIP; 1861 } 1862 1863 xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx); 1864 return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx); 1865 } 1866 1867 static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj) 1868 { 1869 return __testapp_validate_traffic(test, ifobj, NULL); 1870 } 1871 1872 static int testapp_teardown(struct test_spec *test) 1873 { 1874 int i; 1875 1876 for (i = 0; i < MAX_TEARDOWN_ITER; i++) { 1877 if (testapp_validate_traffic(test)) 1878 return TEST_FAILURE; 1879 test_spec_reset(test); 1880 } 1881 1882 return TEST_PASS; 1883 } 1884 1885 static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2) 1886 { 1887 thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr; 1888 struct ifobject *tmp_ifobj = (*ifobj1); 1889 1890 (*ifobj1)->func_ptr = (*ifobj2)->func_ptr; 1891 (*ifobj2)->func_ptr = tmp_func_ptr; 1892 1893 *ifobj1 = *ifobj2; 1894 *ifobj2 = tmp_ifobj; 1895 } 1896 1897 static int testapp_bidirectional(struct test_spec *test) 1898 { 1899 int res; 1900 1901 test->ifobj_tx->rx_on = true; 1902 test->ifobj_rx->tx_on = true; 1903 test->total_steps = 2; 1904 if (testapp_validate_traffic(test)) 1905 return TEST_FAILURE; 1906 1907 print_verbose("Switching Tx/Rx direction\n"); 1908 swap_directions(&test->ifobj_rx, &test->ifobj_tx); 1909 res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx); 1910 1911 swap_directions(&test->ifobj_rx, &test->ifobj_tx); 1912 return res; 1913 } 1914 1915 static int swap_xsk_resources(struct test_spec *test) 1916 { 1917 int ret; 1918 1919 test->ifobj_tx->xsk_arr[0].pkt_stream = NULL; 1920 test->ifobj_rx->xsk_arr[0].pkt_stream = NULL; 1921 test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default; 1922 test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default; 1923 test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1]; 1924 test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1]; 1925 1926 ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0); 1927 if (ret) 1928 return TEST_FAILURE; 1929 1930 return TEST_PASS; 1931 } 1932 1933 static int testapp_xdp_prog_cleanup(struct test_spec *test) 1934 { 1935 test->total_steps = 2; 1936 test->nb_sockets = 2; 1937 if (testapp_validate_traffic(test)) 1938 return TEST_FAILURE; 1939 1940 if (swap_xsk_resources(test)) 1941 return TEST_FAILURE; 1942 return testapp_validate_traffic(test); 1943 } 1944 1945 static int testapp_headroom(struct test_spec *test) 1946 { 1947 test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; 1948 return testapp_validate_traffic(test); 1949 } 1950 1951 static int testapp_stats_rx_dropped(struct test_spec *test) 1952 { 1953 if (test->mode == TEST_MODE_ZC) { 1954 ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n"); 1955 return TEST_SKIP; 1956 } 1957 1958 pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0); 1959 test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - 1960 XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3; 1961 pkt_stream_receive_half(test); 1962 test->ifobj_rx->validation_func = validate_rx_dropped; 1963 return testapp_validate_traffic(test); 1964 } 1965 1966 static int testapp_stats_tx_invalid_descs(struct test_spec *test) 1967 { 1968 pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0); 1969 test->ifobj_tx->validation_func = validate_tx_invalid_descs; 1970 return testapp_validate_traffic(test); 1971 } 1972 1973 static int testapp_stats_rx_full(struct test_spec *test) 1974 { 1975 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); 1976 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); 1977 1978 test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS; 1979 test->ifobj_rx->release_rx = false; 1980 test->ifobj_rx->validation_func = validate_rx_full; 1981 return testapp_validate_traffic(test); 1982 } 1983 1984 static int testapp_stats_fill_empty(struct test_spec *test) 1985 { 1986 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); 1987 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); 1988 1989 test->ifobj_rx->use_fill_ring = false; 1990 test->ifobj_rx->validation_func = validate_fill_empty; 1991 return testapp_validate_traffic(test); 1992 } 1993 1994 static int testapp_send_receive_unaligned(struct test_spec *test) 1995 { 1996 test->ifobj_tx->umem->unaligned_mode = true; 1997 test->ifobj_rx->umem->unaligned_mode = true; 1998 /* Let half of the packets straddle a 4K buffer boundary */ 1999 pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2); 2000 2001 return testapp_validate_traffic(test); 2002 } 2003 2004 static int testapp_send_receive_unaligned_mb(struct test_spec *test) 2005 { 2006 test->mtu = MAX_ETH_JUMBO_SIZE; 2007 test->ifobj_tx->umem->unaligned_mode = true; 2008 test->ifobj_rx->umem->unaligned_mode = true; 2009 pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); 2010 return testapp_validate_traffic(test); 2011 } 2012 2013 static int testapp_single_pkt(struct test_spec *test) 2014 { 2015 struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}}; 2016 2017 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); 2018 return testapp_validate_traffic(test); 2019 } 2020 2021 static int testapp_send_receive_mb(struct test_spec *test) 2022 { 2023 test->mtu = MAX_ETH_JUMBO_SIZE; 2024 pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); 2025 2026 return testapp_validate_traffic(test); 2027 } 2028 2029 static int testapp_invalid_desc_mb(struct test_spec *test) 2030 { 2031 struct xsk_umem_info *umem = test->ifobj_tx->umem; 2032 u64 umem_size = umem->num_frames * umem->frame_size; 2033 struct pkt pkts[] = { 2034 /* Valid packet for synch to start with */ 2035 {0, MIN_PKT_SIZE, 0, true, 0}, 2036 /* Zero frame len is not legal */ 2037 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2038 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2039 {0, 0, 0, false, 0}, 2040 /* Invalid address in the second frame */ 2041 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2042 {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2043 /* Invalid len in the middle */ 2044 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2045 {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2046 /* Invalid options in the middle */ 2047 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2048 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION}, 2049 /* Transmit 2 frags, receive 3 */ 2050 {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD}, 2051 {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0}, 2052 /* Middle frame crosses chunk boundary with small length */ 2053 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2054 {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0}, 2055 /* Valid packet for synch so that something is received */ 2056 {0, MIN_PKT_SIZE, 0, true, 0}}; 2057 2058 if (umem->unaligned_mode) { 2059 /* Crossing a chunk boundary allowed */ 2060 pkts[12].valid = true; 2061 pkts[13].valid = true; 2062 } 2063 2064 test->mtu = MAX_ETH_JUMBO_SIZE; 2065 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); 2066 return testapp_validate_traffic(test); 2067 } 2068 2069 static int testapp_invalid_desc(struct test_spec *test) 2070 { 2071 struct xsk_umem_info *umem = test->ifobj_tx->umem; 2072 u64 umem_size = umem->num_frames * umem->frame_size; 2073 struct pkt pkts[] = { 2074 /* Zero packet address allowed */ 2075 {0, MIN_PKT_SIZE, 0, true}, 2076 /* Allowed packet */ 2077 {0, MIN_PKT_SIZE, 0, true}, 2078 /* Straddling the start of umem */ 2079 {-2, MIN_PKT_SIZE, 0, false}, 2080 /* Packet too large */ 2081 {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, 2082 /* Up to end of umem allowed */ 2083 {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true}, 2084 /* After umem ends */ 2085 {umem_size, MIN_PKT_SIZE, 0, false}, 2086 /* Straddle the end of umem */ 2087 {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, 2088 /* Straddle a 4K boundary */ 2089 {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, 2090 /* Straddle a 2K boundary */ 2091 {0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true}, 2092 /* Valid packet for synch so that something is received */ 2093 {0, MIN_PKT_SIZE, 0, true}}; 2094 2095 if (umem->unaligned_mode) { 2096 /* Crossing a page boundary allowed */ 2097 pkts[7].valid = true; 2098 } 2099 if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) { 2100 /* Crossing a 2K frame size boundary not allowed */ 2101 pkts[8].valid = false; 2102 } 2103 2104 if (test->ifobj_tx->shared_umem) { 2105 pkts[4].offset += umem_size; 2106 pkts[5].offset += umem_size; 2107 pkts[6].offset += umem_size; 2108 } 2109 2110 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); 2111 return testapp_validate_traffic(test); 2112 } 2113 2114 static int testapp_xdp_drop(struct test_spec *test) 2115 { 2116 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; 2117 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; 2118 2119 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop, 2120 skel_rx->maps.xsk, skel_tx->maps.xsk); 2121 2122 pkt_stream_receive_half(test); 2123 return testapp_validate_traffic(test); 2124 } 2125 2126 static int testapp_xdp_metadata_copy(struct test_spec *test) 2127 { 2128 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; 2129 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; 2130 struct bpf_map *data_map; 2131 int count = 0; 2132 int key = 0; 2133 2134 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata, 2135 skel_tx->progs.xsk_xdp_populate_metadata, 2136 skel_rx->maps.xsk, skel_tx->maps.xsk); 2137 test->ifobj_rx->use_metadata = true; 2138 2139 data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss"); 2140 if (!data_map || !bpf_map__is_internal(data_map)) { 2141 ksft_print_msg("Error: could not find bss section of XDP program\n"); 2142 return TEST_FAILURE; 2143 } 2144 2145 if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY)) { 2146 ksft_print_msg("Error: could not update count element\n"); 2147 return TEST_FAILURE; 2148 } 2149 2150 return testapp_validate_traffic(test); 2151 } 2152 2153 static int testapp_xdp_shared_umem(struct test_spec *test) 2154 { 2155 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; 2156 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; 2157 2158 test->total_steps = 1; 2159 test->nb_sockets = 2; 2160 2161 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem, 2162 skel_tx->progs.xsk_xdp_shared_umem, 2163 skel_rx->maps.xsk, skel_tx->maps.xsk); 2164 2165 pkt_stream_even_odd_sequence(test); 2166 2167 return testapp_validate_traffic(test); 2168 } 2169 2170 static int testapp_poll_txq_tmout(struct test_spec *test) 2171 { 2172 test->ifobj_tx->use_poll = true; 2173 /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ 2174 test->ifobj_tx->umem->frame_size = 2048; 2175 pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048); 2176 return testapp_validate_traffic_single_thread(test, test->ifobj_tx); 2177 } 2178 2179 static int testapp_poll_rxq_tmout(struct test_spec *test) 2180 { 2181 test->ifobj_rx->use_poll = true; 2182 return testapp_validate_traffic_single_thread(test, test->ifobj_rx); 2183 } 2184 2185 static int testapp_too_many_frags(struct test_spec *test) 2186 { 2187 struct pkt pkts[2 * XSK_DESC__MAX_SKB_FRAGS + 2] = {}; 2188 u32 max_frags, i; 2189 2190 if (test->mode == TEST_MODE_ZC) 2191 max_frags = test->ifobj_tx->xdp_zc_max_segs; 2192 else 2193 max_frags = XSK_DESC__MAX_SKB_FRAGS; 2194 2195 test->mtu = MAX_ETH_JUMBO_SIZE; 2196 2197 /* Valid packet for synch */ 2198 pkts[0].len = MIN_PKT_SIZE; 2199 pkts[0].valid = true; 2200 2201 /* One valid packet with the max amount of frags */ 2202 for (i = 1; i < max_frags + 1; i++) { 2203 pkts[i].len = MIN_PKT_SIZE; 2204 pkts[i].options = XDP_PKT_CONTD; 2205 pkts[i].valid = true; 2206 } 2207 pkts[max_frags].options = 0; 2208 2209 /* An invalid packet with the max amount of frags but signals packet 2210 * continues on the last frag 2211 */ 2212 for (i = max_frags + 1; i < 2 * max_frags + 1; i++) { 2213 pkts[i].len = MIN_PKT_SIZE; 2214 pkts[i].options = XDP_PKT_CONTD; 2215 pkts[i].valid = false; 2216 } 2217 2218 /* Valid packet for synch */ 2219 pkts[2 * max_frags + 1].len = MIN_PKT_SIZE; 2220 pkts[2 * max_frags + 1].valid = true; 2221 2222 pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2); 2223 return testapp_validate_traffic(test); 2224 } 2225 2226 static int xsk_load_xdp_programs(struct ifobject *ifobj) 2227 { 2228 ifobj->xdp_progs = xsk_xdp_progs__open_and_load(); 2229 if (libbpf_get_error(ifobj->xdp_progs)) 2230 return libbpf_get_error(ifobj->xdp_progs); 2231 2232 return 0; 2233 } 2234 2235 static void xsk_unload_xdp_programs(struct ifobject *ifobj) 2236 { 2237 xsk_xdp_progs__destroy(ifobj->xdp_progs); 2238 } 2239 2240 /* Simple test */ 2241 static bool hugepages_present(void) 2242 { 2243 size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; 2244 void *bufs; 2245 2246 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 2247 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB); 2248 if (bufs == MAP_FAILED) 2249 return false; 2250 2251 mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; 2252 munmap(bufs, mmap_sz); 2253 return true; 2254 } 2255 2256 static void init_iface(struct ifobject *ifobj, thread_func_t func_ptr) 2257 { 2258 LIBBPF_OPTS(bpf_xdp_query_opts, query_opts); 2259 int err; 2260 2261 ifobj->func_ptr = func_ptr; 2262 2263 err = xsk_load_xdp_programs(ifobj); 2264 if (err) { 2265 ksft_print_msg("Error loading XDP program\n"); 2266 exit_with_error(err); 2267 } 2268 2269 if (hugepages_present()) 2270 ifobj->unaligned_supp = true; 2271 2272 err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts); 2273 if (err) { 2274 ksft_print_msg("Error querying XDP capabilities\n"); 2275 exit_with_error(-err); 2276 } 2277 if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG) 2278 ifobj->multi_buff_supp = true; 2279 if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) { 2280 if (query_opts.xdp_zc_max_segs > 1) { 2281 ifobj->multi_buff_zc_supp = true; 2282 ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs; 2283 } else { 2284 ifobj->xdp_zc_max_segs = 0; 2285 } 2286 } 2287 } 2288 2289 static int testapp_send_receive(struct test_spec *test) 2290 { 2291 return testapp_validate_traffic(test); 2292 } 2293 2294 static int testapp_send_receive_2k_frame(struct test_spec *test) 2295 { 2296 test->ifobj_tx->umem->frame_size = 2048; 2297 test->ifobj_rx->umem->frame_size = 2048; 2298 pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE); 2299 return testapp_validate_traffic(test); 2300 } 2301 2302 static int testapp_poll_rx(struct test_spec *test) 2303 { 2304 test->ifobj_rx->use_poll = true; 2305 return testapp_validate_traffic(test); 2306 } 2307 2308 static int testapp_poll_tx(struct test_spec *test) 2309 { 2310 test->ifobj_tx->use_poll = true; 2311 return testapp_validate_traffic(test); 2312 } 2313 2314 static int testapp_aligned_inv_desc(struct test_spec *test) 2315 { 2316 return testapp_invalid_desc(test); 2317 } 2318 2319 static int testapp_aligned_inv_desc_2k_frame(struct test_spec *test) 2320 { 2321 test->ifobj_tx->umem->frame_size = 2048; 2322 test->ifobj_rx->umem->frame_size = 2048; 2323 return testapp_invalid_desc(test); 2324 } 2325 2326 static int testapp_unaligned_inv_desc(struct test_spec *test) 2327 { 2328 test->ifobj_tx->umem->unaligned_mode = true; 2329 test->ifobj_rx->umem->unaligned_mode = true; 2330 return testapp_invalid_desc(test); 2331 } 2332 2333 static int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test) 2334 { 2335 u64 page_size, umem_size; 2336 2337 /* Odd frame size so the UMEM doesn't end near a page boundary. */ 2338 test->ifobj_tx->umem->frame_size = 4001; 2339 test->ifobj_rx->umem->frame_size = 4001; 2340 test->ifobj_tx->umem->unaligned_mode = true; 2341 test->ifobj_rx->umem->unaligned_mode = true; 2342 /* This test exists to test descriptors that staddle the end of 2343 * the UMEM but not a page. 2344 */ 2345 page_size = sysconf(_SC_PAGESIZE); 2346 umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; 2347 assert(umem_size % page_size > MIN_PKT_SIZE); 2348 assert(umem_size % page_size < page_size - MIN_PKT_SIZE); 2349 2350 return testapp_invalid_desc(test); 2351 } 2352 2353 static int testapp_aligned_inv_desc_mb(struct test_spec *test) 2354 { 2355 return testapp_invalid_desc_mb(test); 2356 } 2357 2358 static int testapp_unaligned_inv_desc_mb(struct test_spec *test) 2359 { 2360 test->ifobj_tx->umem->unaligned_mode = true; 2361 test->ifobj_rx->umem->unaligned_mode = true; 2362 return testapp_invalid_desc_mb(test); 2363 } 2364 2365 static int testapp_xdp_metadata(struct test_spec *test) 2366 { 2367 return testapp_xdp_metadata_copy(test); 2368 } 2369 2370 static int testapp_xdp_metadata_mb(struct test_spec *test) 2371 { 2372 test->mtu = MAX_ETH_JUMBO_SIZE; 2373 return testapp_xdp_metadata_copy(test); 2374 } 2375 2376 static void run_pkt_test(struct test_spec *test) 2377 { 2378 int ret; 2379 2380 ret = test->test_func(test); 2381 2382 if (ret == TEST_PASS) 2383 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), 2384 test->name); 2385 pkt_stream_restore_default(test); 2386 } 2387 2388 static struct ifobject *ifobject_create(void) 2389 { 2390 struct ifobject *ifobj; 2391 2392 ifobj = calloc(1, sizeof(struct ifobject)); 2393 if (!ifobj) 2394 return NULL; 2395 2396 ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr)); 2397 if (!ifobj->xsk_arr) 2398 goto out_xsk_arr; 2399 2400 ifobj->umem = calloc(1, sizeof(*ifobj->umem)); 2401 if (!ifobj->umem) 2402 goto out_umem; 2403 2404 return ifobj; 2405 2406 out_umem: 2407 free(ifobj->xsk_arr); 2408 out_xsk_arr: 2409 free(ifobj); 2410 return NULL; 2411 } 2412 2413 static void ifobject_delete(struct ifobject *ifobj) 2414 { 2415 free(ifobj->umem); 2416 free(ifobj->xsk_arr); 2417 free(ifobj); 2418 } 2419 2420 static bool is_xdp_supported(int ifindex) 2421 { 2422 int flags = XDP_FLAGS_DRV_MODE; 2423 2424 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); 2425 struct bpf_insn insns[2] = { 2426 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), 2427 BPF_EXIT_INSN() 2428 }; 2429 int prog_fd, insn_cnt = ARRAY_SIZE(insns); 2430 int err; 2431 2432 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); 2433 if (prog_fd < 0) 2434 return false; 2435 2436 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); 2437 if (err) { 2438 close(prog_fd); 2439 return false; 2440 } 2441 2442 bpf_xdp_detach(ifindex, flags, NULL); 2443 close(prog_fd); 2444 2445 return true; 2446 } 2447 2448 static const struct test_spec tests[] = { 2449 {.name = "SEND_RECEIVE", .test_func = testapp_send_receive}, 2450 {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame}, 2451 {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt}, 2452 {.name = "POLL_RX", .test_func = testapp_poll_rx}, 2453 {.name = "POLL_TX", .test_func = testapp_poll_tx}, 2454 {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout}, 2455 {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout}, 2456 {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned}, 2457 {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc}, 2458 {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame}, 2459 {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc}, 2460 {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE", 2461 .test_func = testapp_unaligned_inv_desc_4001_frame}, 2462 {.name = "UMEM_HEADROOM", .test_func = testapp_headroom}, 2463 {.name = "TEARDOWN", .test_func = testapp_teardown}, 2464 {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional}, 2465 {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped}, 2466 {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs}, 2467 {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full}, 2468 {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty}, 2469 {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup}, 2470 {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop}, 2471 {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem}, 2472 {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata}, 2473 {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb}, 2474 {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb}, 2475 {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS", 2476 .test_func = testapp_send_receive_unaligned_mb}, 2477 {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb}, 2478 {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb}, 2479 {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags}, 2480 }; 2481 2482 static void print_tests(void) 2483 { 2484 u32 i; 2485 2486 printf("Tests:\n"); 2487 for (i = 0; i < ARRAY_SIZE(tests); i++) 2488 printf("%u: %s\n", i, tests[i].name); 2489 } 2490 2491 int main(int argc, char **argv) 2492 { 2493 struct pkt_stream *rx_pkt_stream_default; 2494 struct pkt_stream *tx_pkt_stream_default; 2495 struct ifobject *ifobj_tx, *ifobj_rx; 2496 u32 i, j, failed_tests = 0, nb_tests; 2497 int modes = TEST_MODE_SKB + 1; 2498 struct test_spec test; 2499 bool shared_netdev; 2500 2501 /* Use libbpf 1.0 API mode */ 2502 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 2503 2504 ifobj_tx = ifobject_create(); 2505 if (!ifobj_tx) 2506 exit_with_error(ENOMEM); 2507 ifobj_rx = ifobject_create(); 2508 if (!ifobj_rx) 2509 exit_with_error(ENOMEM); 2510 2511 setlocale(LC_ALL, ""); 2512 2513 parse_command_line(ifobj_tx, ifobj_rx, argc, argv); 2514 2515 if (opt_print_tests) { 2516 print_tests(); 2517 ksft_exit_xpass(); 2518 } 2519 if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= ARRAY_SIZE(tests)) { 2520 ksft_print_msg("Error: test %u does not exist.\n", opt_run_test); 2521 ksft_exit_xfail(); 2522 } 2523 2524 shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex); 2525 ifobj_tx->shared_umem = shared_netdev; 2526 ifobj_rx->shared_umem = shared_netdev; 2527 2528 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) 2529 print_usage(argv); 2530 2531 if (is_xdp_supported(ifobj_tx->ifindex)) { 2532 modes++; 2533 if (ifobj_zc_avail(ifobj_tx)) 2534 modes++; 2535 } 2536 2537 init_iface(ifobj_rx, worker_testapp_validate_rx); 2538 init_iface(ifobj_tx, worker_testapp_validate_tx); 2539 2540 test_spec_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]); 2541 tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); 2542 rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); 2543 if (!tx_pkt_stream_default || !rx_pkt_stream_default) 2544 exit_with_error(ENOMEM); 2545 test.tx_pkt_stream_default = tx_pkt_stream_default; 2546 test.rx_pkt_stream_default = rx_pkt_stream_default; 2547 2548 if (opt_run_test == RUN_ALL_TESTS) 2549 nb_tests = ARRAY_SIZE(tests); 2550 else 2551 nb_tests = 1; 2552 if (opt_mode == TEST_MODE_ALL) { 2553 ksft_set_plan(modes * nb_tests); 2554 } else { 2555 if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) { 2556 ksft_print_msg("Error: XDP_DRV mode not supported.\n"); 2557 ksft_exit_xfail(); 2558 } 2559 if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) { 2560 ksft_print_msg("Error: zero-copy mode not supported.\n"); 2561 ksft_exit_xfail(); 2562 } 2563 2564 ksft_set_plan(nb_tests); 2565 } 2566 2567 for (i = 0; i < modes; i++) { 2568 if (opt_mode != TEST_MODE_ALL && i != opt_mode) 2569 continue; 2570 2571 for (j = 0; j < ARRAY_SIZE(tests); j++) { 2572 if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test) 2573 continue; 2574 2575 test_spec_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]); 2576 run_pkt_test(&test); 2577 usleep(USLEEP_MAX); 2578 2579 if (test.fail) 2580 failed_tests++; 2581 } 2582 } 2583 2584 pkt_stream_delete(tx_pkt_stream_default); 2585 pkt_stream_delete(rx_pkt_stream_default); 2586 xsk_unload_xdp_programs(ifobj_tx); 2587 xsk_unload_xdp_programs(ifobj_rx); 2588 ifobject_delete(ifobj_tx); 2589 ifobject_delete(ifobj_rx); 2590 2591 if (failed_tests) 2592 ksft_exit_fail(); 2593 else 2594 ksft_exit_pass(); 2595 } 2596