1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 /* 5 * Some functions in this program are taken from 6 * Linux kernel samples/bpf/xdpsock* and modified 7 * for use. 8 * 9 * See test_xsk.sh for detailed information on test topology 10 * and prerequisite network setup. 11 * 12 * This test program contains two threads, each thread is single socket with 13 * a unique UMEM. It validates in-order packet delivery and packet content 14 * by sending packets to each other. 15 * 16 * Tests Information: 17 * ------------------ 18 * These selftests test AF_XDP SKB and Native/DRV modes using veth 19 * Virtual Ethernet interfaces. 20 * 21 * For each mode, the following tests are run: 22 * a. nopoll - soft-irq processing in run-to-completion mode 23 * b. poll - using poll() syscall 24 * c. Socket Teardown 25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy 26 * both sockets, then repeat multiple times. Only nopoll mode is used 27 * d. Bi-directional sockets 28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and 29 * completion rings on each socket, tx/rx in both directions. Only nopoll 30 * mode is used 31 * e. Statistics 32 * Trigger some error conditions and ensure that the appropriate statistics 33 * are incremented. Within this test, the following statistics are tested: 34 * i. rx dropped 35 * Increase the UMEM frame headroom to a value which results in 36 * insufficient space in the rx buffer for both the packet and the headroom. 37 * ii. tx invalid 38 * Set the 'len' field of tx descriptors to an invalid value (umem frame 39 * size + 1). 40 * iii. rx ring full 41 * Reduce the size of the RX ring to a fraction of the fill ring size. 42 * iv. fill queue empty 43 * Do not populate the fill queue and then try to receive pkts. 44 * f. bpf_link resource persistence 45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0, 46 * then remove xsk sockets from queue 0 on both veth interfaces and 47 * finally run a traffic on queues ids 1 48 * g. unaligned mode 49 * h. tests for invalid and corner case Tx descriptors so that the correct ones 50 * are discarded and let through, respectively. 51 * i. 2K frame size tests 52 * j. If multi-buffer is supported, send 9k packets divided into 3 frames 53 * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame 54 * using unaligned mode 55 * l. If multi-buffer is supported, try various nasty combinations of descriptors to 56 * check if they pass the validation or not 57 * 58 * Flow: 59 * ----- 60 * - Single process spawns two threads: Tx and Rx 61 * - Each of these two threads attach to a veth interface 62 * - Each thread creates one AF_XDP socket connected to a unique umem for each 63 * veth interface 64 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy> 65 * - Rx thread verifies if all packets were received and delivered in-order, 66 * and have the right content 67 * 68 * Enable/disable packet dump mode: 69 * -------------------------- 70 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add 71 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D") 72 */ 73 74 #define _GNU_SOURCE 75 #include <assert.h> 76 #include <fcntl.h> 77 #include <errno.h> 78 #include <getopt.h> 79 #include <linux/if_link.h> 80 #include <linux/if_ether.h> 81 #include <linux/mman.h> 82 #include <linux/netdev.h> 83 #include <linux/bitmap.h> 84 #include <linux/ethtool.h> 85 #include <arpa/inet.h> 86 #include <net/if.h> 87 #include <locale.h> 88 #include <poll.h> 89 #include <pthread.h> 90 #include <signal.h> 91 #include <stdio.h> 92 #include <stdlib.h> 93 #include <libgen.h> 94 #include <string.h> 95 #include <stddef.h> 96 #include <sys/mman.h> 97 #include <sys/socket.h> 98 #include <sys/time.h> 99 #include <sys/types.h> 100 #include <unistd.h> 101 102 #include "xsk_xdp_progs.skel.h" 103 #include "xsk.h" 104 #include "xskxceiver.h" 105 #include <bpf/bpf.h> 106 #include <linux/filter.h> 107 #include "../kselftest.h" 108 #include "xsk_xdp_common.h" 109 110 #include <network_helpers.h> 111 112 #define MAX_TX_BUDGET_DEFAULT 32 113 114 static bool opt_verbose; 115 static bool opt_print_tests; 116 static enum test_mode opt_mode = TEST_MODE_ALL; 117 static u32 opt_run_test = RUN_ALL_TESTS; 118 119 void test__fail(void) { /* for network_helpers.c */ } 120 121 static void __exit_with_error(int error, const char *file, const char *func, int line) 122 { 123 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, 124 strerror(error)); 125 ksft_exit_xfail(); 126 } 127 128 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) 129 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" 130 static char *mode_string(struct test_spec *test) 131 { 132 switch (test->mode) { 133 case TEST_MODE_SKB: 134 return "SKB"; 135 case TEST_MODE_DRV: 136 return "DRV"; 137 case TEST_MODE_ZC: 138 return "ZC"; 139 default: 140 return "BOGUS"; 141 } 142 } 143 144 static void report_failure(struct test_spec *test) 145 { 146 if (test->fail) 147 return; 148 149 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test), 150 test->name); 151 test->fail = true; 152 } 153 154 /* The payload is a word consisting of a packet sequence number in the upper 155 * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's 156 * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0. 157 */ 158 static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size) 159 { 160 u32 *ptr = (u32 *)dest, i; 161 162 start /= sizeof(*ptr); 163 size /= sizeof(*ptr); 164 for (i = 0; i < size; i++) 165 ptr[i] = htonl(pkt_nb << 16 | (i + start)); 166 } 167 168 static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr) 169 { 170 memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN); 171 memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN); 172 eth_hdr->h_proto = htons(ETH_P_LOOPBACK); 173 } 174 175 static bool is_umem_valid(struct ifobject *ifobj) 176 { 177 return !!ifobj->umem->umem; 178 } 179 180 static u32 mode_to_xdp_flags(enum test_mode mode) 181 { 182 return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; 183 } 184 185 static u64 umem_size(struct xsk_umem_info *umem) 186 { 187 return umem->num_frames * umem->frame_size; 188 } 189 190 static int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, 191 u64 size) 192 { 193 struct xsk_umem_config cfg = { 194 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, 195 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, 196 .frame_size = umem->frame_size, 197 .frame_headroom = umem->frame_headroom, 198 .flags = XSK_UMEM__DEFAULT_FLAGS 199 }; 200 int ret; 201 202 if (umem->fill_size) 203 cfg.fill_size = umem->fill_size; 204 205 if (umem->comp_size) 206 cfg.comp_size = umem->comp_size; 207 208 if (umem->unaligned_mode) 209 cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; 210 211 ret = xsk_umem__create(&umem->umem, buffer, size, 212 &umem->fq, &umem->cq, &cfg); 213 if (ret) 214 return ret; 215 216 umem->buffer = buffer; 217 if (ifobj->shared_umem && ifobj->rx_on) { 218 umem->base_addr = umem_size(umem); 219 umem->next_buffer = umem_size(umem); 220 } 221 222 return 0; 223 } 224 225 static u64 umem_alloc_buffer(struct xsk_umem_info *umem) 226 { 227 u64 addr; 228 229 addr = umem->next_buffer; 230 umem->next_buffer += umem->frame_size; 231 if (umem->next_buffer >= umem->base_addr + umem_size(umem)) 232 umem->next_buffer = umem->base_addr; 233 234 return addr; 235 } 236 237 static void umem_reset_alloc(struct xsk_umem_info *umem) 238 { 239 umem->next_buffer = 0; 240 } 241 242 static void enable_busy_poll(struct xsk_socket_info *xsk) 243 { 244 int sock_opt; 245 246 sock_opt = 1; 247 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL, 248 (void *)&sock_opt, sizeof(sock_opt)) < 0) 249 exit_with_error(errno); 250 251 sock_opt = 20; 252 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL, 253 (void *)&sock_opt, sizeof(sock_opt)) < 0) 254 exit_with_error(errno); 255 256 sock_opt = xsk->batch_size; 257 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET, 258 (void *)&sock_opt, sizeof(sock_opt)) < 0) 259 exit_with_error(errno); 260 } 261 262 static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, 263 struct ifobject *ifobject, bool shared) 264 { 265 struct xsk_socket_config cfg = {}; 266 struct xsk_ring_cons *rxr; 267 struct xsk_ring_prod *txr; 268 269 xsk->umem = umem; 270 cfg.rx_size = xsk->rxqsize; 271 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; 272 cfg.bind_flags = ifobject->bind_flags; 273 if (shared) 274 cfg.bind_flags |= XDP_SHARED_UMEM; 275 if (ifobject->mtu > MAX_ETH_PKT_SIZE) 276 cfg.bind_flags |= XDP_USE_SG; 277 if (umem->comp_size) 278 cfg.tx_size = umem->comp_size; 279 if (umem->fill_size) 280 cfg.rx_size = umem->fill_size; 281 282 txr = ifobject->tx_on ? &xsk->tx : NULL; 283 rxr = ifobject->rx_on ? &xsk->rx : NULL; 284 return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg); 285 } 286 287 static bool ifobj_zc_avail(struct ifobject *ifobject) 288 { 289 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; 290 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; 291 struct xsk_socket_info *xsk; 292 struct xsk_umem_info *umem; 293 bool zc_avail = false; 294 void *bufs; 295 int ret; 296 297 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); 298 if (bufs == MAP_FAILED) 299 exit_with_error(errno); 300 301 umem = calloc(1, sizeof(struct xsk_umem_info)); 302 if (!umem) { 303 munmap(bufs, umem_sz); 304 exit_with_error(ENOMEM); 305 } 306 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; 307 ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz); 308 if (ret) 309 exit_with_error(-ret); 310 311 xsk = calloc(1, sizeof(struct xsk_socket_info)); 312 if (!xsk) 313 goto out; 314 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY; 315 ifobject->rx_on = true; 316 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 317 ret = __xsk_configure_socket(xsk, umem, ifobject, false); 318 if (!ret) 319 zc_avail = true; 320 321 xsk_socket__delete(xsk->xsk); 322 free(xsk); 323 out: 324 munmap(umem->buffer, umem_sz); 325 xsk_umem__delete(umem->umem); 326 free(umem); 327 return zc_avail; 328 } 329 330 #define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags" 331 static unsigned int get_max_skb_frags(void) 332 { 333 unsigned int max_skb_frags = 0; 334 FILE *file; 335 336 file = fopen(MAX_SKB_FRAGS_PATH, "r"); 337 if (!file) { 338 ksft_print_msg("Error opening %s\n", MAX_SKB_FRAGS_PATH); 339 return 0; 340 } 341 342 if (fscanf(file, "%u", &max_skb_frags) != 1) 343 ksft_print_msg("Error reading %s\n", MAX_SKB_FRAGS_PATH); 344 345 fclose(file); 346 return max_skb_frags; 347 } 348 349 static struct option long_options[] = { 350 {"interface", required_argument, 0, 'i'}, 351 {"busy-poll", no_argument, 0, 'b'}, 352 {"verbose", no_argument, 0, 'v'}, 353 {"mode", required_argument, 0, 'm'}, 354 {"list", no_argument, 0, 'l'}, 355 {"test", required_argument, 0, 't'}, 356 {"help", no_argument, 0, 'h'}, 357 {0, 0, 0, 0} 358 }; 359 360 static void print_usage(char **argv) 361 { 362 const char *str = 363 " Usage: xskxceiver [OPTIONS]\n" 364 " Options:\n" 365 " -i, --interface Use interface\n" 366 " -v, --verbose Verbose output\n" 367 " -b, --busy-poll Enable busy poll\n" 368 " -m, --mode Run only mode skb, drv, or zc\n" 369 " -l, --list List all available tests\n" 370 " -t, --test Run a specific test. Enter number from -l option.\n" 371 " -h, --help Display this help and exit\n"; 372 373 ksft_print_msg(str, basename(argv[0])); 374 ksft_exit_xfail(); 375 } 376 377 static bool validate_interface(struct ifobject *ifobj) 378 { 379 if (!strcmp(ifobj->ifname, "")) 380 return false; 381 return true; 382 } 383 384 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc, 385 char **argv) 386 { 387 struct ifobject *ifobj; 388 u32 interface_nb = 0; 389 int option_index, c; 390 391 opterr = 0; 392 393 for (;;) { 394 c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index); 395 if (c == -1) 396 break; 397 398 switch (c) { 399 case 'i': 400 if (interface_nb == 0) 401 ifobj = ifobj_tx; 402 else if (interface_nb == 1) 403 ifobj = ifobj_rx; 404 else 405 break; 406 407 memcpy(ifobj->ifname, optarg, 408 min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg))); 409 410 ifobj->ifindex = if_nametoindex(ifobj->ifname); 411 if (!ifobj->ifindex) 412 exit_with_error(errno); 413 414 interface_nb++; 415 break; 416 case 'v': 417 opt_verbose = true; 418 break; 419 case 'b': 420 ifobj_tx->busy_poll = true; 421 ifobj_rx->busy_poll = true; 422 break; 423 case 'm': 424 if (!strncmp("skb", optarg, strlen(optarg))) 425 opt_mode = TEST_MODE_SKB; 426 else if (!strncmp("drv", optarg, strlen(optarg))) 427 opt_mode = TEST_MODE_DRV; 428 else if (!strncmp("zc", optarg, strlen(optarg))) 429 opt_mode = TEST_MODE_ZC; 430 else 431 print_usage(argv); 432 break; 433 case 'l': 434 opt_print_tests = true; 435 break; 436 case 't': 437 errno = 0; 438 opt_run_test = strtol(optarg, NULL, 0); 439 if (errno) 440 print_usage(argv); 441 break; 442 case 'h': 443 default: 444 print_usage(argv); 445 } 446 } 447 } 448 449 static int set_ring_size(struct ifobject *ifobj) 450 { 451 int ret; 452 u32 ctr = 0; 453 454 while (ctr++ < SOCK_RECONF_CTR) { 455 ret = set_hw_ring_size(ifobj->ifname, &ifobj->ring); 456 if (!ret) 457 break; 458 459 /* Retry if it fails */ 460 if (ctr >= SOCK_RECONF_CTR || errno != EBUSY) 461 return -errno; 462 463 usleep(USLEEP_MAX); 464 } 465 466 return ret; 467 } 468 469 static int hw_ring_size_reset(struct ifobject *ifobj) 470 { 471 ifobj->ring.tx_pending = ifobj->set_ring.default_tx; 472 ifobj->ring.rx_pending = ifobj->set_ring.default_rx; 473 return set_ring_size(ifobj); 474 } 475 476 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, 477 struct ifobject *ifobj_rx) 478 { 479 u32 i, j; 480 481 for (i = 0; i < MAX_INTERFACES; i++) { 482 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; 483 484 ifobj->xsk = &ifobj->xsk_arr[0]; 485 ifobj->use_poll = false; 486 ifobj->use_fill_ring = true; 487 ifobj->release_rx = true; 488 ifobj->validation_func = NULL; 489 ifobj->use_metadata = false; 490 491 if (i == 0) { 492 ifobj->rx_on = false; 493 ifobj->tx_on = true; 494 } else { 495 ifobj->rx_on = true; 496 ifobj->tx_on = false; 497 } 498 499 memset(ifobj->umem, 0, sizeof(*ifobj->umem)); 500 ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; 501 ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; 502 503 for (j = 0; j < MAX_SOCKETS; j++) { 504 memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); 505 ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 506 ifobj->xsk_arr[j].batch_size = DEFAULT_BATCH_SIZE; 507 if (i == 0) 508 ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default; 509 else 510 ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default; 511 512 memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN); 513 memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN); 514 ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0); 515 ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1); 516 } 517 } 518 519 if (ifobj_tx->hw_ring_size_supp) 520 hw_ring_size_reset(ifobj_tx); 521 522 test->ifobj_tx = ifobj_tx; 523 test->ifobj_rx = ifobj_rx; 524 test->current_step = 0; 525 test->total_steps = 1; 526 test->nb_sockets = 1; 527 test->fail = false; 528 test->set_ring = false; 529 test->adjust_tail = false; 530 test->adjust_tail_support = false; 531 test->mtu = MAX_ETH_PKT_SIZE; 532 test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog; 533 test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk; 534 test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog; 535 test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk; 536 } 537 538 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, 539 struct ifobject *ifobj_rx, enum test_mode mode, 540 const struct test_spec *test_to_run) 541 { 542 struct pkt_stream *tx_pkt_stream; 543 struct pkt_stream *rx_pkt_stream; 544 u32 i; 545 546 tx_pkt_stream = test->tx_pkt_stream_default; 547 rx_pkt_stream = test->rx_pkt_stream_default; 548 memset(test, 0, sizeof(*test)); 549 test->tx_pkt_stream_default = tx_pkt_stream; 550 test->rx_pkt_stream_default = rx_pkt_stream; 551 552 for (i = 0; i < MAX_INTERFACES; i++) { 553 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; 554 555 ifobj->bind_flags = XDP_USE_NEED_WAKEUP; 556 if (mode == TEST_MODE_ZC) 557 ifobj->bind_flags |= XDP_ZEROCOPY; 558 else 559 ifobj->bind_flags |= XDP_COPY; 560 } 561 562 strncpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE); 563 test->test_func = test_to_run->test_func; 564 test->mode = mode; 565 __test_spec_init(test, ifobj_tx, ifobj_rx); 566 } 567 568 static void test_spec_reset(struct test_spec *test) 569 { 570 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); 571 } 572 573 static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx, 574 struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx, 575 struct bpf_map *xskmap_tx) 576 { 577 test->xdp_prog_rx = xdp_prog_rx; 578 test->xdp_prog_tx = xdp_prog_tx; 579 test->xskmap_rx = xskmap_rx; 580 test->xskmap_tx = xskmap_tx; 581 } 582 583 static int test_spec_set_mtu(struct test_spec *test, int mtu) 584 { 585 int err; 586 587 if (test->ifobj_rx->mtu != mtu) { 588 err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu); 589 if (err) 590 return err; 591 test->ifobj_rx->mtu = mtu; 592 } 593 if (test->ifobj_tx->mtu != mtu) { 594 err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu); 595 if (err) 596 return err; 597 test->ifobj_tx->mtu = mtu; 598 } 599 600 return 0; 601 } 602 603 static void pkt_stream_reset(struct pkt_stream *pkt_stream) 604 { 605 if (pkt_stream) { 606 pkt_stream->current_pkt_nb = 0; 607 pkt_stream->nb_rx_pkts = 0; 608 } 609 } 610 611 static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream) 612 { 613 if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) 614 return NULL; 615 616 return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; 617 } 618 619 static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent) 620 { 621 while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) { 622 (*pkts_sent)++; 623 if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid) 624 return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; 625 pkt_stream->current_pkt_nb++; 626 } 627 return NULL; 628 } 629 630 static void pkt_stream_delete(struct pkt_stream *pkt_stream) 631 { 632 free(pkt_stream->pkts); 633 free(pkt_stream); 634 } 635 636 static void pkt_stream_restore_default(struct test_spec *test) 637 { 638 struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream; 639 struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream; 640 641 if (tx_pkt_stream != test->tx_pkt_stream_default) { 642 pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream); 643 test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default; 644 } 645 646 if (rx_pkt_stream != test->rx_pkt_stream_default) { 647 pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream); 648 test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default; 649 } 650 } 651 652 static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) 653 { 654 struct pkt_stream *pkt_stream; 655 656 pkt_stream = calloc(1, sizeof(*pkt_stream)); 657 if (!pkt_stream) 658 return NULL; 659 660 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts)); 661 if (!pkt_stream->pkts) { 662 free(pkt_stream); 663 return NULL; 664 } 665 666 pkt_stream->nb_pkts = nb_pkts; 667 return pkt_stream; 668 } 669 670 static bool pkt_continues(u32 options) 671 { 672 return options & XDP_PKT_CONTD; 673 } 674 675 static u32 ceil_u32(u32 a, u32 b) 676 { 677 return (a + b - 1) / b; 678 } 679 680 static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt) 681 { 682 u32 nb_frags = 1, next_frag; 683 684 if (!pkt) 685 return 1; 686 687 if (!pkt_stream->verbatim) { 688 if (!pkt->valid || !pkt->len) 689 return 1; 690 return ceil_u32(pkt->len, frame_size); 691 } 692 693 /* Search for the end of the packet in verbatim mode */ 694 if (!pkt_continues(pkt->options)) 695 return nb_frags; 696 697 next_frag = pkt_stream->current_pkt_nb; 698 pkt++; 699 while (next_frag++ < pkt_stream->nb_pkts) { 700 nb_frags++; 701 if (!pkt_continues(pkt->options) || !pkt->valid) 702 break; 703 pkt++; 704 } 705 return nb_frags; 706 } 707 708 static bool set_pkt_valid(int offset, u32 len) 709 { 710 return len <= MAX_ETH_JUMBO_SIZE; 711 } 712 713 static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) 714 { 715 pkt->offset = offset; 716 pkt->len = len; 717 pkt->valid = set_pkt_valid(offset, len); 718 } 719 720 static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) 721 { 722 bool prev_pkt_valid = pkt->valid; 723 724 pkt_set(pkt_stream, pkt, offset, len); 725 pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid; 726 } 727 728 static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len) 729 { 730 return ceil_u32(len, umem->frame_size) * umem->frame_size; 731 } 732 733 static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off) 734 { 735 struct pkt_stream *pkt_stream; 736 u32 i; 737 738 pkt_stream = __pkt_stream_alloc(nb_pkts); 739 if (!pkt_stream) 740 exit_with_error(ENOMEM); 741 742 pkt_stream->nb_pkts = nb_pkts; 743 pkt_stream->max_pkt_len = pkt_len; 744 for (i = 0; i < nb_pkts; i++) { 745 struct pkt *pkt = &pkt_stream->pkts[i]; 746 747 pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len); 748 pkt->pkt_nb = nb_start + i * nb_off; 749 } 750 751 return pkt_stream; 752 } 753 754 static struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len) 755 { 756 return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1); 757 } 758 759 static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream) 760 { 761 return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len); 762 } 763 764 static void pkt_stream_replace_ifobject(struct ifobject *ifobj, u32 nb_pkts, u32 pkt_len) 765 { 766 ifobj->xsk->pkt_stream = pkt_stream_generate(nb_pkts, pkt_len); 767 } 768 769 static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) 770 { 771 pkt_stream_replace_ifobject(test->ifobj_tx, nb_pkts, pkt_len); 772 pkt_stream_replace_ifobject(test->ifobj_rx, nb_pkts, pkt_len); 773 } 774 775 static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len, 776 int offset) 777 { 778 struct pkt_stream *pkt_stream; 779 u32 i; 780 781 pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream); 782 for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2) 783 pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len); 784 785 ifobj->xsk->pkt_stream = pkt_stream; 786 } 787 788 static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) 789 { 790 __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset); 791 __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset); 792 } 793 794 static void pkt_stream_receive_half(struct test_spec *test) 795 { 796 struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream; 797 u32 i; 798 799 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts, 800 pkt_stream->pkts[0].len); 801 pkt_stream = test->ifobj_rx->xsk->pkt_stream; 802 for (i = 1; i < pkt_stream->nb_pkts; i += 2) 803 pkt_stream->pkts[i].valid = false; 804 805 pkt_stream->nb_valid_entries /= 2; 806 } 807 808 static void pkt_stream_even_odd_sequence(struct test_spec *test) 809 { 810 struct pkt_stream *pkt_stream; 811 u32 i; 812 813 for (i = 0; i < test->nb_sockets; i++) { 814 pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream; 815 pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, 816 pkt_stream->pkts[0].len, i, 2); 817 test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream; 818 819 pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream; 820 pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, 821 pkt_stream->pkts[0].len, i, 2); 822 test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream; 823 } 824 } 825 826 static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem) 827 { 828 if (!pkt->valid) 829 return pkt->offset; 830 return pkt->offset + umem_alloc_buffer(umem); 831 } 832 833 static void pkt_stream_cancel(struct pkt_stream *pkt_stream) 834 { 835 pkt_stream->current_pkt_nb--; 836 } 837 838 static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len, 839 u32 pkt_nb, u32 bytes_written) 840 { 841 void *data = xsk_umem__get_data(umem->buffer, addr); 842 843 if (len < MIN_PKT_SIZE) 844 return; 845 846 if (!bytes_written) { 847 gen_eth_hdr(xsk, data); 848 849 len -= PKT_HDR_SIZE; 850 data += PKT_HDR_SIZE; 851 } else { 852 bytes_written -= PKT_HDR_SIZE; 853 } 854 855 write_payload(data, pkt_nb, bytes_written, len); 856 } 857 858 static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames, 859 u32 nb_frames, bool verbatim) 860 { 861 u32 i, len = 0, pkt_nb = 0, payload = 0; 862 struct pkt_stream *pkt_stream; 863 864 pkt_stream = __pkt_stream_alloc(nb_frames); 865 if (!pkt_stream) 866 exit_with_error(ENOMEM); 867 868 for (i = 0; i < nb_frames; i++) { 869 struct pkt *pkt = &pkt_stream->pkts[pkt_nb]; 870 struct pkt *frame = &frames[i]; 871 872 pkt->offset = frame->offset; 873 if (verbatim) { 874 *pkt = *frame; 875 pkt->pkt_nb = payload; 876 if (!frame->valid || !pkt_continues(frame->options)) 877 payload++; 878 } else { 879 if (frame->valid) 880 len += frame->len; 881 if (frame->valid && pkt_continues(frame->options)) 882 continue; 883 884 pkt->pkt_nb = pkt_nb; 885 pkt->len = len; 886 pkt->valid = frame->valid; 887 pkt->options = 0; 888 889 len = 0; 890 } 891 892 print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n", 893 pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb); 894 895 if (pkt->valid && pkt->len > pkt_stream->max_pkt_len) 896 pkt_stream->max_pkt_len = pkt->len; 897 898 if (pkt->valid) 899 pkt_stream->nb_valid_entries++; 900 901 pkt_nb++; 902 } 903 904 pkt_stream->nb_pkts = pkt_nb; 905 pkt_stream->verbatim = verbatim; 906 return pkt_stream; 907 } 908 909 static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) 910 { 911 struct pkt_stream *pkt_stream; 912 913 pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true); 914 test->ifobj_tx->xsk->pkt_stream = pkt_stream; 915 916 pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false); 917 test->ifobj_rx->xsk->pkt_stream = pkt_stream; 918 } 919 920 static void pkt_print_data(u32 *data, u32 cnt) 921 { 922 u32 i; 923 924 for (i = 0; i < cnt; i++) { 925 u32 seqnum, pkt_nb; 926 927 seqnum = ntohl(*data) & 0xffff; 928 pkt_nb = ntohl(*data) >> 16; 929 ksft_print_msg("%u:%u ", pkt_nb, seqnum); 930 data++; 931 } 932 } 933 934 static void pkt_dump(void *pkt, u32 len, bool eth_header) 935 { 936 struct ethhdr *ethhdr = pkt; 937 u32 i, *data; 938 939 if (eth_header) { 940 /*extract L2 frame */ 941 ksft_print_msg("DEBUG>> L2: dst mac: "); 942 for (i = 0; i < ETH_ALEN; i++) 943 ksft_print_msg("%02X", ethhdr->h_dest[i]); 944 945 ksft_print_msg("\nDEBUG>> L2: src mac: "); 946 for (i = 0; i < ETH_ALEN; i++) 947 ksft_print_msg("%02X", ethhdr->h_source[i]); 948 949 data = pkt + PKT_HDR_SIZE; 950 } else { 951 data = pkt; 952 } 953 954 /*extract L5 frame */ 955 ksft_print_msg("\nDEBUG>> L5: seqnum: "); 956 pkt_print_data(data, PKT_DUMP_NB_TO_PRINT); 957 ksft_print_msg("...."); 958 if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) { 959 ksft_print_msg("\n.... "); 960 pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT, 961 PKT_DUMP_NB_TO_PRINT); 962 } 963 ksft_print_msg("\n---------------------------------------\n"); 964 } 965 966 static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr) 967 { 968 u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom; 969 u32 offset = addr % umem->frame_size, expected_offset; 970 int pkt_offset = pkt->valid ? pkt->offset : 0; 971 972 if (!umem->unaligned_mode) 973 pkt_offset = 0; 974 975 expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size; 976 977 if (offset == expected_offset) 978 return true; 979 980 ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset); 981 return false; 982 } 983 984 static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr) 985 { 986 void *data = xsk_umem__get_data(buffer, addr); 987 struct xdp_info *meta = data - sizeof(struct xdp_info); 988 989 if (meta->count != pkt->pkt_nb) { 990 ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%llu]\n", 991 __func__, pkt->pkt_nb, 992 (unsigned long long)meta->count); 993 return false; 994 } 995 996 return true; 997 } 998 999 static bool is_adjust_tail_supported(struct xsk_xdp_progs *skel_rx) 1000 { 1001 struct bpf_map *data_map; 1002 int adjust_value = 0; 1003 int key = 0; 1004 int ret; 1005 1006 data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss"); 1007 if (!data_map || !bpf_map__is_internal(data_map)) { 1008 ksft_print_msg("Error: could not find bss section of XDP program\n"); 1009 exit_with_error(errno); 1010 } 1011 1012 ret = bpf_map_lookup_elem(bpf_map__fd(data_map), &key, &adjust_value); 1013 if (ret) { 1014 ksft_print_msg("Error: bpf_map_lookup_elem failed with error %d\n", ret); 1015 exit_with_error(errno); 1016 } 1017 1018 /* Set the 'adjust_value' variable to -EOPNOTSUPP in the XDP program if the adjust_tail 1019 * helper is not supported. Skip the adjust_tail test case in this scenario. 1020 */ 1021 return adjust_value != -EOPNOTSUPP; 1022 } 1023 1024 static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb, 1025 u32 bytes_processed) 1026 { 1027 u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum; 1028 void *data = xsk_umem__get_data(umem->buffer, addr); 1029 1030 addr -= umem->base_addr; 1031 1032 if (addr >= umem->num_frames * umem->frame_size || 1033 addr + len > umem->num_frames * umem->frame_size) { 1034 ksft_print_msg("Frag invalid addr: %llx len: %u\n", 1035 (unsigned long long)addr, len); 1036 return false; 1037 } 1038 if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) { 1039 ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n", 1040 (unsigned long long)addr, len); 1041 return false; 1042 } 1043 1044 pkt_data = data; 1045 if (!bytes_processed) { 1046 pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data); 1047 len -= PKT_HDR_SIZE; 1048 } else { 1049 bytes_processed -= PKT_HDR_SIZE; 1050 } 1051 1052 expected_seqnum = bytes_processed / sizeof(*pkt_data); 1053 seqnum = ntohl(*pkt_data) & 0xffff; 1054 pkt_nb = ntohl(*pkt_data) >> 16; 1055 1056 if (expected_pkt_nb != pkt_nb) { 1057 ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n", 1058 __func__, expected_pkt_nb, pkt_nb); 1059 goto error; 1060 } 1061 if (expected_seqnum != seqnum) { 1062 ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n", 1063 __func__, expected_seqnum, seqnum); 1064 goto error; 1065 } 1066 1067 words_to_end = len / sizeof(*pkt_data) - 1; 1068 pkt_data += words_to_end; 1069 seqnum = ntohl(*pkt_data) & 0xffff; 1070 expected_seqnum += words_to_end; 1071 if (expected_seqnum != seqnum) { 1072 ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n", 1073 __func__, expected_seqnum, seqnum); 1074 goto error; 1075 } 1076 1077 return true; 1078 1079 error: 1080 pkt_dump(data, len, !bytes_processed); 1081 return false; 1082 } 1083 1084 static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) 1085 { 1086 if (pkt->len != len) { 1087 ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n", 1088 __func__, pkt->len, len); 1089 pkt_dump(xsk_umem__get_data(buffer, addr), len, true); 1090 return false; 1091 } 1092 1093 return true; 1094 } 1095 1096 static u32 load_value(u32 *counter) 1097 { 1098 return __atomic_load_n(counter, __ATOMIC_ACQUIRE); 1099 } 1100 1101 static bool kick_tx_with_check(struct xsk_socket_info *xsk, int *ret) 1102 { 1103 u32 max_budget = MAX_TX_BUDGET_DEFAULT; 1104 u32 cons, ready_to_send; 1105 int delta; 1106 1107 cons = load_value(xsk->tx.consumer); 1108 ready_to_send = load_value(xsk->tx.producer) - cons; 1109 *ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); 1110 1111 delta = load_value(xsk->tx.consumer) - cons; 1112 /* By default, xsk should consume exact @max_budget descs at one 1113 * send in this case where hitting the max budget limit in while 1114 * loop is triggered in __xsk_generic_xmit(). Please make sure that 1115 * the number of descs to be sent is larger than @max_budget, or 1116 * else the tx.consumer will be updated in xskq_cons_peek_desc() 1117 * in time which hides the issue we try to verify. 1118 */ 1119 if (ready_to_send > max_budget && delta != max_budget) 1120 return false; 1121 1122 return true; 1123 } 1124 1125 static int kick_tx(struct xsk_socket_info *xsk) 1126 { 1127 int ret; 1128 1129 if (xsk->check_consumer) { 1130 if (!kick_tx_with_check(xsk, &ret)) 1131 return TEST_FAILURE; 1132 } else { 1133 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); 1134 } 1135 if (ret >= 0) 1136 return TEST_PASS; 1137 if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) { 1138 usleep(100); 1139 return TEST_PASS; 1140 } 1141 return TEST_FAILURE; 1142 } 1143 1144 static int kick_rx(struct xsk_socket_info *xsk) 1145 { 1146 int ret; 1147 1148 ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); 1149 if (ret < 0) 1150 return TEST_FAILURE; 1151 1152 return TEST_PASS; 1153 } 1154 1155 static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) 1156 { 1157 unsigned int rcvd; 1158 u32 idx; 1159 int ret; 1160 1161 if (xsk_ring_prod__needs_wakeup(&xsk->tx)) { 1162 ret = kick_tx(xsk); 1163 if (ret) 1164 return TEST_FAILURE; 1165 } 1166 1167 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); 1168 if (rcvd) { 1169 if (rcvd > xsk->outstanding_tx) { 1170 u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1); 1171 1172 ksft_print_msg("[%s] Too many packets completed\n", __func__); 1173 ksft_print_msg("Last completion address: %llx\n", 1174 (unsigned long long)addr); 1175 return TEST_FAILURE; 1176 } 1177 1178 xsk_ring_cons__release(&xsk->umem->cq, rcvd); 1179 xsk->outstanding_tx -= rcvd; 1180 } 1181 1182 return TEST_PASS; 1183 } 1184 1185 static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) 1186 { 1187 u32 frags_processed = 0, nb_frags = 0, pkt_len = 0; 1188 u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0; 1189 struct pkt_stream *pkt_stream = xsk->pkt_stream; 1190 struct ifobject *ifobj = test->ifobj_rx; 1191 struct xsk_umem_info *umem = xsk->umem; 1192 struct pollfd fds = { }; 1193 struct pkt *pkt; 1194 u64 first_addr = 0; 1195 int ret; 1196 1197 fds.fd = xsk_socket__fd(xsk->xsk); 1198 fds.events = POLLIN; 1199 1200 ret = kick_rx(xsk); 1201 if (ret) 1202 return TEST_FAILURE; 1203 1204 if (ifobj->use_poll) { 1205 ret = poll(&fds, 1, POLL_TMOUT); 1206 if (ret < 0) 1207 return TEST_FAILURE; 1208 1209 if (!ret) { 1210 if (!is_umem_valid(test->ifobj_tx)) 1211 return TEST_PASS; 1212 1213 ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); 1214 return TEST_CONTINUE; 1215 } 1216 1217 if (!(fds.revents & POLLIN)) 1218 return TEST_CONTINUE; 1219 } 1220 1221 rcvd = xsk_ring_cons__peek(&xsk->rx, xsk->batch_size, &idx_rx); 1222 if (!rcvd) 1223 return TEST_CONTINUE; 1224 1225 if (ifobj->use_fill_ring) { 1226 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); 1227 while (ret != rcvd) { 1228 if (xsk_ring_prod__needs_wakeup(&umem->fq)) { 1229 ret = poll(&fds, 1, POLL_TMOUT); 1230 if (ret < 0) 1231 return TEST_FAILURE; 1232 } 1233 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); 1234 } 1235 } 1236 1237 while (frags_processed < rcvd) { 1238 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++); 1239 u64 addr = desc->addr, orig; 1240 1241 orig = xsk_umem__extract_addr(addr); 1242 addr = xsk_umem__add_offset_to_addr(addr); 1243 1244 if (!nb_frags) { 1245 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); 1246 if (!pkt) { 1247 ksft_print_msg("[%s] received too many packets addr: %lx len %u\n", 1248 __func__, addr, desc->len); 1249 return TEST_FAILURE; 1250 } 1251 } 1252 1253 print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n", 1254 addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid); 1255 1256 if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) || 1257 !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata && 1258 !is_metadata_correct(pkt, umem->buffer, addr))) 1259 return TEST_FAILURE; 1260 1261 if (!nb_frags++) 1262 first_addr = addr; 1263 frags_processed++; 1264 pkt_len += desc->len; 1265 if (ifobj->use_fill_ring) 1266 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig; 1267 1268 if (pkt_continues(desc->options)) 1269 continue; 1270 1271 /* The complete packet has been received */ 1272 if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) || 1273 !is_offset_correct(umem, pkt, addr)) 1274 return TEST_FAILURE; 1275 1276 pkt_stream->nb_rx_pkts++; 1277 nb_frags = 0; 1278 pkt_len = 0; 1279 } 1280 1281 if (nb_frags) { 1282 /* In the middle of a packet. Start over from beginning of packet. */ 1283 idx_rx -= nb_frags; 1284 xsk_ring_cons__cancel(&xsk->rx, nb_frags); 1285 if (ifobj->use_fill_ring) { 1286 idx_fq -= nb_frags; 1287 xsk_ring_prod__cancel(&umem->fq, nb_frags); 1288 } 1289 frags_processed -= nb_frags; 1290 } 1291 1292 if (ifobj->use_fill_ring) 1293 xsk_ring_prod__submit(&umem->fq, frags_processed); 1294 if (ifobj->release_rx) 1295 xsk_ring_cons__release(&xsk->rx, frags_processed); 1296 1297 pthread_mutex_lock(&pacing_mutex); 1298 pkts_in_flight -= pkts_sent; 1299 pthread_mutex_unlock(&pacing_mutex); 1300 pkts_sent = 0; 1301 1302 return TEST_CONTINUE; 1303 } 1304 1305 bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num, 1306 unsigned long *bitmap) 1307 { 1308 struct pkt_stream *pkt_stream = xsk->pkt_stream; 1309 1310 if (!pkt_stream) { 1311 __set_bit(sock_num, bitmap); 1312 return false; 1313 } 1314 1315 if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) { 1316 __set_bit(sock_num, bitmap); 1317 if (bitmap_full(bitmap, test->nb_sockets)) 1318 return true; 1319 } 1320 1321 return false; 1322 } 1323 1324 static int receive_pkts(struct test_spec *test) 1325 { 1326 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; 1327 DECLARE_BITMAP(bitmap, test->nb_sockets); 1328 struct xsk_socket_info *xsk; 1329 u32 sock_num = 0; 1330 int res, ret; 1331 1332 ret = gettimeofday(&tv_now, NULL); 1333 if (ret) 1334 exit_with_error(errno); 1335 1336 timeradd(&tv_now, &tv_timeout, &tv_end); 1337 1338 while (1) { 1339 xsk = &test->ifobj_rx->xsk_arr[sock_num]; 1340 1341 if ((all_packets_received(test, xsk, sock_num, bitmap))) 1342 break; 1343 1344 res = __receive_pkts(test, xsk); 1345 if (!(res == TEST_PASS || res == TEST_CONTINUE)) 1346 return res; 1347 1348 ret = gettimeofday(&tv_now, NULL); 1349 if (ret) 1350 exit_with_error(errno); 1351 1352 if (timercmp(&tv_now, &tv_end, >)) { 1353 ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__); 1354 return TEST_FAILURE; 1355 } 1356 sock_num = (sock_num + 1) % test->nb_sockets; 1357 } 1358 1359 return TEST_PASS; 1360 } 1361 1362 static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout) 1363 { 1364 u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len; 1365 struct pkt_stream *pkt_stream = xsk->pkt_stream; 1366 struct xsk_umem_info *umem = ifobject->umem; 1367 bool use_poll = ifobject->use_poll; 1368 struct pollfd fds = { }; 1369 int ret; 1370 1371 buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len); 1372 /* pkts_in_flight might be negative if many invalid packets are sent */ 1373 if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) / 1374 buffer_len)) { 1375 ret = kick_tx(xsk); 1376 if (ret) 1377 return TEST_FAILURE; 1378 return TEST_CONTINUE; 1379 } 1380 1381 fds.fd = xsk_socket__fd(xsk->xsk); 1382 fds.events = POLLOUT; 1383 1384 while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) { 1385 if (use_poll) { 1386 ret = poll(&fds, 1, POLL_TMOUT); 1387 if (timeout) { 1388 if (ret < 0) { 1389 ksft_print_msg("ERROR: [%s] Poll error %d\n", 1390 __func__, errno); 1391 return TEST_FAILURE; 1392 } 1393 if (ret == 0) 1394 return TEST_PASS; 1395 break; 1396 } 1397 if (ret <= 0) { 1398 ksft_print_msg("ERROR: [%s] Poll error %d\n", 1399 __func__, errno); 1400 return TEST_FAILURE; 1401 } 1402 } 1403 1404 complete_pkts(xsk, xsk->batch_size); 1405 } 1406 1407 for (i = 0; i < xsk->batch_size; i++) { 1408 struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream); 1409 u32 nb_frags_left, nb_frags, bytes_written = 0; 1410 1411 if (!pkt) 1412 break; 1413 1414 nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt); 1415 if (nb_frags > xsk->batch_size - i) { 1416 pkt_stream_cancel(pkt_stream); 1417 xsk_ring_prod__cancel(&xsk->tx, xsk->batch_size - i); 1418 break; 1419 } 1420 nb_frags_left = nb_frags; 1421 1422 while (nb_frags_left--) { 1423 struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); 1424 1425 tx_desc->addr = pkt_get_addr(pkt, ifobject->umem); 1426 if (pkt_stream->verbatim) { 1427 tx_desc->len = pkt->len; 1428 tx_desc->options = pkt->options; 1429 } else if (nb_frags_left) { 1430 tx_desc->len = umem->frame_size; 1431 tx_desc->options = XDP_PKT_CONTD; 1432 } else { 1433 tx_desc->len = pkt->len - bytes_written; 1434 tx_desc->options = 0; 1435 } 1436 if (pkt->valid) 1437 pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb, 1438 bytes_written); 1439 bytes_written += tx_desc->len; 1440 1441 print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n", 1442 tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb); 1443 1444 if (nb_frags_left) { 1445 i++; 1446 if (pkt_stream->verbatim) 1447 pkt = pkt_stream_get_next_tx_pkt(pkt_stream); 1448 } 1449 } 1450 1451 if (pkt && pkt->valid) { 1452 valid_pkts++; 1453 valid_frags += nb_frags; 1454 } 1455 } 1456 1457 pthread_mutex_lock(&pacing_mutex); 1458 pkts_in_flight += valid_pkts; 1459 pthread_mutex_unlock(&pacing_mutex); 1460 1461 xsk_ring_prod__submit(&xsk->tx, i); 1462 xsk->outstanding_tx += valid_frags; 1463 1464 if (use_poll) { 1465 ret = poll(&fds, 1, POLL_TMOUT); 1466 if (ret <= 0) { 1467 if (ret == 0 && timeout) 1468 return TEST_PASS; 1469 1470 ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); 1471 return TEST_FAILURE; 1472 } 1473 } 1474 1475 if (!timeout) { 1476 if (complete_pkts(xsk, i)) 1477 return TEST_FAILURE; 1478 1479 usleep(10); 1480 return TEST_PASS; 1481 } 1482 1483 return TEST_CONTINUE; 1484 } 1485 1486 static int wait_for_tx_completion(struct xsk_socket_info *xsk) 1487 { 1488 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; 1489 int ret; 1490 1491 ret = gettimeofday(&tv_now, NULL); 1492 if (ret) 1493 exit_with_error(errno); 1494 timeradd(&tv_now, &tv_timeout, &tv_end); 1495 1496 while (xsk->outstanding_tx) { 1497 ret = gettimeofday(&tv_now, NULL); 1498 if (ret) 1499 exit_with_error(errno); 1500 if (timercmp(&tv_now, &tv_end, >)) { 1501 ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__); 1502 return TEST_FAILURE; 1503 } 1504 1505 complete_pkts(xsk, xsk->batch_size); 1506 } 1507 1508 return TEST_PASS; 1509 } 1510 1511 bool all_packets_sent(struct test_spec *test, unsigned long *bitmap) 1512 { 1513 return bitmap_full(bitmap, test->nb_sockets); 1514 } 1515 1516 static int send_pkts(struct test_spec *test, struct ifobject *ifobject) 1517 { 1518 bool timeout = !is_umem_valid(test->ifobj_rx); 1519 DECLARE_BITMAP(bitmap, test->nb_sockets); 1520 u32 i, ret; 1521 1522 while (!(all_packets_sent(test, bitmap))) { 1523 for (i = 0; i < test->nb_sockets; i++) { 1524 struct pkt_stream *pkt_stream; 1525 1526 pkt_stream = ifobject->xsk_arr[i].pkt_stream; 1527 if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) { 1528 __set_bit(i, bitmap); 1529 continue; 1530 } 1531 ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout); 1532 if (ret == TEST_CONTINUE && !test->fail) 1533 continue; 1534 1535 if ((ret || test->fail) && !timeout) 1536 return TEST_FAILURE; 1537 1538 if (ret == TEST_PASS && timeout) 1539 return ret; 1540 1541 ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); 1542 if (ret) 1543 return TEST_FAILURE; 1544 } 1545 } 1546 1547 return TEST_PASS; 1548 } 1549 1550 static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats) 1551 { 1552 int fd = xsk_socket__fd(xsk), err; 1553 socklen_t optlen, expected_len; 1554 1555 optlen = sizeof(*stats); 1556 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen); 1557 if (err) { 1558 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", 1559 __func__, -err, strerror(-err)); 1560 return TEST_FAILURE; 1561 } 1562 1563 expected_len = sizeof(struct xdp_statistics); 1564 if (optlen != expected_len) { 1565 ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n", 1566 __func__, expected_len, optlen); 1567 return TEST_FAILURE; 1568 } 1569 1570 return TEST_PASS; 1571 } 1572 1573 static int validate_rx_dropped(struct ifobject *ifobject) 1574 { 1575 struct xsk_socket *xsk = ifobject->xsk->xsk; 1576 struct xdp_statistics stats; 1577 int err; 1578 1579 err = kick_rx(ifobject->xsk); 1580 if (err) 1581 return TEST_FAILURE; 1582 1583 err = get_xsk_stats(xsk, &stats); 1584 if (err) 1585 return TEST_FAILURE; 1586 1587 /* The receiver calls getsockopt after receiving the last (valid) 1588 * packet which is not the final packet sent in this test (valid and 1589 * invalid packets are sent in alternating fashion with the final 1590 * packet being invalid). Since the last packet may or may not have 1591 * been dropped already, both outcomes must be allowed. 1592 */ 1593 if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 || 1594 stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1) 1595 return TEST_PASS; 1596 1597 return TEST_FAILURE; 1598 } 1599 1600 static int validate_rx_full(struct ifobject *ifobject) 1601 { 1602 struct xsk_socket *xsk = ifobject->xsk->xsk; 1603 struct xdp_statistics stats; 1604 int err; 1605 1606 usleep(1000); 1607 err = kick_rx(ifobject->xsk); 1608 if (err) 1609 return TEST_FAILURE; 1610 1611 err = get_xsk_stats(xsk, &stats); 1612 if (err) 1613 return TEST_FAILURE; 1614 1615 if (stats.rx_ring_full) 1616 return TEST_PASS; 1617 1618 return TEST_FAILURE; 1619 } 1620 1621 static int validate_fill_empty(struct ifobject *ifobject) 1622 { 1623 struct xsk_socket *xsk = ifobject->xsk->xsk; 1624 struct xdp_statistics stats; 1625 int err; 1626 1627 usleep(1000); 1628 err = kick_rx(ifobject->xsk); 1629 if (err) 1630 return TEST_FAILURE; 1631 1632 err = get_xsk_stats(xsk, &stats); 1633 if (err) 1634 return TEST_FAILURE; 1635 1636 if (stats.rx_fill_ring_empty_descs) 1637 return TEST_PASS; 1638 1639 return TEST_FAILURE; 1640 } 1641 1642 static int validate_tx_invalid_descs(struct ifobject *ifobject) 1643 { 1644 struct xsk_socket *xsk = ifobject->xsk->xsk; 1645 int fd = xsk_socket__fd(xsk); 1646 struct xdp_statistics stats; 1647 socklen_t optlen; 1648 int err; 1649 1650 optlen = sizeof(stats); 1651 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); 1652 if (err) { 1653 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", 1654 __func__, -err, strerror(-err)); 1655 return TEST_FAILURE; 1656 } 1657 1658 if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) { 1659 ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n", 1660 __func__, 1661 (unsigned long long)stats.tx_invalid_descs, 1662 ifobject->xsk->pkt_stream->nb_pkts); 1663 return TEST_FAILURE; 1664 } 1665 1666 return TEST_PASS; 1667 } 1668 1669 static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject, 1670 struct xsk_umem_info *umem, bool tx) 1671 { 1672 int i, ret; 1673 1674 for (i = 0; i < test->nb_sockets; i++) { 1675 bool shared = (ifobject->shared_umem && tx) ? true : !!i; 1676 u32 ctr = 0; 1677 1678 while (ctr++ < SOCK_RECONF_CTR) { 1679 ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem, 1680 ifobject, shared); 1681 if (!ret) 1682 break; 1683 1684 /* Retry if it fails as xsk_socket__create() is asynchronous */ 1685 if (ctr >= SOCK_RECONF_CTR) 1686 exit_with_error(-ret); 1687 usleep(USLEEP_MAX); 1688 } 1689 if (ifobject->busy_poll) 1690 enable_busy_poll(&ifobject->xsk_arr[i]); 1691 } 1692 } 1693 1694 static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) 1695 { 1696 xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true); 1697 ifobject->xsk = &ifobject->xsk_arr[0]; 1698 ifobject->xskmap = test->ifobj_rx->xskmap; 1699 memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); 1700 ifobject->umem->base_addr = 0; 1701 } 1702 1703 static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, 1704 bool fill_up) 1705 { 1706 u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM; 1707 u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts; 1708 int ret; 1709 1710 if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) 1711 buffers_to_fill = umem->num_frames; 1712 else 1713 buffers_to_fill = umem->fill_size; 1714 1715 ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx); 1716 if (ret != buffers_to_fill) 1717 exit_with_error(ENOSPC); 1718 1719 while (filled < buffers_to_fill) { 1720 struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts); 1721 u64 addr; 1722 u32 i; 1723 1724 for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) { 1725 if (!pkt) { 1726 if (!fill_up) 1727 break; 1728 addr = filled * umem->frame_size + umem->base_addr; 1729 } else if (pkt->offset >= 0) { 1730 addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem); 1731 } else { 1732 addr = pkt->offset + umem_alloc_buffer(umem); 1733 } 1734 1735 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr; 1736 if (++filled >= buffers_to_fill) 1737 break; 1738 } 1739 } 1740 xsk_ring_prod__submit(&umem->fq, filled); 1741 xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled); 1742 1743 pkt_stream_reset(pkt_stream); 1744 umem_reset_alloc(umem); 1745 } 1746 1747 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) 1748 { 1749 u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; 1750 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; 1751 LIBBPF_OPTS(bpf_xdp_query_opts, opts); 1752 void *bufs; 1753 int ret; 1754 u32 i; 1755 1756 if (ifobject->umem->unaligned_mode) 1757 mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB; 1758 1759 if (ifobject->shared_umem) 1760 umem_sz *= 2; 1761 1762 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); 1763 if (bufs == MAP_FAILED) 1764 exit_with_error(errno); 1765 1766 ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz); 1767 if (ret) 1768 exit_with_error(-ret); 1769 1770 xsk_configure_socket(test, ifobject, ifobject->umem, false); 1771 1772 ifobject->xsk = &ifobject->xsk_arr[0]; 1773 1774 if (!ifobject->rx_on) 1775 return; 1776 1777 xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring); 1778 1779 for (i = 0; i < test->nb_sockets; i++) { 1780 ifobject->xsk = &ifobject->xsk_arr[i]; 1781 ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i); 1782 if (ret) 1783 exit_with_error(errno); 1784 } 1785 } 1786 1787 static void *worker_testapp_validate_tx(void *arg) 1788 { 1789 struct test_spec *test = (struct test_spec *)arg; 1790 struct ifobject *ifobject = test->ifobj_tx; 1791 int err; 1792 1793 if (test->current_step == 1) { 1794 if (!ifobject->shared_umem) 1795 thread_common_ops(test, ifobject); 1796 else 1797 thread_common_ops_tx(test, ifobject); 1798 } 1799 1800 err = send_pkts(test, ifobject); 1801 1802 if (!err && ifobject->validation_func) 1803 err = ifobject->validation_func(ifobject); 1804 if (err) 1805 report_failure(test); 1806 1807 pthread_exit(NULL); 1808 } 1809 1810 static void *worker_testapp_validate_rx(void *arg) 1811 { 1812 struct test_spec *test = (struct test_spec *)arg; 1813 struct ifobject *ifobject = test->ifobj_rx; 1814 int err; 1815 1816 if (test->current_step == 1) { 1817 thread_common_ops(test, ifobject); 1818 } else { 1819 xsk_clear_xskmap(ifobject->xskmap); 1820 err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0); 1821 if (err) { 1822 ksft_print_msg("Error: Failed to update xskmap, error %s\n", 1823 strerror(-err)); 1824 exit_with_error(-err); 1825 } 1826 } 1827 1828 pthread_barrier_wait(&barr); 1829 1830 err = receive_pkts(test); 1831 1832 if (!err && ifobject->validation_func) 1833 err = ifobject->validation_func(ifobject); 1834 1835 if (err) { 1836 if (test->adjust_tail && !is_adjust_tail_supported(ifobject->xdp_progs)) 1837 test->adjust_tail_support = false; 1838 else 1839 report_failure(test); 1840 } 1841 1842 pthread_exit(NULL); 1843 } 1844 1845 static u64 ceil_u64(u64 a, u64 b) 1846 { 1847 return (a + b - 1) / b; 1848 } 1849 1850 static void testapp_clean_xsk_umem(struct ifobject *ifobj) 1851 { 1852 u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; 1853 1854 if (ifobj->shared_umem) 1855 umem_sz *= 2; 1856 1857 umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; 1858 xsk_umem__delete(ifobj->umem->umem); 1859 munmap(ifobj->umem->buffer, umem_sz); 1860 } 1861 1862 static void handler(int signum) 1863 { 1864 pthread_exit(NULL); 1865 } 1866 1867 static bool xdp_prog_changed_rx(struct test_spec *test) 1868 { 1869 struct ifobject *ifobj = test->ifobj_rx; 1870 1871 return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode; 1872 } 1873 1874 static bool xdp_prog_changed_tx(struct test_spec *test) 1875 { 1876 struct ifobject *ifobj = test->ifobj_tx; 1877 1878 return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode; 1879 } 1880 1881 static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog, 1882 struct bpf_map *xskmap, enum test_mode mode) 1883 { 1884 int err; 1885 1886 xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode)); 1887 err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode)); 1888 if (err) { 1889 ksft_print_msg("Error attaching XDP program\n"); 1890 exit_with_error(-err); 1891 } 1892 1893 if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC)) 1894 if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) { 1895 ksft_print_msg("ERROR: XDP prog not in DRV mode\n"); 1896 exit_with_error(EINVAL); 1897 } 1898 1899 ifobj->xdp_prog = xdp_prog; 1900 ifobj->xskmap = xskmap; 1901 ifobj->mode = mode; 1902 } 1903 1904 static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx, 1905 struct ifobject *ifobj_tx) 1906 { 1907 if (xdp_prog_changed_rx(test)) 1908 xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode); 1909 1910 if (!ifobj_tx || ifobj_tx->shared_umem) 1911 return; 1912 1913 if (xdp_prog_changed_tx(test)) 1914 xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode); 1915 } 1916 1917 static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1, 1918 struct ifobject *ifobj2) 1919 { 1920 pthread_t t0, t1; 1921 int err; 1922 1923 if (test->mtu > MAX_ETH_PKT_SIZE) { 1924 if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp || 1925 (ifobj2 && !ifobj2->multi_buff_zc_supp))) { 1926 ksft_test_result_skip("Multi buffer for zero-copy not supported.\n"); 1927 return TEST_SKIP; 1928 } 1929 if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp || 1930 (ifobj2 && !ifobj2->multi_buff_supp))) { 1931 ksft_test_result_skip("Multi buffer not supported.\n"); 1932 return TEST_SKIP; 1933 } 1934 } 1935 err = test_spec_set_mtu(test, test->mtu); 1936 if (err) { 1937 ksft_print_msg("Error, could not set mtu.\n"); 1938 exit_with_error(err); 1939 } 1940 1941 if (ifobj2) { 1942 if (pthread_barrier_init(&barr, NULL, 2)) 1943 exit_with_error(errno); 1944 pkt_stream_reset(ifobj2->xsk->pkt_stream); 1945 } 1946 1947 test->current_step++; 1948 pkt_stream_reset(ifobj1->xsk->pkt_stream); 1949 pkts_in_flight = 0; 1950 1951 signal(SIGUSR1, handler); 1952 /*Spawn RX thread */ 1953 pthread_create(&t0, NULL, ifobj1->func_ptr, test); 1954 1955 if (ifobj2) { 1956 pthread_barrier_wait(&barr); 1957 if (pthread_barrier_destroy(&barr)) 1958 exit_with_error(errno); 1959 1960 /*Spawn TX thread */ 1961 pthread_create(&t1, NULL, ifobj2->func_ptr, test); 1962 1963 pthread_join(t1, NULL); 1964 } 1965 1966 if (!ifobj2) 1967 pthread_kill(t0, SIGUSR1); 1968 else 1969 pthread_join(t0, NULL); 1970 1971 if (test->total_steps == test->current_step || test->fail) { 1972 u32 i; 1973 1974 if (ifobj2) 1975 for (i = 0; i < test->nb_sockets; i++) 1976 xsk_socket__delete(ifobj2->xsk_arr[i].xsk); 1977 1978 for (i = 0; i < test->nb_sockets; i++) 1979 xsk_socket__delete(ifobj1->xsk_arr[i].xsk); 1980 1981 testapp_clean_xsk_umem(ifobj1); 1982 if (ifobj2 && !ifobj2->shared_umem) 1983 testapp_clean_xsk_umem(ifobj2); 1984 } 1985 1986 return !!test->fail; 1987 } 1988 1989 static int testapp_validate_traffic(struct test_spec *test) 1990 { 1991 struct ifobject *ifobj_rx = test->ifobj_rx; 1992 struct ifobject *ifobj_tx = test->ifobj_tx; 1993 1994 if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) || 1995 (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) { 1996 ksft_test_result_skip("No huge pages present.\n"); 1997 return TEST_SKIP; 1998 } 1999 2000 if (test->set_ring) { 2001 if (ifobj_tx->hw_ring_size_supp) { 2002 if (set_ring_size(ifobj_tx)) { 2003 ksft_test_result_skip("Failed to change HW ring size.\n"); 2004 return TEST_FAILURE; 2005 } 2006 } else { 2007 ksft_test_result_skip("Changing HW ring size not supported.\n"); 2008 return TEST_SKIP; 2009 } 2010 } 2011 2012 xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx); 2013 return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx); 2014 } 2015 2016 static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj) 2017 { 2018 return __testapp_validate_traffic(test, ifobj, NULL); 2019 } 2020 2021 static int testapp_teardown(struct test_spec *test) 2022 { 2023 int i; 2024 2025 for (i = 0; i < MAX_TEARDOWN_ITER; i++) { 2026 if (testapp_validate_traffic(test)) 2027 return TEST_FAILURE; 2028 test_spec_reset(test); 2029 } 2030 2031 return TEST_PASS; 2032 } 2033 2034 static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2) 2035 { 2036 thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr; 2037 struct ifobject *tmp_ifobj = (*ifobj1); 2038 2039 (*ifobj1)->func_ptr = (*ifobj2)->func_ptr; 2040 (*ifobj2)->func_ptr = tmp_func_ptr; 2041 2042 *ifobj1 = *ifobj2; 2043 *ifobj2 = tmp_ifobj; 2044 } 2045 2046 static int testapp_bidirectional(struct test_spec *test) 2047 { 2048 int res; 2049 2050 test->ifobj_tx->rx_on = true; 2051 test->ifobj_rx->tx_on = true; 2052 test->total_steps = 2; 2053 if (testapp_validate_traffic(test)) 2054 return TEST_FAILURE; 2055 2056 print_verbose("Switching Tx/Rx direction\n"); 2057 swap_directions(&test->ifobj_rx, &test->ifobj_tx); 2058 res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx); 2059 2060 swap_directions(&test->ifobj_rx, &test->ifobj_tx); 2061 return res; 2062 } 2063 2064 static int swap_xsk_resources(struct test_spec *test) 2065 { 2066 int ret; 2067 2068 test->ifobj_tx->xsk_arr[0].pkt_stream = NULL; 2069 test->ifobj_rx->xsk_arr[0].pkt_stream = NULL; 2070 test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default; 2071 test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default; 2072 test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1]; 2073 test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1]; 2074 2075 ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0); 2076 if (ret) 2077 return TEST_FAILURE; 2078 2079 return TEST_PASS; 2080 } 2081 2082 static int testapp_xdp_prog_cleanup(struct test_spec *test) 2083 { 2084 test->total_steps = 2; 2085 test->nb_sockets = 2; 2086 if (testapp_validate_traffic(test)) 2087 return TEST_FAILURE; 2088 2089 if (swap_xsk_resources(test)) 2090 return TEST_FAILURE; 2091 return testapp_validate_traffic(test); 2092 } 2093 2094 static int testapp_headroom(struct test_spec *test) 2095 { 2096 test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; 2097 return testapp_validate_traffic(test); 2098 } 2099 2100 static int testapp_stats_rx_dropped(struct test_spec *test) 2101 { 2102 if (test->mode == TEST_MODE_ZC) { 2103 ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n"); 2104 return TEST_SKIP; 2105 } 2106 2107 pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0); 2108 test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - 2109 XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3; 2110 pkt_stream_receive_half(test); 2111 test->ifobj_rx->validation_func = validate_rx_dropped; 2112 return testapp_validate_traffic(test); 2113 } 2114 2115 static int testapp_stats_tx_invalid_descs(struct test_spec *test) 2116 { 2117 pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0); 2118 test->ifobj_tx->validation_func = validate_tx_invalid_descs; 2119 return testapp_validate_traffic(test); 2120 } 2121 2122 static int testapp_stats_rx_full(struct test_spec *test) 2123 { 2124 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); 2125 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); 2126 2127 test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS; 2128 test->ifobj_rx->release_rx = false; 2129 test->ifobj_rx->validation_func = validate_rx_full; 2130 return testapp_validate_traffic(test); 2131 } 2132 2133 static int testapp_stats_fill_empty(struct test_spec *test) 2134 { 2135 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); 2136 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); 2137 2138 test->ifobj_rx->use_fill_ring = false; 2139 test->ifobj_rx->validation_func = validate_fill_empty; 2140 return testapp_validate_traffic(test); 2141 } 2142 2143 static int testapp_send_receive_unaligned(struct test_spec *test) 2144 { 2145 test->ifobj_tx->umem->unaligned_mode = true; 2146 test->ifobj_rx->umem->unaligned_mode = true; 2147 /* Let half of the packets straddle a 4K buffer boundary */ 2148 pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2); 2149 2150 return testapp_validate_traffic(test); 2151 } 2152 2153 static int testapp_send_receive_unaligned_mb(struct test_spec *test) 2154 { 2155 test->mtu = MAX_ETH_JUMBO_SIZE; 2156 test->ifobj_tx->umem->unaligned_mode = true; 2157 test->ifobj_rx->umem->unaligned_mode = true; 2158 pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); 2159 return testapp_validate_traffic(test); 2160 } 2161 2162 static int testapp_single_pkt(struct test_spec *test) 2163 { 2164 struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}}; 2165 2166 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); 2167 return testapp_validate_traffic(test); 2168 } 2169 2170 static int testapp_send_receive_mb(struct test_spec *test) 2171 { 2172 test->mtu = MAX_ETH_JUMBO_SIZE; 2173 pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); 2174 2175 return testapp_validate_traffic(test); 2176 } 2177 2178 static int testapp_invalid_desc_mb(struct test_spec *test) 2179 { 2180 struct xsk_umem_info *umem = test->ifobj_tx->umem; 2181 u64 umem_size = umem->num_frames * umem->frame_size; 2182 struct pkt pkts[] = { 2183 /* Valid packet for synch to start with */ 2184 {0, MIN_PKT_SIZE, 0, true, 0}, 2185 /* Zero frame len is not legal */ 2186 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2187 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2188 {0, 0, 0, false, 0}, 2189 /* Invalid address in the second frame */ 2190 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2191 {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2192 /* Invalid len in the middle */ 2193 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2194 {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2195 /* Invalid options in the middle */ 2196 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2197 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION}, 2198 /* Transmit 2 frags, receive 3 */ 2199 {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD}, 2200 {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0}, 2201 /* Middle frame crosses chunk boundary with small length */ 2202 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, 2203 {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0}, 2204 /* Valid packet for synch so that something is received */ 2205 {0, MIN_PKT_SIZE, 0, true, 0}}; 2206 2207 if (umem->unaligned_mode) { 2208 /* Crossing a chunk boundary allowed */ 2209 pkts[12].valid = true; 2210 pkts[13].valid = true; 2211 } 2212 2213 test->mtu = MAX_ETH_JUMBO_SIZE; 2214 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); 2215 return testapp_validate_traffic(test); 2216 } 2217 2218 static int testapp_invalid_desc(struct test_spec *test) 2219 { 2220 struct xsk_umem_info *umem = test->ifobj_tx->umem; 2221 u64 umem_size = umem->num_frames * umem->frame_size; 2222 struct pkt pkts[] = { 2223 /* Zero packet address allowed */ 2224 {0, MIN_PKT_SIZE, 0, true}, 2225 /* Allowed packet */ 2226 {0, MIN_PKT_SIZE, 0, true}, 2227 /* Straddling the start of umem */ 2228 {-2, MIN_PKT_SIZE, 0, false}, 2229 /* Packet too large */ 2230 {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, 2231 /* Up to end of umem allowed */ 2232 {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true}, 2233 /* After umem ends */ 2234 {umem_size, MIN_PKT_SIZE, 0, false}, 2235 /* Straddle the end of umem */ 2236 {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, 2237 /* Straddle a 4K boundary */ 2238 {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, 2239 /* Straddle a 2K boundary */ 2240 {0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true}, 2241 /* Valid packet for synch so that something is received */ 2242 {0, MIN_PKT_SIZE, 0, true}}; 2243 2244 if (umem->unaligned_mode) { 2245 /* Crossing a page boundary allowed */ 2246 pkts[7].valid = true; 2247 } 2248 if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) { 2249 /* Crossing a 2K frame size boundary not allowed */ 2250 pkts[8].valid = false; 2251 } 2252 2253 if (test->ifobj_tx->shared_umem) { 2254 pkts[4].offset += umem_size; 2255 pkts[5].offset += umem_size; 2256 pkts[6].offset += umem_size; 2257 } 2258 2259 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); 2260 return testapp_validate_traffic(test); 2261 } 2262 2263 static int testapp_xdp_drop(struct test_spec *test) 2264 { 2265 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; 2266 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; 2267 2268 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop, 2269 skel_rx->maps.xsk, skel_tx->maps.xsk); 2270 2271 pkt_stream_receive_half(test); 2272 return testapp_validate_traffic(test); 2273 } 2274 2275 static int testapp_xdp_metadata_copy(struct test_spec *test) 2276 { 2277 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; 2278 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; 2279 struct bpf_map *data_map; 2280 int count = 0; 2281 int key = 0; 2282 2283 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata, 2284 skel_tx->progs.xsk_xdp_populate_metadata, 2285 skel_rx->maps.xsk, skel_tx->maps.xsk); 2286 test->ifobj_rx->use_metadata = true; 2287 2288 data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss"); 2289 if (!data_map || !bpf_map__is_internal(data_map)) { 2290 ksft_print_msg("Error: could not find bss section of XDP program\n"); 2291 return TEST_FAILURE; 2292 } 2293 2294 if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY)) { 2295 ksft_print_msg("Error: could not update count element\n"); 2296 return TEST_FAILURE; 2297 } 2298 2299 return testapp_validate_traffic(test); 2300 } 2301 2302 static int testapp_xdp_shared_umem(struct test_spec *test) 2303 { 2304 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; 2305 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; 2306 2307 test->total_steps = 1; 2308 test->nb_sockets = 2; 2309 2310 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem, 2311 skel_tx->progs.xsk_xdp_shared_umem, 2312 skel_rx->maps.xsk, skel_tx->maps.xsk); 2313 2314 pkt_stream_even_odd_sequence(test); 2315 2316 return testapp_validate_traffic(test); 2317 } 2318 2319 static int testapp_poll_txq_tmout(struct test_spec *test) 2320 { 2321 test->ifobj_tx->use_poll = true; 2322 /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ 2323 test->ifobj_tx->umem->frame_size = 2048; 2324 pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048); 2325 return testapp_validate_traffic_single_thread(test, test->ifobj_tx); 2326 } 2327 2328 static int testapp_poll_rxq_tmout(struct test_spec *test) 2329 { 2330 test->ifobj_rx->use_poll = true; 2331 return testapp_validate_traffic_single_thread(test, test->ifobj_rx); 2332 } 2333 2334 static int testapp_too_many_frags(struct test_spec *test) 2335 { 2336 struct pkt *pkts; 2337 u32 max_frags, i; 2338 int ret; 2339 2340 if (test->mode == TEST_MODE_ZC) { 2341 max_frags = test->ifobj_tx->xdp_zc_max_segs; 2342 } else { 2343 max_frags = get_max_skb_frags(); 2344 if (!max_frags) { 2345 ksft_print_msg("Couldn't retrieve MAX_SKB_FRAGS from system, using default (17) value\n"); 2346 max_frags = 17; 2347 } 2348 max_frags += 1; 2349 } 2350 2351 pkts = calloc(2 * max_frags + 2, sizeof(struct pkt)); 2352 if (!pkts) 2353 return TEST_FAILURE; 2354 2355 test->mtu = MAX_ETH_JUMBO_SIZE; 2356 2357 /* Valid packet for synch */ 2358 pkts[0].len = MIN_PKT_SIZE; 2359 pkts[0].valid = true; 2360 2361 /* One valid packet with the max amount of frags */ 2362 for (i = 1; i < max_frags + 1; i++) { 2363 pkts[i].len = MIN_PKT_SIZE; 2364 pkts[i].options = XDP_PKT_CONTD; 2365 pkts[i].valid = true; 2366 } 2367 pkts[max_frags].options = 0; 2368 2369 /* An invalid packet with the max amount of frags but signals packet 2370 * continues on the last frag 2371 */ 2372 for (i = max_frags + 1; i < 2 * max_frags + 1; i++) { 2373 pkts[i].len = MIN_PKT_SIZE; 2374 pkts[i].options = XDP_PKT_CONTD; 2375 pkts[i].valid = false; 2376 } 2377 2378 /* Valid packet for synch */ 2379 pkts[2 * max_frags + 1].len = MIN_PKT_SIZE; 2380 pkts[2 * max_frags + 1].valid = true; 2381 2382 pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2); 2383 ret = testapp_validate_traffic(test); 2384 2385 free(pkts); 2386 return ret; 2387 } 2388 2389 static int xsk_load_xdp_programs(struct ifobject *ifobj) 2390 { 2391 ifobj->xdp_progs = xsk_xdp_progs__open_and_load(); 2392 if (libbpf_get_error(ifobj->xdp_progs)) 2393 return libbpf_get_error(ifobj->xdp_progs); 2394 2395 return 0; 2396 } 2397 2398 static void xsk_unload_xdp_programs(struct ifobject *ifobj) 2399 { 2400 xsk_xdp_progs__destroy(ifobj->xdp_progs); 2401 } 2402 2403 /* Simple test */ 2404 static bool hugepages_present(void) 2405 { 2406 size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; 2407 void *bufs; 2408 2409 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 2410 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB); 2411 if (bufs == MAP_FAILED) 2412 return false; 2413 2414 mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; 2415 munmap(bufs, mmap_sz); 2416 return true; 2417 } 2418 2419 static void init_iface(struct ifobject *ifobj, thread_func_t func_ptr) 2420 { 2421 LIBBPF_OPTS(bpf_xdp_query_opts, query_opts); 2422 int err; 2423 2424 ifobj->func_ptr = func_ptr; 2425 2426 err = xsk_load_xdp_programs(ifobj); 2427 if (err) { 2428 ksft_print_msg("Error loading XDP program\n"); 2429 exit_with_error(err); 2430 } 2431 2432 if (hugepages_present()) 2433 ifobj->unaligned_supp = true; 2434 2435 err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts); 2436 if (err) { 2437 ksft_print_msg("Error querying XDP capabilities\n"); 2438 exit_with_error(-err); 2439 } 2440 if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG) 2441 ifobj->multi_buff_supp = true; 2442 if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) { 2443 if (query_opts.xdp_zc_max_segs > 1) { 2444 ifobj->multi_buff_zc_supp = true; 2445 ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs; 2446 } else { 2447 ifobj->xdp_zc_max_segs = 0; 2448 } 2449 } 2450 } 2451 2452 static int testapp_send_receive(struct test_spec *test) 2453 { 2454 return testapp_validate_traffic(test); 2455 } 2456 2457 static int testapp_send_receive_2k_frame(struct test_spec *test) 2458 { 2459 test->ifobj_tx->umem->frame_size = 2048; 2460 test->ifobj_rx->umem->frame_size = 2048; 2461 pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE); 2462 return testapp_validate_traffic(test); 2463 } 2464 2465 static int testapp_poll_rx(struct test_spec *test) 2466 { 2467 test->ifobj_rx->use_poll = true; 2468 return testapp_validate_traffic(test); 2469 } 2470 2471 static int testapp_poll_tx(struct test_spec *test) 2472 { 2473 test->ifobj_tx->use_poll = true; 2474 return testapp_validate_traffic(test); 2475 } 2476 2477 static int testapp_aligned_inv_desc(struct test_spec *test) 2478 { 2479 return testapp_invalid_desc(test); 2480 } 2481 2482 static int testapp_aligned_inv_desc_2k_frame(struct test_spec *test) 2483 { 2484 test->ifobj_tx->umem->frame_size = 2048; 2485 test->ifobj_rx->umem->frame_size = 2048; 2486 return testapp_invalid_desc(test); 2487 } 2488 2489 static int testapp_unaligned_inv_desc(struct test_spec *test) 2490 { 2491 test->ifobj_tx->umem->unaligned_mode = true; 2492 test->ifobj_rx->umem->unaligned_mode = true; 2493 return testapp_invalid_desc(test); 2494 } 2495 2496 static int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test) 2497 { 2498 u64 page_size, umem_size; 2499 2500 /* Odd frame size so the UMEM doesn't end near a page boundary. */ 2501 test->ifobj_tx->umem->frame_size = 4001; 2502 test->ifobj_rx->umem->frame_size = 4001; 2503 test->ifobj_tx->umem->unaligned_mode = true; 2504 test->ifobj_rx->umem->unaligned_mode = true; 2505 /* This test exists to test descriptors that staddle the end of 2506 * the UMEM but not a page. 2507 */ 2508 page_size = sysconf(_SC_PAGESIZE); 2509 umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; 2510 assert(umem_size % page_size > MIN_PKT_SIZE); 2511 assert(umem_size % page_size < page_size - MIN_PKT_SIZE); 2512 2513 return testapp_invalid_desc(test); 2514 } 2515 2516 static int testapp_aligned_inv_desc_mb(struct test_spec *test) 2517 { 2518 return testapp_invalid_desc_mb(test); 2519 } 2520 2521 static int testapp_unaligned_inv_desc_mb(struct test_spec *test) 2522 { 2523 test->ifobj_tx->umem->unaligned_mode = true; 2524 test->ifobj_rx->umem->unaligned_mode = true; 2525 return testapp_invalid_desc_mb(test); 2526 } 2527 2528 static int testapp_xdp_metadata(struct test_spec *test) 2529 { 2530 return testapp_xdp_metadata_copy(test); 2531 } 2532 2533 static int testapp_xdp_metadata_mb(struct test_spec *test) 2534 { 2535 test->mtu = MAX_ETH_JUMBO_SIZE; 2536 return testapp_xdp_metadata_copy(test); 2537 } 2538 2539 static int testapp_hw_sw_min_ring_size(struct test_spec *test) 2540 { 2541 int ret; 2542 2543 test->set_ring = true; 2544 test->total_steps = 2; 2545 test->ifobj_tx->ring.tx_pending = DEFAULT_BATCH_SIZE; 2546 test->ifobj_tx->ring.rx_pending = DEFAULT_BATCH_SIZE * 2; 2547 test->ifobj_tx->xsk->batch_size = 1; 2548 test->ifobj_rx->xsk->batch_size = 1; 2549 ret = testapp_validate_traffic(test); 2550 if (ret) 2551 return ret; 2552 2553 /* Set batch size to hw_ring_size - 1 */ 2554 test->ifobj_tx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1; 2555 test->ifobj_rx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1; 2556 return testapp_validate_traffic(test); 2557 } 2558 2559 static int testapp_hw_sw_max_ring_size(struct test_spec *test) 2560 { 2561 u32 max_descs = XSK_RING_PROD__DEFAULT_NUM_DESCS * 4; 2562 int ret; 2563 2564 test->set_ring = true; 2565 test->total_steps = 2; 2566 test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending; 2567 test->ifobj_tx->ring.rx_pending = test->ifobj_tx->ring.rx_max_pending; 2568 test->ifobj_rx->umem->num_frames = max_descs; 2569 test->ifobj_rx->umem->fill_size = max_descs; 2570 test->ifobj_rx->umem->comp_size = max_descs; 2571 test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; 2572 test->ifobj_rx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; 2573 2574 ret = testapp_validate_traffic(test); 2575 if (ret) 2576 return ret; 2577 2578 /* Set batch_size to 8152 for testing, as the ice HW ignores the 3 lowest bits when 2579 * updating the Rx HW tail register. 2580 */ 2581 test->ifobj_tx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8; 2582 test->ifobj_rx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8; 2583 pkt_stream_replace(test, max_descs, MIN_PKT_SIZE); 2584 return testapp_validate_traffic(test); 2585 } 2586 2587 static int testapp_xdp_adjust_tail(struct test_spec *test, int adjust_value) 2588 { 2589 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; 2590 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; 2591 2592 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_adjust_tail, 2593 skel_tx->progs.xsk_xdp_adjust_tail, 2594 skel_rx->maps.xsk, skel_tx->maps.xsk); 2595 2596 skel_rx->bss->adjust_value = adjust_value; 2597 2598 return testapp_validate_traffic(test); 2599 } 2600 2601 static int testapp_adjust_tail(struct test_spec *test, u32 value, u32 pkt_len) 2602 { 2603 int ret; 2604 2605 test->adjust_tail_support = true; 2606 test->adjust_tail = true; 2607 test->total_steps = 1; 2608 2609 pkt_stream_replace_ifobject(test->ifobj_tx, DEFAULT_BATCH_SIZE, pkt_len); 2610 pkt_stream_replace_ifobject(test->ifobj_rx, DEFAULT_BATCH_SIZE, pkt_len + value); 2611 2612 ret = testapp_xdp_adjust_tail(test, value); 2613 if (ret) 2614 return ret; 2615 2616 if (!test->adjust_tail_support) { 2617 ksft_test_result_skip("%s %sResize pkt with bpf_xdp_adjust_tail() not supported\n", 2618 mode_string(test), busy_poll_string(test)); 2619 return TEST_SKIP; 2620 } 2621 2622 return 0; 2623 } 2624 2625 static int testapp_adjust_tail_shrink(struct test_spec *test) 2626 { 2627 /* Shrink by 4 bytes for testing purpose */ 2628 return testapp_adjust_tail(test, -4, MIN_PKT_SIZE * 2); 2629 } 2630 2631 static int testapp_adjust_tail_shrink_mb(struct test_spec *test) 2632 { 2633 test->mtu = MAX_ETH_JUMBO_SIZE; 2634 /* Shrink by the frag size */ 2635 return testapp_adjust_tail(test, -XSK_UMEM__MAX_FRAME_SIZE, XSK_UMEM__LARGE_FRAME_SIZE * 2); 2636 } 2637 2638 static int testapp_adjust_tail_grow(struct test_spec *test) 2639 { 2640 /* Grow by 4 bytes for testing purpose */ 2641 return testapp_adjust_tail(test, 4, MIN_PKT_SIZE * 2); 2642 } 2643 2644 static int testapp_adjust_tail_grow_mb(struct test_spec *test) 2645 { 2646 test->mtu = MAX_ETH_JUMBO_SIZE; 2647 /* Grow by (frag_size - last_frag_Size) - 1 to stay inside the last fragment */ 2648 return testapp_adjust_tail(test, (XSK_UMEM__MAX_FRAME_SIZE / 2) - 1, 2649 XSK_UMEM__LARGE_FRAME_SIZE * 2); 2650 } 2651 2652 static int testapp_tx_queue_consumer(struct test_spec *test) 2653 { 2654 int nr_packets; 2655 2656 if (test->mode == TEST_MODE_ZC) { 2657 ksft_test_result_skip("Can not run TX_QUEUE_CONSUMER test for ZC mode\n"); 2658 return TEST_SKIP; 2659 } 2660 2661 nr_packets = MAX_TX_BUDGET_DEFAULT + 1; 2662 pkt_stream_replace(test, nr_packets, MIN_PKT_SIZE); 2663 test->ifobj_tx->xsk->batch_size = nr_packets; 2664 test->ifobj_tx->xsk->check_consumer = true; 2665 2666 return testapp_validate_traffic(test); 2667 } 2668 2669 static void run_pkt_test(struct test_spec *test) 2670 { 2671 int ret; 2672 2673 ret = test->test_func(test); 2674 2675 if (ret == TEST_PASS) 2676 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), 2677 test->name); 2678 pkt_stream_restore_default(test); 2679 } 2680 2681 static struct ifobject *ifobject_create(void) 2682 { 2683 struct ifobject *ifobj; 2684 2685 ifobj = calloc(1, sizeof(struct ifobject)); 2686 if (!ifobj) 2687 return NULL; 2688 2689 ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr)); 2690 if (!ifobj->xsk_arr) 2691 goto out_xsk_arr; 2692 2693 ifobj->umem = calloc(1, sizeof(*ifobj->umem)); 2694 if (!ifobj->umem) 2695 goto out_umem; 2696 2697 return ifobj; 2698 2699 out_umem: 2700 free(ifobj->xsk_arr); 2701 out_xsk_arr: 2702 free(ifobj); 2703 return NULL; 2704 } 2705 2706 static void ifobject_delete(struct ifobject *ifobj) 2707 { 2708 free(ifobj->umem); 2709 free(ifobj->xsk_arr); 2710 free(ifobj); 2711 } 2712 2713 static bool is_xdp_supported(int ifindex) 2714 { 2715 int flags = XDP_FLAGS_DRV_MODE; 2716 2717 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); 2718 struct bpf_insn insns[2] = { 2719 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), 2720 BPF_EXIT_INSN() 2721 }; 2722 int prog_fd, insn_cnt = ARRAY_SIZE(insns); 2723 int err; 2724 2725 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); 2726 if (prog_fd < 0) 2727 return false; 2728 2729 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); 2730 if (err) { 2731 close(prog_fd); 2732 return false; 2733 } 2734 2735 bpf_xdp_detach(ifindex, flags, NULL); 2736 close(prog_fd); 2737 2738 return true; 2739 } 2740 2741 static const struct test_spec tests[] = { 2742 {.name = "SEND_RECEIVE", .test_func = testapp_send_receive}, 2743 {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame}, 2744 {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt}, 2745 {.name = "POLL_RX", .test_func = testapp_poll_rx}, 2746 {.name = "POLL_TX", .test_func = testapp_poll_tx}, 2747 {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout}, 2748 {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout}, 2749 {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned}, 2750 {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc}, 2751 {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame}, 2752 {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc}, 2753 {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE", 2754 .test_func = testapp_unaligned_inv_desc_4001_frame}, 2755 {.name = "UMEM_HEADROOM", .test_func = testapp_headroom}, 2756 {.name = "TEARDOWN", .test_func = testapp_teardown}, 2757 {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional}, 2758 {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped}, 2759 {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs}, 2760 {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full}, 2761 {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty}, 2762 {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup}, 2763 {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop}, 2764 {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem}, 2765 {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata}, 2766 {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb}, 2767 {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb}, 2768 {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS", 2769 .test_func = testapp_send_receive_unaligned_mb}, 2770 {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb}, 2771 {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb}, 2772 {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags}, 2773 {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size}, 2774 {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size}, 2775 {.name = "XDP_ADJUST_TAIL_SHRINK", .test_func = testapp_adjust_tail_shrink}, 2776 {.name = "XDP_ADJUST_TAIL_SHRINK_MULTI_BUFF", .test_func = testapp_adjust_tail_shrink_mb}, 2777 {.name = "XDP_ADJUST_TAIL_GROW", .test_func = testapp_adjust_tail_grow}, 2778 {.name = "XDP_ADJUST_TAIL_GROW_MULTI_BUFF", .test_func = testapp_adjust_tail_grow_mb}, 2779 {.name = "TX_QUEUE_CONSUMER", .test_func = testapp_tx_queue_consumer}, 2780 }; 2781 2782 static void print_tests(void) 2783 { 2784 u32 i; 2785 2786 printf("Tests:\n"); 2787 for (i = 0; i < ARRAY_SIZE(tests); i++) 2788 printf("%u: %s\n", i, tests[i].name); 2789 } 2790 2791 int main(int argc, char **argv) 2792 { 2793 struct pkt_stream *rx_pkt_stream_default; 2794 struct pkt_stream *tx_pkt_stream_default; 2795 struct ifobject *ifobj_tx, *ifobj_rx; 2796 u32 i, j, failed_tests = 0, nb_tests; 2797 int modes = TEST_MODE_SKB + 1; 2798 struct test_spec test; 2799 bool shared_netdev; 2800 int ret; 2801 2802 /* Use libbpf 1.0 API mode */ 2803 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 2804 2805 ifobj_tx = ifobject_create(); 2806 if (!ifobj_tx) 2807 exit_with_error(ENOMEM); 2808 ifobj_rx = ifobject_create(); 2809 if (!ifobj_rx) 2810 exit_with_error(ENOMEM); 2811 2812 setlocale(LC_ALL, ""); 2813 2814 parse_command_line(ifobj_tx, ifobj_rx, argc, argv); 2815 2816 if (opt_print_tests) { 2817 print_tests(); 2818 ksft_exit_xpass(); 2819 } 2820 if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= ARRAY_SIZE(tests)) { 2821 ksft_print_msg("Error: test %u does not exist.\n", opt_run_test); 2822 ksft_exit_xfail(); 2823 } 2824 2825 shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex); 2826 ifobj_tx->shared_umem = shared_netdev; 2827 ifobj_rx->shared_umem = shared_netdev; 2828 2829 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) 2830 print_usage(argv); 2831 2832 if (is_xdp_supported(ifobj_tx->ifindex)) { 2833 modes++; 2834 if (ifobj_zc_avail(ifobj_tx)) 2835 modes++; 2836 } 2837 2838 ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring); 2839 if (!ret) { 2840 ifobj_tx->hw_ring_size_supp = true; 2841 ifobj_tx->set_ring.default_tx = ifobj_tx->ring.tx_pending; 2842 ifobj_tx->set_ring.default_rx = ifobj_tx->ring.rx_pending; 2843 } 2844 2845 init_iface(ifobj_rx, worker_testapp_validate_rx); 2846 init_iface(ifobj_tx, worker_testapp_validate_tx); 2847 2848 test_spec_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]); 2849 tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); 2850 rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); 2851 if (!tx_pkt_stream_default || !rx_pkt_stream_default) 2852 exit_with_error(ENOMEM); 2853 test.tx_pkt_stream_default = tx_pkt_stream_default; 2854 test.rx_pkt_stream_default = rx_pkt_stream_default; 2855 2856 if (opt_run_test == RUN_ALL_TESTS) 2857 nb_tests = ARRAY_SIZE(tests); 2858 else 2859 nb_tests = 1; 2860 if (opt_mode == TEST_MODE_ALL) { 2861 ksft_set_plan(modes * nb_tests); 2862 } else { 2863 if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) { 2864 ksft_print_msg("Error: XDP_DRV mode not supported.\n"); 2865 ksft_exit_xfail(); 2866 } 2867 if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) { 2868 ksft_print_msg("Error: zero-copy mode not supported.\n"); 2869 ksft_exit_xfail(); 2870 } 2871 2872 ksft_set_plan(nb_tests); 2873 } 2874 2875 for (i = 0; i < modes; i++) { 2876 if (opt_mode != TEST_MODE_ALL && i != opt_mode) 2877 continue; 2878 2879 for (j = 0; j < ARRAY_SIZE(tests); j++) { 2880 if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test) 2881 continue; 2882 2883 test_spec_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]); 2884 run_pkt_test(&test); 2885 usleep(USLEEP_MAX); 2886 2887 if (test.fail) 2888 failed_tests++; 2889 } 2890 } 2891 2892 if (ifobj_tx->hw_ring_size_supp) 2893 hw_ring_size_reset(ifobj_tx); 2894 2895 pkt_stream_delete(tx_pkt_stream_default); 2896 pkt_stream_delete(rx_pkt_stream_default); 2897 xsk_unload_xdp_programs(ifobj_tx); 2898 xsk_unload_xdp_programs(ifobj_rx); 2899 ifobject_delete(ifobj_tx); 2900 ifobject_delete(ifobj_rx); 2901 2902 if (failed_tests) 2903 ksft_exit_fail(); 2904 else 2905 ksft_exit_pass(); 2906 } 2907