1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <sys/socket.h> 6 #include <sys/ioctl.h> 7 #include <sys/select.h> 8 #include <netinet/in.h> 9 #include <arpa/inet.h> 10 #include <unistd.h> 11 #include <string.h> 12 #include <errno.h> 13 #include <stdbool.h> 14 #include <signal.h> 15 #include <fcntl.h> 16 #include <sys/wait.h> 17 #include <time.h> 18 #include <sched.h> 19 20 #include <sys/time.h> 21 #include <sys/types.h> 22 #include <sys/sendfile.h> 23 24 #include <linux/netlink.h> 25 #include <linux/socket.h> 26 #include <linux/sock_diag.h> 27 #include <linux/bpf.h> 28 #include <linux/if_link.h> 29 #include <linux/tls.h> 30 #include <assert.h> 31 #include <libgen.h> 32 33 #include <getopt.h> 34 35 #include <bpf/bpf.h> 36 #include <bpf/libbpf.h> 37 38 #include "bpf_util.h" 39 #include "cgroup_helpers.h" 40 41 int running; 42 static void running_handler(int a); 43 44 #ifndef TCP_ULP 45 # define TCP_ULP 31 46 #endif 47 #ifndef SOL_TLS 48 # define SOL_TLS 282 49 #endif 50 51 /* randomly selected ports for testing on lo */ 52 #define S1_PORT 10000 53 #define S2_PORT 10001 54 55 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o" 56 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o" 57 #define CG_PATH "/sockmap" 58 59 #define EDATAINTEGRITY 2001 60 61 /* global sockets */ 62 int s1, s2, c1, c2, p1, p2; 63 int test_cnt; 64 int passed; 65 int failed; 66 int map_fd[9]; 67 struct bpf_map *maps[9]; 68 struct bpf_program *progs[9]; 69 struct bpf_link *links[9]; 70 71 int txmsg_pass; 72 int txmsg_redir; 73 int txmsg_drop; 74 int txmsg_apply; 75 int txmsg_cork; 76 int txmsg_start; 77 int txmsg_end; 78 int txmsg_start_push; 79 int txmsg_end_push; 80 int txmsg_start_pop; 81 int txmsg_pop; 82 int txmsg_ingress; 83 int txmsg_redir_skb; 84 int txmsg_ktls_skb; 85 int txmsg_ktls_skb_drop; 86 int txmsg_ktls_skb_redir; 87 int ktls; 88 int peek_flag; 89 int skb_use_parser; 90 int txmsg_omit_skb_parser; 91 int verify_push_start; 92 int verify_push_len; 93 int verify_pop_start; 94 int verify_pop_len; 95 96 static const struct option long_options[] = { 97 {"help", no_argument, NULL, 'h' }, 98 {"cgroup", required_argument, NULL, 'c' }, 99 {"rate", required_argument, NULL, 'r' }, 100 {"verbose", optional_argument, NULL, 'v' }, 101 {"iov_count", required_argument, NULL, 'i' }, 102 {"length", required_argument, NULL, 'l' }, 103 {"test", required_argument, NULL, 't' }, 104 {"data_test", no_argument, NULL, 'd' }, 105 {"txmsg", no_argument, &txmsg_pass, 1 }, 106 {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 107 {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 108 {"txmsg_apply", required_argument, NULL, 'a'}, 109 {"txmsg_cork", required_argument, NULL, 'k'}, 110 {"txmsg_start", required_argument, NULL, 's'}, 111 {"txmsg_end", required_argument, NULL, 'e'}, 112 {"txmsg_start_push", required_argument, NULL, 'p'}, 113 {"txmsg_end_push", required_argument, NULL, 'q'}, 114 {"txmsg_start_pop", required_argument, NULL, 'w'}, 115 {"txmsg_pop", required_argument, NULL, 'x'}, 116 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 117 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 }, 118 {"ktls", no_argument, &ktls, 1 }, 119 {"peek", no_argument, &peek_flag, 1 }, 120 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1}, 121 {"whitelist", required_argument, NULL, 'n' }, 122 {"blacklist", required_argument, NULL, 'b' }, 123 {0, 0, NULL, 0 } 124 }; 125 126 struct test_env { 127 const char *type; 128 const char *subtest; 129 const char *prepend; 130 131 int test_num; 132 int subtest_num; 133 134 int succ_cnt; 135 int fail_cnt; 136 int fail_last; 137 }; 138 139 struct test_env env; 140 141 struct sockmap_options { 142 int verbose; 143 bool base; 144 bool sendpage; 145 bool data_test; 146 bool drop_expected; 147 bool check_recved_len; 148 bool tx_wait_mem; 149 int iov_count; 150 int iov_length; 151 int rate; 152 char *map; 153 char *whitelist; 154 char *blacklist; 155 char *prepend; 156 }; 157 158 struct _test { 159 char *title; 160 void (*tester)(int cg_fd, struct sockmap_options *opt); 161 }; 162 163 static void test_start(void) 164 { 165 env.subtest_num++; 166 } 167 168 static void test_fail(void) 169 { 170 env.fail_cnt++; 171 } 172 173 static void test_pass(void) 174 { 175 env.succ_cnt++; 176 } 177 178 static void test_reset(void) 179 { 180 txmsg_start = txmsg_end = 0; 181 txmsg_start_pop = txmsg_pop = 0; 182 txmsg_start_push = txmsg_end_push = 0; 183 txmsg_pass = txmsg_drop = txmsg_redir = 0; 184 txmsg_apply = txmsg_cork = 0; 185 txmsg_ingress = txmsg_redir_skb = 0; 186 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0; 187 txmsg_omit_skb_parser = 0; 188 skb_use_parser = 0; 189 } 190 191 static int test_start_subtest(const struct _test *t, struct sockmap_options *o) 192 { 193 env.type = o->map; 194 env.subtest = t->title; 195 env.prepend = o->prepend; 196 env.test_num++; 197 env.subtest_num = 0; 198 env.fail_last = env.fail_cnt; 199 test_reset(); 200 return 0; 201 } 202 203 static void test_end_subtest(void) 204 { 205 int error = env.fail_cnt - env.fail_last; 206 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME); 207 208 if (!error) 209 test_pass(); 210 211 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n", 212 env.test_num, env.subtest_num, 213 !type ? "sockmap" : "sockhash", 214 env.prepend ? : "", 215 env.subtest, error ? "FAIL" : "OK"); 216 } 217 218 static void test_print_results(void) 219 { 220 fprintf(stdout, "Pass: %d Fail: %d\n", 221 env.succ_cnt, env.fail_cnt); 222 } 223 224 static void usage(char *argv[]) 225 { 226 int i; 227 228 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 229 printf(" options:\n"); 230 for (i = 0; long_options[i].name != 0; i++) { 231 printf(" --%-12s", long_options[i].name); 232 if (long_options[i].flag != NULL) 233 printf(" flag (internal value:%d)\n", 234 *long_options[i].flag); 235 else 236 printf(" -%c\n", long_options[i].val); 237 } 238 printf("\n"); 239 } 240 241 char *sock_to_string(int s) 242 { 243 if (s == c1) 244 return "client1"; 245 else if (s == c2) 246 return "client2"; 247 else if (s == s1) 248 return "server1"; 249 else if (s == s2) 250 return "server2"; 251 else if (s == p1) 252 return "peer1"; 253 else if (s == p2) 254 return "peer2"; 255 else 256 return "unknown"; 257 } 258 259 static int sockmap_init_ktls(int verbose, int s) 260 { 261 struct tls12_crypto_info_aes_gcm_128 tls_tx = { 262 .info = { 263 .version = TLS_1_2_VERSION, 264 .cipher_type = TLS_CIPHER_AES_GCM_128, 265 }, 266 }; 267 struct tls12_crypto_info_aes_gcm_128 tls_rx = { 268 .info = { 269 .version = TLS_1_2_VERSION, 270 .cipher_type = TLS_CIPHER_AES_GCM_128, 271 }, 272 }; 273 int so_buf = 6553500; 274 int err; 275 276 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls")); 277 if (err) { 278 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err); 279 return -EINVAL; 280 } 281 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); 282 if (err) { 283 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err); 284 return -EINVAL; 285 } 286 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); 287 if (err) { 288 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err); 289 return -EINVAL; 290 } 291 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); 292 if (err) { 293 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err); 294 return -EINVAL; 295 } 296 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); 297 if (err) { 298 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err); 299 return -EINVAL; 300 } 301 302 if (verbose) 303 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s)); 304 return 0; 305 } 306 static int sockmap_init_sockets(int verbose) 307 { 308 int i, err, one = 1; 309 struct sockaddr_in addr; 310 int *fds[4] = {&s1, &s2, &c1, &c2}; 311 312 s1 = s2 = p1 = p2 = c1 = c2 = 0; 313 314 /* Init sockets */ 315 for (i = 0; i < 4; i++) { 316 *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 317 if (*fds[i] < 0) { 318 perror("socket s1 failed()"); 319 return errno; 320 } 321 } 322 323 /* Allow reuse */ 324 for (i = 0; i < 2; i++) { 325 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 326 (char *)&one, sizeof(one)); 327 if (err) { 328 perror("setsockopt failed()"); 329 return errno; 330 } 331 } 332 333 /* Non-blocking sockets */ 334 for (i = 0; i < 2; i++) { 335 err = ioctl(*fds[i], FIONBIO, (char *)&one); 336 if (err < 0) { 337 perror("ioctl s1 failed()"); 338 return errno; 339 } 340 } 341 342 /* Bind server sockets */ 343 memset(&addr, 0, sizeof(struct sockaddr_in)); 344 addr.sin_family = AF_INET; 345 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 346 347 addr.sin_port = htons(S1_PORT); 348 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 349 if (err < 0) { 350 perror("bind s1 failed()"); 351 return errno; 352 } 353 354 addr.sin_port = htons(S2_PORT); 355 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 356 if (err < 0) { 357 perror("bind s2 failed()"); 358 return errno; 359 } 360 361 /* Listen server sockets */ 362 addr.sin_port = htons(S1_PORT); 363 err = listen(s1, 32); 364 if (err < 0) { 365 perror("listen s1 failed()"); 366 return errno; 367 } 368 369 addr.sin_port = htons(S2_PORT); 370 err = listen(s2, 32); 371 if (err < 0) { 372 perror("listen s1 failed()"); 373 return errno; 374 } 375 376 /* Initiate Connect */ 377 addr.sin_port = htons(S1_PORT); 378 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 379 if (err < 0 && errno != EINPROGRESS) { 380 perror("connect c1 failed()"); 381 return errno; 382 } 383 384 addr.sin_port = htons(S2_PORT); 385 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 386 if (err < 0 && errno != EINPROGRESS) { 387 perror("connect c2 failed()"); 388 return errno; 389 } else if (err < 0) { 390 err = 0; 391 } 392 393 /* Accept Connecrtions */ 394 p1 = accept(s1, NULL, NULL); 395 if (p1 < 0) { 396 perror("accept s1 failed()"); 397 return errno; 398 } 399 400 p2 = accept(s2, NULL, NULL); 401 if (p2 < 0) { 402 perror("accept s1 failed()"); 403 return errno; 404 } 405 406 if (verbose > 1) { 407 printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 408 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 409 c1, s1, c2, s2); 410 } 411 return 0; 412 } 413 414 struct msg_stats { 415 size_t bytes_sent; 416 size_t bytes_recvd; 417 struct timespec start; 418 struct timespec end; 419 }; 420 421 static int msg_loop_sendpage(int fd, int iov_length, int cnt, 422 struct msg_stats *s, 423 struct sockmap_options *opt) 424 { 425 bool drop = opt->drop_expected; 426 unsigned char k = 0; 427 int i, j, fp; 428 FILE *file; 429 430 file = tmpfile(); 431 if (!file) { 432 perror("create file for sendpage"); 433 return 1; 434 } 435 for (i = 0; i < cnt; i++, k = 0) { 436 for (j = 0; j < iov_length; j++, k++) 437 fwrite(&k, sizeof(char), 1, file); 438 } 439 fflush(file); 440 fseek(file, 0, SEEK_SET); 441 442 fp = fileno(file); 443 444 clock_gettime(CLOCK_MONOTONIC, &s->start); 445 for (i = 0; i < cnt; i++) { 446 int sent; 447 448 errno = 0; 449 sent = sendfile(fd, fp, NULL, iov_length); 450 451 if (!drop && sent < 0) { 452 perror("sendpage loop error"); 453 fclose(file); 454 return sent; 455 } else if (drop && sent >= 0) { 456 printf("sendpage loop error expected: %i errno %i\n", 457 sent, errno); 458 fclose(file); 459 return -EIO; 460 } 461 462 if (sent > 0) 463 s->bytes_sent += sent; 464 } 465 clock_gettime(CLOCK_MONOTONIC, &s->end); 466 fclose(file); 467 return 0; 468 } 469 470 static void msg_free_iov(struct msghdr *msg) 471 { 472 int i; 473 474 for (i = 0; i < msg->msg_iovlen; i++) 475 free(msg->msg_iov[i].iov_base); 476 free(msg->msg_iov); 477 msg->msg_iov = NULL; 478 msg->msg_iovlen = 0; 479 } 480 481 static int msg_alloc_iov(struct msghdr *msg, 482 int iov_count, int iov_length, 483 bool data, bool xmit) 484 { 485 unsigned char k = 0; 486 struct iovec *iov; 487 int i; 488 489 iov = calloc(iov_count, sizeof(struct iovec)); 490 if (!iov) 491 return errno; 492 493 for (i = 0; i < iov_count; i++) { 494 unsigned char *d = calloc(iov_length, sizeof(char)); 495 496 if (!d) { 497 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 498 goto unwind_iov; 499 } 500 iov[i].iov_base = d; 501 iov[i].iov_len = iov_length; 502 503 if (data && xmit) { 504 int j; 505 506 for (j = 0; j < iov_length; j++) 507 d[j] = k++; 508 } 509 } 510 511 msg->msg_iov = iov; 512 msg->msg_iovlen = iov_count; 513 514 return 0; 515 unwind_iov: 516 for (i--; i >= 0 ; i--) 517 free(msg->msg_iov[i].iov_base); 518 return -ENOMEM; 519 } 520 521 /* In push or pop test, we need to do some calculations for msg_verify_data */ 522 static void msg_verify_date_prep(void) 523 { 524 int push_range_end = txmsg_start_push + txmsg_end_push - 1; 525 int pop_range_end = txmsg_start_pop + txmsg_pop - 1; 526 527 if (txmsg_end_push && txmsg_pop && 528 txmsg_start_push <= pop_range_end && txmsg_start_pop <= push_range_end) { 529 /* The push range and the pop range overlap */ 530 int overlap_len; 531 532 verify_push_start = txmsg_start_push; 533 verify_pop_start = txmsg_start_pop; 534 if (txmsg_start_push < txmsg_start_pop) 535 overlap_len = min(push_range_end - txmsg_start_pop + 1, txmsg_pop); 536 else 537 overlap_len = min(pop_range_end - txmsg_start_push + 1, txmsg_end_push); 538 verify_push_len = max(txmsg_end_push - overlap_len, 0); 539 verify_pop_len = max(txmsg_pop - overlap_len, 0); 540 } else { 541 /* Otherwise */ 542 verify_push_start = txmsg_start_push; 543 verify_pop_start = txmsg_start_pop; 544 verify_push_len = txmsg_end_push; 545 verify_pop_len = txmsg_pop; 546 } 547 } 548 549 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz, 550 unsigned char *k_p, int *bytes_cnt_p, 551 int *check_cnt_p, int *push_p) 552 { 553 int bytes_cnt = *bytes_cnt_p, check_cnt = *check_cnt_p, push = *push_p; 554 unsigned char k = *k_p; 555 int i, j; 556 557 for (i = 0, j = 0; i < msg->msg_iovlen && size; i++, j = 0) { 558 unsigned char *d = msg->msg_iov[i].iov_base; 559 560 /* Special case test for skb ingress + ktls */ 561 if (i == 0 && txmsg_ktls_skb) { 562 if (msg->msg_iov[i].iov_len < 4) 563 return -EDATAINTEGRITY; 564 if (memcmp(d, "PASS", 4) != 0) { 565 fprintf(stderr, 566 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", 567 i, 0, d[0], d[1], d[2], d[3]); 568 return -EDATAINTEGRITY; 569 } 570 j = 4; /* advance index past PASS header */ 571 } 572 573 for (; j < msg->msg_iov[i].iov_len && size; j++) { 574 if (push > 0 && 575 check_cnt == verify_push_start + verify_push_len - push) { 576 int skipped; 577 revisit_push: 578 skipped = push; 579 if (j + push >= msg->msg_iov[i].iov_len) 580 skipped = msg->msg_iov[i].iov_len - j; 581 push -= skipped; 582 size -= skipped; 583 j += skipped - 1; 584 check_cnt += skipped; 585 continue; 586 } 587 588 if (verify_pop_len > 0 && check_cnt == verify_pop_start) { 589 bytes_cnt += verify_pop_len; 590 check_cnt += verify_pop_len; 591 k += verify_pop_len; 592 593 if (bytes_cnt == chunk_sz) { 594 k = 0; 595 bytes_cnt = 0; 596 check_cnt = 0; 597 push = verify_push_len; 598 } 599 600 if (push > 0 && 601 check_cnt == verify_push_start + verify_push_len - push) 602 goto revisit_push; 603 } 604 605 if (d[j] != k++) { 606 fprintf(stderr, 607 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 608 i, j, d[j], k - 1, d[j+1], k); 609 return -EDATAINTEGRITY; 610 } 611 bytes_cnt++; 612 check_cnt++; 613 if (bytes_cnt == chunk_sz) { 614 k = 0; 615 bytes_cnt = 0; 616 check_cnt = 0; 617 push = verify_push_len; 618 } 619 size--; 620 } 621 } 622 *k_p = k; 623 *bytes_cnt_p = bytes_cnt; 624 *check_cnt_p = check_cnt; 625 *push_p = push; 626 return 0; 627 } 628 629 static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 630 struct msg_stats *s, bool tx, 631 struct sockmap_options *opt) 632 { 633 struct msghdr msg = {0}, msg_peek = {0}; 634 int err, i, flags = MSG_NOSIGNAL; 635 bool drop = opt->drop_expected; 636 bool data = opt->data_test; 637 int iov_alloc_length = iov_length; 638 639 if (!tx && opt->check_recved_len) 640 iov_alloc_length *= 2; 641 642 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx); 643 if (err) 644 goto out_errno; 645 if (peek_flag) { 646 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx); 647 if (err) 648 goto out_errno; 649 } 650 651 if (tx) { 652 clock_gettime(CLOCK_MONOTONIC, &s->start); 653 for (i = 0; i < cnt; i++) { 654 int sent; 655 656 errno = 0; 657 sent = sendmsg(fd, &msg, flags); 658 659 if (!drop && sent < 0) { 660 if (opt->tx_wait_mem && errno == EACCES) { 661 errno = 0; 662 goto out_errno; 663 } 664 perror("sendmsg loop error"); 665 goto out_errno; 666 } else if (drop && sent >= 0) { 667 fprintf(stderr, 668 "sendmsg loop error expected: %i errno %i\n", 669 sent, errno); 670 errno = -EIO; 671 goto out_errno; 672 } 673 if (sent > 0) 674 s->bytes_sent += sent; 675 } 676 clock_gettime(CLOCK_MONOTONIC, &s->end); 677 } else { 678 float total_bytes, txmsg_pop_total, txmsg_push_total; 679 int slct, recvp = 0, recv, max_fd = fd; 680 int fd_flags = O_NONBLOCK; 681 struct timeval timeout; 682 unsigned char k = 0; 683 int bytes_cnt = 0; 684 int check_cnt = 0; 685 int push = 0; 686 fd_set w; 687 688 fcntl(fd, fd_flags); 689 /* Account for pop bytes noting each iteration of apply will 690 * call msg_pop_data helper so we need to account for this 691 * by calculating the number of apply iterations. Note user 692 * of the tool can create cases where no data is sent by 693 * manipulating pop/push/pull/etc. For example txmsg_apply 1 694 * with txmsg_pop 1 will try to apply 1B at a time but each 695 * iteration will then pop 1B so no data will ever be sent. 696 * This is really only useful for testing edge cases in code 697 * paths. 698 */ 699 total_bytes = (float)iov_length * (float)cnt; 700 if (!opt->sendpage) 701 total_bytes *= (float)iov_count; 702 if (txmsg_apply) { 703 txmsg_push_total = txmsg_end_push * (total_bytes / txmsg_apply); 704 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply); 705 } else { 706 txmsg_push_total = txmsg_end_push * cnt; 707 txmsg_pop_total = txmsg_pop * cnt; 708 } 709 total_bytes += txmsg_push_total; 710 total_bytes -= txmsg_pop_total; 711 if (data) { 712 msg_verify_date_prep(); 713 push = verify_push_len; 714 } 715 err = clock_gettime(CLOCK_MONOTONIC, &s->start); 716 if (err < 0) 717 perror("recv start time"); 718 while (s->bytes_recvd < total_bytes) { 719 if (txmsg_cork) { 720 timeout.tv_sec = 0; 721 timeout.tv_usec = 300000; 722 } else { 723 timeout.tv_sec = 3; 724 timeout.tv_usec = 0; 725 } 726 727 /* FD sets */ 728 FD_ZERO(&w); 729 FD_SET(fd, &w); 730 731 slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 732 if (slct == -1) { 733 perror("select()"); 734 clock_gettime(CLOCK_MONOTONIC, &s->end); 735 goto out_errno; 736 } else if (!slct) { 737 if (opt->verbose) 738 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total); 739 errno = -EIO; 740 clock_gettime(CLOCK_MONOTONIC, &s->end); 741 goto out_errno; 742 } 743 744 if (opt->tx_wait_mem) { 745 FD_ZERO(&w); 746 FD_SET(fd, &w); 747 slct = select(max_fd + 1, NULL, NULL, &w, &timeout); 748 errno = 0; 749 close(fd); 750 goto out_errno; 751 } 752 753 errno = 0; 754 if (peek_flag) { 755 flags |= MSG_PEEK; 756 recvp = recvmsg(fd, &msg_peek, flags); 757 if (recvp < 0) { 758 if (errno != EWOULDBLOCK) { 759 clock_gettime(CLOCK_MONOTONIC, &s->end); 760 goto out_errno; 761 } 762 } 763 flags = 0; 764 } 765 766 recv = recvmsg(fd, &msg, flags); 767 if (recv < 0) { 768 if (errno != EWOULDBLOCK) { 769 clock_gettime(CLOCK_MONOTONIC, &s->end); 770 perror("recv failed()"); 771 goto out_errno; 772 } 773 } 774 775 if (recv > 0) 776 s->bytes_recvd += recv; 777 778 if (opt->check_recved_len && s->bytes_recvd > total_bytes) { 779 errno = EMSGSIZE; 780 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n", 781 s->bytes_recvd, total_bytes); 782 goto out_errno; 783 } 784 785 if (data) { 786 int chunk_sz = opt->sendpage ? 787 iov_length : 788 iov_length * iov_count; 789 790 errno = msg_verify_data(&msg, recv, chunk_sz, &k, &bytes_cnt, 791 &check_cnt, &push); 792 if (errno) { 793 perror("data verify msg failed"); 794 goto out_errno; 795 } 796 if (recvp) { 797 errno = msg_verify_data(&msg_peek, 798 recvp, 799 chunk_sz, 800 &k, 801 &bytes_cnt, 802 &check_cnt, 803 &push); 804 if (errno) { 805 perror("data verify msg_peek failed"); 806 goto out_errno; 807 } 808 } 809 } 810 } 811 clock_gettime(CLOCK_MONOTONIC, &s->end); 812 } 813 814 msg_free_iov(&msg); 815 msg_free_iov(&msg_peek); 816 return err; 817 out_errno: 818 msg_free_iov(&msg); 819 msg_free_iov(&msg_peek); 820 return errno; 821 } 822 823 static float giga = 1000000000; 824 825 static inline float sentBps(struct msg_stats s) 826 { 827 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 828 } 829 830 static inline float recvdBps(struct msg_stats s) 831 { 832 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 833 } 834 835 static int sendmsg_test(struct sockmap_options *opt) 836 { 837 float sent_Bps = 0, recvd_Bps = 0; 838 int rx_fd, txpid, rxpid, err = 0; 839 struct msg_stats s = {0}; 840 int iov_count = opt->iov_count; 841 int iov_buf = opt->iov_length; 842 int rx_status, tx_status; 843 int cnt = opt->rate; 844 845 errno = 0; 846 847 if (opt->base) 848 rx_fd = p1; 849 else 850 rx_fd = p2; 851 852 if (ktls) { 853 /* Redirecting into non-TLS socket which sends into a TLS 854 * socket is not a valid test. So in this case lets not 855 * enable kTLS but still run the test. 856 */ 857 if (!txmsg_redir || txmsg_ingress) { 858 err = sockmap_init_ktls(opt->verbose, rx_fd); 859 if (err) 860 return err; 861 } 862 err = sockmap_init_ktls(opt->verbose, c1); 863 if (err) 864 return err; 865 } 866 867 if (opt->tx_wait_mem) { 868 struct timeval timeout; 869 int rxtx_buf_len = 1024; 870 871 timeout.tv_sec = 3; 872 timeout.tv_usec = 0; 873 874 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval)); 875 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int)); 876 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int)); 877 if (err) { 878 perror("setsockopt failed()"); 879 return errno; 880 } 881 } 882 883 rxpid = fork(); 884 if (rxpid == 0) { 885 if (opt->drop_expected || txmsg_ktls_skb_drop) 886 _exit(0); 887 888 if (!iov_buf) /* zero bytes sent case */ 889 _exit(0); 890 891 if (opt->sendpage) 892 iov_count = 1; 893 err = msg_loop(rx_fd, iov_count, iov_buf, 894 cnt, &s, false, opt); 895 if (opt->verbose > 1) 896 fprintf(stderr, 897 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 898 iov_count, iov_buf, cnt, err); 899 if (s.end.tv_sec - s.start.tv_sec) { 900 sent_Bps = sentBps(s); 901 recvd_Bps = recvdBps(s); 902 } 903 if (opt->verbose > 1) 904 fprintf(stdout, 905 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n", 906 s.bytes_sent, sent_Bps, sent_Bps/giga, 907 s.bytes_recvd, recvd_Bps, recvd_Bps/giga, 908 peek_flag ? "(peek_msg)" : ""); 909 if (err && err != -EDATAINTEGRITY && txmsg_cork) 910 err = 0; 911 exit(err ? 1 : 0); 912 } else if (rxpid == -1) { 913 perror("msg_loop_rx"); 914 return errno; 915 } 916 917 if (opt->tx_wait_mem) 918 close(c2); 919 920 txpid = fork(); 921 if (txpid == 0) { 922 if (opt->sendpage) 923 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 924 else 925 err = msg_loop(c1, iov_count, iov_buf, 926 cnt, &s, true, opt); 927 928 if (err) 929 fprintf(stderr, 930 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 931 iov_count, iov_buf, cnt, err); 932 if (s.end.tv_sec - s.start.tv_sec) { 933 sent_Bps = sentBps(s); 934 recvd_Bps = recvdBps(s); 935 } 936 if (opt->verbose > 1) 937 fprintf(stdout, 938 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 939 s.bytes_sent, sent_Bps, sent_Bps/giga, 940 s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 941 exit(err ? 1 : 0); 942 } else if (txpid == -1) { 943 perror("msg_loop_tx"); 944 return errno; 945 } 946 947 assert(waitpid(rxpid, &rx_status, 0) == rxpid); 948 assert(waitpid(txpid, &tx_status, 0) == txpid); 949 if (WIFEXITED(rx_status)) { 950 err = WEXITSTATUS(rx_status); 951 if (err) { 952 fprintf(stderr, "rx thread exited with err %d.\n", err); 953 goto out; 954 } 955 } 956 if (WIFEXITED(tx_status)) { 957 err = WEXITSTATUS(tx_status); 958 if (err) 959 fprintf(stderr, "tx thread exited with err %d.\n", err); 960 } 961 out: 962 return err; 963 } 964 965 static int forever_ping_pong(int rate, struct sockmap_options *opt) 966 { 967 struct timeval timeout; 968 char buf[1024] = {0}; 969 int sc; 970 971 timeout.tv_sec = 10; 972 timeout.tv_usec = 0; 973 974 /* Ping/Pong data from client to server */ 975 sc = send(c1, buf, sizeof(buf), 0); 976 if (sc < 0) { 977 perror("send failed()"); 978 return sc; 979 } 980 981 do { 982 int s, rc, i, max_fd = p2; 983 fd_set w; 984 985 /* FD sets */ 986 FD_ZERO(&w); 987 FD_SET(c1, &w); 988 FD_SET(c2, &w); 989 FD_SET(p1, &w); 990 FD_SET(p2, &w); 991 992 s = select(max_fd + 1, &w, NULL, NULL, &timeout); 993 if (s == -1) { 994 perror("select()"); 995 break; 996 } else if (!s) { 997 fprintf(stderr, "unexpected timeout\n"); 998 break; 999 } 1000 1001 for (i = 0; i <= max_fd && s > 0; ++i) { 1002 if (!FD_ISSET(i, &w)) 1003 continue; 1004 1005 s--; 1006 1007 rc = recv(i, buf, sizeof(buf), 0); 1008 if (rc < 0) { 1009 if (errno != EWOULDBLOCK) { 1010 perror("recv failed()"); 1011 return rc; 1012 } 1013 } 1014 1015 if (rc == 0) { 1016 close(i); 1017 break; 1018 } 1019 1020 sc = send(i, buf, rc, 0); 1021 if (sc < 0) { 1022 perror("send failed()"); 1023 return sc; 1024 } 1025 } 1026 1027 if (rate) 1028 sleep(rate); 1029 1030 if (opt->verbose) { 1031 printf("."); 1032 fflush(stdout); 1033 1034 } 1035 } while (running); 1036 1037 return 0; 1038 } 1039 1040 enum { 1041 SELFTESTS, 1042 PING_PONG, 1043 SENDMSG, 1044 BASE, 1045 BASE_SENDPAGE, 1046 SENDPAGE, 1047 }; 1048 1049 static int run_options(struct sockmap_options *options, int cg_fd, int test) 1050 { 1051 int i, key, next_key, err, zero = 0; 1052 struct bpf_program *tx_prog; 1053 1054 /* If base test skip BPF setup */ 1055 if (test == BASE || test == BASE_SENDPAGE) 1056 goto run; 1057 1058 /* Attach programs to sockmap */ 1059 if (!txmsg_omit_skb_parser) { 1060 links[0] = bpf_program__attach_sockmap(progs[0], map_fd[0]); 1061 if (!links[0]) { 1062 fprintf(stderr, 1063 "ERROR: bpf_program__attach_sockmap (sockmap %i->%i): (%s)\n", 1064 bpf_program__fd(progs[0]), map_fd[0], strerror(errno)); 1065 return -1; 1066 } 1067 } 1068 1069 links[1] = bpf_program__attach_sockmap(progs[1], map_fd[0]); 1070 if (!links[1]) { 1071 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (sockmap): (%s)\n", 1072 strerror(errno)); 1073 return -1; 1074 } 1075 1076 /* Attach programs to TLS sockmap */ 1077 if (txmsg_ktls_skb) { 1078 if (!txmsg_omit_skb_parser) { 1079 links[2] = bpf_program__attach_sockmap(progs[0], map_fd[8]); 1080 if (!links[2]) { 1081 fprintf(stderr, 1082 "ERROR: bpf_program__attach_sockmap (TLS sockmap %i->%i): (%s)\n", 1083 bpf_program__fd(progs[0]), map_fd[8], strerror(errno)); 1084 return -1; 1085 } 1086 } 1087 1088 links[3] = bpf_program__attach_sockmap(progs[2], map_fd[8]); 1089 if (!links[3]) { 1090 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (TLS sockmap): (%s)\n", 1091 strerror(errno)); 1092 return -1; 1093 } 1094 } 1095 1096 /* Attach to cgroups */ 1097 err = bpf_prog_attach(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS, 0); 1098 if (err) { 1099 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 1100 err, strerror(errno)); 1101 return err; 1102 } 1103 1104 run: 1105 err = sockmap_init_sockets(options->verbose); 1106 if (err) { 1107 fprintf(stderr, "ERROR: test socket failed: %d\n", err); 1108 goto out; 1109 } 1110 1111 /* Attach txmsg program to sockmap */ 1112 if (txmsg_pass) 1113 tx_prog = progs[4]; 1114 else if (txmsg_redir) 1115 tx_prog = progs[5]; 1116 else if (txmsg_apply) 1117 tx_prog = progs[6]; 1118 else if (txmsg_cork) 1119 tx_prog = progs[7]; 1120 else if (txmsg_drop) 1121 tx_prog = progs[8]; 1122 else 1123 tx_prog = NULL; 1124 1125 if (tx_prog) { 1126 int redir_fd; 1127 1128 links[4] = bpf_program__attach_sockmap(tx_prog, map_fd[1]); 1129 if (!links[4]) { 1130 fprintf(stderr, 1131 "ERROR: bpf_program__attach_sockmap (txmsg): (%s)\n", 1132 strerror(errno)); 1133 err = -1; 1134 goto out; 1135 } 1136 1137 i = 0; 1138 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 1139 if (err) { 1140 fprintf(stderr, 1141 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 1142 err, strerror(errno)); 1143 goto out; 1144 } 1145 1146 if (txmsg_redir) 1147 redir_fd = c2; 1148 else 1149 redir_fd = c1; 1150 1151 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 1152 if (err) { 1153 fprintf(stderr, 1154 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 1155 err, strerror(errno)); 1156 goto out; 1157 } 1158 1159 if (txmsg_apply) { 1160 err = bpf_map_update_elem(map_fd[3], 1161 &i, &txmsg_apply, BPF_ANY); 1162 if (err) { 1163 fprintf(stderr, 1164 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 1165 err, strerror(errno)); 1166 goto out; 1167 } 1168 } 1169 1170 if (txmsg_cork) { 1171 err = bpf_map_update_elem(map_fd[4], 1172 &i, &txmsg_cork, BPF_ANY); 1173 if (err) { 1174 fprintf(stderr, 1175 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 1176 err, strerror(errno)); 1177 goto out; 1178 } 1179 } 1180 1181 if (txmsg_start) { 1182 err = bpf_map_update_elem(map_fd[5], 1183 &i, &txmsg_start, BPF_ANY); 1184 if (err) { 1185 fprintf(stderr, 1186 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 1187 err, strerror(errno)); 1188 goto out; 1189 } 1190 } 1191 1192 if (txmsg_end) { 1193 i = 1; 1194 err = bpf_map_update_elem(map_fd[5], 1195 &i, &txmsg_end, BPF_ANY); 1196 if (err) { 1197 fprintf(stderr, 1198 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 1199 err, strerror(errno)); 1200 goto out; 1201 } 1202 } 1203 1204 if (txmsg_start_push) { 1205 i = 2; 1206 err = bpf_map_update_elem(map_fd[5], 1207 &i, &txmsg_start_push, BPF_ANY); 1208 if (err) { 1209 fprintf(stderr, 1210 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n", 1211 err, strerror(errno)); 1212 goto out; 1213 } 1214 } 1215 1216 if (txmsg_end_push) { 1217 i = 3; 1218 err = bpf_map_update_elem(map_fd[5], 1219 &i, &txmsg_end_push, BPF_ANY); 1220 if (err) { 1221 fprintf(stderr, 1222 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n", 1223 txmsg_end_push, i, err, strerror(errno)); 1224 goto out; 1225 } 1226 } 1227 1228 if (txmsg_start_pop) { 1229 i = 4; 1230 err = bpf_map_update_elem(map_fd[5], 1231 &i, &txmsg_start_pop, BPF_ANY); 1232 if (err) { 1233 fprintf(stderr, 1234 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n", 1235 txmsg_start_pop, i, err, strerror(errno)); 1236 goto out; 1237 } 1238 } else { 1239 i = 4; 1240 bpf_map_update_elem(map_fd[5], 1241 &i, &txmsg_start_pop, BPF_ANY); 1242 } 1243 1244 if (txmsg_pop) { 1245 i = 5; 1246 err = bpf_map_update_elem(map_fd[5], 1247 &i, &txmsg_pop, BPF_ANY); 1248 if (err) { 1249 fprintf(stderr, 1250 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n", 1251 txmsg_pop, i, err, strerror(errno)); 1252 goto out; 1253 } 1254 } else { 1255 i = 5; 1256 bpf_map_update_elem(map_fd[5], 1257 &i, &txmsg_pop, BPF_ANY); 1258 1259 } 1260 1261 if (txmsg_ingress) { 1262 int in = BPF_F_INGRESS; 1263 1264 i = 0; 1265 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 1266 if (err) { 1267 fprintf(stderr, 1268 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1269 err, strerror(errno)); 1270 } 1271 i = 1; 1272 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 1273 if (err) { 1274 fprintf(stderr, 1275 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 1276 err, strerror(errno)); 1277 } 1278 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 1279 if (err) { 1280 fprintf(stderr, 1281 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 1282 err, strerror(errno)); 1283 } 1284 1285 i = 2; 1286 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 1287 if (err) { 1288 fprintf(stderr, 1289 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 1290 err, strerror(errno)); 1291 } 1292 } 1293 1294 if (txmsg_ktls_skb) { 1295 int ingress = BPF_F_INGRESS; 1296 1297 i = 0; 1298 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY); 1299 if (err) { 1300 fprintf(stderr, 1301 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1302 err, strerror(errno)); 1303 } 1304 1305 if (txmsg_ktls_skb_redir) { 1306 i = 1; 1307 err = bpf_map_update_elem(map_fd[7], 1308 &i, &ingress, BPF_ANY); 1309 if (err) { 1310 fprintf(stderr, 1311 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1312 err, strerror(errno)); 1313 } 1314 } 1315 1316 if (txmsg_ktls_skb_drop) { 1317 i = 1; 1318 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY); 1319 } 1320 } 1321 1322 if (txmsg_redir_skb) { 1323 int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 1324 p2 : p1; 1325 int ingress = BPF_F_INGRESS; 1326 1327 i = 0; 1328 err = bpf_map_update_elem(map_fd[7], 1329 &i, &ingress, BPF_ANY); 1330 if (err) { 1331 fprintf(stderr, 1332 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1333 err, strerror(errno)); 1334 } 1335 1336 i = 3; 1337 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY); 1338 if (err) { 1339 fprintf(stderr, 1340 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1341 err, strerror(errno)); 1342 } 1343 } 1344 } 1345 1346 if (skb_use_parser) { 1347 i = 2; 1348 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY); 1349 } 1350 1351 if (txmsg_drop) 1352 options->drop_expected = true; 1353 1354 if (test == PING_PONG) 1355 err = forever_ping_pong(options->rate, options); 1356 else if (test == SENDMSG) { 1357 options->base = false; 1358 options->sendpage = false; 1359 err = sendmsg_test(options); 1360 } else if (test == SENDPAGE) { 1361 options->base = false; 1362 options->sendpage = true; 1363 err = sendmsg_test(options); 1364 } else if (test == BASE) { 1365 options->base = true; 1366 options->sendpage = false; 1367 err = sendmsg_test(options); 1368 } else if (test == BASE_SENDPAGE) { 1369 options->base = true; 1370 options->sendpage = true; 1371 err = sendmsg_test(options); 1372 } else 1373 fprintf(stderr, "unknown test\n"); 1374 out: 1375 /* Detatch and zero all the maps */ 1376 bpf_prog_detach2(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS); 1377 1378 for (i = 0; i < ARRAY_SIZE(links); i++) { 1379 if (links[i]) 1380 bpf_link__detach(links[i]); 1381 } 1382 1383 for (i = 0; i < ARRAY_SIZE(map_fd); i++) { 1384 key = next_key = 0; 1385 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1386 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 1387 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1388 key = next_key; 1389 } 1390 } 1391 1392 close(s1); 1393 close(s2); 1394 close(p1); 1395 close(p2); 1396 close(c1); 1397 close(c2); 1398 return err; 1399 } 1400 1401 static char *test_to_str(int test) 1402 { 1403 switch (test) { 1404 case SENDMSG: 1405 return "sendmsg"; 1406 case SENDPAGE: 1407 return "sendpage"; 1408 } 1409 return "unknown"; 1410 } 1411 1412 static void append_str(char *dst, const char *src, size_t dst_cap) 1413 { 1414 size_t avail = dst_cap - strlen(dst); 1415 1416 if (avail <= 1) /* just zero byte could be written */ 1417 return; 1418 1419 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */ 1420 } 1421 1422 #define OPTSTRING 60 1423 static void test_options(char *options) 1424 { 1425 char tstr[OPTSTRING]; 1426 1427 memset(options, 0, OPTSTRING); 1428 1429 if (txmsg_pass) 1430 append_str(options, "pass,", OPTSTRING); 1431 if (txmsg_redir) 1432 append_str(options, "redir,", OPTSTRING); 1433 if (txmsg_drop) 1434 append_str(options, "drop,", OPTSTRING); 1435 if (txmsg_apply) { 1436 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 1437 append_str(options, tstr, OPTSTRING); 1438 } 1439 if (txmsg_cork) { 1440 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 1441 append_str(options, tstr, OPTSTRING); 1442 } 1443 if (txmsg_start) { 1444 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 1445 append_str(options, tstr, OPTSTRING); 1446 } 1447 if (txmsg_end) { 1448 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 1449 append_str(options, tstr, OPTSTRING); 1450 } 1451 if (txmsg_start_pop) { 1452 snprintf(tstr, OPTSTRING, "pop (%d,%d),", 1453 txmsg_start_pop, txmsg_start_pop + txmsg_pop); 1454 append_str(options, tstr, OPTSTRING); 1455 } 1456 if (txmsg_ingress) 1457 append_str(options, "ingress,", OPTSTRING); 1458 if (txmsg_redir_skb) 1459 append_str(options, "redir_skb,", OPTSTRING); 1460 if (txmsg_ktls_skb) 1461 append_str(options, "ktls_skb,", OPTSTRING); 1462 if (ktls) 1463 append_str(options, "ktls,", OPTSTRING); 1464 if (peek_flag) 1465 append_str(options, "peek,", OPTSTRING); 1466 } 1467 1468 static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 1469 { 1470 char *options = calloc(OPTSTRING, sizeof(char)); 1471 int err; 1472 1473 if (test == SENDPAGE) 1474 opt->sendpage = true; 1475 else 1476 opt->sendpage = false; 1477 1478 if (txmsg_drop) 1479 opt->drop_expected = true; 1480 else 1481 opt->drop_expected = false; 1482 1483 test_options(options); 1484 1485 if (opt->verbose) { 1486 fprintf(stdout, 1487 " [TEST %i]: (%i, %i, %i, %s, %s): ", 1488 test_cnt, opt->rate, opt->iov_count, opt->iov_length, 1489 test_to_str(test), options); 1490 fflush(stdout); 1491 } 1492 err = run_options(opt, cgrp, test); 1493 if (opt->verbose) 1494 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED"); 1495 test_cnt++; 1496 !err ? passed++ : failed++; 1497 free(options); 1498 return err; 1499 } 1500 1501 static void test_exec(int cgrp, struct sockmap_options *opt) 1502 { 1503 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME); 1504 int err; 1505 1506 if (type == 0) { 1507 test_start(); 1508 err = __test_exec(cgrp, SENDMSG, opt); 1509 if (err) 1510 test_fail(); 1511 } else { 1512 test_start(); 1513 err = __test_exec(cgrp, SENDPAGE, opt); 1514 if (err) 1515 test_fail(); 1516 } 1517 } 1518 1519 static void test_send_one(struct sockmap_options *opt, int cgrp) 1520 { 1521 opt->iov_length = 1; 1522 opt->iov_count = 1; 1523 opt->rate = 1; 1524 test_exec(cgrp, opt); 1525 1526 opt->iov_length = 1; 1527 opt->iov_count = 1024; 1528 opt->rate = 1; 1529 test_exec(cgrp, opt); 1530 1531 opt->iov_length = 1024; 1532 opt->iov_count = 1; 1533 opt->rate = 1; 1534 test_exec(cgrp, opt); 1535 1536 } 1537 1538 static void test_send_many(struct sockmap_options *opt, int cgrp) 1539 { 1540 opt->iov_length = 3; 1541 opt->iov_count = 1; 1542 opt->rate = 512; 1543 test_exec(cgrp, opt); 1544 1545 opt->rate = 100; 1546 opt->iov_count = 1; 1547 opt->iov_length = 5; 1548 test_exec(cgrp, opt); 1549 } 1550 1551 static void test_send_large(struct sockmap_options *opt, int cgrp) 1552 { 1553 opt->iov_length = 8192; 1554 opt->iov_count = 32; 1555 opt->rate = 2; 1556 test_exec(cgrp, opt); 1557 } 1558 1559 static void test_send(struct sockmap_options *opt, int cgrp) 1560 { 1561 test_send_one(opt, cgrp); 1562 test_send_many(opt, cgrp); 1563 test_send_large(opt, cgrp); 1564 sched_yield(); 1565 } 1566 1567 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt) 1568 { 1569 /* Test small and large iov_count values with pass/redir/apply/cork */ 1570 txmsg_pass = 1; 1571 test_send(opt, cgrp); 1572 } 1573 1574 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt) 1575 { 1576 txmsg_redir = 1; 1577 test_send(opt, cgrp); 1578 } 1579 1580 static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt) 1581 { 1582 txmsg_redir = 1; 1583 opt->tx_wait_mem = true; 1584 test_send_large(opt, cgrp); 1585 opt->tx_wait_mem = false; 1586 } 1587 1588 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt) 1589 { 1590 txmsg_drop = 1; 1591 test_send(opt, cgrp); 1592 } 1593 1594 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt) 1595 { 1596 txmsg_pass = txmsg_drop = 0; 1597 txmsg_ingress = txmsg_redir = 1; 1598 test_send(opt, cgrp); 1599 } 1600 1601 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt) 1602 { 1603 bool data = opt->data_test; 1604 int k = ktls; 1605 1606 opt->data_test = true; 1607 ktls = 1; 1608 1609 txmsg_pass = txmsg_drop = 0; 1610 txmsg_ingress = txmsg_redir = 0; 1611 txmsg_ktls_skb = 1; 1612 txmsg_pass = 1; 1613 1614 /* Using data verification so ensure iov layout is 1615 * expected from test receiver side. e.g. has enough 1616 * bytes to write test code. 1617 */ 1618 opt->iov_length = 100; 1619 opt->iov_count = 1; 1620 opt->rate = 1; 1621 test_exec(cgrp, opt); 1622 1623 txmsg_ktls_skb_drop = 1; 1624 test_exec(cgrp, opt); 1625 1626 txmsg_ktls_skb_drop = 0; 1627 txmsg_ktls_skb_redir = 1; 1628 test_exec(cgrp, opt); 1629 txmsg_ktls_skb_redir = 0; 1630 1631 /* Tests that omit skb_parser */ 1632 txmsg_omit_skb_parser = 1; 1633 ktls = 0; 1634 txmsg_ktls_skb = 0; 1635 test_exec(cgrp, opt); 1636 1637 txmsg_ktls_skb_drop = 1; 1638 test_exec(cgrp, opt); 1639 txmsg_ktls_skb_drop = 0; 1640 1641 txmsg_ktls_skb_redir = 1; 1642 test_exec(cgrp, opt); 1643 1644 ktls = 1; 1645 test_exec(cgrp, opt); 1646 txmsg_omit_skb_parser = 0; 1647 1648 opt->data_test = data; 1649 ktls = k; 1650 } 1651 1652 /* Test cork with hung data. This tests poor usage patterns where 1653 * cork can leave data on the ring if user program is buggy and 1654 * doesn't flush them somehow. They do take some time however 1655 * because they wait for a timeout. Test pass, redir and cork with 1656 * apply logic. Use cork size of 4097 with send_large to avoid 1657 * aligning cork size with send size. 1658 */ 1659 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt) 1660 { 1661 txmsg_pass = 1; 1662 txmsg_redir = 0; 1663 txmsg_cork = 4097; 1664 txmsg_apply = 4097; 1665 test_send_large(opt, cgrp); 1666 1667 txmsg_pass = 0; 1668 txmsg_redir = 1; 1669 txmsg_apply = 0; 1670 txmsg_cork = 4097; 1671 test_send_large(opt, cgrp); 1672 1673 txmsg_pass = 0; 1674 txmsg_redir = 1; 1675 txmsg_apply = 4097; 1676 txmsg_cork = 4097; 1677 test_send_large(opt, cgrp); 1678 } 1679 1680 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt) 1681 { 1682 /* Test basic start/end */ 1683 txmsg_pass = 1; 1684 txmsg_start = 1; 1685 txmsg_end = 2; 1686 test_send(opt, cgrp); 1687 1688 /* Test >4k pull */ 1689 txmsg_pass = 1; 1690 txmsg_start = 4096; 1691 txmsg_end = 9182; 1692 test_send_large(opt, cgrp); 1693 1694 /* Test pull + redirect */ 1695 txmsg_redir = 1; 1696 txmsg_start = 1; 1697 txmsg_end = 2; 1698 test_send(opt, cgrp); 1699 1700 /* Test pull + cork */ 1701 txmsg_redir = 0; 1702 txmsg_cork = 512; 1703 txmsg_start = 1; 1704 txmsg_end = 2; 1705 test_send_many(opt, cgrp); 1706 1707 /* Test pull + cork + redirect */ 1708 txmsg_redir = 1; 1709 txmsg_cork = 512; 1710 txmsg_start = 1; 1711 txmsg_end = 2; 1712 test_send_many(opt, cgrp); 1713 } 1714 1715 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt) 1716 { 1717 bool data = opt->data_test; 1718 1719 /* Test basic pop */ 1720 txmsg_pass = 1; 1721 txmsg_start_pop = 1; 1722 txmsg_pop = 2; 1723 test_send_many(opt, cgrp); 1724 1725 /* Test pop with >4k */ 1726 txmsg_pass = 1; 1727 txmsg_start_pop = 4096; 1728 txmsg_pop = 4096; 1729 test_send_large(opt, cgrp); 1730 1731 /* Test pop + redirect */ 1732 txmsg_redir = 1; 1733 txmsg_start_pop = 1; 1734 txmsg_pop = 2; 1735 test_send_many(opt, cgrp); 1736 1737 /* TODO: Test for pop + cork should be different, 1738 * - It makes the layout of the received data difficult 1739 * - It makes it hard to calculate the total_bytes in the recvmsg 1740 * Temporarily skip the data integrity test for this case now. 1741 */ 1742 opt->data_test = false; 1743 /* Test pop + cork */ 1744 txmsg_redir = 0; 1745 txmsg_cork = 512; 1746 txmsg_start_pop = 1; 1747 txmsg_pop = 2; 1748 test_send_many(opt, cgrp); 1749 1750 /* Test pop + redirect + cork */ 1751 txmsg_redir = 1; 1752 txmsg_cork = 4; 1753 txmsg_start_pop = 1; 1754 txmsg_pop = 2; 1755 test_send_many(opt, cgrp); 1756 opt->data_test = data; 1757 } 1758 1759 static void test_txmsg_push(int cgrp, struct sockmap_options *opt) 1760 { 1761 bool data = opt->data_test; 1762 1763 /* Test basic push */ 1764 txmsg_pass = 1; 1765 txmsg_start_push = 1; 1766 txmsg_end_push = 1; 1767 test_send(opt, cgrp); 1768 1769 /* Test push 4kB >4k */ 1770 txmsg_pass = 1; 1771 txmsg_start_push = 4096; 1772 txmsg_end_push = 4096; 1773 test_send_large(opt, cgrp); 1774 1775 /* Test push + redirect */ 1776 txmsg_redir = 1; 1777 txmsg_start_push = 1; 1778 txmsg_end_push = 2; 1779 test_send_many(opt, cgrp); 1780 1781 /* TODO: Test for push + cork should be different, 1782 * - It makes the layout of the received data difficult 1783 * - It makes it hard to calculate the total_bytes in the recvmsg 1784 * Temporarily skip the data integrity test for this case now. 1785 */ 1786 opt->data_test = false; 1787 /* Test push + cork */ 1788 txmsg_redir = 0; 1789 txmsg_cork = 512; 1790 txmsg_start_push = 1; 1791 txmsg_end_push = 2; 1792 test_send_many(opt, cgrp); 1793 opt->data_test = data; 1794 } 1795 1796 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt) 1797 { 1798 /* Test push/pop range overlapping */ 1799 txmsg_pass = 1; 1800 txmsg_start_push = 1; 1801 txmsg_end_push = 10; 1802 txmsg_start_pop = 5; 1803 txmsg_pop = 4; 1804 test_send_large(opt, cgrp); 1805 1806 txmsg_pass = 1; 1807 txmsg_start_push = 1; 1808 txmsg_end_push = 10; 1809 txmsg_start_pop = 5; 1810 txmsg_pop = 16; 1811 test_send_large(opt, cgrp); 1812 1813 txmsg_pass = 1; 1814 txmsg_start_push = 5; 1815 txmsg_end_push = 4; 1816 txmsg_start_pop = 1; 1817 txmsg_pop = 10; 1818 test_send_large(opt, cgrp); 1819 1820 txmsg_pass = 1; 1821 txmsg_start_push = 5; 1822 txmsg_end_push = 16; 1823 txmsg_start_pop = 1; 1824 txmsg_pop = 10; 1825 test_send_large(opt, cgrp); 1826 1827 /* Test push/pop range non-overlapping */ 1828 txmsg_pass = 1; 1829 txmsg_start_push = 1; 1830 txmsg_end_push = 10; 1831 txmsg_start_pop = 16; 1832 txmsg_pop = 4; 1833 test_send_large(opt, cgrp); 1834 1835 txmsg_pass = 1; 1836 txmsg_start_push = 16; 1837 txmsg_end_push = 10; 1838 txmsg_start_pop = 5; 1839 txmsg_pop = 4; 1840 test_send_large(opt, cgrp); 1841 } 1842 1843 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt) 1844 { 1845 txmsg_pass = 1; 1846 txmsg_redir = 0; 1847 txmsg_ingress = 0; 1848 txmsg_apply = 1; 1849 txmsg_cork = 0; 1850 test_send_one(opt, cgrp); 1851 1852 txmsg_pass = 0; 1853 txmsg_redir = 1; 1854 txmsg_ingress = 0; 1855 txmsg_apply = 1; 1856 txmsg_cork = 0; 1857 test_send_one(opt, cgrp); 1858 1859 txmsg_pass = 0; 1860 txmsg_redir = 1; 1861 txmsg_ingress = 1; 1862 txmsg_apply = 1; 1863 txmsg_cork = 0; 1864 test_send_one(opt, cgrp); 1865 1866 txmsg_pass = 1; 1867 txmsg_redir = 0; 1868 txmsg_ingress = 0; 1869 txmsg_apply = 1024; 1870 txmsg_cork = 0; 1871 test_send_large(opt, cgrp); 1872 1873 txmsg_pass = 0; 1874 txmsg_redir = 1; 1875 txmsg_ingress = 0; 1876 txmsg_apply = 1024; 1877 txmsg_cork = 0; 1878 test_send_large(opt, cgrp); 1879 1880 txmsg_pass = 0; 1881 txmsg_redir = 1; 1882 txmsg_ingress = 1; 1883 txmsg_apply = 1024; 1884 txmsg_cork = 0; 1885 test_send_large(opt, cgrp); 1886 } 1887 1888 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt) 1889 { 1890 txmsg_pass = 1; 1891 txmsg_redir = 0; 1892 txmsg_apply = 0; 1893 txmsg_cork = 1; 1894 test_send(opt, cgrp); 1895 1896 txmsg_pass = 1; 1897 txmsg_redir = 0; 1898 txmsg_apply = 1; 1899 txmsg_cork = 1; 1900 test_send(opt, cgrp); 1901 } 1902 1903 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt) 1904 { 1905 txmsg_pass = 1; 1906 skb_use_parser = 512; 1907 if (ktls == 1) 1908 skb_use_parser = 570; 1909 opt->iov_length = 256; 1910 opt->iov_count = 1; 1911 opt->rate = 2; 1912 test_exec(cgrp, opt); 1913 } 1914 1915 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt) 1916 { 1917 if (ktls == 1) 1918 return; 1919 skb_use_parser = 10; 1920 opt->iov_length = 20; 1921 opt->iov_count = 1; 1922 opt->rate = 1; 1923 opt->check_recved_len = true; 1924 test_exec(cgrp, opt); 1925 opt->check_recved_len = false; 1926 } 1927 1928 char *map_names[] = { 1929 "sock_map", 1930 "sock_map_txmsg", 1931 "sock_map_redir", 1932 "sock_apply_bytes", 1933 "sock_cork_bytes", 1934 "sock_bytes", 1935 "sock_redir_flags", 1936 "sock_skb_opts", 1937 "tls_sock_map", 1938 }; 1939 1940 static int populate_progs(char *bpf_file) 1941 { 1942 struct bpf_program *prog; 1943 struct bpf_object *obj; 1944 int i = 0; 1945 long err; 1946 1947 obj = bpf_object__open(bpf_file); 1948 err = libbpf_get_error(obj); 1949 if (err) { 1950 char err_buf[256]; 1951 1952 libbpf_strerror(err, err_buf, sizeof(err_buf)); 1953 printf("Unable to load eBPF objects in file '%s' : %s\n", 1954 bpf_file, err_buf); 1955 return -1; 1956 } 1957 1958 i = bpf_object__load(obj); 1959 i = 0; 1960 bpf_object__for_each_program(prog, obj) { 1961 progs[i] = prog; 1962 i++; 1963 } 1964 1965 for (i = 0; i < ARRAY_SIZE(map_fd); i++) { 1966 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 1967 map_fd[i] = bpf_map__fd(maps[i]); 1968 if (map_fd[i] < 0) { 1969 fprintf(stderr, "load_bpf_file: (%i) %s\n", 1970 map_fd[i], strerror(errno)); 1971 return -1; 1972 } 1973 } 1974 1975 for (i = 0; i < ARRAY_SIZE(links); i++) 1976 links[i] = NULL; 1977 1978 return 0; 1979 } 1980 1981 struct _test test[] = { 1982 {"txmsg test passthrough", test_txmsg_pass}, 1983 {"txmsg test redirect", test_txmsg_redir}, 1984 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem}, 1985 {"txmsg test drop", test_txmsg_drop}, 1986 {"txmsg test ingress redirect", test_txmsg_ingress_redir}, 1987 {"txmsg test skb", test_txmsg_skb}, 1988 {"txmsg test apply", test_txmsg_apply}, 1989 {"txmsg test cork", test_txmsg_cork}, 1990 {"txmsg test hanging corks", test_txmsg_cork_hangs}, 1991 {"txmsg test push_data", test_txmsg_push}, 1992 {"txmsg test pull-data", test_txmsg_pull}, 1993 {"txmsg test pop-data", test_txmsg_pop}, 1994 {"txmsg test push/pop data", test_txmsg_push_pop}, 1995 {"txmsg test ingress parser", test_txmsg_ingress_parser}, 1996 {"txmsg test ingress parser2", test_txmsg_ingress_parser2}, 1997 }; 1998 1999 static int check_whitelist(struct _test *t, struct sockmap_options *opt) 2000 { 2001 char *entry, *ptr; 2002 2003 if (!opt->whitelist) 2004 return 0; 2005 ptr = strdup(opt->whitelist); 2006 if (!ptr) 2007 return -ENOMEM; 2008 entry = strtok(ptr, ","); 2009 while (entry) { 2010 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 2011 strstr(opt->map, entry) != 0 || 2012 strstr(t->title, entry) != 0) { 2013 free(ptr); 2014 return 0; 2015 } 2016 entry = strtok(NULL, ","); 2017 } 2018 free(ptr); 2019 return -EINVAL; 2020 } 2021 2022 static int check_blacklist(struct _test *t, struct sockmap_options *opt) 2023 { 2024 char *entry, *ptr; 2025 2026 if (!opt->blacklist) 2027 return -EINVAL; 2028 ptr = strdup(opt->blacklist); 2029 if (!ptr) 2030 return -ENOMEM; 2031 entry = strtok(ptr, ","); 2032 while (entry) { 2033 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 2034 strstr(opt->map, entry) != 0 || 2035 strstr(t->title, entry) != 0) { 2036 free(ptr); 2037 return 0; 2038 } 2039 entry = strtok(NULL, ","); 2040 } 2041 free(ptr); 2042 return -EINVAL; 2043 } 2044 2045 static int __test_selftests(int cg_fd, struct sockmap_options *opt) 2046 { 2047 int i, err; 2048 2049 err = populate_progs(opt->map); 2050 if (err < 0) { 2051 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 2052 return err; 2053 } 2054 2055 /* Tests basic commands and APIs */ 2056 for (i = 0; i < ARRAY_SIZE(test); i++) { 2057 struct _test t = test[i]; 2058 2059 if (check_whitelist(&t, opt) != 0) 2060 continue; 2061 if (check_blacklist(&t, opt) == 0) 2062 continue; 2063 2064 test_start_subtest(&t, opt); 2065 t.tester(cg_fd, opt); 2066 test_end_subtest(); 2067 } 2068 2069 return err; 2070 } 2071 2072 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt) 2073 { 2074 opt->map = BPF_SOCKMAP_FILENAME; 2075 __test_selftests(cg_fd, opt); 2076 } 2077 2078 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt) 2079 { 2080 opt->map = BPF_SOCKHASH_FILENAME; 2081 __test_selftests(cg_fd, opt); 2082 } 2083 2084 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt) 2085 { 2086 opt->map = BPF_SOCKHASH_FILENAME; 2087 opt->prepend = "ktls"; 2088 ktls = 1; 2089 __test_selftests(cg_fd, opt); 2090 ktls = 0; 2091 } 2092 2093 static int test_selftest(int cg_fd, struct sockmap_options *opt) 2094 { 2095 test_selftests_sockmap(cg_fd, opt); 2096 test_selftests_sockhash(cg_fd, opt); 2097 test_selftests_ktls(cg_fd, opt); 2098 test_print_results(); 2099 return 0; 2100 } 2101 2102 int main(int argc, char **argv) 2103 { 2104 int iov_count = 1, length = 1024, rate = 1; 2105 struct sockmap_options options = {0}; 2106 int opt, longindex, err, cg_fd = 0; 2107 char *bpf_file = BPF_SOCKMAP_FILENAME; 2108 int test = SELFTESTS; 2109 bool cg_created = 0; 2110 2111 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:", 2112 long_options, &longindex)) != -1) { 2113 switch (opt) { 2114 case 's': 2115 txmsg_start = atoi(optarg); 2116 break; 2117 case 'e': 2118 txmsg_end = atoi(optarg); 2119 break; 2120 case 'p': 2121 txmsg_start_push = atoi(optarg); 2122 break; 2123 case 'q': 2124 txmsg_end_push = atoi(optarg); 2125 break; 2126 case 'w': 2127 txmsg_start_pop = atoi(optarg); 2128 break; 2129 case 'x': 2130 txmsg_pop = atoi(optarg); 2131 break; 2132 case 'a': 2133 txmsg_apply = atoi(optarg); 2134 break; 2135 case 'k': 2136 txmsg_cork = atoi(optarg); 2137 break; 2138 case 'c': 2139 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 2140 if (cg_fd < 0) { 2141 fprintf(stderr, 2142 "ERROR: (%i) open cg path failed: %s\n", 2143 cg_fd, optarg); 2144 return cg_fd; 2145 } 2146 break; 2147 case 'r': 2148 rate = atoi(optarg); 2149 break; 2150 case 'v': 2151 options.verbose = 1; 2152 if (optarg) 2153 options.verbose = atoi(optarg); 2154 break; 2155 case 'i': 2156 iov_count = atoi(optarg); 2157 break; 2158 case 'l': 2159 length = atoi(optarg); 2160 break; 2161 case 'd': 2162 options.data_test = true; 2163 break; 2164 case 't': 2165 if (strcmp(optarg, "ping") == 0) { 2166 test = PING_PONG; 2167 } else if (strcmp(optarg, "sendmsg") == 0) { 2168 test = SENDMSG; 2169 } else if (strcmp(optarg, "base") == 0) { 2170 test = BASE; 2171 } else if (strcmp(optarg, "base_sendpage") == 0) { 2172 test = BASE_SENDPAGE; 2173 } else if (strcmp(optarg, "sendpage") == 0) { 2174 test = SENDPAGE; 2175 } else { 2176 usage(argv); 2177 return -1; 2178 } 2179 break; 2180 case 'n': 2181 options.whitelist = strdup(optarg); 2182 if (!options.whitelist) 2183 return -ENOMEM; 2184 break; 2185 case 'b': 2186 options.blacklist = strdup(optarg); 2187 if (!options.blacklist) 2188 return -ENOMEM; 2189 case 0: 2190 break; 2191 case 'h': 2192 default: 2193 usage(argv); 2194 return -1; 2195 } 2196 } 2197 2198 if (!cg_fd) { 2199 cg_fd = cgroup_setup_and_join(CG_PATH); 2200 if (cg_fd < 0) 2201 return cg_fd; 2202 cg_created = 1; 2203 } 2204 2205 /* Use libbpf 1.0 API mode */ 2206 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 2207 2208 if (test == SELFTESTS) { 2209 err = test_selftest(cg_fd, &options); 2210 goto out; 2211 } 2212 2213 err = populate_progs(bpf_file); 2214 if (err) { 2215 fprintf(stderr, "populate program: (%s) %s\n", 2216 bpf_file, strerror(errno)); 2217 return 1; 2218 } 2219 running = 1; 2220 2221 /* catch SIGINT */ 2222 signal(SIGINT, running_handler); 2223 2224 options.iov_count = iov_count; 2225 options.iov_length = length; 2226 options.rate = rate; 2227 2228 err = run_options(&options, cg_fd, test); 2229 out: 2230 if (options.whitelist) 2231 free(options.whitelist); 2232 if (options.blacklist) 2233 free(options.blacklist); 2234 close(cg_fd); 2235 if (cg_created) 2236 cleanup_cgroup_environment(); 2237 return err; 2238 } 2239 2240 void running_handler(int a) 2241 { 2242 running = 0; 2243 } 2244