1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <sys/socket.h> 6 #include <sys/ioctl.h> 7 #include <sys/select.h> 8 #include <netinet/in.h> 9 #include <arpa/inet.h> 10 #include <unistd.h> 11 #include <string.h> 12 #include <errno.h> 13 #include <stdbool.h> 14 #include <signal.h> 15 #include <fcntl.h> 16 #include <sys/wait.h> 17 #include <time.h> 18 #include <sched.h> 19 20 #include <sys/time.h> 21 #include <sys/types.h> 22 #include <sys/sendfile.h> 23 24 #include <linux/netlink.h> 25 #include <linux/socket.h> 26 #include <linux/sock_diag.h> 27 #include <linux/bpf.h> 28 #include <linux/if_link.h> 29 #include <assert.h> 30 #include <libgen.h> 31 32 #include <getopt.h> 33 34 #include <bpf/bpf.h> 35 #include <bpf/libbpf.h> 36 37 #include "bpf_util.h" 38 #include "cgroup_helpers.h" 39 40 int running; 41 static void running_handler(int a); 42 43 /* randomly selected ports for testing on lo */ 44 #define S1_PORT 10000 45 #define S2_PORT 10001 46 47 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o" 48 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o" 49 #define CG_PATH "/sockmap" 50 51 #define EDATAINTEGRITY 2001 52 53 /* global sockets */ 54 int s1, s2, c1, c2, p1, p2; 55 int test_cnt; 56 int passed; 57 int failed; 58 int map_fd[8]; 59 struct bpf_map *maps[8]; 60 struct bpf_program *progs[8]; 61 struct bpf_link *links[8]; 62 63 int txmsg_pass; 64 int txmsg_redir; 65 int txmsg_drop; 66 int txmsg_apply; 67 int txmsg_cork; 68 int txmsg_start; 69 int txmsg_end; 70 int txmsg_start_push; 71 int txmsg_end_push; 72 int txmsg_start_pop; 73 int txmsg_pop; 74 int txmsg_ingress; 75 int txmsg_redir_skb; 76 int peek_flag; 77 int skb_use_parser; 78 int txmsg_omit_skb_parser; 79 int verify_push_start; 80 int verify_push_len; 81 int verify_pop_start; 82 int verify_pop_len; 83 84 static const struct option long_options[] = { 85 {"help", no_argument, NULL, 'h' }, 86 {"cgroup", required_argument, NULL, 'c' }, 87 {"rate", required_argument, NULL, 'r' }, 88 {"verbose", optional_argument, NULL, 'v' }, 89 {"iov_count", required_argument, NULL, 'i' }, 90 {"length", required_argument, NULL, 'l' }, 91 {"test", required_argument, NULL, 't' }, 92 {"data_test", no_argument, NULL, 'd' }, 93 {"txmsg", no_argument, &txmsg_pass, 1 }, 94 {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 95 {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 96 {"txmsg_apply", required_argument, NULL, 'a'}, 97 {"txmsg_cork", required_argument, NULL, 'k'}, 98 {"txmsg_start", required_argument, NULL, 's'}, 99 {"txmsg_end", required_argument, NULL, 'e'}, 100 {"txmsg_start_push", required_argument, NULL, 'p'}, 101 {"txmsg_end_push", required_argument, NULL, 'q'}, 102 {"txmsg_start_pop", required_argument, NULL, 'w'}, 103 {"txmsg_pop", required_argument, NULL, 'x'}, 104 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 105 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 }, 106 {"peek", no_argument, &peek_flag, 1 }, 107 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1}, 108 {"whitelist", required_argument, NULL, 'n' }, 109 {"blacklist", required_argument, NULL, 'b' }, 110 {0, 0, NULL, 0 } 111 }; 112 113 struct test_env { 114 const char *type; 115 const char *subtest; 116 const char *prepend; 117 118 int test_num; 119 int subtest_num; 120 121 int succ_cnt; 122 int fail_cnt; 123 int fail_last; 124 }; 125 126 struct test_env env; 127 128 struct sockmap_options { 129 int verbose; 130 bool base; 131 bool sendpage; 132 bool data_test; 133 bool drop_expected; 134 bool check_recved_len; 135 bool tx_wait_mem; 136 int iov_count; 137 int iov_length; 138 int rate; 139 char *map; 140 char *whitelist; 141 char *blacklist; 142 char *prepend; 143 }; 144 145 struct _test { 146 char *title; 147 void (*tester)(int cg_fd, struct sockmap_options *opt); 148 }; 149 150 static void test_start(void) 151 { 152 env.subtest_num++; 153 } 154 155 static void test_fail(void) 156 { 157 env.fail_cnt++; 158 } 159 160 static void test_pass(void) 161 { 162 env.succ_cnt++; 163 } 164 165 static void test_reset(void) 166 { 167 txmsg_start = txmsg_end = 0; 168 txmsg_start_pop = txmsg_pop = 0; 169 txmsg_start_push = txmsg_end_push = 0; 170 txmsg_pass = txmsg_drop = txmsg_redir = 0; 171 txmsg_apply = txmsg_cork = 0; 172 txmsg_ingress = txmsg_redir_skb = 0; 173 txmsg_omit_skb_parser = 0; 174 skb_use_parser = 0; 175 } 176 177 static int test_start_subtest(const struct _test *t, struct sockmap_options *o) 178 { 179 env.type = o->map; 180 env.subtest = t->title; 181 env.prepend = o->prepend; 182 env.test_num++; 183 env.subtest_num = 0; 184 env.fail_last = env.fail_cnt; 185 test_reset(); 186 return 0; 187 } 188 189 static void test_end_subtest(void) 190 { 191 int error = env.fail_cnt - env.fail_last; 192 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME); 193 194 if (!error) 195 test_pass(); 196 197 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n", 198 env.test_num, env.subtest_num, 199 !type ? "sockmap" : "sockhash", 200 env.prepend ? : "", 201 env.subtest, error ? "FAIL" : "OK"); 202 } 203 204 static void test_print_results(void) 205 { 206 fprintf(stdout, "Pass: %d Fail: %d\n", 207 env.succ_cnt, env.fail_cnt); 208 } 209 210 static void usage(char *argv[]) 211 { 212 int i; 213 214 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 215 printf(" options:\n"); 216 for (i = 0; long_options[i].name != 0; i++) { 217 printf(" --%-12s", long_options[i].name); 218 if (long_options[i].flag != NULL) 219 printf(" flag (internal value:%d)\n", 220 *long_options[i].flag); 221 else 222 printf(" -%c\n", long_options[i].val); 223 } 224 printf("\n"); 225 } 226 227 static int sockmap_init_sockets(int verbose) 228 { 229 int i, err, one = 1; 230 struct sockaddr_in addr; 231 int *fds[4] = {&s1, &s2, &c1, &c2}; 232 233 s1 = s2 = p1 = p2 = c1 = c2 = 0; 234 235 /* Init sockets */ 236 for (i = 0; i < 4; i++) { 237 *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 238 if (*fds[i] < 0) { 239 perror("socket s1 failed()"); 240 return errno; 241 } 242 } 243 244 /* Allow reuse */ 245 for (i = 0; i < 2; i++) { 246 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 247 (char *)&one, sizeof(one)); 248 if (err) { 249 perror("setsockopt failed()"); 250 return errno; 251 } 252 } 253 254 /* Non-blocking sockets */ 255 for (i = 0; i < 2; i++) { 256 err = ioctl(*fds[i], FIONBIO, (char *)&one); 257 if (err < 0) { 258 perror("ioctl s1 failed()"); 259 return errno; 260 } 261 } 262 263 /* Bind server sockets */ 264 memset(&addr, 0, sizeof(struct sockaddr_in)); 265 addr.sin_family = AF_INET; 266 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 267 268 addr.sin_port = htons(S1_PORT); 269 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 270 if (err < 0) { 271 perror("bind s1 failed()"); 272 return errno; 273 } 274 275 addr.sin_port = htons(S2_PORT); 276 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 277 if (err < 0) { 278 perror("bind s2 failed()"); 279 return errno; 280 } 281 282 /* Listen server sockets */ 283 addr.sin_port = htons(S1_PORT); 284 err = listen(s1, 32); 285 if (err < 0) { 286 perror("listen s1 failed()"); 287 return errno; 288 } 289 290 addr.sin_port = htons(S2_PORT); 291 err = listen(s2, 32); 292 if (err < 0) { 293 perror("listen s1 failed()"); 294 return errno; 295 } 296 297 /* Initiate Connect */ 298 addr.sin_port = htons(S1_PORT); 299 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 300 if (err < 0 && errno != EINPROGRESS) { 301 perror("connect c1 failed()"); 302 return errno; 303 } 304 305 addr.sin_port = htons(S2_PORT); 306 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 307 if (err < 0 && errno != EINPROGRESS) { 308 perror("connect c2 failed()"); 309 return errno; 310 } else if (err < 0) { 311 err = 0; 312 } 313 314 /* Accept Connecrtions */ 315 p1 = accept(s1, NULL, NULL); 316 if (p1 < 0) { 317 perror("accept s1 failed()"); 318 return errno; 319 } 320 321 p2 = accept(s2, NULL, NULL); 322 if (p2 < 0) { 323 perror("accept s1 failed()"); 324 return errno; 325 } 326 327 if (verbose > 1) { 328 printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 329 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 330 c1, s1, c2, s2); 331 } 332 return 0; 333 } 334 335 struct msg_stats { 336 size_t bytes_sent; 337 size_t bytes_recvd; 338 struct timespec start; 339 struct timespec end; 340 }; 341 342 static int msg_loop_sendpage(int fd, int iov_length, int cnt, 343 struct msg_stats *s, 344 struct sockmap_options *opt) 345 { 346 bool drop = opt->drop_expected; 347 unsigned char k = 0; 348 int i, j, fp; 349 FILE *file; 350 351 file = tmpfile(); 352 if (!file) { 353 perror("create file for sendpage"); 354 return 1; 355 } 356 for (i = 0; i < cnt; i++, k = 0) { 357 for (j = 0; j < iov_length; j++, k++) 358 fwrite(&k, sizeof(char), 1, file); 359 } 360 fflush(file); 361 fseek(file, 0, SEEK_SET); 362 363 fp = fileno(file); 364 365 clock_gettime(CLOCK_MONOTONIC, &s->start); 366 for (i = 0; i < cnt; i++) { 367 int sent; 368 369 errno = 0; 370 sent = sendfile(fd, fp, NULL, iov_length); 371 372 if (!drop && sent < 0) { 373 perror("sendpage loop error"); 374 fclose(file); 375 return sent; 376 } else if (drop && sent >= 0) { 377 printf("sendpage loop error expected: %i errno %i\n", 378 sent, errno); 379 fclose(file); 380 return -EIO; 381 } 382 383 if (sent > 0) 384 s->bytes_sent += sent; 385 } 386 clock_gettime(CLOCK_MONOTONIC, &s->end); 387 fclose(file); 388 return 0; 389 } 390 391 static void msg_free_iov(struct msghdr *msg) 392 { 393 int i; 394 395 for (i = 0; i < msg->msg_iovlen; i++) 396 free(msg->msg_iov[i].iov_base); 397 free(msg->msg_iov); 398 msg->msg_iov = NULL; 399 msg->msg_iovlen = 0; 400 } 401 402 static int msg_alloc_iov(struct msghdr *msg, 403 int iov_count, int iov_length, 404 bool data, bool xmit) 405 { 406 unsigned char k = 0; 407 struct iovec *iov; 408 int i; 409 410 iov = calloc(iov_count, sizeof(struct iovec)); 411 if (!iov) 412 return errno; 413 414 for (i = 0; i < iov_count; i++) { 415 unsigned char *d = calloc(iov_length, sizeof(char)); 416 417 if (!d) { 418 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 419 goto unwind_iov; 420 } 421 iov[i].iov_base = d; 422 iov[i].iov_len = iov_length; 423 424 if (data && xmit) { 425 int j; 426 427 for (j = 0; j < iov_length; j++) 428 d[j] = k++; 429 } 430 } 431 432 msg->msg_iov = iov; 433 msg->msg_iovlen = iov_count; 434 435 return 0; 436 unwind_iov: 437 for (i--; i >= 0 ; i--) 438 free(iov[i].iov_base); 439 free(iov); 440 return -ENOMEM; 441 } 442 443 /* In push or pop test, we need to do some calculations for msg_verify_data */ 444 static void msg_verify_date_prep(void) 445 { 446 int push_range_end = txmsg_start_push + txmsg_end_push - 1; 447 int pop_range_end = txmsg_start_pop + txmsg_pop - 1; 448 449 if (txmsg_end_push && txmsg_pop && 450 txmsg_start_push <= pop_range_end && txmsg_start_pop <= push_range_end) { 451 /* The push range and the pop range overlap */ 452 int overlap_len; 453 454 verify_push_start = txmsg_start_push; 455 verify_pop_start = txmsg_start_pop; 456 if (txmsg_start_push < txmsg_start_pop) 457 overlap_len = min(push_range_end - txmsg_start_pop + 1, txmsg_pop); 458 else 459 overlap_len = min(pop_range_end - txmsg_start_push + 1, txmsg_end_push); 460 verify_push_len = max(txmsg_end_push - overlap_len, 0); 461 verify_pop_len = max(txmsg_pop - overlap_len, 0); 462 } else { 463 /* Otherwise */ 464 verify_push_start = txmsg_start_push; 465 verify_pop_start = txmsg_start_pop; 466 verify_push_len = txmsg_end_push; 467 verify_pop_len = txmsg_pop; 468 } 469 } 470 471 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz, 472 unsigned char *k_p, int *bytes_cnt_p, 473 int *check_cnt_p, int *push_p) 474 { 475 int bytes_cnt = *bytes_cnt_p, check_cnt = *check_cnt_p, push = *push_p; 476 unsigned char k = *k_p; 477 int i, j; 478 479 for (i = 0, j = 0; i < msg->msg_iovlen && size; i++, j = 0) { 480 unsigned char *d = msg->msg_iov[i].iov_base; 481 482 for (; j < msg->msg_iov[i].iov_len && size; j++) { 483 if (push > 0 && 484 check_cnt == verify_push_start + verify_push_len - push) { 485 int skipped; 486 revisit_push: 487 skipped = push; 488 if (j + push >= msg->msg_iov[i].iov_len) 489 skipped = msg->msg_iov[i].iov_len - j; 490 push -= skipped; 491 size -= skipped; 492 j += skipped - 1; 493 check_cnt += skipped; 494 continue; 495 } 496 497 if (verify_pop_len > 0 && check_cnt == verify_pop_start) { 498 bytes_cnt += verify_pop_len; 499 check_cnt += verify_pop_len; 500 k += verify_pop_len; 501 502 if (bytes_cnt == chunk_sz) { 503 k = 0; 504 bytes_cnt = 0; 505 check_cnt = 0; 506 push = verify_push_len; 507 } 508 509 if (push > 0 && 510 check_cnt == verify_push_start + verify_push_len - push) 511 goto revisit_push; 512 } 513 514 if (d[j] != k++) { 515 fprintf(stderr, 516 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 517 i, j, d[j], k - 1, d[j+1], k); 518 return -EDATAINTEGRITY; 519 } 520 bytes_cnt++; 521 check_cnt++; 522 if (bytes_cnt == chunk_sz) { 523 k = 0; 524 bytes_cnt = 0; 525 check_cnt = 0; 526 push = verify_push_len; 527 } 528 size--; 529 } 530 } 531 *k_p = k; 532 *bytes_cnt_p = bytes_cnt; 533 *check_cnt_p = check_cnt; 534 *push_p = push; 535 return 0; 536 } 537 538 static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 539 struct msg_stats *s, bool tx, 540 struct sockmap_options *opt) 541 { 542 struct msghdr msg = {0}, msg_peek = {0}; 543 int err, i, flags = MSG_NOSIGNAL; 544 bool drop = opt->drop_expected; 545 bool data = opt->data_test; 546 int iov_alloc_length = iov_length; 547 548 if (!tx && opt->check_recved_len) 549 iov_alloc_length *= 2; 550 551 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx); 552 if (err) 553 goto out_errno; 554 if (peek_flag) { 555 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx); 556 if (err) 557 goto out_errno; 558 } 559 560 if (tx) { 561 clock_gettime(CLOCK_MONOTONIC, &s->start); 562 for (i = 0; i < cnt; i++) { 563 int sent; 564 565 errno = 0; 566 sent = sendmsg(fd, &msg, flags); 567 568 if (!drop && sent < 0) { 569 if (opt->tx_wait_mem && errno == EACCES) { 570 errno = 0; 571 goto out_errno; 572 } 573 perror("sendmsg loop error"); 574 goto out_errno; 575 } else if (drop && sent >= 0) { 576 fprintf(stderr, 577 "sendmsg loop error expected: %i errno %i\n", 578 sent, errno); 579 errno = -EIO; 580 goto out_errno; 581 } 582 if (sent > 0) 583 s->bytes_sent += sent; 584 } 585 clock_gettime(CLOCK_MONOTONIC, &s->end); 586 } else { 587 float total_bytes, txmsg_pop_total, txmsg_push_total; 588 int slct, recvp = 0, recv, max_fd = fd; 589 int fd_flags = O_NONBLOCK; 590 struct timeval timeout; 591 unsigned char k = 0; 592 int bytes_cnt = 0; 593 int check_cnt = 0; 594 int push = 0; 595 fd_set w; 596 597 fcntl(fd, fd_flags); 598 /* Account for pop bytes noting each iteration of apply will 599 * call msg_pop_data helper so we need to account for this 600 * by calculating the number of apply iterations. Note user 601 * of the tool can create cases where no data is sent by 602 * manipulating pop/push/pull/etc. For example txmsg_apply 1 603 * with txmsg_pop 1 will try to apply 1B at a time but each 604 * iteration will then pop 1B so no data will ever be sent. 605 * This is really only useful for testing edge cases in code 606 * paths. 607 */ 608 total_bytes = (float)iov_length * (float)cnt; 609 if (!opt->sendpage) 610 total_bytes *= (float)iov_count; 611 if (txmsg_apply) { 612 txmsg_push_total = txmsg_end_push * (total_bytes / txmsg_apply); 613 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply); 614 } else { 615 txmsg_push_total = txmsg_end_push * cnt; 616 txmsg_pop_total = txmsg_pop * cnt; 617 } 618 total_bytes += txmsg_push_total; 619 total_bytes -= txmsg_pop_total; 620 if (data) { 621 msg_verify_date_prep(); 622 push = verify_push_len; 623 } 624 err = clock_gettime(CLOCK_MONOTONIC, &s->start); 625 if (err < 0) 626 perror("recv start time"); 627 while (s->bytes_recvd < total_bytes) { 628 if (txmsg_cork) { 629 timeout.tv_sec = 0; 630 timeout.tv_usec = 300000; 631 } else { 632 timeout.tv_sec = 3; 633 timeout.tv_usec = 0; 634 } 635 636 /* FD sets */ 637 FD_ZERO(&w); 638 FD_SET(fd, &w); 639 640 slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 641 if (slct == -1) { 642 perror("select()"); 643 clock_gettime(CLOCK_MONOTONIC, &s->end); 644 goto out_errno; 645 } else if (!slct) { 646 if (opt->verbose) 647 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total); 648 errno = -EIO; 649 clock_gettime(CLOCK_MONOTONIC, &s->end); 650 goto out_errno; 651 } 652 653 if (opt->tx_wait_mem) { 654 FD_ZERO(&w); 655 FD_SET(fd, &w); 656 slct = select(max_fd + 1, NULL, NULL, &w, &timeout); 657 errno = 0; 658 close(fd); 659 goto out_errno; 660 } 661 662 errno = 0; 663 if (peek_flag) { 664 flags |= MSG_PEEK; 665 recvp = recvmsg(fd, &msg_peek, flags); 666 if (recvp < 0) { 667 if (errno != EWOULDBLOCK) { 668 clock_gettime(CLOCK_MONOTONIC, &s->end); 669 goto out_errno; 670 } 671 } 672 flags = 0; 673 } 674 675 recv = recvmsg(fd, &msg, flags); 676 if (recv < 0) { 677 if (errno != EWOULDBLOCK) { 678 clock_gettime(CLOCK_MONOTONIC, &s->end); 679 perror("recv failed()"); 680 goto out_errno; 681 } 682 } 683 684 if (recv > 0) 685 s->bytes_recvd += recv; 686 687 if (opt->check_recved_len && s->bytes_recvd > total_bytes) { 688 errno = EMSGSIZE; 689 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n", 690 s->bytes_recvd, total_bytes); 691 goto out_errno; 692 } 693 694 if (data) { 695 int chunk_sz = opt->sendpage ? 696 iov_length : 697 iov_length * iov_count; 698 699 errno = msg_verify_data(&msg, recv, chunk_sz, &k, &bytes_cnt, 700 &check_cnt, &push); 701 if (errno) { 702 perror("data verify msg failed"); 703 goto out_errno; 704 } 705 if (recvp) { 706 errno = msg_verify_data(&msg_peek, 707 recvp, 708 chunk_sz, 709 &k, 710 &bytes_cnt, 711 &check_cnt, 712 &push); 713 if (errno) { 714 perror("data verify msg_peek failed"); 715 goto out_errno; 716 } 717 } 718 } 719 } 720 clock_gettime(CLOCK_MONOTONIC, &s->end); 721 } 722 723 msg_free_iov(&msg); 724 msg_free_iov(&msg_peek); 725 return err; 726 out_errno: 727 msg_free_iov(&msg); 728 msg_free_iov(&msg_peek); 729 return errno; 730 } 731 732 static float giga = 1000000000; 733 734 static inline float sentBps(struct msg_stats s) 735 { 736 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 737 } 738 739 static inline float recvdBps(struct msg_stats s) 740 { 741 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 742 } 743 744 static int sendmsg_test(struct sockmap_options *opt) 745 { 746 float sent_Bps = 0, recvd_Bps = 0; 747 int rx_fd, txpid, rxpid, err = 0; 748 struct msg_stats s = {0}; 749 int iov_count = opt->iov_count; 750 int iov_buf = opt->iov_length; 751 int rx_status, tx_status; 752 int cnt = opt->rate; 753 754 errno = 0; 755 756 if (opt->base) 757 rx_fd = p1; 758 else 759 rx_fd = p2; 760 761 if (opt->tx_wait_mem) { 762 struct timeval timeout; 763 int rxtx_buf_len = 1024; 764 765 timeout.tv_sec = 3; 766 timeout.tv_usec = 0; 767 768 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval)); 769 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int)); 770 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int)); 771 if (err) { 772 perror("setsockopt failed()"); 773 return errno; 774 } 775 } 776 777 rxpid = fork(); 778 if (rxpid == 0) { 779 if (opt->drop_expected) 780 _exit(0); 781 782 if (!iov_buf) /* zero bytes sent case */ 783 _exit(0); 784 785 if (opt->sendpage) 786 iov_count = 1; 787 err = msg_loop(rx_fd, iov_count, iov_buf, 788 cnt, &s, false, opt); 789 if (opt->verbose > 1) 790 fprintf(stderr, 791 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 792 iov_count, iov_buf, cnt, err); 793 if (s.end.tv_sec - s.start.tv_sec) { 794 sent_Bps = sentBps(s); 795 recvd_Bps = recvdBps(s); 796 } 797 if (opt->verbose > 1) 798 fprintf(stdout, 799 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n", 800 s.bytes_sent, sent_Bps, sent_Bps/giga, 801 s.bytes_recvd, recvd_Bps, recvd_Bps/giga, 802 peek_flag ? "(peek_msg)" : ""); 803 if (err && err != -EDATAINTEGRITY && txmsg_cork) 804 err = 0; 805 exit(err ? 1 : 0); 806 } else if (rxpid == -1) { 807 perror("msg_loop_rx"); 808 return errno; 809 } 810 811 if (opt->tx_wait_mem) 812 close(c2); 813 814 txpid = fork(); 815 if (txpid == 0) { 816 if (opt->sendpage) 817 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 818 else 819 err = msg_loop(c1, iov_count, iov_buf, 820 cnt, &s, true, opt); 821 822 if (err) 823 fprintf(stderr, 824 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 825 iov_count, iov_buf, cnt, err); 826 if (s.end.tv_sec - s.start.tv_sec) { 827 sent_Bps = sentBps(s); 828 recvd_Bps = recvdBps(s); 829 } 830 if (opt->verbose > 1) 831 fprintf(stdout, 832 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 833 s.bytes_sent, sent_Bps, sent_Bps/giga, 834 s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 835 exit(err ? 1 : 0); 836 } else if (txpid == -1) { 837 perror("msg_loop_tx"); 838 return errno; 839 } 840 841 assert(waitpid(rxpid, &rx_status, 0) == rxpid); 842 assert(waitpid(txpid, &tx_status, 0) == txpid); 843 if (WIFEXITED(rx_status)) { 844 err = WEXITSTATUS(rx_status); 845 if (err) { 846 fprintf(stderr, "rx thread exited with err %d.\n", err); 847 goto out; 848 } 849 } 850 if (WIFEXITED(tx_status)) { 851 err = WEXITSTATUS(tx_status); 852 if (err) 853 fprintf(stderr, "tx thread exited with err %d.\n", err); 854 } 855 out: 856 return err; 857 } 858 859 static int forever_ping_pong(int rate, struct sockmap_options *opt) 860 { 861 struct timeval timeout; 862 char buf[1024] = {0}; 863 int sc; 864 865 timeout.tv_sec = 10; 866 timeout.tv_usec = 0; 867 868 /* Ping/Pong data from client to server */ 869 sc = send(c1, buf, sizeof(buf), 0); 870 if (sc < 0) { 871 perror("send failed()"); 872 return sc; 873 } 874 875 do { 876 int s, rc, i, max_fd = p2; 877 fd_set w; 878 879 /* FD sets */ 880 FD_ZERO(&w); 881 FD_SET(c1, &w); 882 FD_SET(c2, &w); 883 FD_SET(p1, &w); 884 FD_SET(p2, &w); 885 886 s = select(max_fd + 1, &w, NULL, NULL, &timeout); 887 if (s == -1) { 888 perror("select()"); 889 break; 890 } else if (!s) { 891 fprintf(stderr, "unexpected timeout\n"); 892 break; 893 } 894 895 for (i = 0; i <= max_fd && s > 0; ++i) { 896 if (!FD_ISSET(i, &w)) 897 continue; 898 899 s--; 900 901 rc = recv(i, buf, sizeof(buf), 0); 902 if (rc < 0) { 903 if (errno != EWOULDBLOCK) { 904 perror("recv failed()"); 905 return rc; 906 } 907 } 908 909 if (rc == 0) { 910 close(i); 911 break; 912 } 913 914 sc = send(i, buf, rc, 0); 915 if (sc < 0) { 916 perror("send failed()"); 917 return sc; 918 } 919 } 920 921 if (rate) 922 sleep(rate); 923 924 if (opt->verbose) { 925 printf("."); 926 fflush(stdout); 927 928 } 929 } while (running); 930 931 return 0; 932 } 933 934 enum { 935 SELFTESTS, 936 PING_PONG, 937 SENDMSG, 938 BASE, 939 BASE_SENDPAGE, 940 SENDPAGE, 941 }; 942 943 static int run_options(struct sockmap_options *options, int cg_fd, int test) 944 { 945 int i, key, next_key, err, zero = 0; 946 struct bpf_program *tx_prog; 947 948 /* If base test skip BPF setup */ 949 if (test == BASE || test == BASE_SENDPAGE) 950 goto run; 951 952 /* Attach programs to sockmap */ 953 if (!txmsg_omit_skb_parser) { 954 links[0] = bpf_program__attach_sockmap(progs[0], map_fd[0]); 955 if (!links[0]) { 956 fprintf(stderr, 957 "ERROR: bpf_program__attach_sockmap (sockmap %i->%i): (%s)\n", 958 bpf_program__fd(progs[0]), map_fd[0], strerror(errno)); 959 return -1; 960 } 961 } 962 963 links[1] = bpf_program__attach_sockmap(progs[1], map_fd[0]); 964 if (!links[1]) { 965 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (sockmap): (%s)\n", 966 strerror(errno)); 967 return -1; 968 } 969 970 /* Attach to cgroups */ 971 err = bpf_prog_attach(bpf_program__fd(progs[2]), cg_fd, BPF_CGROUP_SOCK_OPS, 0); 972 if (err) { 973 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 974 err, strerror(errno)); 975 return err; 976 } 977 978 run: 979 err = sockmap_init_sockets(options->verbose); 980 if (err) { 981 fprintf(stderr, "ERROR: test socket failed: %d\n", err); 982 goto out; 983 } 984 985 /* Attach txmsg program to sockmap */ 986 if (txmsg_pass) 987 tx_prog = progs[3]; 988 else if (txmsg_redir) 989 tx_prog = progs[4]; 990 else if (txmsg_apply) 991 tx_prog = progs[5]; 992 else if (txmsg_cork) 993 tx_prog = progs[6]; 994 else if (txmsg_drop) 995 tx_prog = progs[7]; 996 else 997 tx_prog = NULL; 998 999 if (tx_prog) { 1000 int redir_fd; 1001 1002 links[4] = bpf_program__attach_sockmap(tx_prog, map_fd[1]); 1003 if (!links[4]) { 1004 fprintf(stderr, 1005 "ERROR: bpf_program__attach_sockmap (txmsg): (%s)\n", 1006 strerror(errno)); 1007 err = -1; 1008 goto out; 1009 } 1010 1011 i = 0; 1012 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 1013 if (err) { 1014 fprintf(stderr, 1015 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 1016 err, strerror(errno)); 1017 goto out; 1018 } 1019 1020 if (txmsg_redir) 1021 redir_fd = c2; 1022 else 1023 redir_fd = c1; 1024 1025 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 1026 if (err) { 1027 fprintf(stderr, 1028 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 1029 err, strerror(errno)); 1030 goto out; 1031 } 1032 1033 if (txmsg_apply) { 1034 err = bpf_map_update_elem(map_fd[3], 1035 &i, &txmsg_apply, BPF_ANY); 1036 if (err) { 1037 fprintf(stderr, 1038 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 1039 err, strerror(errno)); 1040 goto out; 1041 } 1042 } 1043 1044 if (txmsg_cork) { 1045 err = bpf_map_update_elem(map_fd[4], 1046 &i, &txmsg_cork, BPF_ANY); 1047 if (err) { 1048 fprintf(stderr, 1049 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 1050 err, strerror(errno)); 1051 goto out; 1052 } 1053 } 1054 1055 if (txmsg_start) { 1056 err = bpf_map_update_elem(map_fd[5], 1057 &i, &txmsg_start, BPF_ANY); 1058 if (err) { 1059 fprintf(stderr, 1060 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 1061 err, strerror(errno)); 1062 goto out; 1063 } 1064 } 1065 1066 if (txmsg_end) { 1067 i = 1; 1068 err = bpf_map_update_elem(map_fd[5], 1069 &i, &txmsg_end, BPF_ANY); 1070 if (err) { 1071 fprintf(stderr, 1072 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 1073 err, strerror(errno)); 1074 goto out; 1075 } 1076 } 1077 1078 if (txmsg_start_push) { 1079 i = 2; 1080 err = bpf_map_update_elem(map_fd[5], 1081 &i, &txmsg_start_push, BPF_ANY); 1082 if (err) { 1083 fprintf(stderr, 1084 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n", 1085 err, strerror(errno)); 1086 goto out; 1087 } 1088 } 1089 1090 if (txmsg_end_push) { 1091 i = 3; 1092 err = bpf_map_update_elem(map_fd[5], 1093 &i, &txmsg_end_push, BPF_ANY); 1094 if (err) { 1095 fprintf(stderr, 1096 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n", 1097 txmsg_end_push, i, err, strerror(errno)); 1098 goto out; 1099 } 1100 } 1101 1102 if (txmsg_start_pop) { 1103 i = 4; 1104 err = bpf_map_update_elem(map_fd[5], 1105 &i, &txmsg_start_pop, BPF_ANY); 1106 if (err) { 1107 fprintf(stderr, 1108 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n", 1109 txmsg_start_pop, i, err, strerror(errno)); 1110 goto out; 1111 } 1112 } else { 1113 i = 4; 1114 bpf_map_update_elem(map_fd[5], 1115 &i, &txmsg_start_pop, BPF_ANY); 1116 } 1117 1118 if (txmsg_pop) { 1119 i = 5; 1120 err = bpf_map_update_elem(map_fd[5], 1121 &i, &txmsg_pop, BPF_ANY); 1122 if (err) { 1123 fprintf(stderr, 1124 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n", 1125 txmsg_pop, i, err, strerror(errno)); 1126 goto out; 1127 } 1128 } else { 1129 i = 5; 1130 bpf_map_update_elem(map_fd[5], 1131 &i, &txmsg_pop, BPF_ANY); 1132 1133 } 1134 1135 if (txmsg_ingress) { 1136 int in = BPF_F_INGRESS; 1137 1138 i = 0; 1139 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 1140 if (err) { 1141 fprintf(stderr, 1142 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1143 err, strerror(errno)); 1144 } 1145 i = 1; 1146 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 1147 if (err) { 1148 fprintf(stderr, 1149 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 1150 err, strerror(errno)); 1151 } 1152 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 1153 if (err) { 1154 fprintf(stderr, 1155 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 1156 err, strerror(errno)); 1157 } 1158 1159 i = 2; 1160 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 1161 if (err) { 1162 fprintf(stderr, 1163 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 1164 err, strerror(errno)); 1165 } 1166 } 1167 1168 if (txmsg_redir_skb) { 1169 int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 1170 p2 : p1; 1171 int ingress = BPF_F_INGRESS; 1172 1173 i = 0; 1174 err = bpf_map_update_elem(map_fd[7], 1175 &i, &ingress, BPF_ANY); 1176 if (err) { 1177 fprintf(stderr, 1178 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1179 err, strerror(errno)); 1180 } 1181 1182 i = 3; 1183 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY); 1184 if (err) { 1185 fprintf(stderr, 1186 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1187 err, strerror(errno)); 1188 } 1189 } 1190 } 1191 1192 if (skb_use_parser) { 1193 i = 2; 1194 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY); 1195 } 1196 1197 if (txmsg_drop) 1198 options->drop_expected = true; 1199 1200 if (test == PING_PONG) 1201 err = forever_ping_pong(options->rate, options); 1202 else if (test == SENDMSG) { 1203 options->base = false; 1204 options->sendpage = false; 1205 err = sendmsg_test(options); 1206 } else if (test == SENDPAGE) { 1207 options->base = false; 1208 options->sendpage = true; 1209 err = sendmsg_test(options); 1210 } else if (test == BASE) { 1211 options->base = true; 1212 options->sendpage = false; 1213 err = sendmsg_test(options); 1214 } else if (test == BASE_SENDPAGE) { 1215 options->base = true; 1216 options->sendpage = true; 1217 err = sendmsg_test(options); 1218 } else 1219 fprintf(stderr, "unknown test\n"); 1220 out: 1221 /* Detach and zero all the maps */ 1222 bpf_prog_detach2(bpf_program__fd(progs[2]), cg_fd, BPF_CGROUP_SOCK_OPS); 1223 1224 for (i = 0; i < ARRAY_SIZE(links); i++) { 1225 if (links[i]) 1226 bpf_link__detach(links[i]); 1227 } 1228 1229 for (i = 0; i < ARRAY_SIZE(map_fd); i++) { 1230 key = next_key = 0; 1231 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1232 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 1233 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1234 key = next_key; 1235 } 1236 } 1237 1238 close(s1); 1239 close(s2); 1240 close(p1); 1241 close(p2); 1242 close(c1); 1243 close(c2); 1244 return err; 1245 } 1246 1247 static char *test_to_str(int test) 1248 { 1249 switch (test) { 1250 case SENDMSG: 1251 return "sendmsg"; 1252 case SENDPAGE: 1253 return "sendpage"; 1254 } 1255 return "unknown"; 1256 } 1257 1258 static void append_str(char *dst, const char *src, size_t dst_cap) 1259 { 1260 size_t avail = dst_cap - strlen(dst); 1261 1262 if (avail <= 1) /* just zero byte could be written */ 1263 return; 1264 1265 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */ 1266 } 1267 1268 #define OPTSTRING 60 1269 static void test_options(char *options) 1270 { 1271 char tstr[OPTSTRING]; 1272 1273 memset(options, 0, OPTSTRING); 1274 1275 if (txmsg_pass) 1276 append_str(options, "pass,", OPTSTRING); 1277 if (txmsg_redir) 1278 append_str(options, "redir,", OPTSTRING); 1279 if (txmsg_drop) 1280 append_str(options, "drop,", OPTSTRING); 1281 if (txmsg_apply) { 1282 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 1283 append_str(options, tstr, OPTSTRING); 1284 } 1285 if (txmsg_cork) { 1286 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 1287 append_str(options, tstr, OPTSTRING); 1288 } 1289 if (txmsg_start) { 1290 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 1291 append_str(options, tstr, OPTSTRING); 1292 } 1293 if (txmsg_end) { 1294 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 1295 append_str(options, tstr, OPTSTRING); 1296 } 1297 if (txmsg_start_pop) { 1298 snprintf(tstr, OPTSTRING, "pop (%d,%d),", 1299 txmsg_start_pop, txmsg_start_pop + txmsg_pop); 1300 append_str(options, tstr, OPTSTRING); 1301 } 1302 if (txmsg_ingress) 1303 append_str(options, "ingress,", OPTSTRING); 1304 if (txmsg_redir_skb) 1305 append_str(options, "redir_skb,", OPTSTRING); 1306 if (peek_flag) 1307 append_str(options, "peek,", OPTSTRING); 1308 } 1309 1310 static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 1311 { 1312 char *options = calloc(OPTSTRING, sizeof(char)); 1313 int err; 1314 1315 if (test == SENDPAGE) 1316 opt->sendpage = true; 1317 else 1318 opt->sendpage = false; 1319 1320 if (txmsg_drop) 1321 opt->drop_expected = true; 1322 else 1323 opt->drop_expected = false; 1324 1325 test_options(options); 1326 1327 if (opt->verbose) { 1328 fprintf(stdout, 1329 " [TEST %i]: (%i, %i, %i, %s, %s): ", 1330 test_cnt, opt->rate, opt->iov_count, opt->iov_length, 1331 test_to_str(test), options); 1332 fflush(stdout); 1333 } 1334 err = run_options(opt, cgrp, test); 1335 if (opt->verbose) 1336 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED"); 1337 test_cnt++; 1338 !err ? passed++ : failed++; 1339 free(options); 1340 return err; 1341 } 1342 1343 static void test_exec(int cgrp, struct sockmap_options *opt) 1344 { 1345 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME); 1346 int err; 1347 1348 if (type == 0) { 1349 test_start(); 1350 err = __test_exec(cgrp, SENDMSG, opt); 1351 if (err) 1352 test_fail(); 1353 } else { 1354 test_start(); 1355 err = __test_exec(cgrp, SENDPAGE, opt); 1356 if (err) 1357 test_fail(); 1358 } 1359 } 1360 1361 static void test_send_one(struct sockmap_options *opt, int cgrp) 1362 { 1363 opt->iov_length = 1; 1364 opt->iov_count = 1; 1365 opt->rate = 1; 1366 test_exec(cgrp, opt); 1367 1368 opt->iov_length = 1; 1369 opt->iov_count = 1024; 1370 opt->rate = 1; 1371 test_exec(cgrp, opt); 1372 1373 opt->iov_length = 1024; 1374 opt->iov_count = 1; 1375 opt->rate = 1; 1376 test_exec(cgrp, opt); 1377 1378 } 1379 1380 static void test_send_many(struct sockmap_options *opt, int cgrp) 1381 { 1382 opt->iov_length = 3; 1383 opt->iov_count = 1; 1384 opt->rate = 512; 1385 test_exec(cgrp, opt); 1386 1387 opt->rate = 100; 1388 opt->iov_count = 1; 1389 opt->iov_length = 5; 1390 test_exec(cgrp, opt); 1391 } 1392 1393 static void test_send_large(struct sockmap_options *opt, int cgrp) 1394 { 1395 opt->iov_length = 8192; 1396 opt->iov_count = 32; 1397 opt->rate = 2; 1398 test_exec(cgrp, opt); 1399 } 1400 1401 static void test_send(struct sockmap_options *opt, int cgrp) 1402 { 1403 test_send_one(opt, cgrp); 1404 test_send_many(opt, cgrp); 1405 test_send_large(opt, cgrp); 1406 sched_yield(); 1407 } 1408 1409 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt) 1410 { 1411 /* Test small and large iov_count values with pass/redir/apply/cork */ 1412 txmsg_pass = 1; 1413 test_send(opt, cgrp); 1414 } 1415 1416 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt) 1417 { 1418 txmsg_redir = 1; 1419 test_send(opt, cgrp); 1420 } 1421 1422 static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt) 1423 { 1424 opt->tx_wait_mem = true; 1425 txmsg_redir = 1; 1426 test_send_large(opt, cgrp); 1427 1428 txmsg_redir = 1; 1429 txmsg_apply = 4097; 1430 test_send_large(opt, cgrp); 1431 opt->tx_wait_mem = false; 1432 } 1433 1434 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt) 1435 { 1436 txmsg_drop = 1; 1437 test_send(opt, cgrp); 1438 } 1439 1440 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt) 1441 { 1442 txmsg_pass = txmsg_drop = 0; 1443 txmsg_ingress = txmsg_redir = 1; 1444 test_send(opt, cgrp); 1445 } 1446 1447 /* Test cork with hung data. This tests poor usage patterns where 1448 * cork can leave data on the ring if user program is buggy and 1449 * doesn't flush them somehow. They do take some time however 1450 * because they wait for a timeout. Test pass, redir and cork with 1451 * apply logic. Use cork size of 4097 with send_large to avoid 1452 * aligning cork size with send size. 1453 */ 1454 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt) 1455 { 1456 txmsg_pass = 1; 1457 txmsg_redir = 0; 1458 txmsg_cork = 4097; 1459 txmsg_apply = 4097; 1460 test_send_large(opt, cgrp); 1461 1462 txmsg_pass = 0; 1463 txmsg_redir = 1; 1464 txmsg_apply = 0; 1465 txmsg_cork = 4097; 1466 test_send_large(opt, cgrp); 1467 1468 txmsg_pass = 0; 1469 txmsg_redir = 1; 1470 txmsg_apply = 4097; 1471 txmsg_cork = 4097; 1472 test_send_large(opt, cgrp); 1473 } 1474 1475 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt) 1476 { 1477 /* Test basic start/end */ 1478 txmsg_pass = 1; 1479 txmsg_start = 1; 1480 txmsg_end = 2; 1481 test_send(opt, cgrp); 1482 1483 /* Test >4k pull */ 1484 txmsg_pass = 1; 1485 txmsg_start = 4096; 1486 txmsg_end = 9182; 1487 test_send_large(opt, cgrp); 1488 1489 /* Test pull + redirect */ 1490 txmsg_redir = 1; 1491 txmsg_start = 1; 1492 txmsg_end = 2; 1493 test_send(opt, cgrp); 1494 1495 /* Test pull + cork */ 1496 txmsg_redir = 0; 1497 txmsg_cork = 512; 1498 txmsg_start = 1; 1499 txmsg_end = 2; 1500 test_send_many(opt, cgrp); 1501 1502 /* Test pull + cork + redirect */ 1503 txmsg_redir = 1; 1504 txmsg_cork = 512; 1505 txmsg_start = 1; 1506 txmsg_end = 2; 1507 test_send_many(opt, cgrp); 1508 } 1509 1510 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt) 1511 { 1512 bool data = opt->data_test; 1513 1514 /* Test basic pop */ 1515 txmsg_pass = 1; 1516 txmsg_start_pop = 1; 1517 txmsg_pop = 2; 1518 test_send_many(opt, cgrp); 1519 1520 /* Test pop with >4k */ 1521 txmsg_pass = 1; 1522 txmsg_start_pop = 4096; 1523 txmsg_pop = 4096; 1524 test_send_large(opt, cgrp); 1525 1526 /* Test pop + redirect */ 1527 txmsg_redir = 1; 1528 txmsg_start_pop = 1; 1529 txmsg_pop = 2; 1530 test_send_many(opt, cgrp); 1531 1532 /* TODO: Test for pop + cork should be different, 1533 * - It makes the layout of the received data difficult 1534 * - It makes it hard to calculate the total_bytes in the recvmsg 1535 * Temporarily skip the data integrity test for this case now. 1536 */ 1537 opt->data_test = false; 1538 /* Test pop + cork */ 1539 txmsg_redir = 0; 1540 txmsg_cork = 512; 1541 txmsg_start_pop = 1; 1542 txmsg_pop = 2; 1543 test_send_many(opt, cgrp); 1544 1545 /* Test pop + redirect + cork */ 1546 txmsg_redir = 1; 1547 txmsg_cork = 4; 1548 txmsg_start_pop = 1; 1549 txmsg_pop = 2; 1550 test_send_many(opt, cgrp); 1551 opt->data_test = data; 1552 } 1553 1554 static void test_txmsg_push(int cgrp, struct sockmap_options *opt) 1555 { 1556 bool data = opt->data_test; 1557 1558 /* Test basic push */ 1559 txmsg_pass = 1; 1560 txmsg_start_push = 1; 1561 txmsg_end_push = 1; 1562 test_send(opt, cgrp); 1563 1564 /* Test push 4kB >4k */ 1565 txmsg_pass = 1; 1566 txmsg_start_push = 4096; 1567 txmsg_end_push = 4096; 1568 test_send_large(opt, cgrp); 1569 1570 /* Test push + redirect */ 1571 txmsg_redir = 1; 1572 txmsg_start_push = 1; 1573 txmsg_end_push = 2; 1574 test_send_many(opt, cgrp); 1575 1576 /* TODO: Test for push + cork should be different, 1577 * - It makes the layout of the received data difficult 1578 * - It makes it hard to calculate the total_bytes in the recvmsg 1579 * Temporarily skip the data integrity test for this case now. 1580 */ 1581 opt->data_test = false; 1582 /* Test push + cork */ 1583 txmsg_redir = 0; 1584 txmsg_cork = 512; 1585 txmsg_start_push = 1; 1586 txmsg_end_push = 2; 1587 test_send_many(opt, cgrp); 1588 opt->data_test = data; 1589 } 1590 1591 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt) 1592 { 1593 /* Test push/pop range overlapping */ 1594 txmsg_pass = 1; 1595 txmsg_start_push = 1; 1596 txmsg_end_push = 10; 1597 txmsg_start_pop = 5; 1598 txmsg_pop = 4; 1599 test_send_large(opt, cgrp); 1600 1601 txmsg_pass = 1; 1602 txmsg_start_push = 1; 1603 txmsg_end_push = 10; 1604 txmsg_start_pop = 5; 1605 txmsg_pop = 16; 1606 test_send_large(opt, cgrp); 1607 1608 txmsg_pass = 1; 1609 txmsg_start_push = 5; 1610 txmsg_end_push = 4; 1611 txmsg_start_pop = 1; 1612 txmsg_pop = 10; 1613 test_send_large(opt, cgrp); 1614 1615 txmsg_pass = 1; 1616 txmsg_start_push = 5; 1617 txmsg_end_push = 16; 1618 txmsg_start_pop = 1; 1619 txmsg_pop = 10; 1620 test_send_large(opt, cgrp); 1621 1622 /* Test push/pop range non-overlapping */ 1623 txmsg_pass = 1; 1624 txmsg_start_push = 1; 1625 txmsg_end_push = 10; 1626 txmsg_start_pop = 16; 1627 txmsg_pop = 4; 1628 test_send_large(opt, cgrp); 1629 1630 txmsg_pass = 1; 1631 txmsg_start_push = 16; 1632 txmsg_end_push = 10; 1633 txmsg_start_pop = 5; 1634 txmsg_pop = 4; 1635 test_send_large(opt, cgrp); 1636 } 1637 1638 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt) 1639 { 1640 txmsg_pass = 1; 1641 txmsg_redir = 0; 1642 txmsg_ingress = 0; 1643 txmsg_apply = 1; 1644 txmsg_cork = 0; 1645 test_send_one(opt, cgrp); 1646 1647 txmsg_pass = 0; 1648 txmsg_redir = 1; 1649 txmsg_ingress = 0; 1650 txmsg_apply = 1; 1651 txmsg_cork = 0; 1652 test_send_one(opt, cgrp); 1653 1654 txmsg_pass = 0; 1655 txmsg_redir = 1; 1656 txmsg_ingress = 1; 1657 txmsg_apply = 1; 1658 txmsg_cork = 0; 1659 test_send_one(opt, cgrp); 1660 1661 txmsg_pass = 1; 1662 txmsg_redir = 0; 1663 txmsg_ingress = 0; 1664 txmsg_apply = 1024; 1665 txmsg_cork = 0; 1666 test_send_large(opt, cgrp); 1667 1668 txmsg_pass = 0; 1669 txmsg_redir = 1; 1670 txmsg_ingress = 0; 1671 txmsg_apply = 1024; 1672 txmsg_cork = 0; 1673 test_send_large(opt, cgrp); 1674 1675 txmsg_pass = 0; 1676 txmsg_redir = 1; 1677 txmsg_ingress = 1; 1678 txmsg_apply = 1024; 1679 txmsg_cork = 0; 1680 test_send_large(opt, cgrp); 1681 } 1682 1683 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt) 1684 { 1685 txmsg_pass = 1; 1686 txmsg_redir = 0; 1687 txmsg_apply = 0; 1688 txmsg_cork = 1; 1689 test_send(opt, cgrp); 1690 1691 txmsg_pass = 1; 1692 txmsg_redir = 0; 1693 txmsg_apply = 1; 1694 txmsg_cork = 1; 1695 test_send(opt, cgrp); 1696 } 1697 1698 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt) 1699 { 1700 txmsg_pass = 1; 1701 skb_use_parser = 512; 1702 opt->iov_length = 256; 1703 opt->iov_count = 1; 1704 opt->rate = 2; 1705 test_exec(cgrp, opt); 1706 } 1707 1708 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt) 1709 { 1710 skb_use_parser = 10; 1711 opt->iov_length = 20; 1712 opt->iov_count = 1; 1713 opt->rate = 1; 1714 opt->check_recved_len = true; 1715 test_exec(cgrp, opt); 1716 opt->check_recved_len = false; 1717 } 1718 1719 char *map_names[] = { 1720 "sock_map", 1721 "sock_map_txmsg", 1722 "sock_map_redir", 1723 "sock_apply_bytes", 1724 "sock_cork_bytes", 1725 "sock_bytes", 1726 "sock_redir_flags", 1727 "sock_skb_opts", 1728 }; 1729 1730 static int populate_progs(char *bpf_file) 1731 { 1732 struct bpf_program *prog; 1733 struct bpf_object *obj; 1734 int i = 0; 1735 long err; 1736 1737 obj = bpf_object__open(bpf_file); 1738 err = libbpf_get_error(obj); 1739 if (err) { 1740 char err_buf[256]; 1741 1742 libbpf_strerror(err, err_buf, sizeof(err_buf)); 1743 printf("Unable to load eBPF objects in file '%s' : %s\n", 1744 bpf_file, err_buf); 1745 return -1; 1746 } 1747 1748 i = bpf_object__load(obj); 1749 i = 0; 1750 bpf_object__for_each_program(prog, obj) { 1751 progs[i] = prog; 1752 i++; 1753 } 1754 1755 for (i = 0; i < ARRAY_SIZE(map_fd); i++) { 1756 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 1757 map_fd[i] = bpf_map__fd(maps[i]); 1758 if (map_fd[i] < 0) { 1759 fprintf(stderr, "load_bpf_file: (%i) %s\n", 1760 map_fd[i], strerror(errno)); 1761 return -1; 1762 } 1763 } 1764 1765 for (i = 0; i < ARRAY_SIZE(links); i++) 1766 links[i] = NULL; 1767 1768 return 0; 1769 } 1770 1771 struct _test test[] = { 1772 {"txmsg test passthrough", test_txmsg_pass}, 1773 {"txmsg test redirect", test_txmsg_redir}, 1774 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem}, 1775 {"txmsg test drop", test_txmsg_drop}, 1776 {"txmsg test ingress redirect", test_txmsg_ingress_redir}, 1777 {"txmsg test apply", test_txmsg_apply}, 1778 {"txmsg test cork", test_txmsg_cork}, 1779 {"txmsg test hanging corks", test_txmsg_cork_hangs}, 1780 {"txmsg test push_data", test_txmsg_push}, 1781 {"txmsg test pull-data", test_txmsg_pull}, 1782 {"txmsg test pop-data", test_txmsg_pop}, 1783 {"txmsg test push/pop data", test_txmsg_push_pop}, 1784 {"txmsg test ingress parser", test_txmsg_ingress_parser}, 1785 {"txmsg test ingress parser2", test_txmsg_ingress_parser2}, 1786 }; 1787 1788 static int check_whitelist(struct _test *t, struct sockmap_options *opt) 1789 { 1790 char *entry, *ptr; 1791 1792 if (!opt->whitelist) 1793 return 0; 1794 ptr = strdup(opt->whitelist); 1795 if (!ptr) 1796 return -ENOMEM; 1797 entry = strtok(ptr, ","); 1798 while (entry) { 1799 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 1800 strstr(opt->map, entry) != 0 || 1801 strstr(t->title, entry) != 0) { 1802 free(ptr); 1803 return 0; 1804 } 1805 entry = strtok(NULL, ","); 1806 } 1807 free(ptr); 1808 return -EINVAL; 1809 } 1810 1811 static int check_blacklist(struct _test *t, struct sockmap_options *opt) 1812 { 1813 char *entry, *ptr; 1814 1815 if (!opt->blacklist) 1816 return -EINVAL; 1817 ptr = strdup(opt->blacklist); 1818 if (!ptr) 1819 return -ENOMEM; 1820 entry = strtok(ptr, ","); 1821 while (entry) { 1822 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 1823 strstr(opt->map, entry) != 0 || 1824 strstr(t->title, entry) != 0) { 1825 free(ptr); 1826 return 0; 1827 } 1828 entry = strtok(NULL, ","); 1829 } 1830 free(ptr); 1831 return -EINVAL; 1832 } 1833 1834 static int __test_selftests(int cg_fd, struct sockmap_options *opt) 1835 { 1836 int i, err; 1837 1838 err = populate_progs(opt->map); 1839 if (err < 0) { 1840 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 1841 return err; 1842 } 1843 1844 /* Tests basic commands and APIs */ 1845 for (i = 0; i < ARRAY_SIZE(test); i++) { 1846 struct _test t = test[i]; 1847 1848 if (check_whitelist(&t, opt) != 0) 1849 continue; 1850 if (check_blacklist(&t, opt) == 0) 1851 continue; 1852 1853 test_start_subtest(&t, opt); 1854 t.tester(cg_fd, opt); 1855 test_end_subtest(); 1856 } 1857 1858 return err; 1859 } 1860 1861 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt) 1862 { 1863 opt->map = BPF_SOCKMAP_FILENAME; 1864 __test_selftests(cg_fd, opt); 1865 } 1866 1867 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt) 1868 { 1869 opt->map = BPF_SOCKHASH_FILENAME; 1870 __test_selftests(cg_fd, opt); 1871 } 1872 1873 static int test_selftest(int cg_fd, struct sockmap_options *opt) 1874 { 1875 test_selftests_sockmap(cg_fd, opt); 1876 test_selftests_sockhash(cg_fd, opt); 1877 test_print_results(); 1878 return 0; 1879 } 1880 1881 int main(int argc, char **argv) 1882 { 1883 int iov_count = 1, length = 1024, rate = 1; 1884 struct sockmap_options options = {0}; 1885 int opt, longindex, err, cg_fd = 0; 1886 char *bpf_file = BPF_SOCKMAP_FILENAME; 1887 int test = SELFTESTS; 1888 bool cg_created = 0; 1889 1890 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:", 1891 long_options, &longindex)) != -1) { 1892 switch (opt) { 1893 case 's': 1894 txmsg_start = atoi(optarg); 1895 break; 1896 case 'e': 1897 txmsg_end = atoi(optarg); 1898 break; 1899 case 'p': 1900 txmsg_start_push = atoi(optarg); 1901 break; 1902 case 'q': 1903 txmsg_end_push = atoi(optarg); 1904 break; 1905 case 'w': 1906 txmsg_start_pop = atoi(optarg); 1907 break; 1908 case 'x': 1909 txmsg_pop = atoi(optarg); 1910 break; 1911 case 'a': 1912 txmsg_apply = atoi(optarg); 1913 break; 1914 case 'k': 1915 txmsg_cork = atoi(optarg); 1916 break; 1917 case 'c': 1918 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 1919 if (cg_fd < 0) { 1920 fprintf(stderr, 1921 "ERROR: (%i) open cg path failed: %s\n", 1922 cg_fd, optarg); 1923 return cg_fd; 1924 } 1925 break; 1926 case 'r': 1927 rate = atoi(optarg); 1928 break; 1929 case 'v': 1930 options.verbose = 1; 1931 if (optarg) 1932 options.verbose = atoi(optarg); 1933 break; 1934 case 'i': 1935 iov_count = atoi(optarg); 1936 break; 1937 case 'l': 1938 length = atoi(optarg); 1939 break; 1940 case 'd': 1941 options.data_test = true; 1942 break; 1943 case 't': 1944 if (strcmp(optarg, "ping") == 0) { 1945 test = PING_PONG; 1946 } else if (strcmp(optarg, "sendmsg") == 0) { 1947 test = SENDMSG; 1948 } else if (strcmp(optarg, "base") == 0) { 1949 test = BASE; 1950 } else if (strcmp(optarg, "base_sendpage") == 0) { 1951 test = BASE_SENDPAGE; 1952 } else if (strcmp(optarg, "sendpage") == 0) { 1953 test = SENDPAGE; 1954 } else { 1955 usage(argv); 1956 return -1; 1957 } 1958 break; 1959 case 'n': 1960 options.whitelist = strdup(optarg); 1961 if (!options.whitelist) 1962 return -ENOMEM; 1963 break; 1964 case 'b': 1965 options.blacklist = strdup(optarg); 1966 if (!options.blacklist) 1967 return -ENOMEM; 1968 case 0: 1969 break; 1970 case 'h': 1971 default: 1972 usage(argv); 1973 return -1; 1974 } 1975 } 1976 1977 if (!cg_fd) { 1978 cg_fd = cgroup_setup_and_join(CG_PATH); 1979 if (cg_fd < 0) 1980 return cg_fd; 1981 cg_created = 1; 1982 } 1983 1984 /* Use libbpf 1.0 API mode */ 1985 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 1986 1987 if (test == SELFTESTS) { 1988 err = test_selftest(cg_fd, &options); 1989 goto out; 1990 } 1991 1992 err = populate_progs(bpf_file); 1993 if (err) { 1994 fprintf(stderr, "populate program: (%s) %s\n", 1995 bpf_file, strerror(errno)); 1996 return 1; 1997 } 1998 running = 1; 1999 2000 /* catch SIGINT */ 2001 signal(SIGINT, running_handler); 2002 2003 options.iov_count = iov_count; 2004 options.iov_length = length; 2005 options.rate = rate; 2006 2007 err = run_options(&options, cg_fd, test); 2008 out: 2009 if (options.whitelist) 2010 free(options.whitelist); 2011 if (options.blacklist) 2012 free(options.blacklist); 2013 close(cg_fd); 2014 if (cg_created) 2015 cleanup_cgroup_environment(); 2016 return err; 2017 } 2018 2019 void running_handler(int a) 2020 { 2021 running = 0; 2022 } 2023