1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <sys/socket.h> 6 #include <sys/ioctl.h> 7 #include <sys/select.h> 8 #include <netinet/in.h> 9 #include <arpa/inet.h> 10 #include <unistd.h> 11 #include <string.h> 12 #include <errno.h> 13 #include <stdbool.h> 14 #include <signal.h> 15 #include <fcntl.h> 16 #include <sys/wait.h> 17 #include <time.h> 18 #include <sched.h> 19 20 #include <sys/time.h> 21 #include <sys/types.h> 22 #include <sys/sendfile.h> 23 24 #include <linux/netlink.h> 25 #include <linux/socket.h> 26 #include <linux/sock_diag.h> 27 #include <linux/bpf.h> 28 #include <linux/if_link.h> 29 #include <linux/tls.h> 30 #include <assert.h> 31 #include <libgen.h> 32 33 #include <getopt.h> 34 35 #include <bpf/bpf.h> 36 #include <bpf/libbpf.h> 37 38 #include "bpf_util.h" 39 #include "cgroup_helpers.h" 40 41 int running; 42 static void running_handler(int a); 43 44 #ifndef TCP_ULP 45 # define TCP_ULP 31 46 #endif 47 #ifndef SOL_TLS 48 # define SOL_TLS 282 49 #endif 50 51 /* randomly selected ports for testing on lo */ 52 #define S1_PORT 10000 53 #define S2_PORT 10001 54 55 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o" 56 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o" 57 #define CG_PATH "/sockmap" 58 59 /* global sockets */ 60 int s1, s2, c1, c2, p1, p2; 61 int test_cnt; 62 int passed; 63 int failed; 64 int map_fd[9]; 65 struct bpf_map *maps[9]; 66 struct bpf_program *progs[9]; 67 struct bpf_link *links[9]; 68 69 int txmsg_pass; 70 int txmsg_redir; 71 int txmsg_drop; 72 int txmsg_apply; 73 int txmsg_cork; 74 int txmsg_start; 75 int txmsg_end; 76 int txmsg_start_push; 77 int txmsg_end_push; 78 int txmsg_start_pop; 79 int txmsg_pop; 80 int txmsg_ingress; 81 int txmsg_redir_skb; 82 int txmsg_ktls_skb; 83 int txmsg_ktls_skb_drop; 84 int txmsg_ktls_skb_redir; 85 int ktls; 86 int peek_flag; 87 int skb_use_parser; 88 int txmsg_omit_skb_parser; 89 90 static const struct option long_options[] = { 91 {"help", no_argument, NULL, 'h' }, 92 {"cgroup", required_argument, NULL, 'c' }, 93 {"rate", required_argument, NULL, 'r' }, 94 {"verbose", optional_argument, NULL, 'v' }, 95 {"iov_count", required_argument, NULL, 'i' }, 96 {"length", required_argument, NULL, 'l' }, 97 {"test", required_argument, NULL, 't' }, 98 {"data_test", no_argument, NULL, 'd' }, 99 {"txmsg", no_argument, &txmsg_pass, 1 }, 100 {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 101 {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 102 {"txmsg_apply", required_argument, NULL, 'a'}, 103 {"txmsg_cork", required_argument, NULL, 'k'}, 104 {"txmsg_start", required_argument, NULL, 's'}, 105 {"txmsg_end", required_argument, NULL, 'e'}, 106 {"txmsg_start_push", required_argument, NULL, 'p'}, 107 {"txmsg_end_push", required_argument, NULL, 'q'}, 108 {"txmsg_start_pop", required_argument, NULL, 'w'}, 109 {"txmsg_pop", required_argument, NULL, 'x'}, 110 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 111 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 }, 112 {"ktls", no_argument, &ktls, 1 }, 113 {"peek", no_argument, &peek_flag, 1 }, 114 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1}, 115 {"whitelist", required_argument, NULL, 'n' }, 116 {"blacklist", required_argument, NULL, 'b' }, 117 {0, 0, NULL, 0 } 118 }; 119 120 struct test_env { 121 const char *type; 122 const char *subtest; 123 const char *prepend; 124 125 int test_num; 126 int subtest_num; 127 128 int succ_cnt; 129 int fail_cnt; 130 int fail_last; 131 }; 132 133 struct test_env env; 134 135 struct sockmap_options { 136 int verbose; 137 bool base; 138 bool sendpage; 139 bool data_test; 140 bool drop_expected; 141 bool check_recved_len; 142 bool tx_wait_mem; 143 int iov_count; 144 int iov_length; 145 int rate; 146 char *map; 147 char *whitelist; 148 char *blacklist; 149 char *prepend; 150 }; 151 152 struct _test { 153 char *title; 154 void (*tester)(int cg_fd, struct sockmap_options *opt); 155 }; 156 157 static void test_start(void) 158 { 159 env.subtest_num++; 160 } 161 162 static void test_fail(void) 163 { 164 env.fail_cnt++; 165 } 166 167 static void test_pass(void) 168 { 169 env.succ_cnt++; 170 } 171 172 static void test_reset(void) 173 { 174 txmsg_start = txmsg_end = 0; 175 txmsg_start_pop = txmsg_pop = 0; 176 txmsg_start_push = txmsg_end_push = 0; 177 txmsg_pass = txmsg_drop = txmsg_redir = 0; 178 txmsg_apply = txmsg_cork = 0; 179 txmsg_ingress = txmsg_redir_skb = 0; 180 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0; 181 txmsg_omit_skb_parser = 0; 182 skb_use_parser = 0; 183 } 184 185 static int test_start_subtest(const struct _test *t, struct sockmap_options *o) 186 { 187 env.type = o->map; 188 env.subtest = t->title; 189 env.prepend = o->prepend; 190 env.test_num++; 191 env.subtest_num = 0; 192 env.fail_last = env.fail_cnt; 193 test_reset(); 194 return 0; 195 } 196 197 static void test_end_subtest(void) 198 { 199 int error = env.fail_cnt - env.fail_last; 200 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME); 201 202 if (!error) 203 test_pass(); 204 205 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n", 206 env.test_num, env.subtest_num, 207 !type ? "sockmap" : "sockhash", 208 env.prepend ? : "", 209 env.subtest, error ? "FAIL" : "OK"); 210 } 211 212 static void test_print_results(void) 213 { 214 fprintf(stdout, "Pass: %d Fail: %d\n", 215 env.succ_cnt, env.fail_cnt); 216 } 217 218 static void usage(char *argv[]) 219 { 220 int i; 221 222 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 223 printf(" options:\n"); 224 for (i = 0; long_options[i].name != 0; i++) { 225 printf(" --%-12s", long_options[i].name); 226 if (long_options[i].flag != NULL) 227 printf(" flag (internal value:%d)\n", 228 *long_options[i].flag); 229 else 230 printf(" -%c\n", long_options[i].val); 231 } 232 printf("\n"); 233 } 234 235 char *sock_to_string(int s) 236 { 237 if (s == c1) 238 return "client1"; 239 else if (s == c2) 240 return "client2"; 241 else if (s == s1) 242 return "server1"; 243 else if (s == s2) 244 return "server2"; 245 else if (s == p1) 246 return "peer1"; 247 else if (s == p2) 248 return "peer2"; 249 else 250 return "unknown"; 251 } 252 253 static int sockmap_init_ktls(int verbose, int s) 254 { 255 struct tls12_crypto_info_aes_gcm_128 tls_tx = { 256 .info = { 257 .version = TLS_1_2_VERSION, 258 .cipher_type = TLS_CIPHER_AES_GCM_128, 259 }, 260 }; 261 struct tls12_crypto_info_aes_gcm_128 tls_rx = { 262 .info = { 263 .version = TLS_1_2_VERSION, 264 .cipher_type = TLS_CIPHER_AES_GCM_128, 265 }, 266 }; 267 int so_buf = 6553500; 268 int err; 269 270 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls")); 271 if (err) { 272 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err); 273 return -EINVAL; 274 } 275 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); 276 if (err) { 277 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err); 278 return -EINVAL; 279 } 280 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); 281 if (err) { 282 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err); 283 return -EINVAL; 284 } 285 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); 286 if (err) { 287 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err); 288 return -EINVAL; 289 } 290 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); 291 if (err) { 292 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err); 293 return -EINVAL; 294 } 295 296 if (verbose) 297 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s)); 298 return 0; 299 } 300 static int sockmap_init_sockets(int verbose) 301 { 302 int i, err, one = 1; 303 struct sockaddr_in addr; 304 int *fds[4] = {&s1, &s2, &c1, &c2}; 305 306 s1 = s2 = p1 = p2 = c1 = c2 = 0; 307 308 /* Init sockets */ 309 for (i = 0; i < 4; i++) { 310 *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 311 if (*fds[i] < 0) { 312 perror("socket s1 failed()"); 313 return errno; 314 } 315 } 316 317 /* Allow reuse */ 318 for (i = 0; i < 2; i++) { 319 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 320 (char *)&one, sizeof(one)); 321 if (err) { 322 perror("setsockopt failed()"); 323 return errno; 324 } 325 } 326 327 /* Non-blocking sockets */ 328 for (i = 0; i < 2; i++) { 329 err = ioctl(*fds[i], FIONBIO, (char *)&one); 330 if (err < 0) { 331 perror("ioctl s1 failed()"); 332 return errno; 333 } 334 } 335 336 /* Bind server sockets */ 337 memset(&addr, 0, sizeof(struct sockaddr_in)); 338 addr.sin_family = AF_INET; 339 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 340 341 addr.sin_port = htons(S1_PORT); 342 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 343 if (err < 0) { 344 perror("bind s1 failed()"); 345 return errno; 346 } 347 348 addr.sin_port = htons(S2_PORT); 349 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 350 if (err < 0) { 351 perror("bind s2 failed()"); 352 return errno; 353 } 354 355 /* Listen server sockets */ 356 addr.sin_port = htons(S1_PORT); 357 err = listen(s1, 32); 358 if (err < 0) { 359 perror("listen s1 failed()"); 360 return errno; 361 } 362 363 addr.sin_port = htons(S2_PORT); 364 err = listen(s2, 32); 365 if (err < 0) { 366 perror("listen s1 failed()"); 367 return errno; 368 } 369 370 /* Initiate Connect */ 371 addr.sin_port = htons(S1_PORT); 372 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 373 if (err < 0 && errno != EINPROGRESS) { 374 perror("connect c1 failed()"); 375 return errno; 376 } 377 378 addr.sin_port = htons(S2_PORT); 379 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 380 if (err < 0 && errno != EINPROGRESS) { 381 perror("connect c2 failed()"); 382 return errno; 383 } else if (err < 0) { 384 err = 0; 385 } 386 387 /* Accept Connecrtions */ 388 p1 = accept(s1, NULL, NULL); 389 if (p1 < 0) { 390 perror("accept s1 failed()"); 391 return errno; 392 } 393 394 p2 = accept(s2, NULL, NULL); 395 if (p2 < 0) { 396 perror("accept s1 failed()"); 397 return errno; 398 } 399 400 if (verbose > 1) { 401 printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 402 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 403 c1, s1, c2, s2); 404 } 405 return 0; 406 } 407 408 struct msg_stats { 409 size_t bytes_sent; 410 size_t bytes_recvd; 411 struct timespec start; 412 struct timespec end; 413 }; 414 415 static int msg_loop_sendpage(int fd, int iov_length, int cnt, 416 struct msg_stats *s, 417 struct sockmap_options *opt) 418 { 419 bool drop = opt->drop_expected; 420 unsigned char k = 0; 421 FILE *file; 422 int i, fp; 423 424 file = tmpfile(); 425 if (!file) { 426 perror("create file for sendpage"); 427 return 1; 428 } 429 for (i = 0; i < iov_length * cnt; i++, k++) 430 fwrite(&k, sizeof(char), 1, file); 431 fflush(file); 432 fseek(file, 0, SEEK_SET); 433 434 fp = fileno(file); 435 436 clock_gettime(CLOCK_MONOTONIC, &s->start); 437 for (i = 0; i < cnt; i++) { 438 int sent; 439 440 errno = 0; 441 sent = sendfile(fd, fp, NULL, iov_length); 442 443 if (!drop && sent < 0) { 444 perror("sendpage loop error"); 445 fclose(file); 446 return sent; 447 } else if (drop && sent >= 0) { 448 printf("sendpage loop error expected: %i errno %i\n", 449 sent, errno); 450 fclose(file); 451 return -EIO; 452 } 453 454 if (sent > 0) 455 s->bytes_sent += sent; 456 } 457 clock_gettime(CLOCK_MONOTONIC, &s->end); 458 fclose(file); 459 return 0; 460 } 461 462 static void msg_free_iov(struct msghdr *msg) 463 { 464 int i; 465 466 for (i = 0; i < msg->msg_iovlen; i++) 467 free(msg->msg_iov[i].iov_base); 468 free(msg->msg_iov); 469 msg->msg_iov = NULL; 470 msg->msg_iovlen = 0; 471 } 472 473 static int msg_alloc_iov(struct msghdr *msg, 474 int iov_count, int iov_length, 475 bool data, bool xmit) 476 { 477 unsigned char k = 0; 478 struct iovec *iov; 479 int i; 480 481 iov = calloc(iov_count, sizeof(struct iovec)); 482 if (!iov) 483 return errno; 484 485 for (i = 0; i < iov_count; i++) { 486 unsigned char *d = calloc(iov_length, sizeof(char)); 487 488 if (!d) { 489 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 490 goto unwind_iov; 491 } 492 iov[i].iov_base = d; 493 iov[i].iov_len = iov_length; 494 495 if (data && xmit) { 496 int j; 497 498 for (j = 0; j < iov_length; j++) 499 d[j] = k++; 500 } 501 } 502 503 msg->msg_iov = iov; 504 msg->msg_iovlen = iov_count; 505 506 return 0; 507 unwind_iov: 508 for (i--; i >= 0 ; i--) 509 free(msg->msg_iov[i].iov_base); 510 return -ENOMEM; 511 } 512 513 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz) 514 { 515 int i, j = 0, bytes_cnt = 0; 516 unsigned char k = 0; 517 518 for (i = 0; i < msg->msg_iovlen; i++) { 519 unsigned char *d = msg->msg_iov[i].iov_base; 520 521 /* Special case test for skb ingress + ktls */ 522 if (i == 0 && txmsg_ktls_skb) { 523 if (msg->msg_iov[i].iov_len < 4) 524 return -EIO; 525 if (memcmp(d, "PASS", 4) != 0) { 526 fprintf(stderr, 527 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", 528 i, 0, d[0], d[1], d[2], d[3]); 529 return -EIO; 530 } 531 j = 4; /* advance index past PASS header */ 532 } 533 534 for (; j < msg->msg_iov[i].iov_len && size; j++) { 535 if (d[j] != k++) { 536 fprintf(stderr, 537 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 538 i, j, d[j], k - 1, d[j+1], k); 539 return -EIO; 540 } 541 bytes_cnt++; 542 if (bytes_cnt == chunk_sz) { 543 k = 0; 544 bytes_cnt = 0; 545 } 546 size--; 547 } 548 } 549 return 0; 550 } 551 552 static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 553 struct msg_stats *s, bool tx, 554 struct sockmap_options *opt) 555 { 556 struct msghdr msg = {0}, msg_peek = {0}; 557 int err, i, flags = MSG_NOSIGNAL; 558 bool drop = opt->drop_expected; 559 bool data = opt->data_test; 560 int iov_alloc_length = iov_length; 561 562 if (!tx && opt->check_recved_len) 563 iov_alloc_length *= 2; 564 565 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx); 566 if (err) 567 goto out_errno; 568 if (peek_flag) { 569 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx); 570 if (err) 571 goto out_errno; 572 } 573 574 if (tx) { 575 clock_gettime(CLOCK_MONOTONIC, &s->start); 576 for (i = 0; i < cnt; i++) { 577 int sent; 578 579 errno = 0; 580 sent = sendmsg(fd, &msg, flags); 581 582 if (!drop && sent < 0) { 583 if (opt->tx_wait_mem && errno == EACCES) { 584 errno = 0; 585 goto out_errno; 586 } 587 perror("sendmsg loop error"); 588 goto out_errno; 589 } else if (drop && sent >= 0) { 590 fprintf(stderr, 591 "sendmsg loop error expected: %i errno %i\n", 592 sent, errno); 593 errno = -EIO; 594 goto out_errno; 595 } 596 if (sent > 0) 597 s->bytes_sent += sent; 598 } 599 clock_gettime(CLOCK_MONOTONIC, &s->end); 600 } else { 601 int slct, recvp = 0, recv, max_fd = fd; 602 float total_bytes, txmsg_pop_total; 603 int fd_flags = O_NONBLOCK; 604 struct timeval timeout; 605 fd_set w; 606 607 fcntl(fd, fd_flags); 608 /* Account for pop bytes noting each iteration of apply will 609 * call msg_pop_data helper so we need to account for this 610 * by calculating the number of apply iterations. Note user 611 * of the tool can create cases where no data is sent by 612 * manipulating pop/push/pull/etc. For example txmsg_apply 1 613 * with txmsg_pop 1 will try to apply 1B at a time but each 614 * iteration will then pop 1B so no data will ever be sent. 615 * This is really only useful for testing edge cases in code 616 * paths. 617 */ 618 total_bytes = (float)iov_count * (float)iov_length * (float)cnt; 619 if (txmsg_apply) 620 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply); 621 else 622 txmsg_pop_total = txmsg_pop * cnt; 623 total_bytes -= txmsg_pop_total; 624 err = clock_gettime(CLOCK_MONOTONIC, &s->start); 625 if (err < 0) 626 perror("recv start time"); 627 while (s->bytes_recvd < total_bytes) { 628 if (txmsg_cork) { 629 timeout.tv_sec = 0; 630 timeout.tv_usec = 300000; 631 } else { 632 timeout.tv_sec = 3; 633 timeout.tv_usec = 0; 634 } 635 636 /* FD sets */ 637 FD_ZERO(&w); 638 FD_SET(fd, &w); 639 640 slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 641 if (slct == -1) { 642 perror("select()"); 643 clock_gettime(CLOCK_MONOTONIC, &s->end); 644 goto out_errno; 645 } else if (!slct) { 646 if (opt->verbose) 647 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total); 648 errno = -EIO; 649 clock_gettime(CLOCK_MONOTONIC, &s->end); 650 goto out_errno; 651 } 652 653 if (opt->tx_wait_mem) { 654 FD_ZERO(&w); 655 FD_SET(fd, &w); 656 slct = select(max_fd + 1, NULL, NULL, &w, &timeout); 657 errno = 0; 658 close(fd); 659 goto out_errno; 660 } 661 662 errno = 0; 663 if (peek_flag) { 664 flags |= MSG_PEEK; 665 recvp = recvmsg(fd, &msg_peek, flags); 666 if (recvp < 0) { 667 if (errno != EWOULDBLOCK) { 668 clock_gettime(CLOCK_MONOTONIC, &s->end); 669 goto out_errno; 670 } 671 } 672 flags = 0; 673 } 674 675 recv = recvmsg(fd, &msg, flags); 676 if (recv < 0) { 677 if (errno != EWOULDBLOCK) { 678 clock_gettime(CLOCK_MONOTONIC, &s->end); 679 perror("recv failed()"); 680 goto out_errno; 681 } 682 } 683 684 if (recv > 0) 685 s->bytes_recvd += recv; 686 687 if (opt->check_recved_len && s->bytes_recvd > total_bytes) { 688 errno = EMSGSIZE; 689 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n", 690 s->bytes_recvd, total_bytes); 691 goto out_errno; 692 } 693 694 if (data) { 695 int chunk_sz = opt->sendpage ? 696 iov_length * cnt : 697 iov_length * iov_count; 698 699 errno = msg_verify_data(&msg, recv, chunk_sz); 700 if (errno) { 701 perror("data verify msg failed"); 702 goto out_errno; 703 } 704 if (recvp) { 705 errno = msg_verify_data(&msg_peek, 706 recvp, 707 chunk_sz); 708 if (errno) { 709 perror("data verify msg_peek failed"); 710 goto out_errno; 711 } 712 } 713 } 714 } 715 clock_gettime(CLOCK_MONOTONIC, &s->end); 716 } 717 718 msg_free_iov(&msg); 719 msg_free_iov(&msg_peek); 720 return err; 721 out_errno: 722 msg_free_iov(&msg); 723 msg_free_iov(&msg_peek); 724 return errno; 725 } 726 727 static float giga = 1000000000; 728 729 static inline float sentBps(struct msg_stats s) 730 { 731 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 732 } 733 734 static inline float recvdBps(struct msg_stats s) 735 { 736 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 737 } 738 739 static int sendmsg_test(struct sockmap_options *opt) 740 { 741 float sent_Bps = 0, recvd_Bps = 0; 742 int rx_fd, txpid, rxpid, err = 0; 743 struct msg_stats s = {0}; 744 int iov_count = opt->iov_count; 745 int iov_buf = opt->iov_length; 746 int rx_status, tx_status; 747 int cnt = opt->rate; 748 749 errno = 0; 750 751 if (opt->base) 752 rx_fd = p1; 753 else 754 rx_fd = p2; 755 756 if (ktls) { 757 /* Redirecting into non-TLS socket which sends into a TLS 758 * socket is not a valid test. So in this case lets not 759 * enable kTLS but still run the test. 760 */ 761 if (!txmsg_redir || txmsg_ingress) { 762 err = sockmap_init_ktls(opt->verbose, rx_fd); 763 if (err) 764 return err; 765 } 766 err = sockmap_init_ktls(opt->verbose, c1); 767 if (err) 768 return err; 769 } 770 771 if (opt->tx_wait_mem) { 772 struct timeval timeout; 773 int rxtx_buf_len = 1024; 774 775 timeout.tv_sec = 3; 776 timeout.tv_usec = 0; 777 778 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval)); 779 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int)); 780 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int)); 781 if (err) { 782 perror("setsockopt failed()"); 783 return errno; 784 } 785 } 786 787 rxpid = fork(); 788 if (rxpid == 0) { 789 if (txmsg_pop || txmsg_start_pop) 790 iov_buf -= (txmsg_pop - txmsg_start_pop + 1); 791 if (opt->drop_expected || txmsg_ktls_skb_drop) 792 _exit(0); 793 794 if (!iov_buf) /* zero bytes sent case */ 795 _exit(0); 796 797 if (opt->sendpage) 798 iov_count = 1; 799 err = msg_loop(rx_fd, iov_count, iov_buf, 800 cnt, &s, false, opt); 801 if (opt->verbose > 1) 802 fprintf(stderr, 803 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 804 iov_count, iov_buf, cnt, err); 805 if (s.end.tv_sec - s.start.tv_sec) { 806 sent_Bps = sentBps(s); 807 recvd_Bps = recvdBps(s); 808 } 809 if (opt->verbose > 1) 810 fprintf(stdout, 811 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n", 812 s.bytes_sent, sent_Bps, sent_Bps/giga, 813 s.bytes_recvd, recvd_Bps, recvd_Bps/giga, 814 peek_flag ? "(peek_msg)" : ""); 815 if (err && txmsg_cork) 816 err = 0; 817 exit(err ? 1 : 0); 818 } else if (rxpid == -1) { 819 perror("msg_loop_rx"); 820 return errno; 821 } 822 823 if (opt->tx_wait_mem) 824 close(c2); 825 826 txpid = fork(); 827 if (txpid == 0) { 828 if (opt->sendpage) 829 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 830 else 831 err = msg_loop(c1, iov_count, iov_buf, 832 cnt, &s, true, opt); 833 834 if (err) 835 fprintf(stderr, 836 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 837 iov_count, iov_buf, cnt, err); 838 if (s.end.tv_sec - s.start.tv_sec) { 839 sent_Bps = sentBps(s); 840 recvd_Bps = recvdBps(s); 841 } 842 if (opt->verbose > 1) 843 fprintf(stdout, 844 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 845 s.bytes_sent, sent_Bps, sent_Bps/giga, 846 s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 847 exit(err ? 1 : 0); 848 } else if (txpid == -1) { 849 perror("msg_loop_tx"); 850 return errno; 851 } 852 853 assert(waitpid(rxpid, &rx_status, 0) == rxpid); 854 assert(waitpid(txpid, &tx_status, 0) == txpid); 855 if (WIFEXITED(rx_status)) { 856 err = WEXITSTATUS(rx_status); 857 if (err) { 858 fprintf(stderr, "rx thread exited with err %d.\n", err); 859 goto out; 860 } 861 } 862 if (WIFEXITED(tx_status)) { 863 err = WEXITSTATUS(tx_status); 864 if (err) 865 fprintf(stderr, "tx thread exited with err %d.\n", err); 866 } 867 out: 868 return err; 869 } 870 871 static int forever_ping_pong(int rate, struct sockmap_options *opt) 872 { 873 struct timeval timeout; 874 char buf[1024] = {0}; 875 int sc; 876 877 timeout.tv_sec = 10; 878 timeout.tv_usec = 0; 879 880 /* Ping/Pong data from client to server */ 881 sc = send(c1, buf, sizeof(buf), 0); 882 if (sc < 0) { 883 perror("send failed()"); 884 return sc; 885 } 886 887 do { 888 int s, rc, i, max_fd = p2; 889 fd_set w; 890 891 /* FD sets */ 892 FD_ZERO(&w); 893 FD_SET(c1, &w); 894 FD_SET(c2, &w); 895 FD_SET(p1, &w); 896 FD_SET(p2, &w); 897 898 s = select(max_fd + 1, &w, NULL, NULL, &timeout); 899 if (s == -1) { 900 perror("select()"); 901 break; 902 } else if (!s) { 903 fprintf(stderr, "unexpected timeout\n"); 904 break; 905 } 906 907 for (i = 0; i <= max_fd && s > 0; ++i) { 908 if (!FD_ISSET(i, &w)) 909 continue; 910 911 s--; 912 913 rc = recv(i, buf, sizeof(buf), 0); 914 if (rc < 0) { 915 if (errno != EWOULDBLOCK) { 916 perror("recv failed()"); 917 return rc; 918 } 919 } 920 921 if (rc == 0) { 922 close(i); 923 break; 924 } 925 926 sc = send(i, buf, rc, 0); 927 if (sc < 0) { 928 perror("send failed()"); 929 return sc; 930 } 931 } 932 933 if (rate) 934 sleep(rate); 935 936 if (opt->verbose) { 937 printf("."); 938 fflush(stdout); 939 940 } 941 } while (running); 942 943 return 0; 944 } 945 946 enum { 947 SELFTESTS, 948 PING_PONG, 949 SENDMSG, 950 BASE, 951 BASE_SENDPAGE, 952 SENDPAGE, 953 }; 954 955 static int run_options(struct sockmap_options *options, int cg_fd, int test) 956 { 957 int i, key, next_key, err, zero = 0; 958 struct bpf_program *tx_prog; 959 960 /* If base test skip BPF setup */ 961 if (test == BASE || test == BASE_SENDPAGE) 962 goto run; 963 964 /* Attach programs to sockmap */ 965 if (!txmsg_omit_skb_parser) { 966 links[0] = bpf_program__attach_sockmap(progs[0], map_fd[0]); 967 if (!links[0]) { 968 fprintf(stderr, 969 "ERROR: bpf_program__attach_sockmap (sockmap %i->%i): (%s)\n", 970 bpf_program__fd(progs[0]), map_fd[0], strerror(errno)); 971 return -1; 972 } 973 } 974 975 links[1] = bpf_program__attach_sockmap(progs[1], map_fd[0]); 976 if (!links[1]) { 977 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (sockmap): (%s)\n", 978 strerror(errno)); 979 return -1; 980 } 981 982 /* Attach programs to TLS sockmap */ 983 if (txmsg_ktls_skb) { 984 if (!txmsg_omit_skb_parser) { 985 links[2] = bpf_program__attach_sockmap(progs[0], map_fd[8]); 986 if (!links[2]) { 987 fprintf(stderr, 988 "ERROR: bpf_program__attach_sockmap (TLS sockmap %i->%i): (%s)\n", 989 bpf_program__fd(progs[0]), map_fd[8], strerror(errno)); 990 return -1; 991 } 992 } 993 994 links[3] = bpf_program__attach_sockmap(progs[2], map_fd[8]); 995 if (!links[3]) { 996 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (TLS sockmap): (%s)\n", 997 strerror(errno)); 998 return -1; 999 } 1000 } 1001 1002 /* Attach to cgroups */ 1003 err = bpf_prog_attach(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS, 0); 1004 if (err) { 1005 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 1006 err, strerror(errno)); 1007 return err; 1008 } 1009 1010 run: 1011 err = sockmap_init_sockets(options->verbose); 1012 if (err) { 1013 fprintf(stderr, "ERROR: test socket failed: %d\n", err); 1014 goto out; 1015 } 1016 1017 /* Attach txmsg program to sockmap */ 1018 if (txmsg_pass) 1019 tx_prog = progs[4]; 1020 else if (txmsg_redir) 1021 tx_prog = progs[5]; 1022 else if (txmsg_apply) 1023 tx_prog = progs[6]; 1024 else if (txmsg_cork) 1025 tx_prog = progs[7]; 1026 else if (txmsg_drop) 1027 tx_prog = progs[8]; 1028 else 1029 tx_prog = NULL; 1030 1031 if (tx_prog) { 1032 int redir_fd; 1033 1034 links[4] = bpf_program__attach_sockmap(tx_prog, map_fd[1]); 1035 if (!links[4]) { 1036 fprintf(stderr, 1037 "ERROR: bpf_program__attach_sockmap (txmsg): (%s)\n", 1038 strerror(errno)); 1039 err = -1; 1040 goto out; 1041 } 1042 1043 i = 0; 1044 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 1045 if (err) { 1046 fprintf(stderr, 1047 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 1048 err, strerror(errno)); 1049 goto out; 1050 } 1051 1052 if (txmsg_redir) 1053 redir_fd = c2; 1054 else 1055 redir_fd = c1; 1056 1057 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 1058 if (err) { 1059 fprintf(stderr, 1060 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 1061 err, strerror(errno)); 1062 goto out; 1063 } 1064 1065 if (txmsg_apply) { 1066 err = bpf_map_update_elem(map_fd[3], 1067 &i, &txmsg_apply, BPF_ANY); 1068 if (err) { 1069 fprintf(stderr, 1070 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 1071 err, strerror(errno)); 1072 goto out; 1073 } 1074 } 1075 1076 if (txmsg_cork) { 1077 err = bpf_map_update_elem(map_fd[4], 1078 &i, &txmsg_cork, BPF_ANY); 1079 if (err) { 1080 fprintf(stderr, 1081 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 1082 err, strerror(errno)); 1083 goto out; 1084 } 1085 } 1086 1087 if (txmsg_start) { 1088 err = bpf_map_update_elem(map_fd[5], 1089 &i, &txmsg_start, BPF_ANY); 1090 if (err) { 1091 fprintf(stderr, 1092 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 1093 err, strerror(errno)); 1094 goto out; 1095 } 1096 } 1097 1098 if (txmsg_end) { 1099 i = 1; 1100 err = bpf_map_update_elem(map_fd[5], 1101 &i, &txmsg_end, BPF_ANY); 1102 if (err) { 1103 fprintf(stderr, 1104 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 1105 err, strerror(errno)); 1106 goto out; 1107 } 1108 } 1109 1110 if (txmsg_start_push) { 1111 i = 2; 1112 err = bpf_map_update_elem(map_fd[5], 1113 &i, &txmsg_start_push, BPF_ANY); 1114 if (err) { 1115 fprintf(stderr, 1116 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n", 1117 err, strerror(errno)); 1118 goto out; 1119 } 1120 } 1121 1122 if (txmsg_end_push) { 1123 i = 3; 1124 err = bpf_map_update_elem(map_fd[5], 1125 &i, &txmsg_end_push, BPF_ANY); 1126 if (err) { 1127 fprintf(stderr, 1128 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n", 1129 txmsg_end_push, i, err, strerror(errno)); 1130 goto out; 1131 } 1132 } 1133 1134 if (txmsg_start_pop) { 1135 i = 4; 1136 err = bpf_map_update_elem(map_fd[5], 1137 &i, &txmsg_start_pop, BPF_ANY); 1138 if (err) { 1139 fprintf(stderr, 1140 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n", 1141 txmsg_start_pop, i, err, strerror(errno)); 1142 goto out; 1143 } 1144 } else { 1145 i = 4; 1146 bpf_map_update_elem(map_fd[5], 1147 &i, &txmsg_start_pop, BPF_ANY); 1148 } 1149 1150 if (txmsg_pop) { 1151 i = 5; 1152 err = bpf_map_update_elem(map_fd[5], 1153 &i, &txmsg_pop, BPF_ANY); 1154 if (err) { 1155 fprintf(stderr, 1156 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n", 1157 txmsg_pop, i, err, strerror(errno)); 1158 goto out; 1159 } 1160 } else { 1161 i = 5; 1162 bpf_map_update_elem(map_fd[5], 1163 &i, &txmsg_pop, BPF_ANY); 1164 1165 } 1166 1167 if (txmsg_ingress) { 1168 int in = BPF_F_INGRESS; 1169 1170 i = 0; 1171 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 1172 if (err) { 1173 fprintf(stderr, 1174 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1175 err, strerror(errno)); 1176 } 1177 i = 1; 1178 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 1179 if (err) { 1180 fprintf(stderr, 1181 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 1182 err, strerror(errno)); 1183 } 1184 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 1185 if (err) { 1186 fprintf(stderr, 1187 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 1188 err, strerror(errno)); 1189 } 1190 1191 i = 2; 1192 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 1193 if (err) { 1194 fprintf(stderr, 1195 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 1196 err, strerror(errno)); 1197 } 1198 } 1199 1200 if (txmsg_ktls_skb) { 1201 int ingress = BPF_F_INGRESS; 1202 1203 i = 0; 1204 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY); 1205 if (err) { 1206 fprintf(stderr, 1207 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1208 err, strerror(errno)); 1209 } 1210 1211 if (txmsg_ktls_skb_redir) { 1212 i = 1; 1213 err = bpf_map_update_elem(map_fd[7], 1214 &i, &ingress, BPF_ANY); 1215 if (err) { 1216 fprintf(stderr, 1217 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1218 err, strerror(errno)); 1219 } 1220 } 1221 1222 if (txmsg_ktls_skb_drop) { 1223 i = 1; 1224 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY); 1225 } 1226 } 1227 1228 if (txmsg_redir_skb) { 1229 int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 1230 p2 : p1; 1231 int ingress = BPF_F_INGRESS; 1232 1233 i = 0; 1234 err = bpf_map_update_elem(map_fd[7], 1235 &i, &ingress, BPF_ANY); 1236 if (err) { 1237 fprintf(stderr, 1238 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1239 err, strerror(errno)); 1240 } 1241 1242 i = 3; 1243 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY); 1244 if (err) { 1245 fprintf(stderr, 1246 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1247 err, strerror(errno)); 1248 } 1249 } 1250 } 1251 1252 if (skb_use_parser) { 1253 i = 2; 1254 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY); 1255 } 1256 1257 if (txmsg_drop) 1258 options->drop_expected = true; 1259 1260 if (test == PING_PONG) 1261 err = forever_ping_pong(options->rate, options); 1262 else if (test == SENDMSG) { 1263 options->base = false; 1264 options->sendpage = false; 1265 err = sendmsg_test(options); 1266 } else if (test == SENDPAGE) { 1267 options->base = false; 1268 options->sendpage = true; 1269 err = sendmsg_test(options); 1270 } else if (test == BASE) { 1271 options->base = true; 1272 options->sendpage = false; 1273 err = sendmsg_test(options); 1274 } else if (test == BASE_SENDPAGE) { 1275 options->base = true; 1276 options->sendpage = true; 1277 err = sendmsg_test(options); 1278 } else 1279 fprintf(stderr, "unknown test\n"); 1280 out: 1281 /* Detatch and zero all the maps */ 1282 bpf_prog_detach2(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS); 1283 1284 for (i = 0; i < ARRAY_SIZE(links); i++) { 1285 if (links[i]) 1286 bpf_link__detach(links[i]); 1287 } 1288 1289 for (i = 0; i < ARRAY_SIZE(map_fd); i++) { 1290 key = next_key = 0; 1291 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1292 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 1293 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1294 key = next_key; 1295 } 1296 } 1297 1298 close(s1); 1299 close(s2); 1300 close(p1); 1301 close(p2); 1302 close(c1); 1303 close(c2); 1304 return err; 1305 } 1306 1307 static char *test_to_str(int test) 1308 { 1309 switch (test) { 1310 case SENDMSG: 1311 return "sendmsg"; 1312 case SENDPAGE: 1313 return "sendpage"; 1314 } 1315 return "unknown"; 1316 } 1317 1318 static void append_str(char *dst, const char *src, size_t dst_cap) 1319 { 1320 size_t avail = dst_cap - strlen(dst); 1321 1322 if (avail <= 1) /* just zero byte could be written */ 1323 return; 1324 1325 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */ 1326 } 1327 1328 #define OPTSTRING 60 1329 static void test_options(char *options) 1330 { 1331 char tstr[OPTSTRING]; 1332 1333 memset(options, 0, OPTSTRING); 1334 1335 if (txmsg_pass) 1336 append_str(options, "pass,", OPTSTRING); 1337 if (txmsg_redir) 1338 append_str(options, "redir,", OPTSTRING); 1339 if (txmsg_drop) 1340 append_str(options, "drop,", OPTSTRING); 1341 if (txmsg_apply) { 1342 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 1343 append_str(options, tstr, OPTSTRING); 1344 } 1345 if (txmsg_cork) { 1346 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 1347 append_str(options, tstr, OPTSTRING); 1348 } 1349 if (txmsg_start) { 1350 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 1351 append_str(options, tstr, OPTSTRING); 1352 } 1353 if (txmsg_end) { 1354 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 1355 append_str(options, tstr, OPTSTRING); 1356 } 1357 if (txmsg_start_pop) { 1358 snprintf(tstr, OPTSTRING, "pop (%d,%d),", 1359 txmsg_start_pop, txmsg_start_pop + txmsg_pop); 1360 append_str(options, tstr, OPTSTRING); 1361 } 1362 if (txmsg_ingress) 1363 append_str(options, "ingress,", OPTSTRING); 1364 if (txmsg_redir_skb) 1365 append_str(options, "redir_skb,", OPTSTRING); 1366 if (txmsg_ktls_skb) 1367 append_str(options, "ktls_skb,", OPTSTRING); 1368 if (ktls) 1369 append_str(options, "ktls,", OPTSTRING); 1370 if (peek_flag) 1371 append_str(options, "peek,", OPTSTRING); 1372 } 1373 1374 static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 1375 { 1376 char *options = calloc(OPTSTRING, sizeof(char)); 1377 int err; 1378 1379 if (test == SENDPAGE) 1380 opt->sendpage = true; 1381 else 1382 opt->sendpage = false; 1383 1384 if (txmsg_drop) 1385 opt->drop_expected = true; 1386 else 1387 opt->drop_expected = false; 1388 1389 test_options(options); 1390 1391 if (opt->verbose) { 1392 fprintf(stdout, 1393 " [TEST %i]: (%i, %i, %i, %s, %s): ", 1394 test_cnt, opt->rate, opt->iov_count, opt->iov_length, 1395 test_to_str(test), options); 1396 fflush(stdout); 1397 } 1398 err = run_options(opt, cgrp, test); 1399 if (opt->verbose) 1400 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED"); 1401 test_cnt++; 1402 !err ? passed++ : failed++; 1403 free(options); 1404 return err; 1405 } 1406 1407 static void test_exec(int cgrp, struct sockmap_options *opt) 1408 { 1409 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME); 1410 int err; 1411 1412 if (type == 0) { 1413 test_start(); 1414 err = __test_exec(cgrp, SENDMSG, opt); 1415 if (err) 1416 test_fail(); 1417 } else { 1418 test_start(); 1419 err = __test_exec(cgrp, SENDPAGE, opt); 1420 if (err) 1421 test_fail(); 1422 } 1423 } 1424 1425 static void test_send_one(struct sockmap_options *opt, int cgrp) 1426 { 1427 opt->iov_length = 1; 1428 opt->iov_count = 1; 1429 opt->rate = 1; 1430 test_exec(cgrp, opt); 1431 1432 opt->iov_length = 1; 1433 opt->iov_count = 1024; 1434 opt->rate = 1; 1435 test_exec(cgrp, opt); 1436 1437 opt->iov_length = 1024; 1438 opt->iov_count = 1; 1439 opt->rate = 1; 1440 test_exec(cgrp, opt); 1441 1442 } 1443 1444 static void test_send_many(struct sockmap_options *opt, int cgrp) 1445 { 1446 opt->iov_length = 3; 1447 opt->iov_count = 1; 1448 opt->rate = 512; 1449 test_exec(cgrp, opt); 1450 1451 opt->rate = 100; 1452 opt->iov_count = 1; 1453 opt->iov_length = 5; 1454 test_exec(cgrp, opt); 1455 } 1456 1457 static void test_send_large(struct sockmap_options *opt, int cgrp) 1458 { 1459 opt->iov_length = 256; 1460 opt->iov_count = 1024; 1461 opt->rate = 2; 1462 test_exec(cgrp, opt); 1463 } 1464 1465 static void test_send(struct sockmap_options *opt, int cgrp) 1466 { 1467 test_send_one(opt, cgrp); 1468 test_send_many(opt, cgrp); 1469 test_send_large(opt, cgrp); 1470 sched_yield(); 1471 } 1472 1473 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt) 1474 { 1475 /* Test small and large iov_count values with pass/redir/apply/cork */ 1476 txmsg_pass = 1; 1477 test_send(opt, cgrp); 1478 } 1479 1480 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt) 1481 { 1482 txmsg_redir = 1; 1483 test_send(opt, cgrp); 1484 } 1485 1486 static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt) 1487 { 1488 txmsg_redir = 1; 1489 opt->tx_wait_mem = true; 1490 test_send_large(opt, cgrp); 1491 opt->tx_wait_mem = false; 1492 } 1493 1494 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt) 1495 { 1496 txmsg_drop = 1; 1497 test_send(opt, cgrp); 1498 } 1499 1500 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt) 1501 { 1502 txmsg_pass = txmsg_drop = 0; 1503 txmsg_ingress = txmsg_redir = 1; 1504 test_send(opt, cgrp); 1505 } 1506 1507 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt) 1508 { 1509 bool data = opt->data_test; 1510 int k = ktls; 1511 1512 opt->data_test = true; 1513 ktls = 1; 1514 1515 txmsg_pass = txmsg_drop = 0; 1516 txmsg_ingress = txmsg_redir = 0; 1517 txmsg_ktls_skb = 1; 1518 txmsg_pass = 1; 1519 1520 /* Using data verification so ensure iov layout is 1521 * expected from test receiver side. e.g. has enough 1522 * bytes to write test code. 1523 */ 1524 opt->iov_length = 100; 1525 opt->iov_count = 1; 1526 opt->rate = 1; 1527 test_exec(cgrp, opt); 1528 1529 txmsg_ktls_skb_drop = 1; 1530 test_exec(cgrp, opt); 1531 1532 txmsg_ktls_skb_drop = 0; 1533 txmsg_ktls_skb_redir = 1; 1534 test_exec(cgrp, opt); 1535 txmsg_ktls_skb_redir = 0; 1536 1537 /* Tests that omit skb_parser */ 1538 txmsg_omit_skb_parser = 1; 1539 ktls = 0; 1540 txmsg_ktls_skb = 0; 1541 test_exec(cgrp, opt); 1542 1543 txmsg_ktls_skb_drop = 1; 1544 test_exec(cgrp, opt); 1545 txmsg_ktls_skb_drop = 0; 1546 1547 txmsg_ktls_skb_redir = 1; 1548 test_exec(cgrp, opt); 1549 1550 ktls = 1; 1551 test_exec(cgrp, opt); 1552 txmsg_omit_skb_parser = 0; 1553 1554 opt->data_test = data; 1555 ktls = k; 1556 } 1557 1558 /* Test cork with hung data. This tests poor usage patterns where 1559 * cork can leave data on the ring if user program is buggy and 1560 * doesn't flush them somehow. They do take some time however 1561 * because they wait for a timeout. Test pass, redir and cork with 1562 * apply logic. Use cork size of 4097 with send_large to avoid 1563 * aligning cork size with send size. 1564 */ 1565 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt) 1566 { 1567 txmsg_pass = 1; 1568 txmsg_redir = 0; 1569 txmsg_cork = 4097; 1570 txmsg_apply = 4097; 1571 test_send_large(opt, cgrp); 1572 1573 txmsg_pass = 0; 1574 txmsg_redir = 1; 1575 txmsg_apply = 0; 1576 txmsg_cork = 4097; 1577 test_send_large(opt, cgrp); 1578 1579 txmsg_pass = 0; 1580 txmsg_redir = 1; 1581 txmsg_apply = 4097; 1582 txmsg_cork = 4097; 1583 test_send_large(opt, cgrp); 1584 } 1585 1586 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt) 1587 { 1588 /* Test basic start/end */ 1589 txmsg_start = 1; 1590 txmsg_end = 2; 1591 test_send(opt, cgrp); 1592 1593 /* Test >4k pull */ 1594 txmsg_start = 4096; 1595 txmsg_end = 9182; 1596 test_send_large(opt, cgrp); 1597 1598 /* Test pull + redirect */ 1599 txmsg_redir = 0; 1600 txmsg_start = 1; 1601 txmsg_end = 2; 1602 test_send(opt, cgrp); 1603 1604 /* Test pull + cork */ 1605 txmsg_redir = 0; 1606 txmsg_cork = 512; 1607 txmsg_start = 1; 1608 txmsg_end = 2; 1609 test_send_many(opt, cgrp); 1610 1611 /* Test pull + cork + redirect */ 1612 txmsg_redir = 1; 1613 txmsg_cork = 512; 1614 txmsg_start = 1; 1615 txmsg_end = 2; 1616 test_send_many(opt, cgrp); 1617 } 1618 1619 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt) 1620 { 1621 /* Test basic pop */ 1622 txmsg_start_pop = 1; 1623 txmsg_pop = 2; 1624 test_send_many(opt, cgrp); 1625 1626 /* Test pop with >4k */ 1627 txmsg_start_pop = 4096; 1628 txmsg_pop = 4096; 1629 test_send_large(opt, cgrp); 1630 1631 /* Test pop + redirect */ 1632 txmsg_redir = 1; 1633 txmsg_start_pop = 1; 1634 txmsg_pop = 2; 1635 test_send_many(opt, cgrp); 1636 1637 /* Test pop + cork */ 1638 txmsg_redir = 0; 1639 txmsg_cork = 512; 1640 txmsg_start_pop = 1; 1641 txmsg_pop = 2; 1642 test_send_many(opt, cgrp); 1643 1644 /* Test pop + redirect + cork */ 1645 txmsg_redir = 1; 1646 txmsg_cork = 4; 1647 txmsg_start_pop = 1; 1648 txmsg_pop = 2; 1649 test_send_many(opt, cgrp); 1650 } 1651 1652 static void test_txmsg_push(int cgrp, struct sockmap_options *opt) 1653 { 1654 /* Test basic push */ 1655 txmsg_start_push = 1; 1656 txmsg_end_push = 1; 1657 test_send(opt, cgrp); 1658 1659 /* Test push 4kB >4k */ 1660 txmsg_start_push = 4096; 1661 txmsg_end_push = 4096; 1662 test_send_large(opt, cgrp); 1663 1664 /* Test push + redirect */ 1665 txmsg_redir = 1; 1666 txmsg_start_push = 1; 1667 txmsg_end_push = 2; 1668 test_send_many(opt, cgrp); 1669 1670 /* Test push + cork */ 1671 txmsg_redir = 0; 1672 txmsg_cork = 512; 1673 txmsg_start_push = 1; 1674 txmsg_end_push = 2; 1675 test_send_many(opt, cgrp); 1676 } 1677 1678 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt) 1679 { 1680 txmsg_start_push = 1; 1681 txmsg_end_push = 10; 1682 txmsg_start_pop = 5; 1683 txmsg_pop = 4; 1684 test_send_large(opt, cgrp); 1685 } 1686 1687 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt) 1688 { 1689 txmsg_pass = 1; 1690 txmsg_redir = 0; 1691 txmsg_ingress = 0; 1692 txmsg_apply = 1; 1693 txmsg_cork = 0; 1694 test_send_one(opt, cgrp); 1695 1696 txmsg_pass = 0; 1697 txmsg_redir = 1; 1698 txmsg_ingress = 0; 1699 txmsg_apply = 1; 1700 txmsg_cork = 0; 1701 test_send_one(opt, cgrp); 1702 1703 txmsg_pass = 0; 1704 txmsg_redir = 1; 1705 txmsg_ingress = 1; 1706 txmsg_apply = 1; 1707 txmsg_cork = 0; 1708 test_send_one(opt, cgrp); 1709 1710 txmsg_pass = 1; 1711 txmsg_redir = 0; 1712 txmsg_ingress = 0; 1713 txmsg_apply = 1024; 1714 txmsg_cork = 0; 1715 test_send_large(opt, cgrp); 1716 1717 txmsg_pass = 0; 1718 txmsg_redir = 1; 1719 txmsg_ingress = 0; 1720 txmsg_apply = 1024; 1721 txmsg_cork = 0; 1722 test_send_large(opt, cgrp); 1723 1724 txmsg_pass = 0; 1725 txmsg_redir = 1; 1726 txmsg_ingress = 1; 1727 txmsg_apply = 1024; 1728 txmsg_cork = 0; 1729 test_send_large(opt, cgrp); 1730 } 1731 1732 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt) 1733 { 1734 txmsg_pass = 1; 1735 txmsg_redir = 0; 1736 txmsg_apply = 0; 1737 txmsg_cork = 1; 1738 test_send(opt, cgrp); 1739 1740 txmsg_pass = 1; 1741 txmsg_redir = 0; 1742 txmsg_apply = 1; 1743 txmsg_cork = 1; 1744 test_send(opt, cgrp); 1745 } 1746 1747 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt) 1748 { 1749 txmsg_pass = 1; 1750 skb_use_parser = 512; 1751 if (ktls == 1) 1752 skb_use_parser = 570; 1753 opt->iov_length = 256; 1754 opt->iov_count = 1; 1755 opt->rate = 2; 1756 test_exec(cgrp, opt); 1757 } 1758 1759 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt) 1760 { 1761 if (ktls == 1) 1762 return; 1763 skb_use_parser = 10; 1764 opt->iov_length = 20; 1765 opt->iov_count = 1; 1766 opt->rate = 1; 1767 opt->check_recved_len = true; 1768 test_exec(cgrp, opt); 1769 opt->check_recved_len = false; 1770 } 1771 1772 char *map_names[] = { 1773 "sock_map", 1774 "sock_map_txmsg", 1775 "sock_map_redir", 1776 "sock_apply_bytes", 1777 "sock_cork_bytes", 1778 "sock_bytes", 1779 "sock_redir_flags", 1780 "sock_skb_opts", 1781 "tls_sock_map", 1782 }; 1783 1784 static int populate_progs(char *bpf_file) 1785 { 1786 struct bpf_program *prog; 1787 struct bpf_object *obj; 1788 int i = 0; 1789 long err; 1790 1791 obj = bpf_object__open(bpf_file); 1792 err = libbpf_get_error(obj); 1793 if (err) { 1794 char err_buf[256]; 1795 1796 libbpf_strerror(err, err_buf, sizeof(err_buf)); 1797 printf("Unable to load eBPF objects in file '%s' : %s\n", 1798 bpf_file, err_buf); 1799 return -1; 1800 } 1801 1802 i = bpf_object__load(obj); 1803 i = 0; 1804 bpf_object__for_each_program(prog, obj) { 1805 progs[i] = prog; 1806 i++; 1807 } 1808 1809 for (i = 0; i < ARRAY_SIZE(map_fd); i++) { 1810 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 1811 map_fd[i] = bpf_map__fd(maps[i]); 1812 if (map_fd[i] < 0) { 1813 fprintf(stderr, "load_bpf_file: (%i) %s\n", 1814 map_fd[i], strerror(errno)); 1815 return -1; 1816 } 1817 } 1818 1819 for (i = 0; i < ARRAY_SIZE(links); i++) 1820 links[i] = NULL; 1821 1822 return 0; 1823 } 1824 1825 struct _test test[] = { 1826 {"txmsg test passthrough", test_txmsg_pass}, 1827 {"txmsg test redirect", test_txmsg_redir}, 1828 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem}, 1829 {"txmsg test drop", test_txmsg_drop}, 1830 {"txmsg test ingress redirect", test_txmsg_ingress_redir}, 1831 {"txmsg test skb", test_txmsg_skb}, 1832 {"txmsg test apply", test_txmsg_apply}, 1833 {"txmsg test cork", test_txmsg_cork}, 1834 {"txmsg test hanging corks", test_txmsg_cork_hangs}, 1835 {"txmsg test push_data", test_txmsg_push}, 1836 {"txmsg test pull-data", test_txmsg_pull}, 1837 {"txmsg test pop-data", test_txmsg_pop}, 1838 {"txmsg test push/pop data", test_txmsg_push_pop}, 1839 {"txmsg test ingress parser", test_txmsg_ingress_parser}, 1840 {"txmsg test ingress parser2", test_txmsg_ingress_parser2}, 1841 }; 1842 1843 static int check_whitelist(struct _test *t, struct sockmap_options *opt) 1844 { 1845 char *entry, *ptr; 1846 1847 if (!opt->whitelist) 1848 return 0; 1849 ptr = strdup(opt->whitelist); 1850 if (!ptr) 1851 return -ENOMEM; 1852 entry = strtok(ptr, ","); 1853 while (entry) { 1854 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 1855 strstr(opt->map, entry) != 0 || 1856 strstr(t->title, entry) != 0) { 1857 free(ptr); 1858 return 0; 1859 } 1860 entry = strtok(NULL, ","); 1861 } 1862 free(ptr); 1863 return -EINVAL; 1864 } 1865 1866 static int check_blacklist(struct _test *t, struct sockmap_options *opt) 1867 { 1868 char *entry, *ptr; 1869 1870 if (!opt->blacklist) 1871 return -EINVAL; 1872 ptr = strdup(opt->blacklist); 1873 if (!ptr) 1874 return -ENOMEM; 1875 entry = strtok(ptr, ","); 1876 while (entry) { 1877 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 1878 strstr(opt->map, entry) != 0 || 1879 strstr(t->title, entry) != 0) { 1880 free(ptr); 1881 return 0; 1882 } 1883 entry = strtok(NULL, ","); 1884 } 1885 free(ptr); 1886 return -EINVAL; 1887 } 1888 1889 static int __test_selftests(int cg_fd, struct sockmap_options *opt) 1890 { 1891 int i, err; 1892 1893 err = populate_progs(opt->map); 1894 if (err < 0) { 1895 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 1896 return err; 1897 } 1898 1899 /* Tests basic commands and APIs */ 1900 for (i = 0; i < ARRAY_SIZE(test); i++) { 1901 struct _test t = test[i]; 1902 1903 if (check_whitelist(&t, opt) != 0) 1904 continue; 1905 if (check_blacklist(&t, opt) == 0) 1906 continue; 1907 1908 test_start_subtest(&t, opt); 1909 t.tester(cg_fd, opt); 1910 test_end_subtest(); 1911 } 1912 1913 return err; 1914 } 1915 1916 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt) 1917 { 1918 opt->map = BPF_SOCKMAP_FILENAME; 1919 __test_selftests(cg_fd, opt); 1920 } 1921 1922 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt) 1923 { 1924 opt->map = BPF_SOCKHASH_FILENAME; 1925 __test_selftests(cg_fd, opt); 1926 } 1927 1928 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt) 1929 { 1930 opt->map = BPF_SOCKHASH_FILENAME; 1931 opt->prepend = "ktls"; 1932 ktls = 1; 1933 __test_selftests(cg_fd, opt); 1934 ktls = 0; 1935 } 1936 1937 static int test_selftest(int cg_fd, struct sockmap_options *opt) 1938 { 1939 test_selftests_sockmap(cg_fd, opt); 1940 test_selftests_sockhash(cg_fd, opt); 1941 test_selftests_ktls(cg_fd, opt); 1942 test_print_results(); 1943 return 0; 1944 } 1945 1946 int main(int argc, char **argv) 1947 { 1948 int iov_count = 1, length = 1024, rate = 1; 1949 struct sockmap_options options = {0}; 1950 int opt, longindex, err, cg_fd = 0; 1951 char *bpf_file = BPF_SOCKMAP_FILENAME; 1952 int test = SELFTESTS; 1953 bool cg_created = 0; 1954 1955 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:", 1956 long_options, &longindex)) != -1) { 1957 switch (opt) { 1958 case 's': 1959 txmsg_start = atoi(optarg); 1960 break; 1961 case 'e': 1962 txmsg_end = atoi(optarg); 1963 break; 1964 case 'p': 1965 txmsg_start_push = atoi(optarg); 1966 break; 1967 case 'q': 1968 txmsg_end_push = atoi(optarg); 1969 break; 1970 case 'w': 1971 txmsg_start_pop = atoi(optarg); 1972 break; 1973 case 'x': 1974 txmsg_pop = atoi(optarg); 1975 break; 1976 case 'a': 1977 txmsg_apply = atoi(optarg); 1978 break; 1979 case 'k': 1980 txmsg_cork = atoi(optarg); 1981 break; 1982 case 'c': 1983 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 1984 if (cg_fd < 0) { 1985 fprintf(stderr, 1986 "ERROR: (%i) open cg path failed: %s\n", 1987 cg_fd, optarg); 1988 return cg_fd; 1989 } 1990 break; 1991 case 'r': 1992 rate = atoi(optarg); 1993 break; 1994 case 'v': 1995 options.verbose = 1; 1996 if (optarg) 1997 options.verbose = atoi(optarg); 1998 break; 1999 case 'i': 2000 iov_count = atoi(optarg); 2001 break; 2002 case 'l': 2003 length = atoi(optarg); 2004 break; 2005 case 'd': 2006 options.data_test = true; 2007 break; 2008 case 't': 2009 if (strcmp(optarg, "ping") == 0) { 2010 test = PING_PONG; 2011 } else if (strcmp(optarg, "sendmsg") == 0) { 2012 test = SENDMSG; 2013 } else if (strcmp(optarg, "base") == 0) { 2014 test = BASE; 2015 } else if (strcmp(optarg, "base_sendpage") == 0) { 2016 test = BASE_SENDPAGE; 2017 } else if (strcmp(optarg, "sendpage") == 0) { 2018 test = SENDPAGE; 2019 } else { 2020 usage(argv); 2021 return -1; 2022 } 2023 break; 2024 case 'n': 2025 options.whitelist = strdup(optarg); 2026 if (!options.whitelist) 2027 return -ENOMEM; 2028 break; 2029 case 'b': 2030 options.blacklist = strdup(optarg); 2031 if (!options.blacklist) 2032 return -ENOMEM; 2033 case 0: 2034 break; 2035 case 'h': 2036 default: 2037 usage(argv); 2038 return -1; 2039 } 2040 } 2041 2042 if (!cg_fd) { 2043 cg_fd = cgroup_setup_and_join(CG_PATH); 2044 if (cg_fd < 0) 2045 return cg_fd; 2046 cg_created = 1; 2047 } 2048 2049 /* Use libbpf 1.0 API mode */ 2050 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 2051 2052 if (test == SELFTESTS) { 2053 err = test_selftest(cg_fd, &options); 2054 goto out; 2055 } 2056 2057 err = populate_progs(bpf_file); 2058 if (err) { 2059 fprintf(stderr, "populate program: (%s) %s\n", 2060 bpf_file, strerror(errno)); 2061 return 1; 2062 } 2063 running = 1; 2064 2065 /* catch SIGINT */ 2066 signal(SIGINT, running_handler); 2067 2068 options.iov_count = iov_count; 2069 options.iov_length = length; 2070 options.rate = rate; 2071 2072 err = run_options(&options, cg_fd, test); 2073 out: 2074 if (options.whitelist) 2075 free(options.whitelist); 2076 if (options.blacklist) 2077 free(options.blacklist); 2078 close(cg_fd); 2079 if (cg_created) 2080 cleanup_cgroup_environment(); 2081 return err; 2082 } 2083 2084 void running_handler(int a) 2085 { 2086 running = 0; 2087 } 2088