1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <sys/socket.h> 6 #include <sys/ioctl.h> 7 #include <sys/select.h> 8 #include <netinet/in.h> 9 #include <arpa/inet.h> 10 #include <unistd.h> 11 #include <string.h> 12 #include <errno.h> 13 #include <stdbool.h> 14 #include <signal.h> 15 #include <fcntl.h> 16 #include <sys/wait.h> 17 #include <time.h> 18 #include <sched.h> 19 20 #include <sys/time.h> 21 #include <sys/types.h> 22 #include <sys/sendfile.h> 23 24 #include <linux/netlink.h> 25 #include <linux/socket.h> 26 #include <linux/sock_diag.h> 27 #include <linux/bpf.h> 28 #include <linux/if_link.h> 29 #include <linux/tls.h> 30 #include <assert.h> 31 #include <libgen.h> 32 33 #include <getopt.h> 34 35 #include <bpf/bpf.h> 36 #include <bpf/libbpf.h> 37 38 #include "bpf_util.h" 39 #include "cgroup_helpers.h" 40 41 int running; 42 static void running_handler(int a); 43 44 #ifndef TCP_ULP 45 # define TCP_ULP 31 46 #endif 47 #ifndef SOL_TLS 48 # define SOL_TLS 282 49 #endif 50 51 /* randomly selected ports for testing on lo */ 52 #define S1_PORT 10000 53 #define S2_PORT 10001 54 55 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o" 56 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o" 57 #define CG_PATH "/sockmap" 58 59 #define EDATAINTEGRITY 2001 60 61 /* global sockets */ 62 int s1, s2, c1, c2, p1, p2; 63 int test_cnt; 64 int passed; 65 int failed; 66 int map_fd[9]; 67 struct bpf_map *maps[9]; 68 struct bpf_program *progs[9]; 69 struct bpf_link *links[9]; 70 71 int txmsg_pass; 72 int txmsg_redir; 73 int txmsg_drop; 74 int txmsg_apply; 75 int txmsg_cork; 76 int txmsg_start; 77 int txmsg_end; 78 int txmsg_start_push; 79 int txmsg_end_push; 80 int txmsg_start_pop; 81 int txmsg_pop; 82 int txmsg_ingress; 83 int txmsg_redir_skb; 84 int txmsg_ktls_skb; 85 int txmsg_ktls_skb_drop; 86 int txmsg_ktls_skb_redir; 87 int ktls; 88 int peek_flag; 89 int skb_use_parser; 90 int txmsg_omit_skb_parser; 91 92 static const struct option long_options[] = { 93 {"help", no_argument, NULL, 'h' }, 94 {"cgroup", required_argument, NULL, 'c' }, 95 {"rate", required_argument, NULL, 'r' }, 96 {"verbose", optional_argument, NULL, 'v' }, 97 {"iov_count", required_argument, NULL, 'i' }, 98 {"length", required_argument, NULL, 'l' }, 99 {"test", required_argument, NULL, 't' }, 100 {"data_test", no_argument, NULL, 'd' }, 101 {"txmsg", no_argument, &txmsg_pass, 1 }, 102 {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 103 {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 104 {"txmsg_apply", required_argument, NULL, 'a'}, 105 {"txmsg_cork", required_argument, NULL, 'k'}, 106 {"txmsg_start", required_argument, NULL, 's'}, 107 {"txmsg_end", required_argument, NULL, 'e'}, 108 {"txmsg_start_push", required_argument, NULL, 'p'}, 109 {"txmsg_end_push", required_argument, NULL, 'q'}, 110 {"txmsg_start_pop", required_argument, NULL, 'w'}, 111 {"txmsg_pop", required_argument, NULL, 'x'}, 112 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 113 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 }, 114 {"ktls", no_argument, &ktls, 1 }, 115 {"peek", no_argument, &peek_flag, 1 }, 116 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1}, 117 {"whitelist", required_argument, NULL, 'n' }, 118 {"blacklist", required_argument, NULL, 'b' }, 119 {0, 0, NULL, 0 } 120 }; 121 122 struct test_env { 123 const char *type; 124 const char *subtest; 125 const char *prepend; 126 127 int test_num; 128 int subtest_num; 129 130 int succ_cnt; 131 int fail_cnt; 132 int fail_last; 133 }; 134 135 struct test_env env; 136 137 struct sockmap_options { 138 int verbose; 139 bool base; 140 bool sendpage; 141 bool data_test; 142 bool drop_expected; 143 bool check_recved_len; 144 bool tx_wait_mem; 145 int iov_count; 146 int iov_length; 147 int rate; 148 char *map; 149 char *whitelist; 150 char *blacklist; 151 char *prepend; 152 }; 153 154 struct _test { 155 char *title; 156 void (*tester)(int cg_fd, struct sockmap_options *opt); 157 }; 158 159 static void test_start(void) 160 { 161 env.subtest_num++; 162 } 163 164 static void test_fail(void) 165 { 166 env.fail_cnt++; 167 } 168 169 static void test_pass(void) 170 { 171 env.succ_cnt++; 172 } 173 174 static void test_reset(void) 175 { 176 txmsg_start = txmsg_end = 0; 177 txmsg_start_pop = txmsg_pop = 0; 178 txmsg_start_push = txmsg_end_push = 0; 179 txmsg_pass = txmsg_drop = txmsg_redir = 0; 180 txmsg_apply = txmsg_cork = 0; 181 txmsg_ingress = txmsg_redir_skb = 0; 182 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0; 183 txmsg_omit_skb_parser = 0; 184 skb_use_parser = 0; 185 } 186 187 static int test_start_subtest(const struct _test *t, struct sockmap_options *o) 188 { 189 env.type = o->map; 190 env.subtest = t->title; 191 env.prepend = o->prepend; 192 env.test_num++; 193 env.subtest_num = 0; 194 env.fail_last = env.fail_cnt; 195 test_reset(); 196 return 0; 197 } 198 199 static void test_end_subtest(void) 200 { 201 int error = env.fail_cnt - env.fail_last; 202 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME); 203 204 if (!error) 205 test_pass(); 206 207 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n", 208 env.test_num, env.subtest_num, 209 !type ? "sockmap" : "sockhash", 210 env.prepend ? : "", 211 env.subtest, error ? "FAIL" : "OK"); 212 } 213 214 static void test_print_results(void) 215 { 216 fprintf(stdout, "Pass: %d Fail: %d\n", 217 env.succ_cnt, env.fail_cnt); 218 } 219 220 static void usage(char *argv[]) 221 { 222 int i; 223 224 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 225 printf(" options:\n"); 226 for (i = 0; long_options[i].name != 0; i++) { 227 printf(" --%-12s", long_options[i].name); 228 if (long_options[i].flag != NULL) 229 printf(" flag (internal value:%d)\n", 230 *long_options[i].flag); 231 else 232 printf(" -%c\n", long_options[i].val); 233 } 234 printf("\n"); 235 } 236 237 char *sock_to_string(int s) 238 { 239 if (s == c1) 240 return "client1"; 241 else if (s == c2) 242 return "client2"; 243 else if (s == s1) 244 return "server1"; 245 else if (s == s2) 246 return "server2"; 247 else if (s == p1) 248 return "peer1"; 249 else if (s == p2) 250 return "peer2"; 251 else 252 return "unknown"; 253 } 254 255 static int sockmap_init_ktls(int verbose, int s) 256 { 257 struct tls12_crypto_info_aes_gcm_128 tls_tx = { 258 .info = { 259 .version = TLS_1_2_VERSION, 260 .cipher_type = TLS_CIPHER_AES_GCM_128, 261 }, 262 }; 263 struct tls12_crypto_info_aes_gcm_128 tls_rx = { 264 .info = { 265 .version = TLS_1_2_VERSION, 266 .cipher_type = TLS_CIPHER_AES_GCM_128, 267 }, 268 }; 269 int so_buf = 6553500; 270 int err; 271 272 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls")); 273 if (err) { 274 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err); 275 return -EINVAL; 276 } 277 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); 278 if (err) { 279 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err); 280 return -EINVAL; 281 } 282 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); 283 if (err) { 284 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err); 285 return -EINVAL; 286 } 287 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); 288 if (err) { 289 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err); 290 return -EINVAL; 291 } 292 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); 293 if (err) { 294 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err); 295 return -EINVAL; 296 } 297 298 if (verbose) 299 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s)); 300 return 0; 301 } 302 static int sockmap_init_sockets(int verbose) 303 { 304 int i, err, one = 1; 305 struct sockaddr_in addr; 306 int *fds[4] = {&s1, &s2, &c1, &c2}; 307 308 s1 = s2 = p1 = p2 = c1 = c2 = 0; 309 310 /* Init sockets */ 311 for (i = 0; i < 4; i++) { 312 *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 313 if (*fds[i] < 0) { 314 perror("socket s1 failed()"); 315 return errno; 316 } 317 } 318 319 /* Allow reuse */ 320 for (i = 0; i < 2; i++) { 321 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 322 (char *)&one, sizeof(one)); 323 if (err) { 324 perror("setsockopt failed()"); 325 return errno; 326 } 327 } 328 329 /* Non-blocking sockets */ 330 for (i = 0; i < 2; i++) { 331 err = ioctl(*fds[i], FIONBIO, (char *)&one); 332 if (err < 0) { 333 perror("ioctl s1 failed()"); 334 return errno; 335 } 336 } 337 338 /* Bind server sockets */ 339 memset(&addr, 0, sizeof(struct sockaddr_in)); 340 addr.sin_family = AF_INET; 341 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 342 343 addr.sin_port = htons(S1_PORT); 344 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 345 if (err < 0) { 346 perror("bind s1 failed()"); 347 return errno; 348 } 349 350 addr.sin_port = htons(S2_PORT); 351 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 352 if (err < 0) { 353 perror("bind s2 failed()"); 354 return errno; 355 } 356 357 /* Listen server sockets */ 358 addr.sin_port = htons(S1_PORT); 359 err = listen(s1, 32); 360 if (err < 0) { 361 perror("listen s1 failed()"); 362 return errno; 363 } 364 365 addr.sin_port = htons(S2_PORT); 366 err = listen(s2, 32); 367 if (err < 0) { 368 perror("listen s1 failed()"); 369 return errno; 370 } 371 372 /* Initiate Connect */ 373 addr.sin_port = htons(S1_PORT); 374 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 375 if (err < 0 && errno != EINPROGRESS) { 376 perror("connect c1 failed()"); 377 return errno; 378 } 379 380 addr.sin_port = htons(S2_PORT); 381 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 382 if (err < 0 && errno != EINPROGRESS) { 383 perror("connect c2 failed()"); 384 return errno; 385 } else if (err < 0) { 386 err = 0; 387 } 388 389 /* Accept Connecrtions */ 390 p1 = accept(s1, NULL, NULL); 391 if (p1 < 0) { 392 perror("accept s1 failed()"); 393 return errno; 394 } 395 396 p2 = accept(s2, NULL, NULL); 397 if (p2 < 0) { 398 perror("accept s1 failed()"); 399 return errno; 400 } 401 402 if (verbose > 1) { 403 printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 404 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 405 c1, s1, c2, s2); 406 } 407 return 0; 408 } 409 410 struct msg_stats { 411 size_t bytes_sent; 412 size_t bytes_recvd; 413 struct timespec start; 414 struct timespec end; 415 }; 416 417 static int msg_loop_sendpage(int fd, int iov_length, int cnt, 418 struct msg_stats *s, 419 struct sockmap_options *opt) 420 { 421 bool drop = opt->drop_expected; 422 unsigned char k = 0; 423 FILE *file; 424 int i, fp; 425 426 file = tmpfile(); 427 if (!file) { 428 perror("create file for sendpage"); 429 return 1; 430 } 431 for (i = 0; i < iov_length * cnt; i++, k++) 432 fwrite(&k, sizeof(char), 1, file); 433 fflush(file); 434 fseek(file, 0, SEEK_SET); 435 436 fp = fileno(file); 437 438 clock_gettime(CLOCK_MONOTONIC, &s->start); 439 for (i = 0; i < cnt; i++) { 440 int sent; 441 442 errno = 0; 443 sent = sendfile(fd, fp, NULL, iov_length); 444 445 if (!drop && sent < 0) { 446 perror("sendpage loop error"); 447 fclose(file); 448 return sent; 449 } else if (drop && sent >= 0) { 450 printf("sendpage loop error expected: %i errno %i\n", 451 sent, errno); 452 fclose(file); 453 return -EIO; 454 } 455 456 if (sent > 0) 457 s->bytes_sent += sent; 458 } 459 clock_gettime(CLOCK_MONOTONIC, &s->end); 460 fclose(file); 461 return 0; 462 } 463 464 static void msg_free_iov(struct msghdr *msg) 465 { 466 int i; 467 468 for (i = 0; i < msg->msg_iovlen; i++) 469 free(msg->msg_iov[i].iov_base); 470 free(msg->msg_iov); 471 msg->msg_iov = NULL; 472 msg->msg_iovlen = 0; 473 } 474 475 static int msg_alloc_iov(struct msghdr *msg, 476 int iov_count, int iov_length, 477 bool data, bool xmit) 478 { 479 unsigned char k = 0; 480 struct iovec *iov; 481 int i; 482 483 iov = calloc(iov_count, sizeof(struct iovec)); 484 if (!iov) 485 return errno; 486 487 for (i = 0; i < iov_count; i++) { 488 unsigned char *d = calloc(iov_length, sizeof(char)); 489 490 if (!d) { 491 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 492 goto unwind_iov; 493 } 494 iov[i].iov_base = d; 495 iov[i].iov_len = iov_length; 496 497 if (data && xmit) { 498 int j; 499 500 for (j = 0; j < iov_length; j++) 501 d[j] = k++; 502 } 503 } 504 505 msg->msg_iov = iov; 506 msg->msg_iovlen = iov_count; 507 508 return 0; 509 unwind_iov: 510 for (i--; i >= 0 ; i--) 511 free(msg->msg_iov[i].iov_base); 512 return -ENOMEM; 513 } 514 515 /* TODO: Add verification logic for push, pull and pop data */ 516 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz, 517 unsigned char *k_p, int *bytes_cnt_p) 518 { 519 int i, j, bytes_cnt = *bytes_cnt_p; 520 unsigned char k = *k_p; 521 522 for (i = 0, j = 0; i < msg->msg_iovlen && size; i++, j = 0) { 523 unsigned char *d = msg->msg_iov[i].iov_base; 524 525 /* Special case test for skb ingress + ktls */ 526 if (i == 0 && txmsg_ktls_skb) { 527 if (msg->msg_iov[i].iov_len < 4) 528 return -EDATAINTEGRITY; 529 if (memcmp(d, "PASS", 4) != 0) { 530 fprintf(stderr, 531 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", 532 i, 0, d[0], d[1], d[2], d[3]); 533 return -EDATAINTEGRITY; 534 } 535 j = 4; /* advance index past PASS header */ 536 } 537 538 for (; j < msg->msg_iov[i].iov_len && size; j++) { 539 if (d[j] != k++) { 540 fprintf(stderr, 541 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 542 i, j, d[j], k - 1, d[j+1], k); 543 return -EDATAINTEGRITY; 544 } 545 bytes_cnt++; 546 if (bytes_cnt == chunk_sz) { 547 k = 0; 548 bytes_cnt = 0; 549 } 550 size--; 551 } 552 } 553 *k_p = k; 554 *bytes_cnt_p = bytes_cnt; 555 return 0; 556 } 557 558 static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 559 struct msg_stats *s, bool tx, 560 struct sockmap_options *opt) 561 { 562 struct msghdr msg = {0}, msg_peek = {0}; 563 int err, i, flags = MSG_NOSIGNAL; 564 bool drop = opt->drop_expected; 565 bool data = opt->data_test; 566 int iov_alloc_length = iov_length; 567 568 if (!tx && opt->check_recved_len) 569 iov_alloc_length *= 2; 570 571 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx); 572 if (err) 573 goto out_errno; 574 if (peek_flag) { 575 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx); 576 if (err) 577 goto out_errno; 578 } 579 580 if (tx) { 581 clock_gettime(CLOCK_MONOTONIC, &s->start); 582 for (i = 0; i < cnt; i++) { 583 int sent; 584 585 errno = 0; 586 sent = sendmsg(fd, &msg, flags); 587 588 if (!drop && sent < 0) { 589 if (opt->tx_wait_mem && errno == EACCES) { 590 errno = 0; 591 goto out_errno; 592 } 593 perror("sendmsg loop error"); 594 goto out_errno; 595 } else if (drop && sent >= 0) { 596 fprintf(stderr, 597 "sendmsg loop error expected: %i errno %i\n", 598 sent, errno); 599 errno = -EIO; 600 goto out_errno; 601 } 602 if (sent > 0) 603 s->bytes_sent += sent; 604 } 605 clock_gettime(CLOCK_MONOTONIC, &s->end); 606 } else { 607 int slct, recvp = 0, recv, max_fd = fd; 608 float total_bytes, txmsg_pop_total; 609 int fd_flags = O_NONBLOCK; 610 struct timeval timeout; 611 unsigned char k = 0; 612 int bytes_cnt = 0; 613 fd_set w; 614 615 fcntl(fd, fd_flags); 616 /* Account for pop bytes noting each iteration of apply will 617 * call msg_pop_data helper so we need to account for this 618 * by calculating the number of apply iterations. Note user 619 * of the tool can create cases where no data is sent by 620 * manipulating pop/push/pull/etc. For example txmsg_apply 1 621 * with txmsg_pop 1 will try to apply 1B at a time but each 622 * iteration will then pop 1B so no data will ever be sent. 623 * This is really only useful for testing edge cases in code 624 * paths. 625 */ 626 total_bytes = (float)iov_count * (float)iov_length * (float)cnt; 627 if (txmsg_apply) 628 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply); 629 else 630 txmsg_pop_total = txmsg_pop * cnt; 631 total_bytes -= txmsg_pop_total; 632 err = clock_gettime(CLOCK_MONOTONIC, &s->start); 633 if (err < 0) 634 perror("recv start time"); 635 while (s->bytes_recvd < total_bytes) { 636 if (txmsg_cork) { 637 timeout.tv_sec = 0; 638 timeout.tv_usec = 300000; 639 } else { 640 timeout.tv_sec = 3; 641 timeout.tv_usec = 0; 642 } 643 644 /* FD sets */ 645 FD_ZERO(&w); 646 FD_SET(fd, &w); 647 648 slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 649 if (slct == -1) { 650 perror("select()"); 651 clock_gettime(CLOCK_MONOTONIC, &s->end); 652 goto out_errno; 653 } else if (!slct) { 654 if (opt->verbose) 655 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total); 656 errno = -EIO; 657 clock_gettime(CLOCK_MONOTONIC, &s->end); 658 goto out_errno; 659 } 660 661 if (opt->tx_wait_mem) { 662 FD_ZERO(&w); 663 FD_SET(fd, &w); 664 slct = select(max_fd + 1, NULL, NULL, &w, &timeout); 665 errno = 0; 666 close(fd); 667 goto out_errno; 668 } 669 670 errno = 0; 671 if (peek_flag) { 672 flags |= MSG_PEEK; 673 recvp = recvmsg(fd, &msg_peek, flags); 674 if (recvp < 0) { 675 if (errno != EWOULDBLOCK) { 676 clock_gettime(CLOCK_MONOTONIC, &s->end); 677 goto out_errno; 678 } 679 } 680 flags = 0; 681 } 682 683 recv = recvmsg(fd, &msg, flags); 684 if (recv < 0) { 685 if (errno != EWOULDBLOCK) { 686 clock_gettime(CLOCK_MONOTONIC, &s->end); 687 perror("recv failed()"); 688 goto out_errno; 689 } 690 } 691 692 if (recv > 0) 693 s->bytes_recvd += recv; 694 695 if (opt->check_recved_len && s->bytes_recvd > total_bytes) { 696 errno = EMSGSIZE; 697 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n", 698 s->bytes_recvd, total_bytes); 699 goto out_errno; 700 } 701 702 if (data) { 703 int chunk_sz = opt->sendpage ? 704 iov_length * cnt : 705 iov_length * iov_count; 706 707 errno = msg_verify_data(&msg, recv, chunk_sz, &k, &bytes_cnt); 708 if (errno) { 709 perror("data verify msg failed"); 710 goto out_errno; 711 } 712 if (recvp) { 713 errno = msg_verify_data(&msg_peek, 714 recvp, 715 chunk_sz, 716 &k, 717 &bytes_cnt); 718 if (errno) { 719 perror("data verify msg_peek failed"); 720 goto out_errno; 721 } 722 } 723 } 724 } 725 clock_gettime(CLOCK_MONOTONIC, &s->end); 726 } 727 728 msg_free_iov(&msg); 729 msg_free_iov(&msg_peek); 730 return err; 731 out_errno: 732 msg_free_iov(&msg); 733 msg_free_iov(&msg_peek); 734 return errno; 735 } 736 737 static float giga = 1000000000; 738 739 static inline float sentBps(struct msg_stats s) 740 { 741 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 742 } 743 744 static inline float recvdBps(struct msg_stats s) 745 { 746 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 747 } 748 749 static int sendmsg_test(struct sockmap_options *opt) 750 { 751 float sent_Bps = 0, recvd_Bps = 0; 752 int rx_fd, txpid, rxpid, err = 0; 753 struct msg_stats s = {0}; 754 int iov_count = opt->iov_count; 755 int iov_buf = opt->iov_length; 756 int rx_status, tx_status; 757 int cnt = opt->rate; 758 759 errno = 0; 760 761 if (opt->base) 762 rx_fd = p1; 763 else 764 rx_fd = p2; 765 766 if (ktls) { 767 /* Redirecting into non-TLS socket which sends into a TLS 768 * socket is not a valid test. So in this case lets not 769 * enable kTLS but still run the test. 770 */ 771 if (!txmsg_redir || txmsg_ingress) { 772 err = sockmap_init_ktls(opt->verbose, rx_fd); 773 if (err) 774 return err; 775 } 776 err = sockmap_init_ktls(opt->verbose, c1); 777 if (err) 778 return err; 779 } 780 781 if (opt->tx_wait_mem) { 782 struct timeval timeout; 783 int rxtx_buf_len = 1024; 784 785 timeout.tv_sec = 3; 786 timeout.tv_usec = 0; 787 788 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval)); 789 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int)); 790 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int)); 791 if (err) { 792 perror("setsockopt failed()"); 793 return errno; 794 } 795 } 796 797 rxpid = fork(); 798 if (rxpid == 0) { 799 if (txmsg_pop || txmsg_start_pop) 800 iov_buf -= (txmsg_pop - txmsg_start_pop + 1); 801 if (opt->drop_expected || txmsg_ktls_skb_drop) 802 _exit(0); 803 804 if (!iov_buf) /* zero bytes sent case */ 805 _exit(0); 806 807 if (opt->sendpage) 808 iov_count = 1; 809 err = msg_loop(rx_fd, iov_count, iov_buf, 810 cnt, &s, false, opt); 811 if (opt->verbose > 1) 812 fprintf(stderr, 813 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 814 iov_count, iov_buf, cnt, err); 815 if (s.end.tv_sec - s.start.tv_sec) { 816 sent_Bps = sentBps(s); 817 recvd_Bps = recvdBps(s); 818 } 819 if (opt->verbose > 1) 820 fprintf(stdout, 821 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n", 822 s.bytes_sent, sent_Bps, sent_Bps/giga, 823 s.bytes_recvd, recvd_Bps, recvd_Bps/giga, 824 peek_flag ? "(peek_msg)" : ""); 825 if (err && err != -EDATAINTEGRITY && txmsg_cork) 826 err = 0; 827 exit(err ? 1 : 0); 828 } else if (rxpid == -1) { 829 perror("msg_loop_rx"); 830 return errno; 831 } 832 833 if (opt->tx_wait_mem) 834 close(c2); 835 836 txpid = fork(); 837 if (txpid == 0) { 838 if (opt->sendpage) 839 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 840 else 841 err = msg_loop(c1, iov_count, iov_buf, 842 cnt, &s, true, opt); 843 844 if (err) 845 fprintf(stderr, 846 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 847 iov_count, iov_buf, cnt, err); 848 if (s.end.tv_sec - s.start.tv_sec) { 849 sent_Bps = sentBps(s); 850 recvd_Bps = recvdBps(s); 851 } 852 if (opt->verbose > 1) 853 fprintf(stdout, 854 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 855 s.bytes_sent, sent_Bps, sent_Bps/giga, 856 s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 857 exit(err ? 1 : 0); 858 } else if (txpid == -1) { 859 perror("msg_loop_tx"); 860 return errno; 861 } 862 863 assert(waitpid(rxpid, &rx_status, 0) == rxpid); 864 assert(waitpid(txpid, &tx_status, 0) == txpid); 865 if (WIFEXITED(rx_status)) { 866 err = WEXITSTATUS(rx_status); 867 if (err) { 868 fprintf(stderr, "rx thread exited with err %d.\n", err); 869 goto out; 870 } 871 } 872 if (WIFEXITED(tx_status)) { 873 err = WEXITSTATUS(tx_status); 874 if (err) 875 fprintf(stderr, "tx thread exited with err %d.\n", err); 876 } 877 out: 878 return err; 879 } 880 881 static int forever_ping_pong(int rate, struct sockmap_options *opt) 882 { 883 struct timeval timeout; 884 char buf[1024] = {0}; 885 int sc; 886 887 timeout.tv_sec = 10; 888 timeout.tv_usec = 0; 889 890 /* Ping/Pong data from client to server */ 891 sc = send(c1, buf, sizeof(buf), 0); 892 if (sc < 0) { 893 perror("send failed()"); 894 return sc; 895 } 896 897 do { 898 int s, rc, i, max_fd = p2; 899 fd_set w; 900 901 /* FD sets */ 902 FD_ZERO(&w); 903 FD_SET(c1, &w); 904 FD_SET(c2, &w); 905 FD_SET(p1, &w); 906 FD_SET(p2, &w); 907 908 s = select(max_fd + 1, &w, NULL, NULL, &timeout); 909 if (s == -1) { 910 perror("select()"); 911 break; 912 } else if (!s) { 913 fprintf(stderr, "unexpected timeout\n"); 914 break; 915 } 916 917 for (i = 0; i <= max_fd && s > 0; ++i) { 918 if (!FD_ISSET(i, &w)) 919 continue; 920 921 s--; 922 923 rc = recv(i, buf, sizeof(buf), 0); 924 if (rc < 0) { 925 if (errno != EWOULDBLOCK) { 926 perror("recv failed()"); 927 return rc; 928 } 929 } 930 931 if (rc == 0) { 932 close(i); 933 break; 934 } 935 936 sc = send(i, buf, rc, 0); 937 if (sc < 0) { 938 perror("send failed()"); 939 return sc; 940 } 941 } 942 943 if (rate) 944 sleep(rate); 945 946 if (opt->verbose) { 947 printf("."); 948 fflush(stdout); 949 950 } 951 } while (running); 952 953 return 0; 954 } 955 956 enum { 957 SELFTESTS, 958 PING_PONG, 959 SENDMSG, 960 BASE, 961 BASE_SENDPAGE, 962 SENDPAGE, 963 }; 964 965 static int run_options(struct sockmap_options *options, int cg_fd, int test) 966 { 967 int i, key, next_key, err, zero = 0; 968 struct bpf_program *tx_prog; 969 970 /* If base test skip BPF setup */ 971 if (test == BASE || test == BASE_SENDPAGE) 972 goto run; 973 974 /* Attach programs to sockmap */ 975 if (!txmsg_omit_skb_parser) { 976 links[0] = bpf_program__attach_sockmap(progs[0], map_fd[0]); 977 if (!links[0]) { 978 fprintf(stderr, 979 "ERROR: bpf_program__attach_sockmap (sockmap %i->%i): (%s)\n", 980 bpf_program__fd(progs[0]), map_fd[0], strerror(errno)); 981 return -1; 982 } 983 } 984 985 links[1] = bpf_program__attach_sockmap(progs[1], map_fd[0]); 986 if (!links[1]) { 987 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (sockmap): (%s)\n", 988 strerror(errno)); 989 return -1; 990 } 991 992 /* Attach programs to TLS sockmap */ 993 if (txmsg_ktls_skb) { 994 if (!txmsg_omit_skb_parser) { 995 links[2] = bpf_program__attach_sockmap(progs[0], map_fd[8]); 996 if (!links[2]) { 997 fprintf(stderr, 998 "ERROR: bpf_program__attach_sockmap (TLS sockmap %i->%i): (%s)\n", 999 bpf_program__fd(progs[0]), map_fd[8], strerror(errno)); 1000 return -1; 1001 } 1002 } 1003 1004 links[3] = bpf_program__attach_sockmap(progs[2], map_fd[8]); 1005 if (!links[3]) { 1006 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (TLS sockmap): (%s)\n", 1007 strerror(errno)); 1008 return -1; 1009 } 1010 } 1011 1012 /* Attach to cgroups */ 1013 err = bpf_prog_attach(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS, 0); 1014 if (err) { 1015 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 1016 err, strerror(errno)); 1017 return err; 1018 } 1019 1020 run: 1021 err = sockmap_init_sockets(options->verbose); 1022 if (err) { 1023 fprintf(stderr, "ERROR: test socket failed: %d\n", err); 1024 goto out; 1025 } 1026 1027 /* Attach txmsg program to sockmap */ 1028 if (txmsg_pass) 1029 tx_prog = progs[4]; 1030 else if (txmsg_redir) 1031 tx_prog = progs[5]; 1032 else if (txmsg_apply) 1033 tx_prog = progs[6]; 1034 else if (txmsg_cork) 1035 tx_prog = progs[7]; 1036 else if (txmsg_drop) 1037 tx_prog = progs[8]; 1038 else 1039 tx_prog = NULL; 1040 1041 if (tx_prog) { 1042 int redir_fd; 1043 1044 links[4] = bpf_program__attach_sockmap(tx_prog, map_fd[1]); 1045 if (!links[4]) { 1046 fprintf(stderr, 1047 "ERROR: bpf_program__attach_sockmap (txmsg): (%s)\n", 1048 strerror(errno)); 1049 err = -1; 1050 goto out; 1051 } 1052 1053 i = 0; 1054 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 1055 if (err) { 1056 fprintf(stderr, 1057 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 1058 err, strerror(errno)); 1059 goto out; 1060 } 1061 1062 if (txmsg_redir) 1063 redir_fd = c2; 1064 else 1065 redir_fd = c1; 1066 1067 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 1068 if (err) { 1069 fprintf(stderr, 1070 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 1071 err, strerror(errno)); 1072 goto out; 1073 } 1074 1075 if (txmsg_apply) { 1076 err = bpf_map_update_elem(map_fd[3], 1077 &i, &txmsg_apply, BPF_ANY); 1078 if (err) { 1079 fprintf(stderr, 1080 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 1081 err, strerror(errno)); 1082 goto out; 1083 } 1084 } 1085 1086 if (txmsg_cork) { 1087 err = bpf_map_update_elem(map_fd[4], 1088 &i, &txmsg_cork, BPF_ANY); 1089 if (err) { 1090 fprintf(stderr, 1091 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 1092 err, strerror(errno)); 1093 goto out; 1094 } 1095 } 1096 1097 if (txmsg_start) { 1098 err = bpf_map_update_elem(map_fd[5], 1099 &i, &txmsg_start, BPF_ANY); 1100 if (err) { 1101 fprintf(stderr, 1102 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 1103 err, strerror(errno)); 1104 goto out; 1105 } 1106 } 1107 1108 if (txmsg_end) { 1109 i = 1; 1110 err = bpf_map_update_elem(map_fd[5], 1111 &i, &txmsg_end, BPF_ANY); 1112 if (err) { 1113 fprintf(stderr, 1114 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 1115 err, strerror(errno)); 1116 goto out; 1117 } 1118 } 1119 1120 if (txmsg_start_push) { 1121 i = 2; 1122 err = bpf_map_update_elem(map_fd[5], 1123 &i, &txmsg_start_push, BPF_ANY); 1124 if (err) { 1125 fprintf(stderr, 1126 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n", 1127 err, strerror(errno)); 1128 goto out; 1129 } 1130 } 1131 1132 if (txmsg_end_push) { 1133 i = 3; 1134 err = bpf_map_update_elem(map_fd[5], 1135 &i, &txmsg_end_push, BPF_ANY); 1136 if (err) { 1137 fprintf(stderr, 1138 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n", 1139 txmsg_end_push, i, err, strerror(errno)); 1140 goto out; 1141 } 1142 } 1143 1144 if (txmsg_start_pop) { 1145 i = 4; 1146 err = bpf_map_update_elem(map_fd[5], 1147 &i, &txmsg_start_pop, BPF_ANY); 1148 if (err) { 1149 fprintf(stderr, 1150 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n", 1151 txmsg_start_pop, i, err, strerror(errno)); 1152 goto out; 1153 } 1154 } else { 1155 i = 4; 1156 bpf_map_update_elem(map_fd[5], 1157 &i, &txmsg_start_pop, BPF_ANY); 1158 } 1159 1160 if (txmsg_pop) { 1161 i = 5; 1162 err = bpf_map_update_elem(map_fd[5], 1163 &i, &txmsg_pop, BPF_ANY); 1164 if (err) { 1165 fprintf(stderr, 1166 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n", 1167 txmsg_pop, i, err, strerror(errno)); 1168 goto out; 1169 } 1170 } else { 1171 i = 5; 1172 bpf_map_update_elem(map_fd[5], 1173 &i, &txmsg_pop, BPF_ANY); 1174 1175 } 1176 1177 if (txmsg_ingress) { 1178 int in = BPF_F_INGRESS; 1179 1180 i = 0; 1181 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 1182 if (err) { 1183 fprintf(stderr, 1184 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1185 err, strerror(errno)); 1186 } 1187 i = 1; 1188 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 1189 if (err) { 1190 fprintf(stderr, 1191 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 1192 err, strerror(errno)); 1193 } 1194 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 1195 if (err) { 1196 fprintf(stderr, 1197 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 1198 err, strerror(errno)); 1199 } 1200 1201 i = 2; 1202 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 1203 if (err) { 1204 fprintf(stderr, 1205 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 1206 err, strerror(errno)); 1207 } 1208 } 1209 1210 if (txmsg_ktls_skb) { 1211 int ingress = BPF_F_INGRESS; 1212 1213 i = 0; 1214 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY); 1215 if (err) { 1216 fprintf(stderr, 1217 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1218 err, strerror(errno)); 1219 } 1220 1221 if (txmsg_ktls_skb_redir) { 1222 i = 1; 1223 err = bpf_map_update_elem(map_fd[7], 1224 &i, &ingress, BPF_ANY); 1225 if (err) { 1226 fprintf(stderr, 1227 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1228 err, strerror(errno)); 1229 } 1230 } 1231 1232 if (txmsg_ktls_skb_drop) { 1233 i = 1; 1234 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY); 1235 } 1236 } 1237 1238 if (txmsg_redir_skb) { 1239 int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 1240 p2 : p1; 1241 int ingress = BPF_F_INGRESS; 1242 1243 i = 0; 1244 err = bpf_map_update_elem(map_fd[7], 1245 &i, &ingress, BPF_ANY); 1246 if (err) { 1247 fprintf(stderr, 1248 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1249 err, strerror(errno)); 1250 } 1251 1252 i = 3; 1253 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY); 1254 if (err) { 1255 fprintf(stderr, 1256 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1257 err, strerror(errno)); 1258 } 1259 } 1260 } 1261 1262 if (skb_use_parser) { 1263 i = 2; 1264 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY); 1265 } 1266 1267 if (txmsg_drop) 1268 options->drop_expected = true; 1269 1270 if (test == PING_PONG) 1271 err = forever_ping_pong(options->rate, options); 1272 else if (test == SENDMSG) { 1273 options->base = false; 1274 options->sendpage = false; 1275 err = sendmsg_test(options); 1276 } else if (test == SENDPAGE) { 1277 options->base = false; 1278 options->sendpage = true; 1279 err = sendmsg_test(options); 1280 } else if (test == BASE) { 1281 options->base = true; 1282 options->sendpage = false; 1283 err = sendmsg_test(options); 1284 } else if (test == BASE_SENDPAGE) { 1285 options->base = true; 1286 options->sendpage = true; 1287 err = sendmsg_test(options); 1288 } else 1289 fprintf(stderr, "unknown test\n"); 1290 out: 1291 /* Detatch and zero all the maps */ 1292 bpf_prog_detach2(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS); 1293 1294 for (i = 0; i < ARRAY_SIZE(links); i++) { 1295 if (links[i]) 1296 bpf_link__detach(links[i]); 1297 } 1298 1299 for (i = 0; i < ARRAY_SIZE(map_fd); i++) { 1300 key = next_key = 0; 1301 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1302 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 1303 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1304 key = next_key; 1305 } 1306 } 1307 1308 close(s1); 1309 close(s2); 1310 close(p1); 1311 close(p2); 1312 close(c1); 1313 close(c2); 1314 return err; 1315 } 1316 1317 static char *test_to_str(int test) 1318 { 1319 switch (test) { 1320 case SENDMSG: 1321 return "sendmsg"; 1322 case SENDPAGE: 1323 return "sendpage"; 1324 } 1325 return "unknown"; 1326 } 1327 1328 static void append_str(char *dst, const char *src, size_t dst_cap) 1329 { 1330 size_t avail = dst_cap - strlen(dst); 1331 1332 if (avail <= 1) /* just zero byte could be written */ 1333 return; 1334 1335 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */ 1336 } 1337 1338 #define OPTSTRING 60 1339 static void test_options(char *options) 1340 { 1341 char tstr[OPTSTRING]; 1342 1343 memset(options, 0, OPTSTRING); 1344 1345 if (txmsg_pass) 1346 append_str(options, "pass,", OPTSTRING); 1347 if (txmsg_redir) 1348 append_str(options, "redir,", OPTSTRING); 1349 if (txmsg_drop) 1350 append_str(options, "drop,", OPTSTRING); 1351 if (txmsg_apply) { 1352 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 1353 append_str(options, tstr, OPTSTRING); 1354 } 1355 if (txmsg_cork) { 1356 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 1357 append_str(options, tstr, OPTSTRING); 1358 } 1359 if (txmsg_start) { 1360 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 1361 append_str(options, tstr, OPTSTRING); 1362 } 1363 if (txmsg_end) { 1364 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 1365 append_str(options, tstr, OPTSTRING); 1366 } 1367 if (txmsg_start_pop) { 1368 snprintf(tstr, OPTSTRING, "pop (%d,%d),", 1369 txmsg_start_pop, txmsg_start_pop + txmsg_pop); 1370 append_str(options, tstr, OPTSTRING); 1371 } 1372 if (txmsg_ingress) 1373 append_str(options, "ingress,", OPTSTRING); 1374 if (txmsg_redir_skb) 1375 append_str(options, "redir_skb,", OPTSTRING); 1376 if (txmsg_ktls_skb) 1377 append_str(options, "ktls_skb,", OPTSTRING); 1378 if (ktls) 1379 append_str(options, "ktls,", OPTSTRING); 1380 if (peek_flag) 1381 append_str(options, "peek,", OPTSTRING); 1382 } 1383 1384 static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 1385 { 1386 char *options = calloc(OPTSTRING, sizeof(char)); 1387 int err; 1388 1389 if (test == SENDPAGE) 1390 opt->sendpage = true; 1391 else 1392 opt->sendpage = false; 1393 1394 if (txmsg_drop) 1395 opt->drop_expected = true; 1396 else 1397 opt->drop_expected = false; 1398 1399 test_options(options); 1400 1401 if (opt->verbose) { 1402 fprintf(stdout, 1403 " [TEST %i]: (%i, %i, %i, %s, %s): ", 1404 test_cnt, opt->rate, opt->iov_count, opt->iov_length, 1405 test_to_str(test), options); 1406 fflush(stdout); 1407 } 1408 err = run_options(opt, cgrp, test); 1409 if (opt->verbose) 1410 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED"); 1411 test_cnt++; 1412 !err ? passed++ : failed++; 1413 free(options); 1414 return err; 1415 } 1416 1417 static void test_exec(int cgrp, struct sockmap_options *opt) 1418 { 1419 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME); 1420 int err; 1421 1422 if (type == 0) { 1423 test_start(); 1424 err = __test_exec(cgrp, SENDMSG, opt); 1425 if (err) 1426 test_fail(); 1427 } else { 1428 test_start(); 1429 err = __test_exec(cgrp, SENDPAGE, opt); 1430 if (err) 1431 test_fail(); 1432 } 1433 } 1434 1435 static void test_send_one(struct sockmap_options *opt, int cgrp) 1436 { 1437 opt->iov_length = 1; 1438 opt->iov_count = 1; 1439 opt->rate = 1; 1440 test_exec(cgrp, opt); 1441 1442 opt->iov_length = 1; 1443 opt->iov_count = 1024; 1444 opt->rate = 1; 1445 test_exec(cgrp, opt); 1446 1447 opt->iov_length = 1024; 1448 opt->iov_count = 1; 1449 opt->rate = 1; 1450 test_exec(cgrp, opt); 1451 1452 } 1453 1454 static void test_send_many(struct sockmap_options *opt, int cgrp) 1455 { 1456 opt->iov_length = 3; 1457 opt->iov_count = 1; 1458 opt->rate = 512; 1459 test_exec(cgrp, opt); 1460 1461 opt->rate = 100; 1462 opt->iov_count = 1; 1463 opt->iov_length = 5; 1464 test_exec(cgrp, opt); 1465 } 1466 1467 static void test_send_large(struct sockmap_options *opt, int cgrp) 1468 { 1469 opt->iov_length = 256; 1470 opt->iov_count = 1024; 1471 opt->rate = 2; 1472 test_exec(cgrp, opt); 1473 } 1474 1475 static void test_send(struct sockmap_options *opt, int cgrp) 1476 { 1477 test_send_one(opt, cgrp); 1478 test_send_many(opt, cgrp); 1479 test_send_large(opt, cgrp); 1480 sched_yield(); 1481 } 1482 1483 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt) 1484 { 1485 /* Test small and large iov_count values with pass/redir/apply/cork */ 1486 txmsg_pass = 1; 1487 test_send(opt, cgrp); 1488 } 1489 1490 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt) 1491 { 1492 txmsg_redir = 1; 1493 test_send(opt, cgrp); 1494 } 1495 1496 static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt) 1497 { 1498 txmsg_redir = 1; 1499 opt->tx_wait_mem = true; 1500 test_send_large(opt, cgrp); 1501 opt->tx_wait_mem = false; 1502 } 1503 1504 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt) 1505 { 1506 txmsg_drop = 1; 1507 test_send(opt, cgrp); 1508 } 1509 1510 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt) 1511 { 1512 txmsg_pass = txmsg_drop = 0; 1513 txmsg_ingress = txmsg_redir = 1; 1514 test_send(opt, cgrp); 1515 } 1516 1517 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt) 1518 { 1519 bool data = opt->data_test; 1520 int k = ktls; 1521 1522 opt->data_test = true; 1523 ktls = 1; 1524 1525 txmsg_pass = txmsg_drop = 0; 1526 txmsg_ingress = txmsg_redir = 0; 1527 txmsg_ktls_skb = 1; 1528 txmsg_pass = 1; 1529 1530 /* Using data verification so ensure iov layout is 1531 * expected from test receiver side. e.g. has enough 1532 * bytes to write test code. 1533 */ 1534 opt->iov_length = 100; 1535 opt->iov_count = 1; 1536 opt->rate = 1; 1537 test_exec(cgrp, opt); 1538 1539 txmsg_ktls_skb_drop = 1; 1540 test_exec(cgrp, opt); 1541 1542 txmsg_ktls_skb_drop = 0; 1543 txmsg_ktls_skb_redir = 1; 1544 test_exec(cgrp, opt); 1545 txmsg_ktls_skb_redir = 0; 1546 1547 /* Tests that omit skb_parser */ 1548 txmsg_omit_skb_parser = 1; 1549 ktls = 0; 1550 txmsg_ktls_skb = 0; 1551 test_exec(cgrp, opt); 1552 1553 txmsg_ktls_skb_drop = 1; 1554 test_exec(cgrp, opt); 1555 txmsg_ktls_skb_drop = 0; 1556 1557 txmsg_ktls_skb_redir = 1; 1558 test_exec(cgrp, opt); 1559 1560 ktls = 1; 1561 test_exec(cgrp, opt); 1562 txmsg_omit_skb_parser = 0; 1563 1564 opt->data_test = data; 1565 ktls = k; 1566 } 1567 1568 /* Test cork with hung data. This tests poor usage patterns where 1569 * cork can leave data on the ring if user program is buggy and 1570 * doesn't flush them somehow. They do take some time however 1571 * because they wait for a timeout. Test pass, redir and cork with 1572 * apply logic. Use cork size of 4097 with send_large to avoid 1573 * aligning cork size with send size. 1574 */ 1575 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt) 1576 { 1577 txmsg_pass = 1; 1578 txmsg_redir = 0; 1579 txmsg_cork = 4097; 1580 txmsg_apply = 4097; 1581 test_send_large(opt, cgrp); 1582 1583 txmsg_pass = 0; 1584 txmsg_redir = 1; 1585 txmsg_apply = 0; 1586 txmsg_cork = 4097; 1587 test_send_large(opt, cgrp); 1588 1589 txmsg_pass = 0; 1590 txmsg_redir = 1; 1591 txmsg_apply = 4097; 1592 txmsg_cork = 4097; 1593 test_send_large(opt, cgrp); 1594 } 1595 1596 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt) 1597 { 1598 /* Test basic start/end */ 1599 txmsg_start = 1; 1600 txmsg_end = 2; 1601 test_send(opt, cgrp); 1602 1603 /* Test >4k pull */ 1604 txmsg_start = 4096; 1605 txmsg_end = 9182; 1606 test_send_large(opt, cgrp); 1607 1608 /* Test pull + redirect */ 1609 txmsg_redir = 1; 1610 txmsg_start = 1; 1611 txmsg_end = 2; 1612 test_send(opt, cgrp); 1613 1614 /* Test pull + cork */ 1615 txmsg_redir = 0; 1616 txmsg_cork = 512; 1617 txmsg_start = 1; 1618 txmsg_end = 2; 1619 test_send_many(opt, cgrp); 1620 1621 /* Test pull + cork + redirect */ 1622 txmsg_redir = 1; 1623 txmsg_cork = 512; 1624 txmsg_start = 1; 1625 txmsg_end = 2; 1626 test_send_many(opt, cgrp); 1627 } 1628 1629 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt) 1630 { 1631 /* Test basic pop */ 1632 txmsg_start_pop = 1; 1633 txmsg_pop = 2; 1634 test_send_many(opt, cgrp); 1635 1636 /* Test pop with >4k */ 1637 txmsg_start_pop = 4096; 1638 txmsg_pop = 4096; 1639 test_send_large(opt, cgrp); 1640 1641 /* Test pop + redirect */ 1642 txmsg_redir = 1; 1643 txmsg_start_pop = 1; 1644 txmsg_pop = 2; 1645 test_send_many(opt, cgrp); 1646 1647 /* Test pop + cork */ 1648 txmsg_redir = 0; 1649 txmsg_cork = 512; 1650 txmsg_start_pop = 1; 1651 txmsg_pop = 2; 1652 test_send_many(opt, cgrp); 1653 1654 /* Test pop + redirect + cork */ 1655 txmsg_redir = 1; 1656 txmsg_cork = 4; 1657 txmsg_start_pop = 1; 1658 txmsg_pop = 2; 1659 test_send_many(opt, cgrp); 1660 } 1661 1662 static void test_txmsg_push(int cgrp, struct sockmap_options *opt) 1663 { 1664 /* Test basic push */ 1665 txmsg_start_push = 1; 1666 txmsg_end_push = 1; 1667 test_send(opt, cgrp); 1668 1669 /* Test push 4kB >4k */ 1670 txmsg_start_push = 4096; 1671 txmsg_end_push = 4096; 1672 test_send_large(opt, cgrp); 1673 1674 /* Test push + redirect */ 1675 txmsg_redir = 1; 1676 txmsg_start_push = 1; 1677 txmsg_end_push = 2; 1678 test_send_many(opt, cgrp); 1679 1680 /* Test push + cork */ 1681 txmsg_redir = 0; 1682 txmsg_cork = 512; 1683 txmsg_start_push = 1; 1684 txmsg_end_push = 2; 1685 test_send_many(opt, cgrp); 1686 } 1687 1688 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt) 1689 { 1690 txmsg_start_push = 1; 1691 txmsg_end_push = 10; 1692 txmsg_start_pop = 5; 1693 txmsg_pop = 4; 1694 test_send_large(opt, cgrp); 1695 } 1696 1697 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt) 1698 { 1699 txmsg_pass = 1; 1700 txmsg_redir = 0; 1701 txmsg_ingress = 0; 1702 txmsg_apply = 1; 1703 txmsg_cork = 0; 1704 test_send_one(opt, cgrp); 1705 1706 txmsg_pass = 0; 1707 txmsg_redir = 1; 1708 txmsg_ingress = 0; 1709 txmsg_apply = 1; 1710 txmsg_cork = 0; 1711 test_send_one(opt, cgrp); 1712 1713 txmsg_pass = 0; 1714 txmsg_redir = 1; 1715 txmsg_ingress = 1; 1716 txmsg_apply = 1; 1717 txmsg_cork = 0; 1718 test_send_one(opt, cgrp); 1719 1720 txmsg_pass = 1; 1721 txmsg_redir = 0; 1722 txmsg_ingress = 0; 1723 txmsg_apply = 1024; 1724 txmsg_cork = 0; 1725 test_send_large(opt, cgrp); 1726 1727 txmsg_pass = 0; 1728 txmsg_redir = 1; 1729 txmsg_ingress = 0; 1730 txmsg_apply = 1024; 1731 txmsg_cork = 0; 1732 test_send_large(opt, cgrp); 1733 1734 txmsg_pass = 0; 1735 txmsg_redir = 1; 1736 txmsg_ingress = 1; 1737 txmsg_apply = 1024; 1738 txmsg_cork = 0; 1739 test_send_large(opt, cgrp); 1740 } 1741 1742 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt) 1743 { 1744 txmsg_pass = 1; 1745 txmsg_redir = 0; 1746 txmsg_apply = 0; 1747 txmsg_cork = 1; 1748 test_send(opt, cgrp); 1749 1750 txmsg_pass = 1; 1751 txmsg_redir = 0; 1752 txmsg_apply = 1; 1753 txmsg_cork = 1; 1754 test_send(opt, cgrp); 1755 } 1756 1757 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt) 1758 { 1759 txmsg_pass = 1; 1760 skb_use_parser = 512; 1761 if (ktls == 1) 1762 skb_use_parser = 570; 1763 opt->iov_length = 256; 1764 opt->iov_count = 1; 1765 opt->rate = 2; 1766 test_exec(cgrp, opt); 1767 } 1768 1769 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt) 1770 { 1771 if (ktls == 1) 1772 return; 1773 skb_use_parser = 10; 1774 opt->iov_length = 20; 1775 opt->iov_count = 1; 1776 opt->rate = 1; 1777 opt->check_recved_len = true; 1778 test_exec(cgrp, opt); 1779 opt->check_recved_len = false; 1780 } 1781 1782 char *map_names[] = { 1783 "sock_map", 1784 "sock_map_txmsg", 1785 "sock_map_redir", 1786 "sock_apply_bytes", 1787 "sock_cork_bytes", 1788 "sock_bytes", 1789 "sock_redir_flags", 1790 "sock_skb_opts", 1791 "tls_sock_map", 1792 }; 1793 1794 static int populate_progs(char *bpf_file) 1795 { 1796 struct bpf_program *prog; 1797 struct bpf_object *obj; 1798 int i = 0; 1799 long err; 1800 1801 obj = bpf_object__open(bpf_file); 1802 err = libbpf_get_error(obj); 1803 if (err) { 1804 char err_buf[256]; 1805 1806 libbpf_strerror(err, err_buf, sizeof(err_buf)); 1807 printf("Unable to load eBPF objects in file '%s' : %s\n", 1808 bpf_file, err_buf); 1809 return -1; 1810 } 1811 1812 i = bpf_object__load(obj); 1813 i = 0; 1814 bpf_object__for_each_program(prog, obj) { 1815 progs[i] = prog; 1816 i++; 1817 } 1818 1819 for (i = 0; i < ARRAY_SIZE(map_fd); i++) { 1820 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 1821 map_fd[i] = bpf_map__fd(maps[i]); 1822 if (map_fd[i] < 0) { 1823 fprintf(stderr, "load_bpf_file: (%i) %s\n", 1824 map_fd[i], strerror(errno)); 1825 return -1; 1826 } 1827 } 1828 1829 for (i = 0; i < ARRAY_SIZE(links); i++) 1830 links[i] = NULL; 1831 1832 return 0; 1833 } 1834 1835 struct _test test[] = { 1836 {"txmsg test passthrough", test_txmsg_pass}, 1837 {"txmsg test redirect", test_txmsg_redir}, 1838 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem}, 1839 {"txmsg test drop", test_txmsg_drop}, 1840 {"txmsg test ingress redirect", test_txmsg_ingress_redir}, 1841 {"txmsg test skb", test_txmsg_skb}, 1842 {"txmsg test apply", test_txmsg_apply}, 1843 {"txmsg test cork", test_txmsg_cork}, 1844 {"txmsg test hanging corks", test_txmsg_cork_hangs}, 1845 {"txmsg test push_data", test_txmsg_push}, 1846 {"txmsg test pull-data", test_txmsg_pull}, 1847 {"txmsg test pop-data", test_txmsg_pop}, 1848 {"txmsg test push/pop data", test_txmsg_push_pop}, 1849 {"txmsg test ingress parser", test_txmsg_ingress_parser}, 1850 {"txmsg test ingress parser2", test_txmsg_ingress_parser2}, 1851 }; 1852 1853 static int check_whitelist(struct _test *t, struct sockmap_options *opt) 1854 { 1855 char *entry, *ptr; 1856 1857 if (!opt->whitelist) 1858 return 0; 1859 ptr = strdup(opt->whitelist); 1860 if (!ptr) 1861 return -ENOMEM; 1862 entry = strtok(ptr, ","); 1863 while (entry) { 1864 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 1865 strstr(opt->map, entry) != 0 || 1866 strstr(t->title, entry) != 0) { 1867 free(ptr); 1868 return 0; 1869 } 1870 entry = strtok(NULL, ","); 1871 } 1872 free(ptr); 1873 return -EINVAL; 1874 } 1875 1876 static int check_blacklist(struct _test *t, struct sockmap_options *opt) 1877 { 1878 char *entry, *ptr; 1879 1880 if (!opt->blacklist) 1881 return -EINVAL; 1882 ptr = strdup(opt->blacklist); 1883 if (!ptr) 1884 return -ENOMEM; 1885 entry = strtok(ptr, ","); 1886 while (entry) { 1887 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 1888 strstr(opt->map, entry) != 0 || 1889 strstr(t->title, entry) != 0) { 1890 free(ptr); 1891 return 0; 1892 } 1893 entry = strtok(NULL, ","); 1894 } 1895 free(ptr); 1896 return -EINVAL; 1897 } 1898 1899 static int __test_selftests(int cg_fd, struct sockmap_options *opt) 1900 { 1901 int i, err; 1902 1903 err = populate_progs(opt->map); 1904 if (err < 0) { 1905 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 1906 return err; 1907 } 1908 1909 /* Tests basic commands and APIs */ 1910 for (i = 0; i < ARRAY_SIZE(test); i++) { 1911 struct _test t = test[i]; 1912 1913 if (check_whitelist(&t, opt) != 0) 1914 continue; 1915 if (check_blacklist(&t, opt) == 0) 1916 continue; 1917 1918 test_start_subtest(&t, opt); 1919 t.tester(cg_fd, opt); 1920 test_end_subtest(); 1921 } 1922 1923 return err; 1924 } 1925 1926 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt) 1927 { 1928 opt->map = BPF_SOCKMAP_FILENAME; 1929 __test_selftests(cg_fd, opt); 1930 } 1931 1932 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt) 1933 { 1934 opt->map = BPF_SOCKHASH_FILENAME; 1935 __test_selftests(cg_fd, opt); 1936 } 1937 1938 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt) 1939 { 1940 opt->map = BPF_SOCKHASH_FILENAME; 1941 opt->prepend = "ktls"; 1942 ktls = 1; 1943 __test_selftests(cg_fd, opt); 1944 ktls = 0; 1945 } 1946 1947 static int test_selftest(int cg_fd, struct sockmap_options *opt) 1948 { 1949 test_selftests_sockmap(cg_fd, opt); 1950 test_selftests_sockhash(cg_fd, opt); 1951 test_selftests_ktls(cg_fd, opt); 1952 test_print_results(); 1953 return 0; 1954 } 1955 1956 int main(int argc, char **argv) 1957 { 1958 int iov_count = 1, length = 1024, rate = 1; 1959 struct sockmap_options options = {0}; 1960 int opt, longindex, err, cg_fd = 0; 1961 char *bpf_file = BPF_SOCKMAP_FILENAME; 1962 int test = SELFTESTS; 1963 bool cg_created = 0; 1964 1965 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:", 1966 long_options, &longindex)) != -1) { 1967 switch (opt) { 1968 case 's': 1969 txmsg_start = atoi(optarg); 1970 break; 1971 case 'e': 1972 txmsg_end = atoi(optarg); 1973 break; 1974 case 'p': 1975 txmsg_start_push = atoi(optarg); 1976 break; 1977 case 'q': 1978 txmsg_end_push = atoi(optarg); 1979 break; 1980 case 'w': 1981 txmsg_start_pop = atoi(optarg); 1982 break; 1983 case 'x': 1984 txmsg_pop = atoi(optarg); 1985 break; 1986 case 'a': 1987 txmsg_apply = atoi(optarg); 1988 break; 1989 case 'k': 1990 txmsg_cork = atoi(optarg); 1991 break; 1992 case 'c': 1993 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 1994 if (cg_fd < 0) { 1995 fprintf(stderr, 1996 "ERROR: (%i) open cg path failed: %s\n", 1997 cg_fd, optarg); 1998 return cg_fd; 1999 } 2000 break; 2001 case 'r': 2002 rate = atoi(optarg); 2003 break; 2004 case 'v': 2005 options.verbose = 1; 2006 if (optarg) 2007 options.verbose = atoi(optarg); 2008 break; 2009 case 'i': 2010 iov_count = atoi(optarg); 2011 break; 2012 case 'l': 2013 length = atoi(optarg); 2014 break; 2015 case 'd': 2016 options.data_test = true; 2017 break; 2018 case 't': 2019 if (strcmp(optarg, "ping") == 0) { 2020 test = PING_PONG; 2021 } else if (strcmp(optarg, "sendmsg") == 0) { 2022 test = SENDMSG; 2023 } else if (strcmp(optarg, "base") == 0) { 2024 test = BASE; 2025 } else if (strcmp(optarg, "base_sendpage") == 0) { 2026 test = BASE_SENDPAGE; 2027 } else if (strcmp(optarg, "sendpage") == 0) { 2028 test = SENDPAGE; 2029 } else { 2030 usage(argv); 2031 return -1; 2032 } 2033 break; 2034 case 'n': 2035 options.whitelist = strdup(optarg); 2036 if (!options.whitelist) 2037 return -ENOMEM; 2038 break; 2039 case 'b': 2040 options.blacklist = strdup(optarg); 2041 if (!options.blacklist) 2042 return -ENOMEM; 2043 case 0: 2044 break; 2045 case 'h': 2046 default: 2047 usage(argv); 2048 return -1; 2049 } 2050 } 2051 2052 if (!cg_fd) { 2053 cg_fd = cgroup_setup_and_join(CG_PATH); 2054 if (cg_fd < 0) 2055 return cg_fd; 2056 cg_created = 1; 2057 } 2058 2059 /* Use libbpf 1.0 API mode */ 2060 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 2061 2062 if (test == SELFTESTS) { 2063 err = test_selftest(cg_fd, &options); 2064 goto out; 2065 } 2066 2067 err = populate_progs(bpf_file); 2068 if (err) { 2069 fprintf(stderr, "populate program: (%s) %s\n", 2070 bpf_file, strerror(errno)); 2071 return 1; 2072 } 2073 running = 1; 2074 2075 /* catch SIGINT */ 2076 signal(SIGINT, running_handler); 2077 2078 options.iov_count = iov_count; 2079 options.iov_length = length; 2080 options.rate = rate; 2081 2082 err = run_options(&options, cg_fd, test); 2083 out: 2084 if (options.whitelist) 2085 free(options.whitelist); 2086 if (options.blacklist) 2087 free(options.blacklist); 2088 close(cg_fd); 2089 if (cg_created) 2090 cleanup_cgroup_environment(); 2091 return err; 2092 } 2093 2094 void running_handler(int a) 2095 { 2096 running = 0; 2097 } 2098