1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <stdlib.h> 5 #include <unistd.h> 6 #include <stdbool.h> 7 #include <string.h> 8 #include <errno.h> 9 #include <assert.h> 10 #include <fcntl.h> 11 #include <linux/bpf.h> 12 #include <linux/err.h> 13 #include <linux/types.h> 14 #include <linux/if_ether.h> 15 #include <sys/types.h> 16 #include <sys/epoll.h> 17 #include <sys/socket.h> 18 #include <netinet/in.h> 19 #include <bpf/bpf.h> 20 #include <bpf/libbpf.h> 21 #include "bpf_rlimit.h" 22 #include "bpf_util.h" 23 24 #include "test_progs.h" 25 #include "test_select_reuseport_common.h" 26 27 #define MAX_TEST_NAME 80 28 #define MIN_TCPHDR_LEN 20 29 #define UDPHDR_LEN 8 30 31 #define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies" 32 #define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen" 33 #define REUSEPORT_ARRAY_SIZE 32 34 35 static int result_map, tmp_index_ovr_map, linum_map, data_check_map; 36 static __u32 expected_results[NR_RESULTS]; 37 static int sk_fds[REUSEPORT_ARRAY_SIZE]; 38 static int reuseport_array = -1, outer_map = -1; 39 static enum bpf_map_type inner_map_type; 40 static int select_by_skb_data_prog; 41 static int saved_tcp_syncookie = -1; 42 static struct bpf_object *obj; 43 static int saved_tcp_fo = -1; 44 static __u32 index_zero; 45 static int epfd; 46 47 static union sa46 { 48 struct sockaddr_in6 v6; 49 struct sockaddr_in v4; 50 sa_family_t family; 51 } srv_sa; 52 53 #define RET_IF(condition, tag, format...) ({ \ 54 if (CHECK_FAIL(condition)) { \ 55 printf(tag " " format); \ 56 return; \ 57 } \ 58 }) 59 60 #define RET_ERR(condition, tag, format...) ({ \ 61 if (CHECK_FAIL(condition)) { \ 62 printf(tag " " format); \ 63 return -1; \ 64 } \ 65 }) 66 67 static int create_maps(enum bpf_map_type inner_type) 68 { 69 struct bpf_create_map_attr attr = {}; 70 71 inner_map_type = inner_type; 72 73 /* Creating reuseport_array */ 74 attr.name = "reuseport_array"; 75 attr.map_type = inner_type; 76 attr.key_size = sizeof(__u32); 77 attr.value_size = sizeof(__u32); 78 attr.max_entries = REUSEPORT_ARRAY_SIZE; 79 80 reuseport_array = bpf_create_map_xattr(&attr); 81 RET_ERR(reuseport_array == -1, "creating reuseport_array", 82 "reuseport_array:%d errno:%d\n", reuseport_array, errno); 83 84 /* Creating outer_map */ 85 attr.name = "outer_map"; 86 attr.map_type = BPF_MAP_TYPE_ARRAY_OF_MAPS; 87 attr.key_size = sizeof(__u32); 88 attr.value_size = sizeof(__u32); 89 attr.max_entries = 1; 90 attr.inner_map_fd = reuseport_array; 91 outer_map = bpf_create_map_xattr(&attr); 92 RET_ERR(outer_map == -1, "creating outer_map", 93 "outer_map:%d errno:%d\n", outer_map, errno); 94 95 return 0; 96 } 97 98 static int prepare_bpf_obj(void) 99 { 100 struct bpf_program *prog; 101 struct bpf_map *map; 102 int err; 103 104 obj = bpf_object__open("test_select_reuseport_kern.o"); 105 RET_ERR(IS_ERR_OR_NULL(obj), "open test_select_reuseport_kern.o", 106 "obj:%p PTR_ERR(obj):%ld\n", obj, PTR_ERR(obj)); 107 108 map = bpf_object__find_map_by_name(obj, "outer_map"); 109 RET_ERR(!map, "find outer_map", "!map\n"); 110 err = bpf_map__reuse_fd(map, outer_map); 111 RET_ERR(err, "reuse outer_map", "err:%d\n", err); 112 113 err = bpf_object__load(obj); 114 RET_ERR(err, "load bpf_object", "err:%d\n", err); 115 116 prog = bpf_program__next(NULL, obj); 117 RET_ERR(!prog, "get first bpf_program", "!prog\n"); 118 select_by_skb_data_prog = bpf_program__fd(prog); 119 RET_ERR(select_by_skb_data_prog == -1, "get prog fd", 120 "select_by_skb_data_prog:%d\n", select_by_skb_data_prog); 121 122 map = bpf_object__find_map_by_name(obj, "result_map"); 123 RET_ERR(!map, "find result_map", "!map\n"); 124 result_map = bpf_map__fd(map); 125 RET_ERR(result_map == -1, "get result_map fd", 126 "result_map:%d\n", result_map); 127 128 map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map"); 129 RET_ERR(!map, "find tmp_index_ovr_map\n", "!map"); 130 tmp_index_ovr_map = bpf_map__fd(map); 131 RET_ERR(tmp_index_ovr_map == -1, "get tmp_index_ovr_map fd", 132 "tmp_index_ovr_map:%d\n", tmp_index_ovr_map); 133 134 map = bpf_object__find_map_by_name(obj, "linum_map"); 135 RET_ERR(!map, "find linum_map", "!map\n"); 136 linum_map = bpf_map__fd(map); 137 RET_ERR(linum_map == -1, "get linum_map fd", 138 "linum_map:%d\n", linum_map); 139 140 map = bpf_object__find_map_by_name(obj, "data_check_map"); 141 RET_ERR(!map, "find data_check_map", "!map\n"); 142 data_check_map = bpf_map__fd(map); 143 RET_ERR(data_check_map == -1, "get data_check_map fd", 144 "data_check_map:%d\n", data_check_map); 145 146 return 0; 147 } 148 149 static void sa46_init_loopback(union sa46 *sa, sa_family_t family) 150 { 151 memset(sa, 0, sizeof(*sa)); 152 sa->family = family; 153 if (sa->family == AF_INET6) 154 sa->v6.sin6_addr = in6addr_loopback; 155 else 156 sa->v4.sin_addr.s_addr = htonl(INADDR_LOOPBACK); 157 } 158 159 static void sa46_init_inany(union sa46 *sa, sa_family_t family) 160 { 161 memset(sa, 0, sizeof(*sa)); 162 sa->family = family; 163 if (sa->family == AF_INET6) 164 sa->v6.sin6_addr = in6addr_any; 165 else 166 sa->v4.sin_addr.s_addr = INADDR_ANY; 167 } 168 169 static int read_int_sysctl(const char *sysctl) 170 { 171 char buf[16]; 172 int fd, ret; 173 174 fd = open(sysctl, 0); 175 RET_ERR(fd == -1, "open(sysctl)", 176 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); 177 178 ret = read(fd, buf, sizeof(buf)); 179 RET_ERR(ret <= 0, "read(sysctl)", 180 "sysctl:%s ret:%d errno:%d\n", sysctl, ret, errno); 181 182 close(fd); 183 return atoi(buf); 184 } 185 186 static int write_int_sysctl(const char *sysctl, int v) 187 { 188 int fd, ret, size; 189 char buf[16]; 190 191 fd = open(sysctl, O_RDWR); 192 RET_ERR(fd == -1, "open(sysctl)", 193 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); 194 195 size = snprintf(buf, sizeof(buf), "%d", v); 196 ret = write(fd, buf, size); 197 RET_ERR(ret != size, "write(sysctl)", 198 "sysctl:%s ret:%d size:%d errno:%d\n", 199 sysctl, ret, size, errno); 200 201 close(fd); 202 return 0; 203 } 204 205 static void restore_sysctls(void) 206 { 207 if (saved_tcp_fo != -1) 208 write_int_sysctl(TCP_FO_SYSCTL, saved_tcp_fo); 209 if (saved_tcp_syncookie != -1) 210 write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, saved_tcp_syncookie); 211 } 212 213 static int enable_fastopen(void) 214 { 215 int fo; 216 217 fo = read_int_sysctl(TCP_FO_SYSCTL); 218 if (fo < 0) 219 return -1; 220 221 return write_int_sysctl(TCP_FO_SYSCTL, fo | 7); 222 } 223 224 static int enable_syncookie(void) 225 { 226 return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2); 227 } 228 229 static int disable_syncookie(void) 230 { 231 return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0); 232 } 233 234 static long get_linum(void) 235 { 236 __u32 linum; 237 int err; 238 239 err = bpf_map_lookup_elem(linum_map, &index_zero, &linum); 240 RET_ERR(err == -1, "lookup_elem(linum_map)", "err:%d errno:%d\n", 241 err, errno); 242 243 return linum; 244 } 245 246 static void check_data(int type, sa_family_t family, const struct cmd *cmd, 247 int cli_fd) 248 { 249 struct data_check expected = {}, result; 250 union sa46 cli_sa; 251 socklen_t addrlen; 252 int err; 253 254 addrlen = sizeof(cli_sa); 255 err = getsockname(cli_fd, (struct sockaddr *)&cli_sa, 256 &addrlen); 257 RET_IF(err == -1, "getsockname(cli_fd)", "err:%d errno:%d\n", 258 err, errno); 259 260 err = bpf_map_lookup_elem(data_check_map, &index_zero, &result); 261 RET_IF(err == -1, "lookup_elem(data_check_map)", "err:%d errno:%d\n", 262 err, errno); 263 264 if (type == SOCK_STREAM) { 265 expected.len = MIN_TCPHDR_LEN; 266 expected.ip_protocol = IPPROTO_TCP; 267 } else { 268 expected.len = UDPHDR_LEN; 269 expected.ip_protocol = IPPROTO_UDP; 270 } 271 272 if (family == AF_INET6) { 273 expected.eth_protocol = htons(ETH_P_IPV6); 274 expected.bind_inany = !srv_sa.v6.sin6_addr.s6_addr32[3] && 275 !srv_sa.v6.sin6_addr.s6_addr32[2] && 276 !srv_sa.v6.sin6_addr.s6_addr32[1] && 277 !srv_sa.v6.sin6_addr.s6_addr32[0]; 278 279 memcpy(&expected.skb_addrs[0], cli_sa.v6.sin6_addr.s6_addr32, 280 sizeof(cli_sa.v6.sin6_addr)); 281 memcpy(&expected.skb_addrs[4], &in6addr_loopback, 282 sizeof(in6addr_loopback)); 283 expected.skb_ports[0] = cli_sa.v6.sin6_port; 284 expected.skb_ports[1] = srv_sa.v6.sin6_port; 285 } else { 286 expected.eth_protocol = htons(ETH_P_IP); 287 expected.bind_inany = !srv_sa.v4.sin_addr.s_addr; 288 289 expected.skb_addrs[0] = cli_sa.v4.sin_addr.s_addr; 290 expected.skb_addrs[1] = htonl(INADDR_LOOPBACK); 291 expected.skb_ports[0] = cli_sa.v4.sin_port; 292 expected.skb_ports[1] = srv_sa.v4.sin_port; 293 } 294 295 if (memcmp(&result, &expected, offsetof(struct data_check, 296 equal_check_end))) { 297 printf("unexpected data_check\n"); 298 printf(" result: (0x%x, %u, %u)\n", 299 result.eth_protocol, result.ip_protocol, 300 result.bind_inany); 301 printf("expected: (0x%x, %u, %u)\n", 302 expected.eth_protocol, expected.ip_protocol, 303 expected.bind_inany); 304 RET_IF(1, "data_check result != expected", 305 "bpf_prog_linum:%ld\n", get_linum()); 306 } 307 308 RET_IF(!result.hash, "data_check result.hash empty", 309 "result.hash:%u", result.hash); 310 311 expected.len += cmd ? sizeof(*cmd) : 0; 312 if (type == SOCK_STREAM) 313 RET_IF(expected.len > result.len, "expected.len > result.len", 314 "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", 315 expected.len, result.len, get_linum()); 316 else 317 RET_IF(expected.len != result.len, "expected.len != result.len", 318 "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", 319 expected.len, result.len, get_linum()); 320 } 321 322 static const char *result_to_str(enum result res) 323 { 324 switch (res) { 325 case DROP_ERR_INNER_MAP: 326 return "DROP_ERR_INNER_MAP"; 327 case DROP_ERR_SKB_DATA: 328 return "DROP_ERR_SKB_DATA"; 329 case DROP_ERR_SK_SELECT_REUSEPORT: 330 return "DROP_ERR_SK_SELECT_REUSEPORT"; 331 case DROP_MISC: 332 return "DROP_MISC"; 333 case PASS: 334 return "PASS"; 335 case PASS_ERR_SK_SELECT_REUSEPORT: 336 return "PASS_ERR_SK_SELECT_REUSEPORT"; 337 default: 338 return "UNKNOWN"; 339 } 340 } 341 342 static void check_results(void) 343 { 344 __u32 results[NR_RESULTS]; 345 __u32 i, broken = 0; 346 int err; 347 348 for (i = 0; i < NR_RESULTS; i++) { 349 err = bpf_map_lookup_elem(result_map, &i, &results[i]); 350 RET_IF(err == -1, "lookup_elem(result_map)", 351 "i:%u err:%d errno:%d\n", i, err, errno); 352 } 353 354 for (i = 0; i < NR_RESULTS; i++) { 355 if (results[i] != expected_results[i]) { 356 broken = i; 357 break; 358 } 359 } 360 361 if (i == NR_RESULTS) 362 return; 363 364 printf("unexpected result\n"); 365 printf(" result: ["); 366 printf("%u", results[0]); 367 for (i = 1; i < NR_RESULTS; i++) 368 printf(", %u", results[i]); 369 printf("]\n"); 370 371 printf("expected: ["); 372 printf("%u", expected_results[0]); 373 for (i = 1; i < NR_RESULTS; i++) 374 printf(", %u", expected_results[i]); 375 printf("]\n"); 376 377 printf("mismatch on %s (bpf_prog_linum:%ld)\n", result_to_str(broken), 378 get_linum()); 379 380 CHECK_FAIL(true); 381 } 382 383 static int send_data(int type, sa_family_t family, void *data, size_t len, 384 enum result expected) 385 { 386 union sa46 cli_sa; 387 int fd, err; 388 389 fd = socket(family, type, 0); 390 RET_ERR(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno); 391 392 sa46_init_loopback(&cli_sa, family); 393 err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa)); 394 RET_ERR(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno); 395 396 err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa, 397 sizeof(srv_sa)); 398 RET_ERR(err != len && expected >= PASS, 399 "sendto()", "family:%u err:%d errno:%d expected:%d\n", 400 family, err, errno, expected); 401 402 return fd; 403 } 404 405 static void do_test(int type, sa_family_t family, struct cmd *cmd, 406 enum result expected) 407 { 408 int nev, srv_fd, cli_fd; 409 struct epoll_event ev; 410 struct cmd rcv_cmd; 411 ssize_t nread; 412 413 cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0, 414 expected); 415 if (cli_fd < 0) 416 return; 417 nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0); 418 RET_IF((nev <= 0 && expected >= PASS) || 419 (nev > 0 && expected < PASS), 420 "nev <> expected", 421 "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n", 422 nev, expected, type, family, 423 cmd ? cmd->reuseport_index : -1, 424 cmd ? cmd->pass_on_failure : -1); 425 check_results(); 426 check_data(type, family, cmd, cli_fd); 427 428 if (expected < PASS) 429 return; 430 431 RET_IF(expected != PASS_ERR_SK_SELECT_REUSEPORT && 432 cmd->reuseport_index != ev.data.u32, 433 "check cmd->reuseport_index", 434 "cmd:(%u, %u) ev.data.u32:%u\n", 435 cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32); 436 437 srv_fd = sk_fds[ev.data.u32]; 438 if (type == SOCK_STREAM) { 439 int new_fd = accept(srv_fd, NULL, 0); 440 441 RET_IF(new_fd == -1, "accept(srv_fd)", 442 "ev.data.u32:%u new_fd:%d errno:%d\n", 443 ev.data.u32, new_fd, errno); 444 445 nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); 446 RET_IF(nread != sizeof(rcv_cmd), 447 "recv(new_fd)", 448 "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", 449 ev.data.u32, nread, sizeof(rcv_cmd), errno); 450 451 close(new_fd); 452 } else { 453 nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); 454 RET_IF(nread != sizeof(rcv_cmd), 455 "recv(sk_fds)", 456 "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", 457 ev.data.u32, nread, sizeof(rcv_cmd), errno); 458 } 459 460 close(cli_fd); 461 } 462 463 static void test_err_inner_map(int type, sa_family_t family) 464 { 465 struct cmd cmd = { 466 .reuseport_index = 0, 467 .pass_on_failure = 0, 468 }; 469 470 expected_results[DROP_ERR_INNER_MAP]++; 471 do_test(type, family, &cmd, DROP_ERR_INNER_MAP); 472 } 473 474 static void test_err_skb_data(int type, sa_family_t family) 475 { 476 expected_results[DROP_ERR_SKB_DATA]++; 477 do_test(type, family, NULL, DROP_ERR_SKB_DATA); 478 } 479 480 static void test_err_sk_select_port(int type, sa_family_t family) 481 { 482 struct cmd cmd = { 483 .reuseport_index = REUSEPORT_ARRAY_SIZE, 484 .pass_on_failure = 0, 485 }; 486 487 expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++; 488 do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT); 489 } 490 491 static void test_pass(int type, sa_family_t family) 492 { 493 struct cmd cmd; 494 int i; 495 496 cmd.pass_on_failure = 0; 497 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { 498 expected_results[PASS]++; 499 cmd.reuseport_index = i; 500 do_test(type, family, &cmd, PASS); 501 } 502 } 503 504 static void test_syncookie(int type, sa_family_t family) 505 { 506 int err, tmp_index = 1; 507 struct cmd cmd = { 508 .reuseport_index = 0, 509 .pass_on_failure = 0, 510 }; 511 512 /* 513 * +1 for TCP-SYN and 514 * +1 for the TCP-ACK (ack the syncookie) 515 */ 516 expected_results[PASS] += 2; 517 enable_syncookie(); 518 /* 519 * Simulate TCP-SYN and TCP-ACK are handled by two different sk: 520 * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the 521 * tmp_index_ovr_map 522 * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index 523 * is from the cmd.reuseport_index 524 */ 525 err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, 526 &tmp_index, BPF_ANY); 527 RET_IF(err == -1, "update_elem(tmp_index_ovr_map, 0, 1)", 528 "err:%d errno:%d\n", err, errno); 529 do_test(type, family, &cmd, PASS); 530 err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero, 531 &tmp_index); 532 RET_IF(err == -1 || tmp_index != -1, 533 "lookup_elem(tmp_index_ovr_map)", 534 "err:%d errno:%d tmp_index:%d\n", 535 err, errno, tmp_index); 536 disable_syncookie(); 537 } 538 539 static void test_pass_on_err(int type, sa_family_t family) 540 { 541 struct cmd cmd = { 542 .reuseport_index = REUSEPORT_ARRAY_SIZE, 543 .pass_on_failure = 1, 544 }; 545 546 expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1; 547 do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT); 548 } 549 550 static void test_detach_bpf(int type, sa_family_t family) 551 { 552 #ifdef SO_DETACH_REUSEPORT_BPF 553 __u32 nr_run_before = 0, nr_run_after = 0, tmp, i; 554 struct epoll_event ev; 555 int cli_fd, err, nev; 556 struct cmd cmd = {}; 557 int optvalue = 0; 558 559 err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, 560 &optvalue, sizeof(optvalue)); 561 RET_IF(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)", 562 "err:%d errno:%d\n", err, errno); 563 564 err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, 565 &optvalue, sizeof(optvalue)); 566 RET_IF(err == 0 || errno != ENOENT, 567 "setsockopt(SO_DETACH_REUSEPORT_BPF)", 568 "err:%d errno:%d\n", err, errno); 569 570 for (i = 0; i < NR_RESULTS; i++) { 571 err = bpf_map_lookup_elem(result_map, &i, &tmp); 572 RET_IF(err == -1, "lookup_elem(result_map)", 573 "i:%u err:%d errno:%d\n", i, err, errno); 574 nr_run_before += tmp; 575 } 576 577 cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS); 578 if (cli_fd < 0) 579 return; 580 nev = epoll_wait(epfd, &ev, 1, 5); 581 RET_IF(nev <= 0, "nev <= 0", 582 "nev:%d expected:1 type:%d family:%d data:(0, 0)\n", 583 nev, type, family); 584 585 for (i = 0; i < NR_RESULTS; i++) { 586 err = bpf_map_lookup_elem(result_map, &i, &tmp); 587 RET_IF(err == -1, "lookup_elem(result_map)", 588 "i:%u err:%d errno:%d\n", i, err, errno); 589 nr_run_after += tmp; 590 } 591 592 RET_IF(nr_run_before != nr_run_after, 593 "nr_run_before != nr_run_after", 594 "nr_run_before:%u nr_run_after:%u\n", 595 nr_run_before, nr_run_after); 596 597 close(cli_fd); 598 #else 599 test__skip(); 600 #endif 601 } 602 603 static void prepare_sk_fds(int type, sa_family_t family, bool inany) 604 { 605 const int first = REUSEPORT_ARRAY_SIZE - 1; 606 int i, err, optval = 1; 607 struct epoll_event ev; 608 socklen_t addrlen; 609 610 if (inany) 611 sa46_init_inany(&srv_sa, family); 612 else 613 sa46_init_loopback(&srv_sa, family); 614 addrlen = sizeof(srv_sa); 615 616 /* 617 * The sk_fds[] is filled from the back such that the order 618 * is exactly opposite to the (struct sock_reuseport *)reuse->socks[]. 619 */ 620 for (i = first; i >= 0; i--) { 621 sk_fds[i] = socket(family, type, 0); 622 RET_IF(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n", 623 i, sk_fds[i], errno); 624 err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT, 625 &optval, sizeof(optval)); 626 RET_IF(err == -1, "setsockopt(SO_REUSEPORT)", 627 "sk_fds[%d] err:%d errno:%d\n", 628 i, err, errno); 629 630 if (i == first) { 631 err = setsockopt(sk_fds[i], SOL_SOCKET, 632 SO_ATTACH_REUSEPORT_EBPF, 633 &select_by_skb_data_prog, 634 sizeof(select_by_skb_data_prog)); 635 RET_IF(err == -1, "setsockopt(SO_ATTACH_REUEPORT_EBPF)", 636 "err:%d errno:%d\n", err, errno); 637 } 638 639 err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen); 640 RET_IF(err == -1, "bind()", "sk_fds[%d] err:%d errno:%d\n", 641 i, err, errno); 642 643 if (type == SOCK_STREAM) { 644 err = listen(sk_fds[i], 10); 645 RET_IF(err == -1, "listen()", 646 "sk_fds[%d] err:%d errno:%d\n", 647 i, err, errno); 648 } 649 650 err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i], 651 BPF_NOEXIST); 652 RET_IF(err == -1, "update_elem(reuseport_array)", 653 "sk_fds[%d] err:%d errno:%d\n", i, err, errno); 654 655 if (i == first) { 656 socklen_t addrlen = sizeof(srv_sa); 657 658 err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa, 659 &addrlen); 660 RET_IF(err == -1, "getsockname()", 661 "sk_fds[%d] err:%d errno:%d\n", i, err, errno); 662 } 663 } 664 665 epfd = epoll_create(1); 666 RET_IF(epfd == -1, "epoll_create(1)", 667 "epfd:%d errno:%d\n", epfd, errno); 668 669 ev.events = EPOLLIN; 670 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { 671 ev.data.u32 = i; 672 err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev); 673 RET_IF(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i); 674 } 675 } 676 677 static void setup_per_test(int type, sa_family_t family, bool inany, 678 bool no_inner_map) 679 { 680 int ovr = -1, err; 681 682 prepare_sk_fds(type, family, inany); 683 err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr, 684 BPF_ANY); 685 RET_IF(err == -1, "update_elem(tmp_index_ovr_map, 0, -1)", 686 "err:%d errno:%d\n", err, errno); 687 688 /* Install reuseport_array to outer_map? */ 689 if (no_inner_map) 690 return; 691 692 err = bpf_map_update_elem(outer_map, &index_zero, &reuseport_array, 693 BPF_ANY); 694 RET_IF(err == -1, "update_elem(outer_map, 0, reuseport_array)", 695 "err:%d errno:%d\n", err, errno); 696 } 697 698 static void cleanup_per_test(bool no_inner_map) 699 { 700 int i, err, zero = 0; 701 702 memset(expected_results, 0, sizeof(expected_results)); 703 704 for (i = 0; i < NR_RESULTS; i++) { 705 err = bpf_map_update_elem(result_map, &i, &zero, BPF_ANY); 706 RET_IF(err, "reset elem in result_map", 707 "i:%u err:%d errno:%d\n", i, err, errno); 708 } 709 710 err = bpf_map_update_elem(linum_map, &zero, &zero, BPF_ANY); 711 RET_IF(err, "reset line number in linum_map", "err:%d errno:%d\n", 712 err, errno); 713 714 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) 715 close(sk_fds[i]); 716 close(epfd); 717 718 /* Delete reuseport_array from outer_map? */ 719 if (no_inner_map) 720 return; 721 722 err = bpf_map_delete_elem(outer_map, &index_zero); 723 RET_IF(err == -1, "delete_elem(outer_map)", 724 "err:%d errno:%d\n", err, errno); 725 } 726 727 static void cleanup(void) 728 { 729 if (outer_map != -1) { 730 close(outer_map); 731 outer_map = -1; 732 } 733 734 if (reuseport_array != -1) { 735 close(reuseport_array); 736 reuseport_array = -1; 737 } 738 739 if (obj) { 740 bpf_object__close(obj); 741 obj = NULL; 742 } 743 744 memset(expected_results, 0, sizeof(expected_results)); 745 } 746 747 static const char *maptype_str(enum bpf_map_type type) 748 { 749 switch (type) { 750 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 751 return "reuseport_sockarray"; 752 case BPF_MAP_TYPE_SOCKMAP: 753 return "sockmap"; 754 case BPF_MAP_TYPE_SOCKHASH: 755 return "sockhash"; 756 default: 757 return "unknown"; 758 } 759 } 760 761 static const char *family_str(sa_family_t family) 762 { 763 switch (family) { 764 case AF_INET: 765 return "IPv4"; 766 case AF_INET6: 767 return "IPv6"; 768 default: 769 return "unknown"; 770 } 771 } 772 773 static const char *sotype_str(int sotype) 774 { 775 switch (sotype) { 776 case SOCK_STREAM: 777 return "TCP"; 778 case SOCK_DGRAM: 779 return "UDP"; 780 default: 781 return "unknown"; 782 } 783 } 784 785 #define TEST_INIT(fn_, ...) { .fn = fn_, .name = #fn_, __VA_ARGS__ } 786 787 static void test_config(int sotype, sa_family_t family, bool inany) 788 { 789 const struct test { 790 void (*fn)(int sotype, sa_family_t family); 791 const char *name; 792 bool no_inner_map; 793 int need_sotype; 794 } tests[] = { 795 TEST_INIT(test_err_inner_map, 796 .no_inner_map = true), 797 TEST_INIT(test_err_skb_data), 798 TEST_INIT(test_err_sk_select_port), 799 TEST_INIT(test_pass), 800 TEST_INIT(test_syncookie, 801 .need_sotype = SOCK_STREAM), 802 TEST_INIT(test_pass_on_err), 803 TEST_INIT(test_detach_bpf), 804 }; 805 char s[MAX_TEST_NAME]; 806 const struct test *t; 807 808 for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { 809 if (t->need_sotype && t->need_sotype != sotype) 810 continue; /* test not compatible with socket type */ 811 812 snprintf(s, sizeof(s), "%s %s/%s %s %s", 813 maptype_str(inner_map_type), 814 family_str(family), sotype_str(sotype), 815 inany ? "INANY" : "LOOPBACK", t->name); 816 817 if (!test__start_subtest(s)) 818 continue; 819 820 setup_per_test(sotype, family, inany, t->no_inner_map); 821 t->fn(sotype, family); 822 cleanup_per_test(t->no_inner_map); 823 } 824 } 825 826 #define BIND_INANY true 827 828 static void test_all(void) 829 { 830 const struct config { 831 int sotype; 832 sa_family_t family; 833 bool inany; 834 } configs[] = { 835 { SOCK_STREAM, AF_INET }, 836 { SOCK_STREAM, AF_INET, BIND_INANY }, 837 { SOCK_STREAM, AF_INET6 }, 838 { SOCK_STREAM, AF_INET6, BIND_INANY }, 839 { SOCK_DGRAM, AF_INET }, 840 { SOCK_DGRAM, AF_INET6 }, 841 }; 842 const struct config *c; 843 844 for (c = configs; c < configs + ARRAY_SIZE(configs); c++) 845 test_config(c->sotype, c->family, c->inany); 846 } 847 848 void test_map_type(enum bpf_map_type mt) 849 { 850 if (create_maps(mt)) 851 goto out; 852 if (prepare_bpf_obj()) 853 goto out; 854 855 test_all(); 856 out: 857 cleanup(); 858 } 859 860 void test_select_reuseport(void) 861 { 862 saved_tcp_fo = read_int_sysctl(TCP_FO_SYSCTL); 863 if (saved_tcp_fo < 0) 864 goto out; 865 saved_tcp_syncookie = read_int_sysctl(TCP_SYNCOOKIE_SYSCTL); 866 if (saved_tcp_syncookie < 0) 867 goto out; 868 869 if (enable_fastopen()) 870 goto out; 871 if (disable_syncookie()) 872 goto out; 873 874 test_map_type(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY); 875 test_map_type(BPF_MAP_TYPE_SOCKMAP); 876 test_map_type(BPF_MAP_TYPE_SOCKHASH); 877 out: 878 restore_sysctls(); 879 } 880