1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <stdlib.h> 5 #include <unistd.h> 6 #include <stdbool.h> 7 #include <string.h> 8 #include <errno.h> 9 #include <error.h> 10 #include <assert.h> 11 #include <fcntl.h> 12 #include <linux/bpf.h> 13 #include <linux/err.h> 14 #include <linux/types.h> 15 #include <linux/if_ether.h> 16 #include <sys/types.h> 17 #include <sys/epoll.h> 18 #include <sys/socket.h> 19 #include <netinet/in.h> 20 #include <bpf/bpf.h> 21 #include <bpf/libbpf.h> 22 #include "bpf_util.h" 23 24 #include "test_progs.h" 25 #include "socket_helpers.h" 26 #include "test_select_reuseport_common.h" 27 28 #define MAX_TEST_NAME 80 29 #define MIN_TCPHDR_LEN 20 30 #define UDPHDR_LEN 8 31 32 #define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies" 33 #define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen" 34 #define REUSEPORT_ARRAY_SIZE 32 35 36 static int result_map, tmp_index_ovr_map, linum_map, data_check_map; 37 static __u32 expected_results[NR_RESULTS]; 38 static int sk_fds[REUSEPORT_ARRAY_SIZE]; 39 static int reuseport_array = -1, outer_map = -1; 40 static enum bpf_map_type inner_map_type; 41 static int select_by_skb_data_prog; 42 static struct bpf_object *obj; 43 static __u32 index_zero; 44 static int epfd; 45 46 static struct sockaddr_storage srv_sa; 47 48 #define RET_IF(condition, tag, format...) ({ \ 49 if (CHECK_FAIL(condition)) { \ 50 printf(tag " " format); \ 51 return; \ 52 } \ 53 }) 54 55 #define RET_ERR(condition, tag, format...) ({ \ 56 if (CHECK_FAIL(condition)) { \ 57 printf(tag " " format); \ 58 return -1; \ 59 } \ 60 }) 61 62 static int create_maps(enum bpf_map_type inner_type) 63 { 64 LIBBPF_OPTS(bpf_map_create_opts, opts); 65 66 inner_map_type = inner_type; 67 68 /* Creating reuseport_array */ 69 reuseport_array = bpf_map_create(inner_type, "reuseport_array", 70 sizeof(__u32), sizeof(__u32), REUSEPORT_ARRAY_SIZE, NULL); 71 RET_ERR(reuseport_array < 0, "creating reuseport_array", 72 "reuseport_array:%d errno:%d\n", reuseport_array, errno); 73 74 /* Creating outer_map */ 75 opts.inner_map_fd = reuseport_array; 76 outer_map = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer_map", 77 sizeof(__u32), sizeof(__u32), 1, &opts); 78 RET_ERR(outer_map < 0, "creating outer_map", 79 "outer_map:%d errno:%d\n", outer_map, errno); 80 81 return 0; 82 } 83 84 static int prepare_bpf_obj(void) 85 { 86 struct bpf_program *prog; 87 struct bpf_map *map; 88 int err; 89 90 obj = bpf_object__open("test_select_reuseport_kern.bpf.o"); 91 err = libbpf_get_error(obj); 92 RET_ERR(err, "open test_select_reuseport_kern.bpf.o", 93 "obj:%p PTR_ERR(obj):%d\n", obj, err); 94 95 map = bpf_object__find_map_by_name(obj, "outer_map"); 96 RET_ERR(!map, "find outer_map", "!map\n"); 97 err = bpf_map__reuse_fd(map, outer_map); 98 RET_ERR(err, "reuse outer_map", "err:%d\n", err); 99 100 err = bpf_object__load(obj); 101 RET_ERR(err, "load bpf_object", "err:%d\n", err); 102 103 prog = bpf_object__next_program(obj, NULL); 104 RET_ERR(!prog, "get first bpf_program", "!prog\n"); 105 select_by_skb_data_prog = bpf_program__fd(prog); 106 RET_ERR(select_by_skb_data_prog < 0, "get prog fd", 107 "select_by_skb_data_prog:%d\n", select_by_skb_data_prog); 108 109 map = bpf_object__find_map_by_name(obj, "result_map"); 110 RET_ERR(!map, "find result_map", "!map\n"); 111 result_map = bpf_map__fd(map); 112 RET_ERR(result_map < 0, "get result_map fd", 113 "result_map:%d\n", result_map); 114 115 map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map"); 116 RET_ERR(!map, "find tmp_index_ovr_map\n", "!map"); 117 tmp_index_ovr_map = bpf_map__fd(map); 118 RET_ERR(tmp_index_ovr_map < 0, "get tmp_index_ovr_map fd", 119 "tmp_index_ovr_map:%d\n", tmp_index_ovr_map); 120 121 map = bpf_object__find_map_by_name(obj, "linum_map"); 122 RET_ERR(!map, "find linum_map", "!map\n"); 123 linum_map = bpf_map__fd(map); 124 RET_ERR(linum_map < 0, "get linum_map fd", 125 "linum_map:%d\n", linum_map); 126 127 map = bpf_object__find_map_by_name(obj, "data_check_map"); 128 RET_ERR(!map, "find data_check_map", "!map\n"); 129 data_check_map = bpf_map__fd(map); 130 RET_ERR(data_check_map < 0, "get data_check_map fd", 131 "data_check_map:%d\n", data_check_map); 132 133 return 0; 134 } 135 136 static void ss_init_loopback(struct sockaddr_storage *sa, sa_family_t family) 137 { 138 memset(sa, 0, sizeof(*sa)); 139 sa->ss_family = family; 140 if (sa->ss_family == AF_INET6) 141 ((struct sockaddr_in6 *)sa)->sin6_addr = in6addr_loopback; 142 else 143 ((struct sockaddr_in *)sa)->sin_addr.s_addr = htonl(INADDR_LOOPBACK); 144 } 145 146 static void ss_init_inany(struct sockaddr_storage *sa, sa_family_t family) 147 { 148 memset(sa, 0, sizeof(*sa)); 149 sa->ss_family = family; 150 if (sa->ss_family == AF_INET6) 151 ((struct sockaddr_in6 *)sa)->sin6_addr = in6addr_any; 152 else 153 ((struct sockaddr_in *)sa)->sin_addr.s_addr = INADDR_ANY; 154 } 155 156 static int read_int_sysctl(const char *sysctl) 157 { 158 char buf[16]; 159 int fd, ret; 160 161 fd = open(sysctl, 0); 162 RET_ERR(fd == -1, "open(sysctl)", 163 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); 164 165 ret = read(fd, buf, sizeof(buf)); 166 RET_ERR(ret <= 0, "read(sysctl)", 167 "sysctl:%s ret:%d errno:%d\n", sysctl, ret, errno); 168 169 close(fd); 170 return atoi(buf); 171 } 172 173 static int write_int_sysctl(const char *sysctl, int v) 174 { 175 int fd, ret, size; 176 char buf[16]; 177 178 fd = open(sysctl, O_RDWR); 179 RET_ERR(fd == -1, "open(sysctl)", 180 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); 181 182 size = snprintf(buf, sizeof(buf), "%d", v); 183 ret = write(fd, buf, size); 184 RET_ERR(ret != size, "write(sysctl)", 185 "sysctl:%s ret:%d size:%d errno:%d\n", 186 sysctl, ret, size, errno); 187 188 close(fd); 189 return 0; 190 } 191 192 static int enable_fastopen(void) 193 { 194 int fo; 195 196 fo = read_int_sysctl(TCP_FO_SYSCTL); 197 if (fo < 0) 198 return -1; 199 200 return write_int_sysctl(TCP_FO_SYSCTL, fo | 7); 201 } 202 203 static int enable_syncookie(void) 204 { 205 return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2); 206 } 207 208 static int disable_syncookie(void) 209 { 210 return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0); 211 } 212 213 static long get_linum(void) 214 { 215 __u32 linum; 216 int err; 217 218 err = bpf_map_lookup_elem(linum_map, &index_zero, &linum); 219 RET_ERR(err < 0, "lookup_elem(linum_map)", "err:%d errno:%d\n", 220 err, errno); 221 222 return linum; 223 } 224 225 static void check_data(int type, sa_family_t family, const struct cmd *cmd, 226 int cli_fd) 227 { 228 struct data_check expected = {}, result; 229 struct sockaddr_storage cli_sa; 230 socklen_t addrlen; 231 int err; 232 233 addrlen = sizeof(cli_sa); 234 err = getsockname(cli_fd, (struct sockaddr *)&cli_sa, 235 &addrlen); 236 RET_IF(err < 0, "getsockname(cli_fd)", "err:%d errno:%d\n", 237 err, errno); 238 239 err = bpf_map_lookup_elem(data_check_map, &index_zero, &result); 240 RET_IF(err < 0, "lookup_elem(data_check_map)", "err:%d errno:%d\n", 241 err, errno); 242 243 if (type == SOCK_STREAM) { 244 expected.len = MIN_TCPHDR_LEN; 245 expected.ip_protocol = IPPROTO_TCP; 246 } else { 247 expected.len = UDPHDR_LEN; 248 expected.ip_protocol = IPPROTO_UDP; 249 } 250 251 if (family == AF_INET6) { 252 struct sockaddr_in6 *srv_v6 = (struct sockaddr_in6 *)&srv_sa; 253 struct sockaddr_in6 *cli_v6 = (struct sockaddr_in6 *)&cli_sa; 254 255 expected.eth_protocol = htons(ETH_P_IPV6); 256 expected.bind_inany = !srv_v6->sin6_addr.s6_addr32[3] && 257 !srv_v6->sin6_addr.s6_addr32[2] && 258 !srv_v6->sin6_addr.s6_addr32[1] && 259 !srv_v6->sin6_addr.s6_addr32[0]; 260 261 memcpy(&expected.skb_addrs[0], cli_v6->sin6_addr.s6_addr32, 262 sizeof(cli_v6->sin6_addr)); 263 memcpy(&expected.skb_addrs[4], &in6addr_loopback, 264 sizeof(in6addr_loopback)); 265 expected.skb_ports[0] = cli_v6->sin6_port; 266 expected.skb_ports[1] = srv_v6->sin6_port; 267 } else { 268 struct sockaddr_in *srv_v4 = (struct sockaddr_in *)&srv_sa; 269 struct sockaddr_in *cli_v4 = (struct sockaddr_in *)&cli_sa; 270 271 expected.eth_protocol = htons(ETH_P_IP); 272 expected.bind_inany = !srv_v4->sin_addr.s_addr; 273 274 expected.skb_addrs[0] = cli_v4->sin_addr.s_addr; 275 expected.skb_addrs[1] = htonl(INADDR_LOOPBACK); 276 expected.skb_ports[0] = cli_v4->sin_port; 277 expected.skb_ports[1] = srv_v4->sin_port; 278 } 279 280 if (memcmp(&result, &expected, offsetof(struct data_check, 281 equal_check_end))) { 282 printf("unexpected data_check\n"); 283 printf(" result: (0x%x, %u, %u)\n", 284 result.eth_protocol, result.ip_protocol, 285 result.bind_inany); 286 printf("expected: (0x%x, %u, %u)\n", 287 expected.eth_protocol, expected.ip_protocol, 288 expected.bind_inany); 289 RET_IF(1, "data_check result != expected", 290 "bpf_prog_linum:%ld\n", get_linum()); 291 } 292 293 RET_IF(!result.hash, "data_check result.hash empty", 294 "result.hash:%u", result.hash); 295 296 expected.len += cmd ? sizeof(*cmd) : 0; 297 if (type == SOCK_STREAM) 298 RET_IF(expected.len > result.len, "expected.len > result.len", 299 "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", 300 expected.len, result.len, get_linum()); 301 else 302 RET_IF(expected.len != result.len, "expected.len != result.len", 303 "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", 304 expected.len, result.len, get_linum()); 305 } 306 307 static const char *result_to_str(enum result res) 308 { 309 switch (res) { 310 case DROP_ERR_INNER_MAP: 311 return "DROP_ERR_INNER_MAP"; 312 case DROP_ERR_SKB_DATA: 313 return "DROP_ERR_SKB_DATA"; 314 case DROP_ERR_SK_SELECT_REUSEPORT: 315 return "DROP_ERR_SK_SELECT_REUSEPORT"; 316 case DROP_MISC: 317 return "DROP_MISC"; 318 case PASS: 319 return "PASS"; 320 case PASS_ERR_SK_SELECT_REUSEPORT: 321 return "PASS_ERR_SK_SELECT_REUSEPORT"; 322 default: 323 return "UNKNOWN"; 324 } 325 } 326 327 static void check_results(void) 328 { 329 __u32 results[NR_RESULTS]; 330 __u32 i, broken = 0; 331 int err; 332 333 for (i = 0; i < NR_RESULTS; i++) { 334 err = bpf_map_lookup_elem(result_map, &i, &results[i]); 335 RET_IF(err < 0, "lookup_elem(result_map)", 336 "i:%u err:%d errno:%d\n", i, err, errno); 337 } 338 339 for (i = 0; i < NR_RESULTS; i++) { 340 if (results[i] != expected_results[i]) { 341 broken = i; 342 break; 343 } 344 } 345 346 if (i == NR_RESULTS) 347 return; 348 349 printf("unexpected result\n"); 350 printf(" result: ["); 351 printf("%u", results[0]); 352 for (i = 1; i < NR_RESULTS; i++) 353 printf(", %u", results[i]); 354 printf("]\n"); 355 356 printf("expected: ["); 357 printf("%u", expected_results[0]); 358 for (i = 1; i < NR_RESULTS; i++) 359 printf(", %u", expected_results[i]); 360 printf("]\n"); 361 362 printf("mismatch on %s (bpf_prog_linum:%ld)\n", result_to_str(broken), 363 get_linum()); 364 365 CHECK_FAIL(true); 366 } 367 368 static int send_data(int type, sa_family_t family, void *data, size_t len, 369 enum result expected) 370 { 371 struct sockaddr_storage cli_sa; 372 int fd, err; 373 374 fd = socket(family, type, 0); 375 RET_ERR(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno); 376 377 ss_init_loopback(&cli_sa, family); 378 err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa)); 379 RET_ERR(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno); 380 err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa, 381 sizeof(srv_sa)); 382 RET_ERR(err != len && expected >= PASS, 383 "sendto()", "family:%u err:%d errno:%d expected:%d\n", 384 family, err, errno, expected); 385 386 return fd; 387 } 388 389 static void do_test(int type, sa_family_t family, struct cmd *cmd, 390 enum result expected) 391 { 392 int nev, srv_fd, cli_fd; 393 struct epoll_event ev; 394 struct cmd rcv_cmd; 395 ssize_t nread; 396 397 cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0, 398 expected); 399 if (cli_fd < 0) 400 return; 401 nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0); 402 RET_IF((nev <= 0 && expected >= PASS) || 403 (nev > 0 && expected < PASS), 404 "nev <> expected", 405 "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n", 406 nev, expected, type, family, 407 cmd ? cmd->reuseport_index : -1, 408 cmd ? cmd->pass_on_failure : -1); 409 check_results(); 410 check_data(type, family, cmd, cli_fd); 411 412 if (expected < PASS) 413 return; 414 415 RET_IF(expected != PASS_ERR_SK_SELECT_REUSEPORT && 416 cmd->reuseport_index != ev.data.u32, 417 "check cmd->reuseport_index", 418 "cmd:(%u, %u) ev.data.u32:%u\n", 419 cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32); 420 421 srv_fd = sk_fds[ev.data.u32]; 422 if (type == SOCK_STREAM) { 423 int new_fd = accept(srv_fd, NULL, 0); 424 425 RET_IF(new_fd == -1, "accept(srv_fd)", 426 "ev.data.u32:%u new_fd:%d errno:%d\n", 427 ev.data.u32, new_fd, errno); 428 429 nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); 430 RET_IF(nread != sizeof(rcv_cmd), 431 "recv(new_fd)", 432 "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", 433 ev.data.u32, nread, sizeof(rcv_cmd), errno); 434 435 close(new_fd); 436 } else { 437 nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); 438 RET_IF(nread != sizeof(rcv_cmd), 439 "recv(sk_fds)", 440 "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", 441 ev.data.u32, nread, sizeof(rcv_cmd), errno); 442 } 443 444 close(cli_fd); 445 } 446 447 static void test_err_inner_map(int type, sa_family_t family) 448 { 449 struct cmd cmd = { 450 .reuseport_index = 0, 451 .pass_on_failure = 0, 452 }; 453 454 expected_results[DROP_ERR_INNER_MAP]++; 455 do_test(type, family, &cmd, DROP_ERR_INNER_MAP); 456 } 457 458 static void test_err_skb_data(int type, sa_family_t family) 459 { 460 expected_results[DROP_ERR_SKB_DATA]++; 461 do_test(type, family, NULL, DROP_ERR_SKB_DATA); 462 } 463 464 static void test_err_sk_select_port(int type, sa_family_t family) 465 { 466 struct cmd cmd = { 467 .reuseport_index = REUSEPORT_ARRAY_SIZE, 468 .pass_on_failure = 0, 469 }; 470 471 expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++; 472 do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT); 473 } 474 475 static void test_pass(int type, sa_family_t family) 476 { 477 struct cmd cmd; 478 int i; 479 480 cmd.pass_on_failure = 0; 481 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { 482 expected_results[PASS]++; 483 cmd.reuseport_index = i; 484 do_test(type, family, &cmd, PASS); 485 } 486 } 487 488 static void test_syncookie(int type, sa_family_t family) 489 { 490 int err, tmp_index = 1; 491 struct cmd cmd = { 492 .reuseport_index = 0, 493 .pass_on_failure = 0, 494 }; 495 496 /* 497 * +1 for TCP-SYN and 498 * +1 for the TCP-ACK (ack the syncookie) 499 */ 500 expected_results[PASS] += 2; 501 enable_syncookie(); 502 /* 503 * Simulate TCP-SYN and TCP-ACK are handled by two different sk: 504 * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the 505 * tmp_index_ovr_map 506 * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index 507 * is from the cmd.reuseport_index 508 */ 509 err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, 510 &tmp_index, BPF_ANY); 511 RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, 1)", 512 "err:%d errno:%d\n", err, errno); 513 do_test(type, family, &cmd, PASS); 514 err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero, 515 &tmp_index); 516 RET_IF(err < 0 || tmp_index >= 0, 517 "lookup_elem(tmp_index_ovr_map)", 518 "err:%d errno:%d tmp_index:%d\n", 519 err, errno, tmp_index); 520 disable_syncookie(); 521 } 522 523 static void test_pass_on_err(int type, sa_family_t family) 524 { 525 struct cmd cmd = { 526 .reuseport_index = REUSEPORT_ARRAY_SIZE, 527 .pass_on_failure = 1, 528 }; 529 530 expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1; 531 do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT); 532 } 533 534 static void test_detach_bpf(int type, sa_family_t family) 535 { 536 #ifdef SO_DETACH_REUSEPORT_BPF 537 __u32 nr_run_before = 0, nr_run_after = 0, tmp, i; 538 struct epoll_event ev; 539 int cli_fd, err, nev; 540 struct cmd cmd = {}; 541 int optvalue = 0; 542 543 err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, 544 &optvalue, sizeof(optvalue)); 545 RET_IF(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)", 546 "err:%d errno:%d\n", err, errno); 547 548 err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, 549 &optvalue, sizeof(optvalue)); 550 RET_IF(err == 0 || errno != ENOENT, 551 "setsockopt(SO_DETACH_REUSEPORT_BPF)", 552 "err:%d errno:%d\n", err, errno); 553 554 for (i = 0; i < NR_RESULTS; i++) { 555 err = bpf_map_lookup_elem(result_map, &i, &tmp); 556 RET_IF(err < 0, "lookup_elem(result_map)", 557 "i:%u err:%d errno:%d\n", i, err, errno); 558 nr_run_before += tmp; 559 } 560 561 cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS); 562 if (cli_fd < 0) 563 return; 564 nev = epoll_wait(epfd, &ev, 1, 5); 565 RET_IF(nev <= 0, "nev <= 0", 566 "nev:%d expected:1 type:%d family:%d data:(0, 0)\n", 567 nev, type, family); 568 569 for (i = 0; i < NR_RESULTS; i++) { 570 err = bpf_map_lookup_elem(result_map, &i, &tmp); 571 RET_IF(err < 0, "lookup_elem(result_map)", 572 "i:%u err:%d errno:%d\n", i, err, errno); 573 nr_run_after += tmp; 574 } 575 576 RET_IF(nr_run_before != nr_run_after, 577 "nr_run_before != nr_run_after", 578 "nr_run_before:%u nr_run_after:%u\n", 579 nr_run_before, nr_run_after); 580 581 close(cli_fd); 582 #else 583 test__skip(); 584 #endif 585 } 586 587 static void prepare_sk_fds(int type, sa_family_t family, bool inany) 588 { 589 const int first = REUSEPORT_ARRAY_SIZE - 1; 590 int i, err, optval = 1; 591 struct epoll_event ev; 592 socklen_t addrlen; 593 594 if (inany) 595 ss_init_inany(&srv_sa, family); 596 else 597 ss_init_loopback(&srv_sa, family); 598 addrlen = sizeof(srv_sa); 599 600 /* 601 * The sk_fds[] is filled from the back such that the order 602 * is exactly opposite to the (struct sock_reuseport *)reuse->socks[]. 603 */ 604 for (i = first; i >= 0; i--) { 605 sk_fds[i] = socket(family, type, 0); 606 RET_IF(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n", 607 i, sk_fds[i], errno); 608 err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT, 609 &optval, sizeof(optval)); 610 RET_IF(err == -1, "setsockopt(SO_REUSEPORT)", 611 "sk_fds[%d] err:%d errno:%d\n", 612 i, err, errno); 613 614 if (i == first) { 615 err = setsockopt(sk_fds[i], SOL_SOCKET, 616 SO_ATTACH_REUSEPORT_EBPF, 617 &select_by_skb_data_prog, 618 sizeof(select_by_skb_data_prog)); 619 RET_IF(err < 0, "setsockopt(SO_ATTACH_REUEPORT_EBPF)", 620 "err:%d errno:%d\n", err, errno); 621 } 622 623 err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen); 624 RET_IF(err < 0, "bind()", "sk_fds[%d] err:%d errno:%d\n", 625 i, err, errno); 626 627 if (type == SOCK_STREAM) { 628 err = listen(sk_fds[i], 10); 629 RET_IF(err < 0, "listen()", 630 "sk_fds[%d] err:%d errno:%d\n", 631 i, err, errno); 632 } 633 634 err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i], 635 BPF_NOEXIST); 636 RET_IF(err < 0, "update_elem(reuseport_array)", 637 "sk_fds[%d] err:%d errno:%d\n", i, err, errno); 638 639 if (i == first) { 640 socklen_t addrlen = sizeof(srv_sa); 641 642 err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa, 643 &addrlen); 644 RET_IF(err == -1, "getsockname()", 645 "sk_fds[%d] err:%d errno:%d\n", i, err, errno); 646 } 647 } 648 649 epfd = epoll_create(1); 650 RET_IF(epfd == -1, "epoll_create(1)", 651 "epfd:%d errno:%d\n", epfd, errno); 652 653 ev.events = EPOLLIN; 654 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { 655 ev.data.u32 = i; 656 err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev); 657 RET_IF(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i); 658 } 659 } 660 661 static void setup_per_test(int type, sa_family_t family, bool inany, 662 bool no_inner_map) 663 { 664 int ovr = -1, err; 665 666 prepare_sk_fds(type, family, inany); 667 err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr, 668 BPF_ANY); 669 RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, -1)", 670 "err:%d errno:%d\n", err, errno); 671 672 /* Install reuseport_array to outer_map? */ 673 if (no_inner_map) 674 return; 675 676 err = bpf_map_update_elem(outer_map, &index_zero, &reuseport_array, 677 BPF_ANY); 678 RET_IF(err < 0, "update_elem(outer_map, 0, reuseport_array)", 679 "err:%d errno:%d\n", err, errno); 680 } 681 682 static void cleanup_per_test(bool no_inner_map) 683 { 684 int i, err, zero = 0; 685 686 memset(expected_results, 0, sizeof(expected_results)); 687 688 for (i = 0; i < NR_RESULTS; i++) { 689 err = bpf_map_update_elem(result_map, &i, &zero, BPF_ANY); 690 RET_IF(err, "reset elem in result_map", 691 "i:%u err:%d errno:%d\n", i, err, errno); 692 } 693 694 err = bpf_map_update_elem(linum_map, &zero, &zero, BPF_ANY); 695 RET_IF(err, "reset line number in linum_map", "err:%d errno:%d\n", 696 err, errno); 697 698 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) 699 close(sk_fds[i]); 700 close(epfd); 701 702 /* Delete reuseport_array from outer_map? */ 703 if (no_inner_map) 704 return; 705 706 err = bpf_map_delete_elem(outer_map, &index_zero); 707 RET_IF(err < 0, "delete_elem(outer_map)", 708 "err:%d errno:%d\n", err, errno); 709 } 710 711 static void cleanup(void) 712 { 713 if (outer_map >= 0) { 714 close(outer_map); 715 outer_map = -1; 716 } 717 718 if (reuseport_array >= 0) { 719 close(reuseport_array); 720 reuseport_array = -1; 721 } 722 723 if (obj) { 724 bpf_object__close(obj); 725 obj = NULL; 726 } 727 728 memset(expected_results, 0, sizeof(expected_results)); 729 } 730 731 static const char *maptype_str(enum bpf_map_type type) 732 { 733 switch (type) { 734 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 735 return "reuseport_sockarray"; 736 case BPF_MAP_TYPE_SOCKMAP: 737 return "sockmap"; 738 case BPF_MAP_TYPE_SOCKHASH: 739 return "sockhash"; 740 default: 741 return "unknown"; 742 } 743 } 744 745 static const char *sotype_str(int sotype) 746 { 747 switch (sotype) { 748 case SOCK_STREAM: 749 return "TCP"; 750 case SOCK_DGRAM: 751 return "UDP"; 752 default: 753 return "unknown"; 754 } 755 } 756 757 #define TEST_INIT(fn_, ...) { .fn = fn_, .name = #fn_, __VA_ARGS__ } 758 759 static void test_config(int sotype, sa_family_t family, bool inany) 760 { 761 const struct test { 762 void (*fn)(int sotype, sa_family_t family); 763 const char *name; 764 bool no_inner_map; 765 int need_sotype; 766 } tests[] = { 767 TEST_INIT(test_err_inner_map, 768 .no_inner_map = true), 769 TEST_INIT(test_err_skb_data), 770 TEST_INIT(test_err_sk_select_port), 771 TEST_INIT(test_pass), 772 TEST_INIT(test_syncookie, 773 .need_sotype = SOCK_STREAM), 774 TEST_INIT(test_pass_on_err), 775 TEST_INIT(test_detach_bpf), 776 }; 777 struct netns_obj *netns; 778 char s[MAX_TEST_NAME]; 779 const struct test *t; 780 781 for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { 782 if (t->need_sotype && t->need_sotype != sotype) 783 continue; /* test not compatible with socket type */ 784 785 snprintf(s, sizeof(s), "%s %s/%s %s %s", 786 maptype_str(inner_map_type), 787 inet_family_str(family), sotype_str(sotype), 788 inany ? "INANY" : "LOOPBACK", t->name); 789 790 if (!test__start_subtest(s)) 791 continue; 792 793 netns = netns_new("select_reuseport", true); 794 if (!ASSERT_OK_PTR(netns, "netns_new")) 795 continue; 796 797 if (CHECK_FAIL(enable_fastopen())) 798 goto out; 799 if (CHECK_FAIL(disable_syncookie())) 800 goto out; 801 802 setup_per_test(sotype, family, inany, t->no_inner_map); 803 t->fn(sotype, family); 804 cleanup_per_test(t->no_inner_map); 805 806 out: 807 netns_free(netns); 808 } 809 } 810 811 #define BIND_INANY true 812 813 static void test_all(void) 814 { 815 const struct config { 816 int sotype; 817 sa_family_t family; 818 bool inany; 819 } configs[] = { 820 { SOCK_STREAM, AF_INET }, 821 { SOCK_STREAM, AF_INET, BIND_INANY }, 822 { SOCK_STREAM, AF_INET6 }, 823 { SOCK_STREAM, AF_INET6, BIND_INANY }, 824 { SOCK_DGRAM, AF_INET }, 825 { SOCK_DGRAM, AF_INET6 }, 826 }; 827 const struct config *c; 828 829 for (c = configs; c < configs + ARRAY_SIZE(configs); c++) 830 test_config(c->sotype, c->family, c->inany); 831 } 832 833 void test_map_type(enum bpf_map_type mt) 834 { 835 if (create_maps(mt)) 836 goto out; 837 if (prepare_bpf_obj()) 838 goto out; 839 840 test_all(); 841 out: 842 cleanup(); 843 } 844 845 void serial_test_select_reuseport(void) 846 { 847 test_map_type(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY); 848 test_map_type(BPF_MAP_TYPE_SOCKMAP); 849 test_map_type(BPF_MAP_TYPE_SOCKHASH); 850 } 851