1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <stdlib.h> 5 #include <unistd.h> 6 #include <stdbool.h> 7 #include <string.h> 8 #include <errno.h> 9 #include <assert.h> 10 #include <fcntl.h> 11 #include <linux/bpf.h> 12 #include <linux/err.h> 13 #include <linux/types.h> 14 #include <linux/if_ether.h> 15 #include <sys/types.h> 16 #include <sys/epoll.h> 17 #include <sys/socket.h> 18 #include <netinet/in.h> 19 #include <bpf/bpf.h> 20 #include <bpf/libbpf.h> 21 #include "bpf_util.h" 22 23 #include "test_progs.h" 24 #include "test_select_reuseport_common.h" 25 26 #define MAX_TEST_NAME 80 27 #define MIN_TCPHDR_LEN 20 28 #define UDPHDR_LEN 8 29 30 #define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies" 31 #define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen" 32 #define REUSEPORT_ARRAY_SIZE 32 33 34 static int result_map, tmp_index_ovr_map, linum_map, data_check_map; 35 static __u32 expected_results[NR_RESULTS]; 36 static int sk_fds[REUSEPORT_ARRAY_SIZE]; 37 static int reuseport_array = -1, outer_map = -1; 38 static enum bpf_map_type inner_map_type; 39 static int select_by_skb_data_prog; 40 static struct bpf_object *obj; 41 static __u32 index_zero; 42 static int epfd; 43 44 static struct sockaddr_storage srv_sa; 45 46 #define RET_IF(condition, tag, format...) ({ \ 47 if (CHECK_FAIL(condition)) { \ 48 printf(tag " " format); \ 49 return; \ 50 } \ 51 }) 52 53 #define RET_ERR(condition, tag, format...) ({ \ 54 if (CHECK_FAIL(condition)) { \ 55 printf(tag " " format); \ 56 return -1; \ 57 } \ 58 }) 59 60 static int create_maps(enum bpf_map_type inner_type) 61 { 62 LIBBPF_OPTS(bpf_map_create_opts, opts); 63 64 inner_map_type = inner_type; 65 66 /* Creating reuseport_array */ 67 reuseport_array = bpf_map_create(inner_type, "reuseport_array", 68 sizeof(__u32), sizeof(__u32), REUSEPORT_ARRAY_SIZE, NULL); 69 RET_ERR(reuseport_array < 0, "creating reuseport_array", 70 "reuseport_array:%d errno:%d\n", reuseport_array, errno); 71 72 /* Creating outer_map */ 73 opts.inner_map_fd = reuseport_array; 74 outer_map = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer_map", 75 sizeof(__u32), sizeof(__u32), 1, &opts); 76 RET_ERR(outer_map < 0, "creating outer_map", 77 "outer_map:%d errno:%d\n", outer_map, errno); 78 79 return 0; 80 } 81 82 static int prepare_bpf_obj(void) 83 { 84 struct bpf_program *prog; 85 struct bpf_map *map; 86 int err; 87 88 obj = bpf_object__open("test_select_reuseport_kern.bpf.o"); 89 err = libbpf_get_error(obj); 90 RET_ERR(err, "open test_select_reuseport_kern.bpf.o", 91 "obj:%p PTR_ERR(obj):%d\n", obj, err); 92 93 map = bpf_object__find_map_by_name(obj, "outer_map"); 94 RET_ERR(!map, "find outer_map", "!map\n"); 95 err = bpf_map__reuse_fd(map, outer_map); 96 RET_ERR(err, "reuse outer_map", "err:%d\n", err); 97 98 err = bpf_object__load(obj); 99 RET_ERR(err, "load bpf_object", "err:%d\n", err); 100 101 prog = bpf_object__next_program(obj, NULL); 102 RET_ERR(!prog, "get first bpf_program", "!prog\n"); 103 select_by_skb_data_prog = bpf_program__fd(prog); 104 RET_ERR(select_by_skb_data_prog < 0, "get prog fd", 105 "select_by_skb_data_prog:%d\n", select_by_skb_data_prog); 106 107 map = bpf_object__find_map_by_name(obj, "result_map"); 108 RET_ERR(!map, "find result_map", "!map\n"); 109 result_map = bpf_map__fd(map); 110 RET_ERR(result_map < 0, "get result_map fd", 111 "result_map:%d\n", result_map); 112 113 map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map"); 114 RET_ERR(!map, "find tmp_index_ovr_map\n", "!map"); 115 tmp_index_ovr_map = bpf_map__fd(map); 116 RET_ERR(tmp_index_ovr_map < 0, "get tmp_index_ovr_map fd", 117 "tmp_index_ovr_map:%d\n", tmp_index_ovr_map); 118 119 map = bpf_object__find_map_by_name(obj, "linum_map"); 120 RET_ERR(!map, "find linum_map", "!map\n"); 121 linum_map = bpf_map__fd(map); 122 RET_ERR(linum_map < 0, "get linum_map fd", 123 "linum_map:%d\n", linum_map); 124 125 map = bpf_object__find_map_by_name(obj, "data_check_map"); 126 RET_ERR(!map, "find data_check_map", "!map\n"); 127 data_check_map = bpf_map__fd(map); 128 RET_ERR(data_check_map < 0, "get data_check_map fd", 129 "data_check_map:%d\n", data_check_map); 130 131 return 0; 132 } 133 134 static void ss_init_loopback(struct sockaddr_storage *sa, sa_family_t family) 135 { 136 memset(sa, 0, sizeof(*sa)); 137 sa->ss_family = family; 138 if (sa->ss_family == AF_INET6) 139 ((struct sockaddr_in6 *)sa)->sin6_addr = in6addr_loopback; 140 else 141 ((struct sockaddr_in *)sa)->sin_addr.s_addr = htonl(INADDR_LOOPBACK); 142 } 143 144 static void ss_init_inany(struct sockaddr_storage *sa, sa_family_t family) 145 { 146 memset(sa, 0, sizeof(*sa)); 147 sa->ss_family = family; 148 if (sa->ss_family == AF_INET6) 149 ((struct sockaddr_in6 *)sa)->sin6_addr = in6addr_any; 150 else 151 ((struct sockaddr_in *)sa)->sin_addr.s_addr = INADDR_ANY; 152 } 153 154 static int read_int_sysctl(const char *sysctl) 155 { 156 char buf[16]; 157 int fd, ret; 158 159 fd = open(sysctl, 0); 160 RET_ERR(fd == -1, "open(sysctl)", 161 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); 162 163 ret = read(fd, buf, sizeof(buf)); 164 RET_ERR(ret <= 0, "read(sysctl)", 165 "sysctl:%s ret:%d errno:%d\n", sysctl, ret, errno); 166 167 close(fd); 168 return atoi(buf); 169 } 170 171 static int write_int_sysctl(const char *sysctl, int v) 172 { 173 int fd, ret, size; 174 char buf[16]; 175 176 fd = open(sysctl, O_RDWR); 177 RET_ERR(fd == -1, "open(sysctl)", 178 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); 179 180 size = snprintf(buf, sizeof(buf), "%d", v); 181 ret = write(fd, buf, size); 182 RET_ERR(ret != size, "write(sysctl)", 183 "sysctl:%s ret:%d size:%d errno:%d\n", 184 sysctl, ret, size, errno); 185 186 close(fd); 187 return 0; 188 } 189 190 static int enable_fastopen(void) 191 { 192 int fo; 193 194 fo = read_int_sysctl(TCP_FO_SYSCTL); 195 if (fo < 0) 196 return -1; 197 198 return write_int_sysctl(TCP_FO_SYSCTL, fo | 7); 199 } 200 201 static int enable_syncookie(void) 202 { 203 return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2); 204 } 205 206 static int disable_syncookie(void) 207 { 208 return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0); 209 } 210 211 static long get_linum(void) 212 { 213 __u32 linum; 214 int err; 215 216 err = bpf_map_lookup_elem(linum_map, &index_zero, &linum); 217 RET_ERR(err < 0, "lookup_elem(linum_map)", "err:%d errno:%d\n", 218 err, errno); 219 220 return linum; 221 } 222 223 static void check_data(int type, sa_family_t family, const struct cmd *cmd, 224 int cli_fd) 225 { 226 struct data_check expected = {}, result; 227 struct sockaddr_storage cli_sa; 228 socklen_t addrlen; 229 int err; 230 231 addrlen = sizeof(cli_sa); 232 err = getsockname(cli_fd, (struct sockaddr *)&cli_sa, 233 &addrlen); 234 RET_IF(err < 0, "getsockname(cli_fd)", "err:%d errno:%d\n", 235 err, errno); 236 237 err = bpf_map_lookup_elem(data_check_map, &index_zero, &result); 238 RET_IF(err < 0, "lookup_elem(data_check_map)", "err:%d errno:%d\n", 239 err, errno); 240 241 if (type == SOCK_STREAM) { 242 expected.len = MIN_TCPHDR_LEN; 243 expected.ip_protocol = IPPROTO_TCP; 244 } else { 245 expected.len = UDPHDR_LEN; 246 expected.ip_protocol = IPPROTO_UDP; 247 } 248 249 if (family == AF_INET6) { 250 struct sockaddr_in6 *srv_v6 = (struct sockaddr_in6 *)&srv_sa; 251 struct sockaddr_in6 *cli_v6 = (struct sockaddr_in6 *)&cli_sa; 252 253 expected.eth_protocol = htons(ETH_P_IPV6); 254 expected.bind_inany = !srv_v6->sin6_addr.s6_addr32[3] && 255 !srv_v6->sin6_addr.s6_addr32[2] && 256 !srv_v6->sin6_addr.s6_addr32[1] && 257 !srv_v6->sin6_addr.s6_addr32[0]; 258 259 memcpy(&expected.skb_addrs[0], cli_v6->sin6_addr.s6_addr32, 260 sizeof(cli_v6->sin6_addr)); 261 memcpy(&expected.skb_addrs[4], &in6addr_loopback, 262 sizeof(in6addr_loopback)); 263 expected.skb_ports[0] = cli_v6->sin6_port; 264 expected.skb_ports[1] = srv_v6->sin6_port; 265 } else { 266 struct sockaddr_in *srv_v4 = (struct sockaddr_in *)&srv_sa; 267 struct sockaddr_in *cli_v4 = (struct sockaddr_in *)&cli_sa; 268 269 expected.eth_protocol = htons(ETH_P_IP); 270 expected.bind_inany = !srv_v4->sin_addr.s_addr; 271 272 expected.skb_addrs[0] = cli_v4->sin_addr.s_addr; 273 expected.skb_addrs[1] = htonl(INADDR_LOOPBACK); 274 expected.skb_ports[0] = cli_v4->sin_port; 275 expected.skb_ports[1] = srv_v4->sin_port; 276 } 277 278 if (memcmp(&result, &expected, offsetof(struct data_check, 279 equal_check_end))) { 280 printf("unexpected data_check\n"); 281 printf(" result: (0x%x, %u, %u)\n", 282 result.eth_protocol, result.ip_protocol, 283 result.bind_inany); 284 printf("expected: (0x%x, %u, %u)\n", 285 expected.eth_protocol, expected.ip_protocol, 286 expected.bind_inany); 287 RET_IF(1, "data_check result != expected", 288 "bpf_prog_linum:%ld\n", get_linum()); 289 } 290 291 RET_IF(!result.hash, "data_check result.hash empty", 292 "result.hash:%u", result.hash); 293 294 expected.len += cmd ? sizeof(*cmd) : 0; 295 if (type == SOCK_STREAM) 296 RET_IF(expected.len > result.len, "expected.len > result.len", 297 "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", 298 expected.len, result.len, get_linum()); 299 else 300 RET_IF(expected.len != result.len, "expected.len != result.len", 301 "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", 302 expected.len, result.len, get_linum()); 303 } 304 305 static const char *result_to_str(enum result res) 306 { 307 switch (res) { 308 case DROP_ERR_INNER_MAP: 309 return "DROP_ERR_INNER_MAP"; 310 case DROP_ERR_SKB_DATA: 311 return "DROP_ERR_SKB_DATA"; 312 case DROP_ERR_SK_SELECT_REUSEPORT: 313 return "DROP_ERR_SK_SELECT_REUSEPORT"; 314 case DROP_MISC: 315 return "DROP_MISC"; 316 case PASS: 317 return "PASS"; 318 case PASS_ERR_SK_SELECT_REUSEPORT: 319 return "PASS_ERR_SK_SELECT_REUSEPORT"; 320 default: 321 return "UNKNOWN"; 322 } 323 } 324 325 static void check_results(void) 326 { 327 __u32 results[NR_RESULTS]; 328 __u32 i, broken = 0; 329 int err; 330 331 for (i = 0; i < NR_RESULTS; i++) { 332 err = bpf_map_lookup_elem(result_map, &i, &results[i]); 333 RET_IF(err < 0, "lookup_elem(result_map)", 334 "i:%u err:%d errno:%d\n", i, err, errno); 335 } 336 337 for (i = 0; i < NR_RESULTS; i++) { 338 if (results[i] != expected_results[i]) { 339 broken = i; 340 break; 341 } 342 } 343 344 if (i == NR_RESULTS) 345 return; 346 347 printf("unexpected result\n"); 348 printf(" result: ["); 349 printf("%u", results[0]); 350 for (i = 1; i < NR_RESULTS; i++) 351 printf(", %u", results[i]); 352 printf("]\n"); 353 354 printf("expected: ["); 355 printf("%u", expected_results[0]); 356 for (i = 1; i < NR_RESULTS; i++) 357 printf(", %u", expected_results[i]); 358 printf("]\n"); 359 360 printf("mismatch on %s (bpf_prog_linum:%ld)\n", result_to_str(broken), 361 get_linum()); 362 363 CHECK_FAIL(true); 364 } 365 366 static int send_data(int type, sa_family_t family, void *data, size_t len, 367 enum result expected) 368 { 369 struct sockaddr_storage cli_sa; 370 int fd, err; 371 372 fd = socket(family, type, 0); 373 RET_ERR(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno); 374 375 ss_init_loopback(&cli_sa, family); 376 err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa)); 377 RET_ERR(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno); 378 err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa, 379 sizeof(srv_sa)); 380 RET_ERR(err != len && expected >= PASS, 381 "sendto()", "family:%u err:%d errno:%d expected:%d\n", 382 family, err, errno, expected); 383 384 return fd; 385 } 386 387 static void do_test(int type, sa_family_t family, struct cmd *cmd, 388 enum result expected) 389 { 390 int nev, srv_fd, cli_fd; 391 struct epoll_event ev; 392 struct cmd rcv_cmd; 393 ssize_t nread; 394 395 cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0, 396 expected); 397 if (cli_fd < 0) 398 return; 399 nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0); 400 RET_IF((nev <= 0 && expected >= PASS) || 401 (nev > 0 && expected < PASS), 402 "nev <> expected", 403 "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n", 404 nev, expected, type, family, 405 cmd ? cmd->reuseport_index : -1, 406 cmd ? cmd->pass_on_failure : -1); 407 check_results(); 408 check_data(type, family, cmd, cli_fd); 409 410 if (expected < PASS) 411 return; 412 413 RET_IF(expected != PASS_ERR_SK_SELECT_REUSEPORT && 414 cmd->reuseport_index != ev.data.u32, 415 "check cmd->reuseport_index", 416 "cmd:(%u, %u) ev.data.u32:%u\n", 417 cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32); 418 419 srv_fd = sk_fds[ev.data.u32]; 420 if (type == SOCK_STREAM) { 421 int new_fd = accept(srv_fd, NULL, 0); 422 423 RET_IF(new_fd == -1, "accept(srv_fd)", 424 "ev.data.u32:%u new_fd:%d errno:%d\n", 425 ev.data.u32, new_fd, errno); 426 427 nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); 428 RET_IF(nread != sizeof(rcv_cmd), 429 "recv(new_fd)", 430 "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", 431 ev.data.u32, nread, sizeof(rcv_cmd), errno); 432 433 close(new_fd); 434 } else { 435 nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); 436 RET_IF(nread != sizeof(rcv_cmd), 437 "recv(sk_fds)", 438 "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", 439 ev.data.u32, nread, sizeof(rcv_cmd), errno); 440 } 441 442 close(cli_fd); 443 } 444 445 static void test_err_inner_map(int type, sa_family_t family) 446 { 447 struct cmd cmd = { 448 .reuseport_index = 0, 449 .pass_on_failure = 0, 450 }; 451 452 expected_results[DROP_ERR_INNER_MAP]++; 453 do_test(type, family, &cmd, DROP_ERR_INNER_MAP); 454 } 455 456 static void test_err_skb_data(int type, sa_family_t family) 457 { 458 expected_results[DROP_ERR_SKB_DATA]++; 459 do_test(type, family, NULL, DROP_ERR_SKB_DATA); 460 } 461 462 static void test_err_sk_select_port(int type, sa_family_t family) 463 { 464 struct cmd cmd = { 465 .reuseport_index = REUSEPORT_ARRAY_SIZE, 466 .pass_on_failure = 0, 467 }; 468 469 expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++; 470 do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT); 471 } 472 473 static void test_pass(int type, sa_family_t family) 474 { 475 struct cmd cmd; 476 int i; 477 478 cmd.pass_on_failure = 0; 479 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { 480 expected_results[PASS]++; 481 cmd.reuseport_index = i; 482 do_test(type, family, &cmd, PASS); 483 } 484 } 485 486 static void test_syncookie(int type, sa_family_t family) 487 { 488 int err, tmp_index = 1; 489 struct cmd cmd = { 490 .reuseport_index = 0, 491 .pass_on_failure = 0, 492 }; 493 494 /* 495 * +1 for TCP-SYN and 496 * +1 for the TCP-ACK (ack the syncookie) 497 */ 498 expected_results[PASS] += 2; 499 enable_syncookie(); 500 /* 501 * Simulate TCP-SYN and TCP-ACK are handled by two different sk: 502 * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the 503 * tmp_index_ovr_map 504 * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index 505 * is from the cmd.reuseport_index 506 */ 507 err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, 508 &tmp_index, BPF_ANY); 509 RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, 1)", 510 "err:%d errno:%d\n", err, errno); 511 do_test(type, family, &cmd, PASS); 512 err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero, 513 &tmp_index); 514 RET_IF(err < 0 || tmp_index >= 0, 515 "lookup_elem(tmp_index_ovr_map)", 516 "err:%d errno:%d tmp_index:%d\n", 517 err, errno, tmp_index); 518 disable_syncookie(); 519 } 520 521 static void test_pass_on_err(int type, sa_family_t family) 522 { 523 struct cmd cmd = { 524 .reuseport_index = REUSEPORT_ARRAY_SIZE, 525 .pass_on_failure = 1, 526 }; 527 528 expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1; 529 do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT); 530 } 531 532 static void test_detach_bpf(int type, sa_family_t family) 533 { 534 #ifdef SO_DETACH_REUSEPORT_BPF 535 __u32 nr_run_before = 0, nr_run_after = 0, tmp, i; 536 struct epoll_event ev; 537 int cli_fd, err, nev; 538 struct cmd cmd = {}; 539 int optvalue = 0; 540 541 err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, 542 &optvalue, sizeof(optvalue)); 543 RET_IF(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)", 544 "err:%d errno:%d\n", err, errno); 545 546 err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, 547 &optvalue, sizeof(optvalue)); 548 RET_IF(err == 0 || errno != ENOENT, 549 "setsockopt(SO_DETACH_REUSEPORT_BPF)", 550 "err:%d errno:%d\n", err, errno); 551 552 for (i = 0; i < NR_RESULTS; i++) { 553 err = bpf_map_lookup_elem(result_map, &i, &tmp); 554 RET_IF(err < 0, "lookup_elem(result_map)", 555 "i:%u err:%d errno:%d\n", i, err, errno); 556 nr_run_before += tmp; 557 } 558 559 cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS); 560 if (cli_fd < 0) 561 return; 562 nev = epoll_wait(epfd, &ev, 1, 5); 563 RET_IF(nev <= 0, "nev <= 0", 564 "nev:%d expected:1 type:%d family:%d data:(0, 0)\n", 565 nev, type, family); 566 567 for (i = 0; i < NR_RESULTS; i++) { 568 err = bpf_map_lookup_elem(result_map, &i, &tmp); 569 RET_IF(err < 0, "lookup_elem(result_map)", 570 "i:%u err:%d errno:%d\n", i, err, errno); 571 nr_run_after += tmp; 572 } 573 574 RET_IF(nr_run_before != nr_run_after, 575 "nr_run_before != nr_run_after", 576 "nr_run_before:%u nr_run_after:%u\n", 577 nr_run_before, nr_run_after); 578 579 close(cli_fd); 580 #else 581 test__skip(); 582 #endif 583 } 584 585 static void prepare_sk_fds(int type, sa_family_t family, bool inany) 586 { 587 const int first = REUSEPORT_ARRAY_SIZE - 1; 588 int i, err, optval = 1; 589 struct epoll_event ev; 590 socklen_t addrlen; 591 592 if (inany) 593 ss_init_inany(&srv_sa, family); 594 else 595 ss_init_loopback(&srv_sa, family); 596 addrlen = sizeof(srv_sa); 597 598 /* 599 * The sk_fds[] is filled from the back such that the order 600 * is exactly opposite to the (struct sock_reuseport *)reuse->socks[]. 601 */ 602 for (i = first; i >= 0; i--) { 603 sk_fds[i] = socket(family, type, 0); 604 RET_IF(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n", 605 i, sk_fds[i], errno); 606 err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT, 607 &optval, sizeof(optval)); 608 RET_IF(err == -1, "setsockopt(SO_REUSEPORT)", 609 "sk_fds[%d] err:%d errno:%d\n", 610 i, err, errno); 611 612 if (i == first) { 613 err = setsockopt(sk_fds[i], SOL_SOCKET, 614 SO_ATTACH_REUSEPORT_EBPF, 615 &select_by_skb_data_prog, 616 sizeof(select_by_skb_data_prog)); 617 RET_IF(err < 0, "setsockopt(SO_ATTACH_REUEPORT_EBPF)", 618 "err:%d errno:%d\n", err, errno); 619 } 620 621 err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen); 622 RET_IF(err < 0, "bind()", "sk_fds[%d] err:%d errno:%d\n", 623 i, err, errno); 624 625 if (type == SOCK_STREAM) { 626 err = listen(sk_fds[i], 10); 627 RET_IF(err < 0, "listen()", 628 "sk_fds[%d] err:%d errno:%d\n", 629 i, err, errno); 630 } 631 632 err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i], 633 BPF_NOEXIST); 634 RET_IF(err < 0, "update_elem(reuseport_array)", 635 "sk_fds[%d] err:%d errno:%d\n", i, err, errno); 636 637 if (i == first) { 638 socklen_t addrlen = sizeof(srv_sa); 639 640 err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa, 641 &addrlen); 642 RET_IF(err == -1, "getsockname()", 643 "sk_fds[%d] err:%d errno:%d\n", i, err, errno); 644 } 645 } 646 647 epfd = epoll_create(1); 648 RET_IF(epfd == -1, "epoll_create(1)", 649 "epfd:%d errno:%d\n", epfd, errno); 650 651 ev.events = EPOLLIN; 652 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { 653 ev.data.u32 = i; 654 err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev); 655 RET_IF(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i); 656 } 657 } 658 659 static void setup_per_test(int type, sa_family_t family, bool inany, 660 bool no_inner_map) 661 { 662 int ovr = -1, err; 663 664 prepare_sk_fds(type, family, inany); 665 err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr, 666 BPF_ANY); 667 RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, -1)", 668 "err:%d errno:%d\n", err, errno); 669 670 /* Install reuseport_array to outer_map? */ 671 if (no_inner_map) 672 return; 673 674 err = bpf_map_update_elem(outer_map, &index_zero, &reuseport_array, 675 BPF_ANY); 676 RET_IF(err < 0, "update_elem(outer_map, 0, reuseport_array)", 677 "err:%d errno:%d\n", err, errno); 678 } 679 680 static void cleanup_per_test(bool no_inner_map) 681 { 682 int i, err, zero = 0; 683 684 memset(expected_results, 0, sizeof(expected_results)); 685 686 for (i = 0; i < NR_RESULTS; i++) { 687 err = bpf_map_update_elem(result_map, &i, &zero, BPF_ANY); 688 RET_IF(err, "reset elem in result_map", 689 "i:%u err:%d errno:%d\n", i, err, errno); 690 } 691 692 err = bpf_map_update_elem(linum_map, &zero, &zero, BPF_ANY); 693 RET_IF(err, "reset line number in linum_map", "err:%d errno:%d\n", 694 err, errno); 695 696 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) 697 close(sk_fds[i]); 698 close(epfd); 699 700 /* Delete reuseport_array from outer_map? */ 701 if (no_inner_map) 702 return; 703 704 err = bpf_map_delete_elem(outer_map, &index_zero); 705 RET_IF(err < 0, "delete_elem(outer_map)", 706 "err:%d errno:%d\n", err, errno); 707 } 708 709 static void cleanup(void) 710 { 711 if (outer_map >= 0) { 712 close(outer_map); 713 outer_map = -1; 714 } 715 716 if (reuseport_array >= 0) { 717 close(reuseport_array); 718 reuseport_array = -1; 719 } 720 721 if (obj) { 722 bpf_object__close(obj); 723 obj = NULL; 724 } 725 726 memset(expected_results, 0, sizeof(expected_results)); 727 } 728 729 static const char *maptype_str(enum bpf_map_type type) 730 { 731 switch (type) { 732 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 733 return "reuseport_sockarray"; 734 case BPF_MAP_TYPE_SOCKMAP: 735 return "sockmap"; 736 case BPF_MAP_TYPE_SOCKHASH: 737 return "sockhash"; 738 default: 739 return "unknown"; 740 } 741 } 742 743 static const char *family_str(sa_family_t family) 744 { 745 switch (family) { 746 case AF_INET: 747 return "IPv4"; 748 case AF_INET6: 749 return "IPv6"; 750 default: 751 return "unknown"; 752 } 753 } 754 755 static const char *sotype_str(int sotype) 756 { 757 switch (sotype) { 758 case SOCK_STREAM: 759 return "TCP"; 760 case SOCK_DGRAM: 761 return "UDP"; 762 default: 763 return "unknown"; 764 } 765 } 766 767 #define TEST_INIT(fn_, ...) { .fn = fn_, .name = #fn_, __VA_ARGS__ } 768 769 static void test_config(int sotype, sa_family_t family, bool inany) 770 { 771 const struct test { 772 void (*fn)(int sotype, sa_family_t family); 773 const char *name; 774 bool no_inner_map; 775 int need_sotype; 776 } tests[] = { 777 TEST_INIT(test_err_inner_map, 778 .no_inner_map = true), 779 TEST_INIT(test_err_skb_data), 780 TEST_INIT(test_err_sk_select_port), 781 TEST_INIT(test_pass), 782 TEST_INIT(test_syncookie, 783 .need_sotype = SOCK_STREAM), 784 TEST_INIT(test_pass_on_err), 785 TEST_INIT(test_detach_bpf), 786 }; 787 struct netns_obj *netns; 788 char s[MAX_TEST_NAME]; 789 const struct test *t; 790 791 for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { 792 if (t->need_sotype && t->need_sotype != sotype) 793 continue; /* test not compatible with socket type */ 794 795 snprintf(s, sizeof(s), "%s %s/%s %s %s", 796 maptype_str(inner_map_type), 797 family_str(family), sotype_str(sotype), 798 inany ? "INANY" : "LOOPBACK", t->name); 799 800 if (!test__start_subtest(s)) 801 continue; 802 803 netns = netns_new("select_reuseport", true); 804 if (!ASSERT_OK_PTR(netns, "netns_new")) 805 continue; 806 807 if (CHECK_FAIL(enable_fastopen())) 808 goto out; 809 if (CHECK_FAIL(disable_syncookie())) 810 goto out; 811 812 setup_per_test(sotype, family, inany, t->no_inner_map); 813 t->fn(sotype, family); 814 cleanup_per_test(t->no_inner_map); 815 816 out: 817 netns_free(netns); 818 } 819 } 820 821 #define BIND_INANY true 822 823 static void test_all(void) 824 { 825 const struct config { 826 int sotype; 827 sa_family_t family; 828 bool inany; 829 } configs[] = { 830 { SOCK_STREAM, AF_INET }, 831 { SOCK_STREAM, AF_INET, BIND_INANY }, 832 { SOCK_STREAM, AF_INET6 }, 833 { SOCK_STREAM, AF_INET6, BIND_INANY }, 834 { SOCK_DGRAM, AF_INET }, 835 { SOCK_DGRAM, AF_INET6 }, 836 }; 837 const struct config *c; 838 839 for (c = configs; c < configs + ARRAY_SIZE(configs); c++) 840 test_config(c->sotype, c->family, c->inany); 841 } 842 843 void test_map_type(enum bpf_map_type mt) 844 { 845 if (create_maps(mt)) 846 goto out; 847 if (prepare_bpf_obj()) 848 goto out; 849 850 test_all(); 851 out: 852 cleanup(); 853 } 854 855 void serial_test_select_reuseport(void) 856 { 857 test_map_type(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY); 858 test_map_type(BPF_MAP_TYPE_SOCKMAP); 859 test_map_type(BPF_MAP_TYPE_SOCKHASH); 860 } 861