1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 Facebook */ 3 4 #include <linux/err.h> 5 #include <netinet/tcp.h> 6 #include <test_progs.h> 7 #include "network_helpers.h" 8 #include "bpf_dctcp.skel.h" 9 #include "bpf_cubic.skel.h" 10 #include "bpf_tcp_nogpl.skel.h" 11 #include "tcp_ca_update.skel.h" 12 #include "bpf_dctcp_release.skel.h" 13 #include "tcp_ca_write_sk_pacing.skel.h" 14 #include "tcp_ca_incompl_cong_ops.skel.h" 15 #include "tcp_ca_unsupp_cong_op.skel.h" 16 17 #ifndef ENOTSUPP 18 #define ENOTSUPP 524 19 #endif 20 21 static const unsigned int total_bytes = 10 * 1024 * 1024; 22 static int expected_stg = 0xeB9F; 23 static int stop; 24 25 static int settcpca(int fd, const char *tcp_ca) 26 { 27 int err; 28 29 err = setsockopt(fd, IPPROTO_TCP, TCP_CONGESTION, tcp_ca, strlen(tcp_ca)); 30 if (!ASSERT_NEQ(err, -1, "setsockopt")) 31 return -1; 32 33 return 0; 34 } 35 36 static void *server(void *arg) 37 { 38 int lfd = (int)(long)arg, err = 0, fd; 39 ssize_t nr_sent = 0, bytes = 0; 40 char batch[1500]; 41 42 fd = accept(lfd, NULL, NULL); 43 while (fd == -1) { 44 if (errno == EINTR) 45 continue; 46 err = -errno; 47 goto done; 48 } 49 50 if (settimeo(fd, 0)) { 51 err = -errno; 52 goto done; 53 } 54 55 while (bytes < total_bytes && !READ_ONCE(stop)) { 56 nr_sent = send(fd, &batch, 57 MIN(total_bytes - bytes, sizeof(batch)), 0); 58 if (nr_sent == -1 && errno == EINTR) 59 continue; 60 if (nr_sent == -1) { 61 err = -errno; 62 break; 63 } 64 bytes += nr_sent; 65 } 66 67 ASSERT_EQ(bytes, total_bytes, "send"); 68 69 done: 70 if (fd >= 0) 71 close(fd); 72 if (err) { 73 WRITE_ONCE(stop, 1); 74 return ERR_PTR(err); 75 } 76 return NULL; 77 } 78 79 static void do_test(const char *tcp_ca, const struct bpf_map *sk_stg_map) 80 { 81 struct sockaddr_in6 sa6 = {}; 82 ssize_t nr_recv = 0, bytes = 0; 83 int lfd = -1, fd = -1; 84 pthread_t srv_thread; 85 socklen_t addrlen = sizeof(sa6); 86 void *thread_ret; 87 char batch[1500]; 88 int err; 89 90 WRITE_ONCE(stop, 0); 91 92 lfd = start_server(AF_INET6, SOCK_STREAM, NULL, 0, 0); 93 if (!ASSERT_NEQ(lfd, -1, "socket")) 94 return; 95 96 fd = socket(AF_INET6, SOCK_STREAM, 0); 97 if (!ASSERT_NEQ(fd, -1, "socket")) { 98 close(lfd); 99 return; 100 } 101 102 if (settcpca(lfd, tcp_ca) || settcpca(fd, tcp_ca) || 103 settimeo(lfd, 0) || settimeo(fd, 0)) 104 goto done; 105 106 err = getsockname(lfd, (struct sockaddr *)&sa6, &addrlen); 107 if (!ASSERT_NEQ(err, -1, "getsockname")) 108 goto done; 109 110 if (sk_stg_map) { 111 err = bpf_map_update_elem(bpf_map__fd(sk_stg_map), &fd, 112 &expected_stg, BPF_NOEXIST); 113 if (!ASSERT_OK(err, "bpf_map_update_elem(sk_stg_map)")) 114 goto done; 115 } 116 117 /* connect to server */ 118 err = connect(fd, (struct sockaddr *)&sa6, addrlen); 119 if (!ASSERT_NEQ(err, -1, "connect")) 120 goto done; 121 122 if (sk_stg_map) { 123 int tmp_stg; 124 125 err = bpf_map_lookup_elem(bpf_map__fd(sk_stg_map), &fd, 126 &tmp_stg); 127 if (!ASSERT_ERR(err, "bpf_map_lookup_elem(sk_stg_map)") || 128 !ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem(sk_stg_map)")) 129 goto done; 130 } 131 132 err = pthread_create(&srv_thread, NULL, server, (void *)(long)lfd); 133 if (!ASSERT_OK(err, "pthread_create")) 134 goto done; 135 136 /* recv total_bytes */ 137 while (bytes < total_bytes && !READ_ONCE(stop)) { 138 nr_recv = recv(fd, &batch, 139 MIN(total_bytes - bytes, sizeof(batch)), 0); 140 if (nr_recv == -1 && errno == EINTR) 141 continue; 142 if (nr_recv == -1) 143 break; 144 bytes += nr_recv; 145 } 146 147 ASSERT_EQ(bytes, total_bytes, "recv"); 148 149 WRITE_ONCE(stop, 1); 150 pthread_join(srv_thread, &thread_ret); 151 ASSERT_OK(IS_ERR(thread_ret), "thread_ret"); 152 153 done: 154 close(lfd); 155 close(fd); 156 } 157 158 static void test_cubic(void) 159 { 160 struct bpf_cubic *cubic_skel; 161 struct bpf_link *link; 162 163 cubic_skel = bpf_cubic__open_and_load(); 164 if (!ASSERT_OK_PTR(cubic_skel, "bpf_cubic__open_and_load")) 165 return; 166 167 link = bpf_map__attach_struct_ops(cubic_skel->maps.cubic); 168 if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 169 bpf_cubic__destroy(cubic_skel); 170 return; 171 } 172 173 do_test("bpf_cubic", NULL); 174 175 ASSERT_EQ(cubic_skel->bss->bpf_cubic_acked_called, 1, "pkts_acked called"); 176 177 bpf_link__destroy(link); 178 bpf_cubic__destroy(cubic_skel); 179 } 180 181 static void test_dctcp(void) 182 { 183 struct bpf_dctcp *dctcp_skel; 184 struct bpf_link *link; 185 186 dctcp_skel = bpf_dctcp__open_and_load(); 187 if (!ASSERT_OK_PTR(dctcp_skel, "bpf_dctcp__open_and_load")) 188 return; 189 190 link = bpf_map__attach_struct_ops(dctcp_skel->maps.dctcp); 191 if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 192 bpf_dctcp__destroy(dctcp_skel); 193 return; 194 } 195 196 do_test("bpf_dctcp", dctcp_skel->maps.sk_stg_map); 197 ASSERT_EQ(dctcp_skel->bss->stg_result, expected_stg, "stg_result"); 198 199 bpf_link__destroy(link); 200 bpf_dctcp__destroy(dctcp_skel); 201 } 202 203 static char *err_str; 204 static bool found; 205 206 static int libbpf_debug_print(enum libbpf_print_level level, 207 const char *format, va_list args) 208 { 209 const char *prog_name, *log_buf; 210 211 if (level != LIBBPF_WARN || 212 !strstr(format, "-- BEGIN PROG LOAD LOG --")) { 213 vprintf(format, args); 214 return 0; 215 } 216 217 prog_name = va_arg(args, char *); 218 log_buf = va_arg(args, char *); 219 if (!log_buf) 220 goto out; 221 if (err_str && strstr(log_buf, err_str) != NULL) 222 found = true; 223 out: 224 printf(format, prog_name, log_buf); 225 return 0; 226 } 227 228 static void test_invalid_license(void) 229 { 230 libbpf_print_fn_t old_print_fn; 231 struct bpf_tcp_nogpl *skel; 232 233 err_str = "struct ops programs must have a GPL compatible license"; 234 found = false; 235 old_print_fn = libbpf_set_print(libbpf_debug_print); 236 237 skel = bpf_tcp_nogpl__open_and_load(); 238 ASSERT_NULL(skel, "bpf_tcp_nogpl"); 239 ASSERT_EQ(found, true, "expected_err_msg"); 240 241 bpf_tcp_nogpl__destroy(skel); 242 libbpf_set_print(old_print_fn); 243 } 244 245 static void test_dctcp_fallback(void) 246 { 247 int err, lfd = -1, cli_fd = -1, srv_fd = -1; 248 struct network_helper_opts opts = { 249 .cc = "cubic", 250 }; 251 struct bpf_dctcp *dctcp_skel; 252 struct bpf_link *link = NULL; 253 char srv_cc[16]; 254 socklen_t cc_len = sizeof(srv_cc); 255 256 dctcp_skel = bpf_dctcp__open(); 257 if (!ASSERT_OK_PTR(dctcp_skel, "dctcp_skel")) 258 return; 259 strcpy(dctcp_skel->rodata->fallback, "cubic"); 260 if (!ASSERT_OK(bpf_dctcp__load(dctcp_skel), "bpf_dctcp__load")) 261 goto done; 262 263 link = bpf_map__attach_struct_ops(dctcp_skel->maps.dctcp); 264 if (!ASSERT_OK_PTR(link, "dctcp link")) 265 goto done; 266 267 lfd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0); 268 if (!ASSERT_GE(lfd, 0, "lfd") || 269 !ASSERT_OK(settcpca(lfd, "bpf_dctcp"), "lfd=>bpf_dctcp")) 270 goto done; 271 272 cli_fd = connect_to_fd_opts(lfd, &opts); 273 if (!ASSERT_GE(cli_fd, 0, "cli_fd")) 274 goto done; 275 276 srv_fd = accept(lfd, NULL, 0); 277 if (!ASSERT_GE(srv_fd, 0, "srv_fd")) 278 goto done; 279 ASSERT_STREQ(dctcp_skel->bss->cc_res, "cubic", "cc_res"); 280 ASSERT_EQ(dctcp_skel->bss->tcp_cdg_res, -ENOTSUPP, "tcp_cdg_res"); 281 /* All setsockopt(TCP_CONGESTION) in the recurred 282 * bpf_dctcp->init() should fail with -EBUSY. 283 */ 284 ASSERT_EQ(dctcp_skel->bss->ebusy_cnt, 3, "ebusy_cnt"); 285 286 err = getsockopt(srv_fd, SOL_TCP, TCP_CONGESTION, srv_cc, &cc_len); 287 if (!ASSERT_OK(err, "getsockopt(srv_fd, TCP_CONGESTION)")) 288 goto done; 289 ASSERT_STREQ(srv_cc, "cubic", "srv_fd cc"); 290 291 done: 292 bpf_link__destroy(link); 293 bpf_dctcp__destroy(dctcp_skel); 294 if (lfd != -1) 295 close(lfd); 296 if (srv_fd != -1) 297 close(srv_fd); 298 if (cli_fd != -1) 299 close(cli_fd); 300 } 301 302 static void test_rel_setsockopt(void) 303 { 304 struct bpf_dctcp_release *rel_skel; 305 libbpf_print_fn_t old_print_fn; 306 307 err_str = "unknown func bpf_setsockopt"; 308 found = false; 309 310 old_print_fn = libbpf_set_print(libbpf_debug_print); 311 rel_skel = bpf_dctcp_release__open_and_load(); 312 libbpf_set_print(old_print_fn); 313 314 ASSERT_ERR_PTR(rel_skel, "rel_skel"); 315 ASSERT_TRUE(found, "expected_err_msg"); 316 317 bpf_dctcp_release__destroy(rel_skel); 318 } 319 320 static void test_write_sk_pacing(void) 321 { 322 struct tcp_ca_write_sk_pacing *skel; 323 struct bpf_link *link; 324 325 skel = tcp_ca_write_sk_pacing__open_and_load(); 326 if (!ASSERT_OK_PTR(skel, "open_and_load")) 327 return; 328 329 link = bpf_map__attach_struct_ops(skel->maps.write_sk_pacing); 330 ASSERT_OK_PTR(link, "attach_struct_ops"); 331 332 bpf_link__destroy(link); 333 tcp_ca_write_sk_pacing__destroy(skel); 334 } 335 336 static void test_incompl_cong_ops(void) 337 { 338 struct tcp_ca_incompl_cong_ops *skel; 339 struct bpf_link *link; 340 341 skel = tcp_ca_incompl_cong_ops__open_and_load(); 342 if (!ASSERT_OK_PTR(skel, "open_and_load")) 343 return; 344 345 /* That cong_avoid() and cong_control() are missing is only reported at 346 * this point: 347 */ 348 link = bpf_map__attach_struct_ops(skel->maps.incompl_cong_ops); 349 ASSERT_ERR_PTR(link, "attach_struct_ops"); 350 351 bpf_link__destroy(link); 352 tcp_ca_incompl_cong_ops__destroy(skel); 353 } 354 355 static void test_unsupp_cong_op(void) 356 { 357 libbpf_print_fn_t old_print_fn; 358 struct tcp_ca_unsupp_cong_op *skel; 359 360 err_str = "attach to unsupported member get_info"; 361 found = false; 362 old_print_fn = libbpf_set_print(libbpf_debug_print); 363 364 skel = tcp_ca_unsupp_cong_op__open_and_load(); 365 ASSERT_NULL(skel, "open_and_load"); 366 ASSERT_EQ(found, true, "expected_err_msg"); 367 368 tcp_ca_unsupp_cong_op__destroy(skel); 369 libbpf_set_print(old_print_fn); 370 } 371 372 static void test_update_ca(void) 373 { 374 struct tcp_ca_update *skel; 375 struct bpf_link *link; 376 int saved_ca1_cnt; 377 int err; 378 379 skel = tcp_ca_update__open_and_load(); 380 if (!ASSERT_OK_PTR(skel, "open")) 381 return; 382 383 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 384 ASSERT_OK_PTR(link, "attach_struct_ops"); 385 386 do_test("tcp_ca_update", NULL); 387 saved_ca1_cnt = skel->bss->ca1_cnt; 388 ASSERT_GT(saved_ca1_cnt, 0, "ca1_ca1_cnt"); 389 390 err = bpf_link__update_map(link, skel->maps.ca_update_2); 391 ASSERT_OK(err, "update_map"); 392 393 do_test("tcp_ca_update", NULL); 394 ASSERT_EQ(skel->bss->ca1_cnt, saved_ca1_cnt, "ca2_ca1_cnt"); 395 ASSERT_GT(skel->bss->ca2_cnt, 0, "ca2_ca2_cnt"); 396 397 bpf_link__destroy(link); 398 tcp_ca_update__destroy(skel); 399 } 400 401 static void test_update_wrong(void) 402 { 403 struct tcp_ca_update *skel; 404 struct bpf_link *link; 405 int saved_ca1_cnt; 406 int err; 407 408 skel = tcp_ca_update__open_and_load(); 409 if (!ASSERT_OK_PTR(skel, "open")) 410 return; 411 412 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 413 ASSERT_OK_PTR(link, "attach_struct_ops"); 414 415 do_test("tcp_ca_update", NULL); 416 saved_ca1_cnt = skel->bss->ca1_cnt; 417 ASSERT_GT(saved_ca1_cnt, 0, "ca1_ca1_cnt"); 418 419 err = bpf_link__update_map(link, skel->maps.ca_wrong); 420 ASSERT_ERR(err, "update_map"); 421 422 do_test("tcp_ca_update", NULL); 423 ASSERT_GT(skel->bss->ca1_cnt, saved_ca1_cnt, "ca2_ca1_cnt"); 424 425 bpf_link__destroy(link); 426 tcp_ca_update__destroy(skel); 427 } 428 429 static void test_mixed_links(void) 430 { 431 struct tcp_ca_update *skel; 432 struct bpf_link *link, *link_nl; 433 int err; 434 435 skel = tcp_ca_update__open_and_load(); 436 if (!ASSERT_OK_PTR(skel, "open")) 437 return; 438 439 link_nl = bpf_map__attach_struct_ops(skel->maps.ca_no_link); 440 ASSERT_OK_PTR(link_nl, "attach_struct_ops_nl"); 441 442 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 443 ASSERT_OK_PTR(link, "attach_struct_ops"); 444 445 do_test("tcp_ca_update", NULL); 446 ASSERT_GT(skel->bss->ca1_cnt, 0, "ca1_ca1_cnt"); 447 448 err = bpf_link__update_map(link, skel->maps.ca_no_link); 449 ASSERT_ERR(err, "update_map"); 450 451 bpf_link__destroy(link); 452 bpf_link__destroy(link_nl); 453 tcp_ca_update__destroy(skel); 454 } 455 456 static void test_multi_links(void) 457 { 458 struct tcp_ca_update *skel; 459 struct bpf_link *link; 460 461 skel = tcp_ca_update__open_and_load(); 462 if (!ASSERT_OK_PTR(skel, "open")) 463 return; 464 465 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 466 ASSERT_OK_PTR(link, "attach_struct_ops_1st"); 467 bpf_link__destroy(link); 468 469 /* A map should be able to be used to create links multiple 470 * times. 471 */ 472 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 473 ASSERT_OK_PTR(link, "attach_struct_ops_2nd"); 474 bpf_link__destroy(link); 475 476 tcp_ca_update__destroy(skel); 477 } 478 479 static void test_link_replace(void) 480 { 481 DECLARE_LIBBPF_OPTS(bpf_link_update_opts, opts); 482 struct tcp_ca_update *skel; 483 struct bpf_link *link; 484 int err; 485 486 skel = tcp_ca_update__open_and_load(); 487 if (!ASSERT_OK_PTR(skel, "open")) 488 return; 489 490 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 491 ASSERT_OK_PTR(link, "attach_struct_ops_1st"); 492 bpf_link__destroy(link); 493 494 link = bpf_map__attach_struct_ops(skel->maps.ca_update_2); 495 ASSERT_OK_PTR(link, "attach_struct_ops_2nd"); 496 497 /* BPF_F_REPLACE with a wrong old map Fd. It should fail! 498 * 499 * With BPF_F_REPLACE, the link should be updated only if the 500 * old map fd given here matches the map backing the link. 501 */ 502 opts.old_map_fd = bpf_map__fd(skel->maps.ca_update_1); 503 opts.flags = BPF_F_REPLACE; 504 err = bpf_link_update(bpf_link__fd(link), 505 bpf_map__fd(skel->maps.ca_update_1), 506 &opts); 507 ASSERT_ERR(err, "bpf_link_update_fail"); 508 509 /* BPF_F_REPLACE with a correct old map Fd. It should success! */ 510 opts.old_map_fd = bpf_map__fd(skel->maps.ca_update_2); 511 err = bpf_link_update(bpf_link__fd(link), 512 bpf_map__fd(skel->maps.ca_update_1), 513 &opts); 514 ASSERT_OK(err, "bpf_link_update_success"); 515 516 bpf_link__destroy(link); 517 518 tcp_ca_update__destroy(skel); 519 } 520 521 void test_bpf_tcp_ca(void) 522 { 523 if (test__start_subtest("dctcp")) 524 test_dctcp(); 525 if (test__start_subtest("cubic")) 526 test_cubic(); 527 if (test__start_subtest("invalid_license")) 528 test_invalid_license(); 529 if (test__start_subtest("dctcp_fallback")) 530 test_dctcp_fallback(); 531 if (test__start_subtest("rel_setsockopt")) 532 test_rel_setsockopt(); 533 if (test__start_subtest("write_sk_pacing")) 534 test_write_sk_pacing(); 535 if (test__start_subtest("incompl_cong_ops")) 536 test_incompl_cong_ops(); 537 if (test__start_subtest("unsupp_cong_op")) 538 test_unsupp_cong_op(); 539 if (test__start_subtest("update_ca")) 540 test_update_ca(); 541 if (test__start_subtest("update_wrong")) 542 test_update_wrong(); 543 if (test__start_subtest("mixed_links")) 544 test_mixed_links(); 545 if (test__start_subtest("multi_links")) 546 test_multi_links(); 547 if (test__start_subtest("link_replace")) 548 test_link_replace(); 549 } 550