1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 Facebook */ 3 4 #include <linux/err.h> 5 #include <netinet/tcp.h> 6 #include <test_progs.h> 7 #include "network_helpers.h" 8 #include "bpf_dctcp.skel.h" 9 #include "bpf_cubic.skel.h" 10 #include "bpf_tcp_nogpl.skel.h" 11 #include "tcp_ca_update.skel.h" 12 #include "bpf_dctcp_release.skel.h" 13 #include "tcp_ca_write_sk_pacing.skel.h" 14 #include "tcp_ca_incompl_cong_ops.skel.h" 15 #include "tcp_ca_unsupp_cong_op.skel.h" 16 #include "tcp_ca_kfunc.skel.h" 17 #include "bpf_cc_cubic.skel.h" 18 19 static const unsigned int total_bytes = 10 * 1024 * 1024; 20 static int expected_stg = 0xeB9F; 21 22 struct cb_opts { 23 const char *cc; 24 int map_fd; 25 }; 26 27 static int settcpca(int fd, const char *tcp_ca) 28 { 29 int err; 30 31 err = setsockopt(fd, IPPROTO_TCP, TCP_CONGESTION, tcp_ca, strlen(tcp_ca)); 32 if (!ASSERT_NEQ(err, -1, "setsockopt")) 33 return -1; 34 35 return 0; 36 } 37 38 static bool start_test(char *addr_str, 39 const struct network_helper_opts *srv_opts, 40 const struct network_helper_opts *cli_opts, 41 int *srv_fd, int *cli_fd) 42 { 43 *srv_fd = start_server_str(AF_INET6, SOCK_STREAM, addr_str, 0, srv_opts); 44 if (!ASSERT_NEQ(*srv_fd, -1, "start_server_str")) 45 goto err; 46 47 /* connect to server */ 48 *cli_fd = connect_to_fd_opts(*srv_fd, cli_opts); 49 if (!ASSERT_NEQ(*cli_fd, -1, "connect_to_fd_opts")) 50 goto err; 51 52 return true; 53 54 err: 55 if (*srv_fd != -1) { 56 close(*srv_fd); 57 *srv_fd = -1; 58 } 59 if (*cli_fd != -1) { 60 close(*cli_fd); 61 *cli_fd = -1; 62 } 63 return false; 64 } 65 66 static void do_test(const struct network_helper_opts *opts) 67 { 68 int lfd = -1, fd = -1; 69 70 if (!start_test(NULL, opts, opts, &lfd, &fd)) 71 goto done; 72 73 ASSERT_OK(send_recv_data(lfd, fd, total_bytes), "send_recv_data"); 74 75 done: 76 if (lfd != -1) 77 close(lfd); 78 if (fd != -1) 79 close(fd); 80 } 81 82 static int cc_cb(int fd, void *opts) 83 { 84 struct cb_opts *cb_opts = (struct cb_opts *)opts; 85 86 return settcpca(fd, cb_opts->cc); 87 } 88 89 static void test_cubic(void) 90 { 91 struct cb_opts cb_opts = { 92 .cc = "bpf_cubic", 93 }; 94 struct network_helper_opts opts = { 95 .post_socket_cb = cc_cb, 96 .cb_opts = &cb_opts, 97 }; 98 struct bpf_cubic *cubic_skel; 99 struct bpf_link *link; 100 101 cubic_skel = bpf_cubic__open_and_load(); 102 if (!ASSERT_OK_PTR(cubic_skel, "bpf_cubic__open_and_load")) 103 return; 104 105 link = bpf_map__attach_struct_ops(cubic_skel->maps.cubic); 106 if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 107 bpf_cubic__destroy(cubic_skel); 108 return; 109 } 110 111 do_test(&opts); 112 113 ASSERT_EQ(cubic_skel->bss->bpf_cubic_acked_called, 1, "pkts_acked called"); 114 115 bpf_link__destroy(link); 116 bpf_cubic__destroy(cubic_skel); 117 } 118 119 static int stg_post_socket_cb(int fd, void *opts) 120 { 121 struct cb_opts *cb_opts = (struct cb_opts *)opts; 122 int err; 123 124 err = settcpca(fd, cb_opts->cc); 125 if (err) 126 return err; 127 128 err = bpf_map_update_elem(cb_opts->map_fd, &fd, 129 &expected_stg, BPF_NOEXIST); 130 if (!ASSERT_OK(err, "bpf_map_update_elem(sk_stg_map)")) 131 return err; 132 133 return 0; 134 } 135 136 static void test_dctcp(void) 137 { 138 struct cb_opts cb_opts = { 139 .cc = "bpf_dctcp", 140 }; 141 struct network_helper_opts opts = { 142 .post_socket_cb = cc_cb, 143 .cb_opts = &cb_opts, 144 }; 145 struct network_helper_opts cli_opts = { 146 .post_socket_cb = stg_post_socket_cb, 147 .cb_opts = &cb_opts, 148 }; 149 int lfd = -1, fd = -1, tmp_stg, err; 150 struct bpf_dctcp *dctcp_skel; 151 struct bpf_link *link; 152 153 dctcp_skel = bpf_dctcp__open_and_load(); 154 if (!ASSERT_OK_PTR(dctcp_skel, "bpf_dctcp__open_and_load")) 155 return; 156 157 link = bpf_map__attach_struct_ops(dctcp_skel->maps.dctcp); 158 if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 159 bpf_dctcp__destroy(dctcp_skel); 160 return; 161 } 162 163 cb_opts.map_fd = bpf_map__fd(dctcp_skel->maps.sk_stg_map); 164 if (!start_test(NULL, &opts, &cli_opts, &lfd, &fd)) 165 goto done; 166 167 err = bpf_map_lookup_elem(cb_opts.map_fd, &fd, &tmp_stg); 168 if (!ASSERT_ERR(err, "bpf_map_lookup_elem(sk_stg_map)") || 169 !ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem(sk_stg_map)")) 170 goto done; 171 172 ASSERT_OK(send_recv_data(lfd, fd, total_bytes), "send_recv_data"); 173 ASSERT_EQ(dctcp_skel->bss->stg_result, expected_stg, "stg_result"); 174 175 done: 176 bpf_link__destroy(link); 177 bpf_dctcp__destroy(dctcp_skel); 178 if (lfd != -1) 179 close(lfd); 180 if (fd != -1) 181 close(fd); 182 } 183 184 static void test_dctcp_autoattach_map(void) 185 { 186 struct cb_opts cb_opts = { 187 .cc = "bpf_dctcp", 188 }; 189 struct network_helper_opts opts = { 190 .post_socket_cb = cc_cb, 191 .cb_opts = &cb_opts, 192 }; 193 struct bpf_dctcp *dctcp_skel; 194 struct bpf_link *link; 195 196 dctcp_skel = bpf_dctcp__open_and_load(); 197 if (!ASSERT_OK_PTR(dctcp_skel, "bpf_dctcp__open_and_load")) 198 return; 199 200 bpf_map__set_autoattach(dctcp_skel->maps.dctcp, true); 201 bpf_map__set_autoattach(dctcp_skel->maps.dctcp_nouse, false); 202 203 if (!ASSERT_OK(bpf_dctcp__attach(dctcp_skel), "bpf_dctcp__attach")) 204 goto destroy; 205 206 /* struct_ops is auto-attached */ 207 link = dctcp_skel->links.dctcp; 208 if (!ASSERT_OK_PTR(link, "link")) 209 goto destroy; 210 211 do_test(&opts); 212 213 destroy: 214 bpf_dctcp__destroy(dctcp_skel); 215 } 216 217 static char *err_str; 218 static bool found; 219 220 static int libbpf_debug_print(enum libbpf_print_level level, 221 const char *format, va_list args) 222 { 223 const char *prog_name, *log_buf; 224 225 if (level != LIBBPF_WARN || 226 !strstr(format, "-- BEGIN PROG LOAD LOG --")) { 227 vprintf(format, args); 228 return 0; 229 } 230 231 prog_name = va_arg(args, char *); 232 log_buf = va_arg(args, char *); 233 if (!log_buf) 234 goto out; 235 if (err_str && strstr(log_buf, err_str) != NULL) 236 found = true; 237 out: 238 printf(format, prog_name, log_buf); 239 return 0; 240 } 241 242 static void test_invalid_license(void) 243 { 244 libbpf_print_fn_t old_print_fn; 245 struct bpf_tcp_nogpl *skel; 246 247 err_str = "struct ops programs must have a GPL compatible license"; 248 found = false; 249 old_print_fn = libbpf_set_print(libbpf_debug_print); 250 251 skel = bpf_tcp_nogpl__open_and_load(); 252 ASSERT_NULL(skel, "bpf_tcp_nogpl"); 253 ASSERT_EQ(found, true, "expected_err_msg"); 254 255 bpf_tcp_nogpl__destroy(skel); 256 libbpf_set_print(old_print_fn); 257 } 258 259 static void test_dctcp_fallback(void) 260 { 261 int err, lfd = -1, cli_fd = -1, srv_fd = -1; 262 struct bpf_dctcp *dctcp_skel; 263 struct bpf_link *link = NULL; 264 struct cb_opts dctcp = { 265 .cc = "bpf_dctcp", 266 }; 267 struct network_helper_opts srv_opts = { 268 .post_socket_cb = cc_cb, 269 .cb_opts = &dctcp, 270 }; 271 struct cb_opts cubic = { 272 .cc = "cubic", 273 }; 274 struct network_helper_opts cli_opts = { 275 .post_socket_cb = cc_cb, 276 .cb_opts = &cubic, 277 }; 278 char srv_cc[16]; 279 socklen_t cc_len = sizeof(srv_cc); 280 281 dctcp_skel = bpf_dctcp__open(); 282 if (!ASSERT_OK_PTR(dctcp_skel, "dctcp_skel")) 283 return; 284 strcpy(dctcp_skel->rodata->fallback_cc, "cubic"); 285 if (!ASSERT_OK(bpf_dctcp__load(dctcp_skel), "bpf_dctcp__load")) 286 goto done; 287 288 link = bpf_map__attach_struct_ops(dctcp_skel->maps.dctcp); 289 if (!ASSERT_OK_PTR(link, "dctcp link")) 290 goto done; 291 292 if (!start_test("::1", &srv_opts, &cli_opts, &lfd, &cli_fd)) 293 goto done; 294 295 srv_fd = accept(lfd, NULL, 0); 296 if (!ASSERT_GE(srv_fd, 0, "srv_fd")) 297 goto done; 298 ASSERT_STREQ(dctcp_skel->bss->cc_res, "cubic", "cc_res"); 299 ASSERT_EQ(dctcp_skel->bss->tcp_cdg_res, -ENOTSUPP, "tcp_cdg_res"); 300 /* All setsockopt(TCP_CONGESTION) in the recurred 301 * bpf_dctcp->init() should fail with -EBUSY. 302 */ 303 ASSERT_EQ(dctcp_skel->bss->ebusy_cnt, 3, "ebusy_cnt"); 304 305 err = getsockopt(srv_fd, SOL_TCP, TCP_CONGESTION, srv_cc, &cc_len); 306 if (!ASSERT_OK(err, "getsockopt(srv_fd, TCP_CONGESTION)")) 307 goto done; 308 ASSERT_STREQ(srv_cc, "cubic", "srv_fd cc"); 309 310 done: 311 bpf_link__destroy(link); 312 bpf_dctcp__destroy(dctcp_skel); 313 if (lfd != -1) 314 close(lfd); 315 if (srv_fd != -1) 316 close(srv_fd); 317 if (cli_fd != -1) 318 close(cli_fd); 319 } 320 321 static void test_rel_setsockopt(void) 322 { 323 struct bpf_dctcp_release *rel_skel; 324 libbpf_print_fn_t old_print_fn; 325 326 err_str = "program of this type cannot use helper bpf_setsockopt"; 327 found = false; 328 329 old_print_fn = libbpf_set_print(libbpf_debug_print); 330 rel_skel = bpf_dctcp_release__open_and_load(); 331 libbpf_set_print(old_print_fn); 332 333 ASSERT_ERR_PTR(rel_skel, "rel_skel"); 334 ASSERT_TRUE(found, "expected_err_msg"); 335 336 bpf_dctcp_release__destroy(rel_skel); 337 } 338 339 static void test_write_sk_pacing(void) 340 { 341 struct tcp_ca_write_sk_pacing *skel; 342 struct bpf_link *link; 343 344 skel = tcp_ca_write_sk_pacing__open_and_load(); 345 if (!ASSERT_OK_PTR(skel, "open_and_load")) 346 return; 347 348 link = bpf_map__attach_struct_ops(skel->maps.write_sk_pacing); 349 ASSERT_OK_PTR(link, "attach_struct_ops"); 350 351 bpf_link__destroy(link); 352 tcp_ca_write_sk_pacing__destroy(skel); 353 } 354 355 static void test_incompl_cong_ops(void) 356 { 357 struct tcp_ca_incompl_cong_ops *skel; 358 struct bpf_link *link; 359 360 skel = tcp_ca_incompl_cong_ops__open_and_load(); 361 if (!ASSERT_OK_PTR(skel, "open_and_load")) 362 return; 363 364 /* That cong_avoid() and cong_control() are missing is only reported at 365 * this point: 366 */ 367 link = bpf_map__attach_struct_ops(skel->maps.incompl_cong_ops); 368 ASSERT_ERR_PTR(link, "attach_struct_ops"); 369 370 bpf_link__destroy(link); 371 tcp_ca_incompl_cong_ops__destroy(skel); 372 } 373 374 static void test_unsupp_cong_op(void) 375 { 376 libbpf_print_fn_t old_print_fn; 377 struct tcp_ca_unsupp_cong_op *skel; 378 379 err_str = "attach to unsupported member get_info"; 380 found = false; 381 old_print_fn = libbpf_set_print(libbpf_debug_print); 382 383 skel = tcp_ca_unsupp_cong_op__open_and_load(); 384 ASSERT_NULL(skel, "open_and_load"); 385 ASSERT_EQ(found, true, "expected_err_msg"); 386 387 tcp_ca_unsupp_cong_op__destroy(skel); 388 libbpf_set_print(old_print_fn); 389 } 390 391 static void test_update_ca(void) 392 { 393 struct cb_opts cb_opts = { 394 .cc = "tcp_ca_update", 395 }; 396 struct network_helper_opts opts = { 397 .post_socket_cb = cc_cb, 398 .cb_opts = &cb_opts, 399 }; 400 struct tcp_ca_update *skel; 401 struct bpf_link *link; 402 int saved_ca1_cnt; 403 int err; 404 405 skel = tcp_ca_update__open_and_load(); 406 if (!ASSERT_OK_PTR(skel, "open")) 407 return; 408 409 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 410 if (!ASSERT_OK_PTR(link, "attach_struct_ops")) 411 goto out; 412 413 do_test(&opts); 414 saved_ca1_cnt = skel->bss->ca1_cnt; 415 ASSERT_GT(saved_ca1_cnt, 0, "ca1_ca1_cnt"); 416 417 err = bpf_link__update_map(link, skel->maps.ca_update_2); 418 ASSERT_OK(err, "update_map"); 419 420 do_test(&opts); 421 ASSERT_EQ(skel->bss->ca1_cnt, saved_ca1_cnt, "ca2_ca1_cnt"); 422 ASSERT_GT(skel->bss->ca2_cnt, 0, "ca2_ca2_cnt"); 423 424 bpf_link__destroy(link); 425 out: 426 tcp_ca_update__destroy(skel); 427 } 428 429 static void test_update_wrong(void) 430 { 431 struct cb_opts cb_opts = { 432 .cc = "tcp_ca_update", 433 }; 434 struct network_helper_opts opts = { 435 .post_socket_cb = cc_cb, 436 .cb_opts = &cb_opts, 437 }; 438 struct tcp_ca_update *skel; 439 struct bpf_link *link; 440 int saved_ca1_cnt; 441 int err; 442 443 skel = tcp_ca_update__open_and_load(); 444 if (!ASSERT_OK_PTR(skel, "open")) 445 return; 446 447 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 448 if (!ASSERT_OK_PTR(link, "attach_struct_ops")) 449 goto out; 450 451 do_test(&opts); 452 saved_ca1_cnt = skel->bss->ca1_cnt; 453 ASSERT_GT(saved_ca1_cnt, 0, "ca1_ca1_cnt"); 454 455 err = bpf_link__update_map(link, skel->maps.ca_wrong); 456 ASSERT_ERR(err, "update_map"); 457 458 do_test(&opts); 459 ASSERT_GT(skel->bss->ca1_cnt, saved_ca1_cnt, "ca2_ca1_cnt"); 460 461 bpf_link__destroy(link); 462 out: 463 tcp_ca_update__destroy(skel); 464 } 465 466 static void test_mixed_links(void) 467 { 468 struct cb_opts cb_opts = { 469 .cc = "tcp_ca_update", 470 }; 471 struct network_helper_opts opts = { 472 .post_socket_cb = cc_cb, 473 .cb_opts = &cb_opts, 474 }; 475 struct tcp_ca_update *skel; 476 struct bpf_link *link, *link_nl; 477 int err; 478 479 skel = tcp_ca_update__open_and_load(); 480 if (!ASSERT_OK_PTR(skel, "open")) 481 return; 482 483 link_nl = bpf_map__attach_struct_ops(skel->maps.ca_no_link); 484 if (!ASSERT_OK_PTR(link_nl, "attach_struct_ops_nl")) 485 goto out; 486 487 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 488 ASSERT_OK_PTR(link, "attach_struct_ops"); 489 490 do_test(&opts); 491 ASSERT_GT(skel->bss->ca1_cnt, 0, "ca1_ca1_cnt"); 492 493 err = bpf_link__update_map(link, skel->maps.ca_no_link); 494 ASSERT_ERR(err, "update_map"); 495 496 bpf_link__destroy(link); 497 bpf_link__destroy(link_nl); 498 out: 499 tcp_ca_update__destroy(skel); 500 } 501 502 static void test_multi_links(void) 503 { 504 struct tcp_ca_update *skel; 505 struct bpf_link *link; 506 507 skel = tcp_ca_update__open_and_load(); 508 if (!ASSERT_OK_PTR(skel, "open")) 509 return; 510 511 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 512 ASSERT_OK_PTR(link, "attach_struct_ops_1st"); 513 bpf_link__destroy(link); 514 515 /* A map should be able to be used to create links multiple 516 * times. 517 */ 518 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 519 ASSERT_OK_PTR(link, "attach_struct_ops_2nd"); 520 bpf_link__destroy(link); 521 522 tcp_ca_update__destroy(skel); 523 } 524 525 static void test_link_replace(void) 526 { 527 DECLARE_LIBBPF_OPTS(bpf_link_update_opts, opts); 528 struct tcp_ca_update *skel; 529 struct bpf_link *link; 530 int err; 531 532 skel = tcp_ca_update__open_and_load(); 533 if (!ASSERT_OK_PTR(skel, "open")) 534 return; 535 536 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 537 ASSERT_OK_PTR(link, "attach_struct_ops_1st"); 538 bpf_link__destroy(link); 539 540 link = bpf_map__attach_struct_ops(skel->maps.ca_update_2); 541 if (!ASSERT_OK_PTR(link, "attach_struct_ops_2nd")) 542 goto out; 543 544 /* BPF_F_REPLACE with a wrong old map Fd. It should fail! 545 * 546 * With BPF_F_REPLACE, the link should be updated only if the 547 * old map fd given here matches the map backing the link. 548 */ 549 opts.old_map_fd = bpf_map__fd(skel->maps.ca_update_1); 550 opts.flags = BPF_F_REPLACE; 551 err = bpf_link_update(bpf_link__fd(link), 552 bpf_map__fd(skel->maps.ca_update_1), 553 &opts); 554 ASSERT_ERR(err, "bpf_link_update_fail"); 555 556 /* BPF_F_REPLACE with a correct old map Fd. It should success! */ 557 opts.old_map_fd = bpf_map__fd(skel->maps.ca_update_2); 558 err = bpf_link_update(bpf_link__fd(link), 559 bpf_map__fd(skel->maps.ca_update_1), 560 &opts); 561 ASSERT_OK(err, "bpf_link_update_success"); 562 563 bpf_link__destroy(link); 564 565 out: 566 tcp_ca_update__destroy(skel); 567 } 568 569 static void test_tcp_ca_kfunc(void) 570 { 571 struct tcp_ca_kfunc *skel; 572 573 skel = tcp_ca_kfunc__open_and_load(); 574 ASSERT_OK_PTR(skel, "tcp_ca_kfunc__open_and_load"); 575 tcp_ca_kfunc__destroy(skel); 576 } 577 578 static void test_cc_cubic(void) 579 { 580 struct cb_opts cb_opts = { 581 .cc = "bpf_cc_cubic", 582 }; 583 struct network_helper_opts opts = { 584 .post_socket_cb = cc_cb, 585 .cb_opts = &cb_opts, 586 }; 587 struct bpf_cc_cubic *cc_cubic_skel; 588 struct bpf_link *link; 589 590 cc_cubic_skel = bpf_cc_cubic__open_and_load(); 591 if (!ASSERT_OK_PTR(cc_cubic_skel, "bpf_cc_cubic__open_and_load")) 592 return; 593 594 link = bpf_map__attach_struct_ops(cc_cubic_skel->maps.cc_cubic); 595 if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 596 bpf_cc_cubic__destroy(cc_cubic_skel); 597 return; 598 } 599 600 do_test(&opts); 601 602 bpf_link__destroy(link); 603 bpf_cc_cubic__destroy(cc_cubic_skel); 604 } 605 606 void test_bpf_tcp_ca(void) 607 { 608 if (test__start_subtest("dctcp")) 609 test_dctcp(); 610 if (test__start_subtest("cubic")) 611 test_cubic(); 612 if (test__start_subtest("invalid_license")) 613 test_invalid_license(); 614 if (test__start_subtest("dctcp_fallback")) 615 test_dctcp_fallback(); 616 if (test__start_subtest("rel_setsockopt")) 617 test_rel_setsockopt(); 618 if (test__start_subtest("write_sk_pacing")) 619 test_write_sk_pacing(); 620 if (test__start_subtest("incompl_cong_ops")) 621 test_incompl_cong_ops(); 622 if (test__start_subtest("unsupp_cong_op")) 623 test_unsupp_cong_op(); 624 if (test__start_subtest("update_ca")) 625 test_update_ca(); 626 if (test__start_subtest("update_wrong")) 627 test_update_wrong(); 628 if (test__start_subtest("mixed_links")) 629 test_mixed_links(); 630 if (test__start_subtest("multi_links")) 631 test_multi_links(); 632 if (test__start_subtest("link_replace")) 633 test_link_replace(); 634 if (test__start_subtest("tcp_ca_kfunc")) 635 test_tcp_ca_kfunc(); 636 if (test__start_subtest("cc_cubic")) 637 test_cc_cubic(); 638 if (test__start_subtest("dctcp_autoattach_map")) 639 test_dctcp_autoattach_map(); 640 } 641