1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 Facebook */ 3 4 #include <linux/err.h> 5 #include <netinet/tcp.h> 6 #include <test_progs.h> 7 #include "network_helpers.h" 8 #include "bpf_dctcp.skel.h" 9 #include "bpf_cubic.skel.h" 10 #include "bpf_tcp_nogpl.skel.h" 11 #include "tcp_ca_update.skel.h" 12 #include "bpf_dctcp_release.skel.h" 13 #include "tcp_ca_write_sk_pacing.skel.h" 14 #include "tcp_ca_incompl_cong_ops.skel.h" 15 #include "tcp_ca_unsupp_cong_op.skel.h" 16 #include "tcp_ca_kfunc.skel.h" 17 #include "bpf_cc_cubic.skel.h" 18 19 #ifndef ENOTSUPP 20 #define ENOTSUPP 524 21 #endif 22 23 static const unsigned int total_bytes = 10 * 1024 * 1024; 24 static int expected_stg = 0xeB9F; 25 26 struct cb_opts { 27 const char *cc; 28 int map_fd; 29 }; 30 31 static int settcpca(int fd, const char *tcp_ca) 32 { 33 int err; 34 35 err = setsockopt(fd, IPPROTO_TCP, TCP_CONGESTION, tcp_ca, strlen(tcp_ca)); 36 if (!ASSERT_NEQ(err, -1, "setsockopt")) 37 return -1; 38 39 return 0; 40 } 41 42 static bool start_test(char *addr_str, 43 const struct network_helper_opts *srv_opts, 44 const struct network_helper_opts *cli_opts, 45 int *srv_fd, int *cli_fd) 46 { 47 *srv_fd = start_server_str(AF_INET6, SOCK_STREAM, addr_str, 0, srv_opts); 48 if (!ASSERT_NEQ(*srv_fd, -1, "start_server_str")) 49 goto err; 50 51 /* connect to server */ 52 *cli_fd = connect_to_fd_opts(*srv_fd, SOCK_STREAM, cli_opts); 53 if (!ASSERT_NEQ(*cli_fd, -1, "connect_to_fd_opts")) 54 goto err; 55 56 return true; 57 58 err: 59 if (*srv_fd != -1) { 60 close(*srv_fd); 61 *srv_fd = -1; 62 } 63 if (*cli_fd != -1) { 64 close(*cli_fd); 65 *cli_fd = -1; 66 } 67 return false; 68 } 69 70 static void do_test(const struct network_helper_opts *opts) 71 { 72 int lfd = -1, fd = -1; 73 74 if (!start_test(NULL, opts, opts, &lfd, &fd)) 75 goto done; 76 77 ASSERT_OK(send_recv_data(lfd, fd, total_bytes), "send_recv_data"); 78 79 done: 80 if (lfd != -1) 81 close(lfd); 82 if (fd != -1) 83 close(fd); 84 } 85 86 static int cc_cb(int fd, void *opts) 87 { 88 struct cb_opts *cb_opts = (struct cb_opts *)opts; 89 90 return settcpca(fd, cb_opts->cc); 91 } 92 93 static void test_cubic(void) 94 { 95 struct cb_opts cb_opts = { 96 .cc = "bpf_cubic", 97 }; 98 struct network_helper_opts opts = { 99 .post_socket_cb = cc_cb, 100 .cb_opts = &cb_opts, 101 }; 102 struct bpf_cubic *cubic_skel; 103 struct bpf_link *link; 104 105 cubic_skel = bpf_cubic__open_and_load(); 106 if (!ASSERT_OK_PTR(cubic_skel, "bpf_cubic__open_and_load")) 107 return; 108 109 link = bpf_map__attach_struct_ops(cubic_skel->maps.cubic); 110 if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 111 bpf_cubic__destroy(cubic_skel); 112 return; 113 } 114 115 do_test(&opts); 116 117 ASSERT_EQ(cubic_skel->bss->bpf_cubic_acked_called, 1, "pkts_acked called"); 118 119 bpf_link__destroy(link); 120 bpf_cubic__destroy(cubic_skel); 121 } 122 123 static int stg_post_socket_cb(int fd, void *opts) 124 { 125 struct cb_opts *cb_opts = (struct cb_opts *)opts; 126 int err; 127 128 err = settcpca(fd, cb_opts->cc); 129 if (err) 130 return err; 131 132 err = bpf_map_update_elem(cb_opts->map_fd, &fd, 133 &expected_stg, BPF_NOEXIST); 134 if (!ASSERT_OK(err, "bpf_map_update_elem(sk_stg_map)")) 135 return err; 136 137 return 0; 138 } 139 140 static void test_dctcp(void) 141 { 142 struct cb_opts cb_opts = { 143 .cc = "bpf_dctcp", 144 }; 145 struct network_helper_opts opts = { 146 .post_socket_cb = cc_cb, 147 .cb_opts = &cb_opts, 148 }; 149 struct network_helper_opts cli_opts = { 150 .post_socket_cb = stg_post_socket_cb, 151 .cb_opts = &cb_opts, 152 }; 153 int lfd = -1, fd = -1, tmp_stg, err; 154 struct bpf_dctcp *dctcp_skel; 155 struct bpf_link *link; 156 157 dctcp_skel = bpf_dctcp__open_and_load(); 158 if (!ASSERT_OK_PTR(dctcp_skel, "bpf_dctcp__open_and_load")) 159 return; 160 161 link = bpf_map__attach_struct_ops(dctcp_skel->maps.dctcp); 162 if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 163 bpf_dctcp__destroy(dctcp_skel); 164 return; 165 } 166 167 cb_opts.map_fd = bpf_map__fd(dctcp_skel->maps.sk_stg_map); 168 if (!start_test(NULL, &opts, &cli_opts, &lfd, &fd)) 169 goto done; 170 171 err = bpf_map_lookup_elem(cb_opts.map_fd, &fd, &tmp_stg); 172 if (!ASSERT_ERR(err, "bpf_map_lookup_elem(sk_stg_map)") || 173 !ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem(sk_stg_map)")) 174 goto done; 175 176 ASSERT_OK(send_recv_data(lfd, fd, total_bytes), "send_recv_data"); 177 ASSERT_EQ(dctcp_skel->bss->stg_result, expected_stg, "stg_result"); 178 179 done: 180 bpf_link__destroy(link); 181 bpf_dctcp__destroy(dctcp_skel); 182 if (lfd != -1) 183 close(lfd); 184 if (fd != -1) 185 close(fd); 186 } 187 188 static void test_dctcp_autoattach_map(void) 189 { 190 struct cb_opts cb_opts = { 191 .cc = "bpf_dctcp", 192 }; 193 struct network_helper_opts opts = { 194 .post_socket_cb = cc_cb, 195 .cb_opts = &cb_opts, 196 }; 197 struct bpf_dctcp *dctcp_skel; 198 struct bpf_link *link; 199 200 dctcp_skel = bpf_dctcp__open_and_load(); 201 if (!ASSERT_OK_PTR(dctcp_skel, "bpf_dctcp__open_and_load")) 202 return; 203 204 bpf_map__set_autoattach(dctcp_skel->maps.dctcp, true); 205 bpf_map__set_autoattach(dctcp_skel->maps.dctcp_nouse, false); 206 207 if (!ASSERT_OK(bpf_dctcp__attach(dctcp_skel), "bpf_dctcp__attach")) 208 goto destroy; 209 210 /* struct_ops is auto-attached */ 211 link = dctcp_skel->links.dctcp; 212 if (!ASSERT_OK_PTR(link, "link")) 213 goto destroy; 214 215 do_test(&opts); 216 217 destroy: 218 bpf_dctcp__destroy(dctcp_skel); 219 } 220 221 static char *err_str; 222 static bool found; 223 224 static int libbpf_debug_print(enum libbpf_print_level level, 225 const char *format, va_list args) 226 { 227 const char *prog_name, *log_buf; 228 229 if (level != LIBBPF_WARN || 230 !strstr(format, "-- BEGIN PROG LOAD LOG --")) { 231 vprintf(format, args); 232 return 0; 233 } 234 235 prog_name = va_arg(args, char *); 236 log_buf = va_arg(args, char *); 237 if (!log_buf) 238 goto out; 239 if (err_str && strstr(log_buf, err_str) != NULL) 240 found = true; 241 out: 242 printf(format, prog_name, log_buf); 243 return 0; 244 } 245 246 static void test_invalid_license(void) 247 { 248 libbpf_print_fn_t old_print_fn; 249 struct bpf_tcp_nogpl *skel; 250 251 err_str = "struct ops programs must have a GPL compatible license"; 252 found = false; 253 old_print_fn = libbpf_set_print(libbpf_debug_print); 254 255 skel = bpf_tcp_nogpl__open_and_load(); 256 ASSERT_NULL(skel, "bpf_tcp_nogpl"); 257 ASSERT_EQ(found, true, "expected_err_msg"); 258 259 bpf_tcp_nogpl__destroy(skel); 260 libbpf_set_print(old_print_fn); 261 } 262 263 static void test_dctcp_fallback(void) 264 { 265 int err, lfd = -1, cli_fd = -1, srv_fd = -1; 266 struct bpf_dctcp *dctcp_skel; 267 struct bpf_link *link = NULL; 268 struct cb_opts dctcp = { 269 .cc = "bpf_dctcp", 270 }; 271 struct network_helper_opts srv_opts = { 272 .post_socket_cb = cc_cb, 273 .cb_opts = &dctcp, 274 }; 275 struct cb_opts cubic = { 276 .cc = "cubic", 277 }; 278 struct network_helper_opts cli_opts = { 279 .post_socket_cb = cc_cb, 280 .cb_opts = &cubic, 281 }; 282 char srv_cc[16]; 283 socklen_t cc_len = sizeof(srv_cc); 284 285 dctcp_skel = bpf_dctcp__open(); 286 if (!ASSERT_OK_PTR(dctcp_skel, "dctcp_skel")) 287 return; 288 strcpy(dctcp_skel->rodata->fallback, "cubic"); 289 if (!ASSERT_OK(bpf_dctcp__load(dctcp_skel), "bpf_dctcp__load")) 290 goto done; 291 292 link = bpf_map__attach_struct_ops(dctcp_skel->maps.dctcp); 293 if (!ASSERT_OK_PTR(link, "dctcp link")) 294 goto done; 295 296 if (!start_test("::1", &srv_opts, &cli_opts, &lfd, &cli_fd)) 297 goto done; 298 299 srv_fd = accept(lfd, NULL, 0); 300 if (!ASSERT_GE(srv_fd, 0, "srv_fd")) 301 goto done; 302 ASSERT_STREQ(dctcp_skel->bss->cc_res, "cubic", "cc_res"); 303 ASSERT_EQ(dctcp_skel->bss->tcp_cdg_res, -ENOTSUPP, "tcp_cdg_res"); 304 /* All setsockopt(TCP_CONGESTION) in the recurred 305 * bpf_dctcp->init() should fail with -EBUSY. 306 */ 307 ASSERT_EQ(dctcp_skel->bss->ebusy_cnt, 3, "ebusy_cnt"); 308 309 err = getsockopt(srv_fd, SOL_TCP, TCP_CONGESTION, srv_cc, &cc_len); 310 if (!ASSERT_OK(err, "getsockopt(srv_fd, TCP_CONGESTION)")) 311 goto done; 312 ASSERT_STREQ(srv_cc, "cubic", "srv_fd cc"); 313 314 done: 315 bpf_link__destroy(link); 316 bpf_dctcp__destroy(dctcp_skel); 317 if (lfd != -1) 318 close(lfd); 319 if (srv_fd != -1) 320 close(srv_fd); 321 if (cli_fd != -1) 322 close(cli_fd); 323 } 324 325 static void test_rel_setsockopt(void) 326 { 327 struct bpf_dctcp_release *rel_skel; 328 libbpf_print_fn_t old_print_fn; 329 330 err_str = "program of this type cannot use helper bpf_setsockopt"; 331 found = false; 332 333 old_print_fn = libbpf_set_print(libbpf_debug_print); 334 rel_skel = bpf_dctcp_release__open_and_load(); 335 libbpf_set_print(old_print_fn); 336 337 ASSERT_ERR_PTR(rel_skel, "rel_skel"); 338 ASSERT_TRUE(found, "expected_err_msg"); 339 340 bpf_dctcp_release__destroy(rel_skel); 341 } 342 343 static void test_write_sk_pacing(void) 344 { 345 struct tcp_ca_write_sk_pacing *skel; 346 struct bpf_link *link; 347 348 skel = tcp_ca_write_sk_pacing__open_and_load(); 349 if (!ASSERT_OK_PTR(skel, "open_and_load")) 350 return; 351 352 link = bpf_map__attach_struct_ops(skel->maps.write_sk_pacing); 353 ASSERT_OK_PTR(link, "attach_struct_ops"); 354 355 bpf_link__destroy(link); 356 tcp_ca_write_sk_pacing__destroy(skel); 357 } 358 359 static void test_incompl_cong_ops(void) 360 { 361 struct tcp_ca_incompl_cong_ops *skel; 362 struct bpf_link *link; 363 364 skel = tcp_ca_incompl_cong_ops__open_and_load(); 365 if (!ASSERT_OK_PTR(skel, "open_and_load")) 366 return; 367 368 /* That cong_avoid() and cong_control() are missing is only reported at 369 * this point: 370 */ 371 link = bpf_map__attach_struct_ops(skel->maps.incompl_cong_ops); 372 ASSERT_ERR_PTR(link, "attach_struct_ops"); 373 374 bpf_link__destroy(link); 375 tcp_ca_incompl_cong_ops__destroy(skel); 376 } 377 378 static void test_unsupp_cong_op(void) 379 { 380 libbpf_print_fn_t old_print_fn; 381 struct tcp_ca_unsupp_cong_op *skel; 382 383 err_str = "attach to unsupported member get_info"; 384 found = false; 385 old_print_fn = libbpf_set_print(libbpf_debug_print); 386 387 skel = tcp_ca_unsupp_cong_op__open_and_load(); 388 ASSERT_NULL(skel, "open_and_load"); 389 ASSERT_EQ(found, true, "expected_err_msg"); 390 391 tcp_ca_unsupp_cong_op__destroy(skel); 392 libbpf_set_print(old_print_fn); 393 } 394 395 static void test_update_ca(void) 396 { 397 struct cb_opts cb_opts = { 398 .cc = "tcp_ca_update", 399 }; 400 struct network_helper_opts opts = { 401 .post_socket_cb = cc_cb, 402 .cb_opts = &cb_opts, 403 }; 404 struct tcp_ca_update *skel; 405 struct bpf_link *link; 406 int saved_ca1_cnt; 407 int err; 408 409 skel = tcp_ca_update__open_and_load(); 410 if (!ASSERT_OK_PTR(skel, "open")) 411 return; 412 413 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 414 if (!ASSERT_OK_PTR(link, "attach_struct_ops")) 415 goto out; 416 417 do_test(&opts); 418 saved_ca1_cnt = skel->bss->ca1_cnt; 419 ASSERT_GT(saved_ca1_cnt, 0, "ca1_ca1_cnt"); 420 421 err = bpf_link__update_map(link, skel->maps.ca_update_2); 422 ASSERT_OK(err, "update_map"); 423 424 do_test(&opts); 425 ASSERT_EQ(skel->bss->ca1_cnt, saved_ca1_cnt, "ca2_ca1_cnt"); 426 ASSERT_GT(skel->bss->ca2_cnt, 0, "ca2_ca2_cnt"); 427 428 bpf_link__destroy(link); 429 out: 430 tcp_ca_update__destroy(skel); 431 } 432 433 static void test_update_wrong(void) 434 { 435 struct cb_opts cb_opts = { 436 .cc = "tcp_ca_update", 437 }; 438 struct network_helper_opts opts = { 439 .post_socket_cb = cc_cb, 440 .cb_opts = &cb_opts, 441 }; 442 struct tcp_ca_update *skel; 443 struct bpf_link *link; 444 int saved_ca1_cnt; 445 int err; 446 447 skel = tcp_ca_update__open_and_load(); 448 if (!ASSERT_OK_PTR(skel, "open")) 449 return; 450 451 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 452 if (!ASSERT_OK_PTR(link, "attach_struct_ops")) 453 goto out; 454 455 do_test(&opts); 456 saved_ca1_cnt = skel->bss->ca1_cnt; 457 ASSERT_GT(saved_ca1_cnt, 0, "ca1_ca1_cnt"); 458 459 err = bpf_link__update_map(link, skel->maps.ca_wrong); 460 ASSERT_ERR(err, "update_map"); 461 462 do_test(&opts); 463 ASSERT_GT(skel->bss->ca1_cnt, saved_ca1_cnt, "ca2_ca1_cnt"); 464 465 bpf_link__destroy(link); 466 out: 467 tcp_ca_update__destroy(skel); 468 } 469 470 static void test_mixed_links(void) 471 { 472 struct cb_opts cb_opts = { 473 .cc = "tcp_ca_update", 474 }; 475 struct network_helper_opts opts = { 476 .post_socket_cb = cc_cb, 477 .cb_opts = &cb_opts, 478 }; 479 struct tcp_ca_update *skel; 480 struct bpf_link *link, *link_nl; 481 int err; 482 483 skel = tcp_ca_update__open_and_load(); 484 if (!ASSERT_OK_PTR(skel, "open")) 485 return; 486 487 link_nl = bpf_map__attach_struct_ops(skel->maps.ca_no_link); 488 if (!ASSERT_OK_PTR(link_nl, "attach_struct_ops_nl")) 489 goto out; 490 491 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 492 ASSERT_OK_PTR(link, "attach_struct_ops"); 493 494 do_test(&opts); 495 ASSERT_GT(skel->bss->ca1_cnt, 0, "ca1_ca1_cnt"); 496 497 err = bpf_link__update_map(link, skel->maps.ca_no_link); 498 ASSERT_ERR(err, "update_map"); 499 500 bpf_link__destroy(link); 501 bpf_link__destroy(link_nl); 502 out: 503 tcp_ca_update__destroy(skel); 504 } 505 506 static void test_multi_links(void) 507 { 508 struct tcp_ca_update *skel; 509 struct bpf_link *link; 510 511 skel = tcp_ca_update__open_and_load(); 512 if (!ASSERT_OK_PTR(skel, "open")) 513 return; 514 515 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 516 ASSERT_OK_PTR(link, "attach_struct_ops_1st"); 517 bpf_link__destroy(link); 518 519 /* A map should be able to be used to create links multiple 520 * times. 521 */ 522 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 523 ASSERT_OK_PTR(link, "attach_struct_ops_2nd"); 524 bpf_link__destroy(link); 525 526 tcp_ca_update__destroy(skel); 527 } 528 529 static void test_link_replace(void) 530 { 531 DECLARE_LIBBPF_OPTS(bpf_link_update_opts, opts); 532 struct tcp_ca_update *skel; 533 struct bpf_link *link; 534 int err; 535 536 skel = tcp_ca_update__open_and_load(); 537 if (!ASSERT_OK_PTR(skel, "open")) 538 return; 539 540 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 541 ASSERT_OK_PTR(link, "attach_struct_ops_1st"); 542 bpf_link__destroy(link); 543 544 link = bpf_map__attach_struct_ops(skel->maps.ca_update_2); 545 if (!ASSERT_OK_PTR(link, "attach_struct_ops_2nd")) 546 goto out; 547 548 /* BPF_F_REPLACE with a wrong old map Fd. It should fail! 549 * 550 * With BPF_F_REPLACE, the link should be updated only if the 551 * old map fd given here matches the map backing the link. 552 */ 553 opts.old_map_fd = bpf_map__fd(skel->maps.ca_update_1); 554 opts.flags = BPF_F_REPLACE; 555 err = bpf_link_update(bpf_link__fd(link), 556 bpf_map__fd(skel->maps.ca_update_1), 557 &opts); 558 ASSERT_ERR(err, "bpf_link_update_fail"); 559 560 /* BPF_F_REPLACE with a correct old map Fd. It should success! */ 561 opts.old_map_fd = bpf_map__fd(skel->maps.ca_update_2); 562 err = bpf_link_update(bpf_link__fd(link), 563 bpf_map__fd(skel->maps.ca_update_1), 564 &opts); 565 ASSERT_OK(err, "bpf_link_update_success"); 566 567 bpf_link__destroy(link); 568 569 out: 570 tcp_ca_update__destroy(skel); 571 } 572 573 static void test_tcp_ca_kfunc(void) 574 { 575 struct tcp_ca_kfunc *skel; 576 577 skel = tcp_ca_kfunc__open_and_load(); 578 ASSERT_OK_PTR(skel, "tcp_ca_kfunc__open_and_load"); 579 tcp_ca_kfunc__destroy(skel); 580 } 581 582 static void test_cc_cubic(void) 583 { 584 struct cb_opts cb_opts = { 585 .cc = "bpf_cc_cubic", 586 }; 587 struct network_helper_opts opts = { 588 .post_socket_cb = cc_cb, 589 .cb_opts = &cb_opts, 590 }; 591 struct bpf_cc_cubic *cc_cubic_skel; 592 struct bpf_link *link; 593 594 cc_cubic_skel = bpf_cc_cubic__open_and_load(); 595 if (!ASSERT_OK_PTR(cc_cubic_skel, "bpf_cc_cubic__open_and_load")) 596 return; 597 598 link = bpf_map__attach_struct_ops(cc_cubic_skel->maps.cc_cubic); 599 if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 600 bpf_cc_cubic__destroy(cc_cubic_skel); 601 return; 602 } 603 604 do_test(&opts); 605 606 bpf_link__destroy(link); 607 bpf_cc_cubic__destroy(cc_cubic_skel); 608 } 609 610 void test_bpf_tcp_ca(void) 611 { 612 if (test__start_subtest("dctcp")) 613 test_dctcp(); 614 if (test__start_subtest("cubic")) 615 test_cubic(); 616 if (test__start_subtest("invalid_license")) 617 test_invalid_license(); 618 if (test__start_subtest("dctcp_fallback")) 619 test_dctcp_fallback(); 620 if (test__start_subtest("rel_setsockopt")) 621 test_rel_setsockopt(); 622 if (test__start_subtest("write_sk_pacing")) 623 test_write_sk_pacing(); 624 if (test__start_subtest("incompl_cong_ops")) 625 test_incompl_cong_ops(); 626 if (test__start_subtest("unsupp_cong_op")) 627 test_unsupp_cong_op(); 628 if (test__start_subtest("update_ca")) 629 test_update_ca(); 630 if (test__start_subtest("update_wrong")) 631 test_update_wrong(); 632 if (test__start_subtest("mixed_links")) 633 test_mixed_links(); 634 if (test__start_subtest("multi_links")) 635 test_multi_links(); 636 if (test__start_subtest("link_replace")) 637 test_link_replace(); 638 if (test__start_subtest("tcp_ca_kfunc")) 639 test_tcp_ca_kfunc(); 640 if (test__start_subtest("cc_cubic")) 641 test_cc_cubic(); 642 if (test__start_subtest("dctcp_autoattach_map")) 643 test_dctcp_autoattach_map(); 644 } 645