1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2020 Cloudflare 3 #include <error.h> 4 #include <netinet/tcp.h> 5 #include <sys/epoll.h> 6 7 #include "test_progs.h" 8 #include "test_skmsg_load_helpers.skel.h" 9 #include "test_sockmap_update.skel.h" 10 #include "test_sockmap_invalid_update.skel.h" 11 #include "test_sockmap_skb_verdict_attach.skel.h" 12 #include "test_sockmap_progs_query.skel.h" 13 #include "test_sockmap_pass_prog.skel.h" 14 #include "test_sockmap_drop_prog.skel.h" 15 #include "bpf_iter_sockmap.skel.h" 16 17 #include "sockmap_helpers.h" 18 19 #define TCP_REPAIR 19 /* TCP sock is under repair right now */ 20 21 #define TCP_REPAIR_ON 1 22 #define TCP_REPAIR_OFF_NO_WP -1 /* Turn off without window probes */ 23 24 static int connected_socket_v4(void) 25 { 26 struct sockaddr_in addr = { 27 .sin_family = AF_INET, 28 .sin_port = htons(80), 29 .sin_addr = { inet_addr("127.0.0.1") }, 30 }; 31 socklen_t len = sizeof(addr); 32 int s, repair, err; 33 34 s = socket(AF_INET, SOCK_STREAM, 0); 35 if (!ASSERT_GE(s, 0, "socket")) 36 goto error; 37 38 repair = TCP_REPAIR_ON; 39 err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair)); 40 if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)")) 41 goto error; 42 43 err = connect(s, (struct sockaddr *)&addr, len); 44 if (!ASSERT_OK(err, "connect")) 45 goto error; 46 47 repair = TCP_REPAIR_OFF_NO_WP; 48 err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair)); 49 if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)")) 50 goto error; 51 52 return s; 53 error: 54 perror(__func__); 55 close(s); 56 return -1; 57 } 58 59 static void compare_cookies(struct bpf_map *src, struct bpf_map *dst) 60 { 61 __u32 i, max_entries = bpf_map__max_entries(src); 62 int err, src_fd, dst_fd; 63 64 src_fd = bpf_map__fd(src); 65 dst_fd = bpf_map__fd(dst); 66 67 for (i = 0; i < max_entries; i++) { 68 __u64 src_cookie, dst_cookie; 69 70 err = bpf_map_lookup_elem(src_fd, &i, &src_cookie); 71 if (err && errno == ENOENT) { 72 err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); 73 ASSERT_ERR(err, "map_lookup_elem(dst)"); 74 ASSERT_EQ(errno, ENOENT, "map_lookup_elem(dst)"); 75 continue; 76 } 77 if (!ASSERT_OK(err, "lookup_elem(src)")) 78 continue; 79 80 err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); 81 if (!ASSERT_OK(err, "lookup_elem(dst)")) 82 continue; 83 84 ASSERT_EQ(dst_cookie, src_cookie, "cookie mismatch"); 85 } 86 } 87 88 /* Create a map, populate it with one socket, and free the map. */ 89 static void test_sockmap_create_update_free(enum bpf_map_type map_type) 90 { 91 const int zero = 0; 92 int s, map, err; 93 94 s = connected_socket_v4(); 95 if (!ASSERT_GE(s, 0, "connected_socket_v4")) 96 return; 97 98 map = bpf_map_create(map_type, NULL, sizeof(int), sizeof(int), 1, NULL); 99 if (!ASSERT_GE(map, 0, "bpf_map_create")) 100 goto out; 101 102 err = bpf_map_update_elem(map, &zero, &s, BPF_NOEXIST); 103 if (!ASSERT_OK(err, "bpf_map_update")) 104 goto out; 105 106 out: 107 close(map); 108 close(s); 109 } 110 111 static void test_sockmap_vsock_delete_on_close(void) 112 { 113 int err, c, p, map; 114 const int zero = 0; 115 116 err = create_pair(AF_VSOCK, SOCK_STREAM, &c, &p); 117 if (!ASSERT_OK(err, "create_pair(AF_VSOCK)")) 118 return; 119 120 map = bpf_map_create(BPF_MAP_TYPE_SOCKMAP, NULL, sizeof(int), 121 sizeof(int), 1, NULL); 122 if (!ASSERT_GE(map, 0, "bpf_map_create")) { 123 close(c); 124 goto out; 125 } 126 127 err = bpf_map_update_elem(map, &zero, &c, BPF_NOEXIST); 128 close(c); 129 if (!ASSERT_OK(err, "bpf_map_update")) 130 goto out; 131 132 err = bpf_map_update_elem(map, &zero, &p, BPF_NOEXIST); 133 ASSERT_OK(err, "after close(), bpf_map_update"); 134 135 out: 136 close(p); 137 close(map); 138 } 139 140 static void test_skmsg_helpers(enum bpf_map_type map_type) 141 { 142 struct test_skmsg_load_helpers *skel; 143 int err, map, verdict; 144 145 skel = test_skmsg_load_helpers__open_and_load(); 146 if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load")) 147 return; 148 149 verdict = bpf_program__fd(skel->progs.prog_msg_verdict); 150 map = bpf_map__fd(skel->maps.sock_map); 151 152 err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0); 153 if (!ASSERT_OK(err, "bpf_prog_attach")) 154 goto out; 155 156 err = bpf_prog_detach2(verdict, map, BPF_SK_MSG_VERDICT); 157 if (!ASSERT_OK(err, "bpf_prog_detach2")) 158 goto out; 159 out: 160 test_skmsg_load_helpers__destroy(skel); 161 } 162 163 static void test_skmsg_helpers_with_link(enum bpf_map_type map_type) 164 { 165 struct bpf_program *prog, *prog_clone, *prog_clone2; 166 DECLARE_LIBBPF_OPTS(bpf_link_update_opts, opts); 167 struct test_skmsg_load_helpers *skel; 168 struct bpf_link *link, *link2; 169 int err, map; 170 171 skel = test_skmsg_load_helpers__open_and_load(); 172 if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load")) 173 return; 174 175 prog = skel->progs.prog_msg_verdict; 176 prog_clone = skel->progs.prog_msg_verdict_clone; 177 prog_clone2 = skel->progs.prog_msg_verdict_clone2; 178 map = bpf_map__fd(skel->maps.sock_map); 179 180 link = bpf_program__attach_sockmap(prog, map); 181 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 182 goto out; 183 184 /* Fail since bpf_link for the same prog has been created. */ 185 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_MSG_VERDICT, 0); 186 if (!ASSERT_ERR(err, "bpf_prog_attach")) 187 goto out; 188 189 /* Fail since bpf_link for the same prog type has been created. */ 190 link2 = bpf_program__attach_sockmap(prog_clone, map); 191 if (!ASSERT_ERR_PTR(link2, "bpf_program__attach_sockmap")) { 192 bpf_link__detach(link2); 193 goto out; 194 } 195 196 err = bpf_link__update_program(link, prog_clone); 197 if (!ASSERT_OK(err, "bpf_link__update_program")) 198 goto out; 199 200 /* Fail since a prog with different type attempts to do update. */ 201 err = bpf_link__update_program(link, skel->progs.prog_skb_verdict); 202 if (!ASSERT_ERR(err, "bpf_link__update_program")) 203 goto out; 204 205 /* Fail since the old prog does not match the one in the kernel. */ 206 opts.old_prog_fd = bpf_program__fd(prog_clone2); 207 opts.flags = BPF_F_REPLACE; 208 err = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), &opts); 209 if (!ASSERT_ERR(err, "bpf_link_update")) 210 goto out; 211 212 opts.old_prog_fd = bpf_program__fd(prog_clone); 213 opts.flags = BPF_F_REPLACE; 214 err = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), &opts); 215 if (!ASSERT_OK(err, "bpf_link_update")) 216 goto out; 217 out: 218 bpf_link__detach(link); 219 test_skmsg_load_helpers__destroy(skel); 220 } 221 222 static void test_sockmap_update(enum bpf_map_type map_type) 223 { 224 int err, prog, src; 225 struct test_sockmap_update *skel; 226 struct bpf_map *dst_map; 227 const __u32 zero = 0; 228 char dummy[14] = {0}; 229 LIBBPF_OPTS(bpf_test_run_opts, topts, 230 .data_in = dummy, 231 .data_size_in = sizeof(dummy), 232 .repeat = 1, 233 ); 234 __s64 sk; 235 236 sk = connected_socket_v4(); 237 if (!ASSERT_NEQ(sk, -1, "connected_socket_v4")) 238 return; 239 240 skel = test_sockmap_update__open_and_load(); 241 if (!ASSERT_OK_PTR(skel, "open_and_load")) 242 goto close_sk; 243 244 prog = bpf_program__fd(skel->progs.copy_sock_map); 245 src = bpf_map__fd(skel->maps.src); 246 if (map_type == BPF_MAP_TYPE_SOCKMAP) 247 dst_map = skel->maps.dst_sock_map; 248 else 249 dst_map = skel->maps.dst_sock_hash; 250 251 err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST); 252 if (!ASSERT_OK(err, "update_elem(src)")) 253 goto out; 254 255 err = bpf_prog_test_run_opts(prog, &topts); 256 if (!ASSERT_OK(err, "test_run")) 257 goto out; 258 if (!ASSERT_NEQ(topts.retval, 0, "test_run retval")) 259 goto out; 260 261 compare_cookies(skel->maps.src, dst_map); 262 263 out: 264 test_sockmap_update__destroy(skel); 265 close_sk: 266 close(sk); 267 } 268 269 static void test_sockmap_invalid_update(void) 270 { 271 struct test_sockmap_invalid_update *skel; 272 273 skel = test_sockmap_invalid_update__open_and_load(); 274 if (!ASSERT_NULL(skel, "open_and_load")) 275 test_sockmap_invalid_update__destroy(skel); 276 } 277 278 static void test_sockmap_copy(enum bpf_map_type map_type) 279 { 280 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 281 int err, len, src_fd, iter_fd; 282 union bpf_iter_link_info linfo = {}; 283 __u32 i, num_sockets, num_elems; 284 struct bpf_iter_sockmap *skel; 285 __s64 *sock_fd = NULL; 286 struct bpf_link *link; 287 struct bpf_map *src; 288 char buf[64]; 289 290 skel = bpf_iter_sockmap__open_and_load(); 291 if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load")) 292 return; 293 294 if (map_type == BPF_MAP_TYPE_SOCKMAP) { 295 src = skel->maps.sockmap; 296 num_elems = bpf_map__max_entries(src); 297 num_sockets = num_elems - 1; 298 } else { 299 src = skel->maps.sockhash; 300 num_elems = bpf_map__max_entries(src) - 1; 301 num_sockets = num_elems; 302 } 303 304 sock_fd = calloc(num_sockets, sizeof(*sock_fd)); 305 if (!ASSERT_OK_PTR(sock_fd, "calloc(sock_fd)")) 306 goto out; 307 308 for (i = 0; i < num_sockets; i++) 309 sock_fd[i] = -1; 310 311 src_fd = bpf_map__fd(src); 312 313 for (i = 0; i < num_sockets; i++) { 314 sock_fd[i] = connected_socket_v4(); 315 if (!ASSERT_NEQ(sock_fd[i], -1, "connected_socket_v4")) 316 goto out; 317 318 err = bpf_map_update_elem(src_fd, &i, &sock_fd[i], BPF_NOEXIST); 319 if (!ASSERT_OK(err, "map_update")) 320 goto out; 321 } 322 323 linfo.map.map_fd = src_fd; 324 opts.link_info = &linfo; 325 opts.link_info_len = sizeof(linfo); 326 link = bpf_program__attach_iter(skel->progs.copy, &opts); 327 if (!ASSERT_OK_PTR(link, "attach_iter")) 328 goto out; 329 330 iter_fd = bpf_iter_create(bpf_link__fd(link)); 331 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 332 goto free_link; 333 334 /* do some tests */ 335 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 336 ; 337 if (!ASSERT_GE(len, 0, "read")) 338 goto close_iter; 339 340 /* test results */ 341 if (!ASSERT_EQ(skel->bss->elems, num_elems, "elems")) 342 goto close_iter; 343 344 if (!ASSERT_EQ(skel->bss->socks, num_sockets, "socks")) 345 goto close_iter; 346 347 compare_cookies(src, skel->maps.dst); 348 349 close_iter: 350 close(iter_fd); 351 free_link: 352 bpf_link__destroy(link); 353 out: 354 for (i = 0; sock_fd && i < num_sockets; i++) 355 if (sock_fd[i] >= 0) 356 close(sock_fd[i]); 357 if (sock_fd) 358 free(sock_fd); 359 bpf_iter_sockmap__destroy(skel); 360 } 361 362 static void test_sockmap_skb_verdict_attach(enum bpf_attach_type first, 363 enum bpf_attach_type second) 364 { 365 struct test_sockmap_skb_verdict_attach *skel; 366 int err, map, verdict; 367 368 skel = test_sockmap_skb_verdict_attach__open_and_load(); 369 if (!ASSERT_OK_PTR(skel, "open_and_load")) 370 return; 371 372 verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 373 map = bpf_map__fd(skel->maps.sock_map); 374 375 err = bpf_prog_attach(verdict, map, first, 0); 376 if (!ASSERT_OK(err, "bpf_prog_attach")) 377 goto out; 378 379 err = bpf_prog_attach(verdict, map, second, 0); 380 ASSERT_EQ(err, -EBUSY, "prog_attach_fail"); 381 382 err = bpf_prog_detach2(verdict, map, first); 383 if (!ASSERT_OK(err, "bpf_prog_detach2")) 384 goto out; 385 out: 386 test_sockmap_skb_verdict_attach__destroy(skel); 387 } 388 389 static void test_sockmap_skb_verdict_attach_with_link(void) 390 { 391 struct test_sockmap_skb_verdict_attach *skel; 392 struct bpf_program *prog; 393 struct bpf_link *link; 394 int err, map; 395 396 skel = test_sockmap_skb_verdict_attach__open_and_load(); 397 if (!ASSERT_OK_PTR(skel, "open_and_load")) 398 return; 399 prog = skel->progs.prog_skb_verdict; 400 map = bpf_map__fd(skel->maps.sock_map); 401 link = bpf_program__attach_sockmap(prog, map); 402 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 403 goto out; 404 405 bpf_link__detach(link); 406 407 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0); 408 if (!ASSERT_OK(err, "bpf_prog_attach")) 409 goto out; 410 411 /* Fail since attaching with the same prog/map has been done. */ 412 link = bpf_program__attach_sockmap(prog, map); 413 if (!ASSERT_ERR_PTR(link, "bpf_program__attach_sockmap")) 414 bpf_link__detach(link); 415 416 err = bpf_prog_detach2(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT); 417 if (!ASSERT_OK(err, "bpf_prog_detach2")) 418 goto out; 419 out: 420 test_sockmap_skb_verdict_attach__destroy(skel); 421 } 422 423 static __u32 query_prog_id(int prog_fd) 424 { 425 struct bpf_prog_info info = {}; 426 __u32 info_len = sizeof(info); 427 int err; 428 429 err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); 430 if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd") || 431 !ASSERT_EQ(info_len, sizeof(info), "bpf_prog_get_info_by_fd")) 432 return 0; 433 434 return info.id; 435 } 436 437 static void test_sockmap_progs_query(enum bpf_attach_type attach_type) 438 { 439 struct test_sockmap_progs_query *skel; 440 int err, map_fd, verdict_fd; 441 __u32 attach_flags = 0; 442 __u32 prog_ids[3] = {}; 443 __u32 prog_cnt = 3; 444 445 skel = test_sockmap_progs_query__open_and_load(); 446 if (!ASSERT_OK_PTR(skel, "test_sockmap_progs_query__open_and_load")) 447 return; 448 449 map_fd = bpf_map__fd(skel->maps.sock_map); 450 451 if (attach_type == BPF_SK_MSG_VERDICT) 452 verdict_fd = bpf_program__fd(skel->progs.prog_skmsg_verdict); 453 else 454 verdict_fd = bpf_program__fd(skel->progs.prog_skb_verdict); 455 456 err = bpf_prog_query(map_fd, attach_type, 0 /* query flags */, 457 &attach_flags, prog_ids, &prog_cnt); 458 ASSERT_OK(err, "bpf_prog_query failed"); 459 ASSERT_EQ(attach_flags, 0, "wrong attach_flags on query"); 460 ASSERT_EQ(prog_cnt, 0, "wrong program count on query"); 461 462 err = bpf_prog_attach(verdict_fd, map_fd, attach_type, 0); 463 if (!ASSERT_OK(err, "bpf_prog_attach failed")) 464 goto out; 465 466 prog_cnt = 1; 467 err = bpf_prog_query(map_fd, attach_type, 0 /* query flags */, 468 &attach_flags, prog_ids, &prog_cnt); 469 ASSERT_OK(err, "bpf_prog_query failed"); 470 ASSERT_EQ(attach_flags, 0, "wrong attach_flags on query"); 471 ASSERT_EQ(prog_cnt, 1, "wrong program count on query"); 472 ASSERT_EQ(prog_ids[0], query_prog_id(verdict_fd), 473 "wrong prog_ids on query"); 474 475 bpf_prog_detach2(verdict_fd, map_fd, attach_type); 476 out: 477 test_sockmap_progs_query__destroy(skel); 478 } 479 480 #define MAX_EVENTS 10 481 static void test_sockmap_skb_verdict_shutdown(void) 482 { 483 int n, err, map, verdict, c1 = -1, p1 = -1; 484 struct epoll_event ev, events[MAX_EVENTS]; 485 struct test_sockmap_pass_prog *skel; 486 int zero = 0; 487 int epollfd; 488 char b; 489 490 skel = test_sockmap_pass_prog__open_and_load(); 491 if (!ASSERT_OK_PTR(skel, "open_and_load")) 492 return; 493 494 verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 495 map = bpf_map__fd(skel->maps.sock_map_rx); 496 497 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 498 if (!ASSERT_OK(err, "bpf_prog_attach")) 499 goto out; 500 501 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); 502 if (err < 0) 503 goto out; 504 505 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 506 if (err < 0) 507 goto out_close; 508 509 shutdown(p1, SHUT_WR); 510 511 ev.events = EPOLLIN; 512 ev.data.fd = c1; 513 514 epollfd = epoll_create1(0); 515 if (!ASSERT_GT(epollfd, -1, "epoll_create(0)")) 516 goto out_close; 517 err = epoll_ctl(epollfd, EPOLL_CTL_ADD, c1, &ev); 518 if (!ASSERT_OK(err, "epoll_ctl(EPOLL_CTL_ADD)")) 519 goto out_close; 520 err = epoll_wait(epollfd, events, MAX_EVENTS, -1); 521 if (!ASSERT_EQ(err, 1, "epoll_wait(fd)")) 522 goto out_close; 523 524 n = recv(c1, &b, 1, SOCK_NONBLOCK); 525 ASSERT_EQ(n, 0, "recv_timeout(fin)"); 526 out_close: 527 close(c1); 528 close(p1); 529 out: 530 test_sockmap_pass_prog__destroy(skel); 531 } 532 533 static void test_sockmap_stream_pass(void) 534 { 535 int zero = 0, sent, recvd; 536 int verdict, parser; 537 int err, map; 538 int c = -1, p = -1; 539 struct test_sockmap_pass_prog *pass = NULL; 540 char snd[256] = "0123456789"; 541 char rcv[256] = "0"; 542 543 pass = test_sockmap_pass_prog__open_and_load(); 544 verdict = bpf_program__fd(pass->progs.prog_skb_verdict); 545 parser = bpf_program__fd(pass->progs.prog_skb_parser); 546 map = bpf_map__fd(pass->maps.sock_map_rx); 547 548 err = bpf_prog_attach(parser, map, BPF_SK_SKB_STREAM_PARSER, 0); 549 if (!ASSERT_OK(err, "bpf_prog_attach stream parser")) 550 goto out; 551 552 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 553 if (!ASSERT_OK(err, "bpf_prog_attach stream verdict")) 554 goto out; 555 556 err = create_pair(AF_INET, SOCK_STREAM, &c, &p); 557 if (err) 558 goto out; 559 560 /* sk_data_ready of 'p' will be replaced by strparser handler */ 561 err = bpf_map_update_elem(map, &zero, &p, BPF_NOEXIST); 562 if (!ASSERT_OK(err, "bpf_map_update_elem(p)")) 563 goto out_close; 564 565 /* 566 * as 'prog_skb_parser' return the original skb len and 567 * 'prog_skb_verdict' return SK_PASS, the kernel will just 568 * pass it through to original socket 'p' 569 */ 570 sent = xsend(c, snd, sizeof(snd), 0); 571 ASSERT_EQ(sent, sizeof(snd), "xsend(c)"); 572 573 recvd = recv_timeout(p, rcv, sizeof(rcv), SOCK_NONBLOCK, 574 IO_TIMEOUT_SEC); 575 ASSERT_EQ(recvd, sizeof(rcv), "recv_timeout(p)"); 576 577 out_close: 578 close(c); 579 close(p); 580 581 out: 582 test_sockmap_pass_prog__destroy(pass); 583 } 584 585 static void test_sockmap_skb_verdict_fionread(bool pass_prog) 586 { 587 int err, map, verdict, c0 = -1, c1 = -1, p0 = -1, p1 = -1; 588 int expected, zero = 0, sent, recvd, avail; 589 struct test_sockmap_pass_prog *pass = NULL; 590 struct test_sockmap_drop_prog *drop = NULL; 591 char buf[256] = "0123456789"; 592 593 if (pass_prog) { 594 pass = test_sockmap_pass_prog__open_and_load(); 595 if (!ASSERT_OK_PTR(pass, "open_and_load")) 596 return; 597 verdict = bpf_program__fd(pass->progs.prog_skb_verdict); 598 map = bpf_map__fd(pass->maps.sock_map_rx); 599 expected = sizeof(buf); 600 } else { 601 drop = test_sockmap_drop_prog__open_and_load(); 602 if (!ASSERT_OK_PTR(drop, "open_and_load")) 603 return; 604 verdict = bpf_program__fd(drop->progs.prog_skb_verdict); 605 map = bpf_map__fd(drop->maps.sock_map_rx); 606 /* On drop data is consumed immediately and copied_seq inc'd */ 607 expected = 0; 608 } 609 610 611 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 612 if (!ASSERT_OK(err, "bpf_prog_attach")) 613 goto out; 614 615 err = create_socket_pairs(AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1); 616 if (!ASSERT_OK(err, "create_socket_pairs()")) 617 goto out; 618 619 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 620 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) 621 goto out_close; 622 623 sent = xsend(p1, &buf, sizeof(buf), 0); 624 ASSERT_EQ(sent, sizeof(buf), "xsend(p0)"); 625 err = ioctl(c1, FIONREAD, &avail); 626 ASSERT_OK(err, "ioctl(FIONREAD) error"); 627 ASSERT_EQ(avail, expected, "ioctl(FIONREAD)"); 628 /* On DROP test there will be no data to read */ 629 if (pass_prog) { 630 recvd = recv_timeout(c1, &buf, sizeof(buf), SOCK_NONBLOCK, IO_TIMEOUT_SEC); 631 ASSERT_EQ(recvd, sizeof(buf), "recv_timeout(c0)"); 632 } 633 634 out_close: 635 close(c0); 636 close(p0); 637 close(c1); 638 close(p1); 639 out: 640 if (pass_prog) 641 test_sockmap_pass_prog__destroy(pass); 642 else 643 test_sockmap_drop_prog__destroy(drop); 644 } 645 646 static void test_sockmap_skb_verdict_peek_helper(int map) 647 { 648 int err, c1, p1, zero = 0, sent, recvd, avail; 649 char snd[256] = "0123456789"; 650 char rcv[256] = "0"; 651 652 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); 653 if (!ASSERT_OK(err, "create_pair()")) 654 return; 655 656 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 657 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) 658 goto out_close; 659 660 sent = xsend(p1, snd, sizeof(snd), 0); 661 ASSERT_EQ(sent, sizeof(snd), "xsend(p1)"); 662 recvd = recv(c1, rcv, sizeof(rcv), MSG_PEEK); 663 ASSERT_EQ(recvd, sizeof(rcv), "recv(c1)"); 664 err = ioctl(c1, FIONREAD, &avail); 665 ASSERT_OK(err, "ioctl(FIONREAD) error"); 666 ASSERT_EQ(avail, sizeof(snd), "after peek ioctl(FIONREAD)"); 667 recvd = recv(c1, rcv, sizeof(rcv), 0); 668 ASSERT_EQ(recvd, sizeof(rcv), "recv(p0)"); 669 err = ioctl(c1, FIONREAD, &avail); 670 ASSERT_OK(err, "ioctl(FIONREAD) error"); 671 ASSERT_EQ(avail, 0, "after read ioctl(FIONREAD)"); 672 673 out_close: 674 close(c1); 675 close(p1); 676 } 677 678 static void test_sockmap_skb_verdict_peek(void) 679 { 680 struct test_sockmap_pass_prog *pass; 681 int err, map, verdict; 682 683 pass = test_sockmap_pass_prog__open_and_load(); 684 if (!ASSERT_OK_PTR(pass, "open_and_load")) 685 return; 686 verdict = bpf_program__fd(pass->progs.prog_skb_verdict); 687 map = bpf_map__fd(pass->maps.sock_map_rx); 688 689 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 690 if (!ASSERT_OK(err, "bpf_prog_attach")) 691 goto out; 692 693 test_sockmap_skb_verdict_peek_helper(map); 694 695 out: 696 test_sockmap_pass_prog__destroy(pass); 697 } 698 699 static void test_sockmap_skb_verdict_peek_with_link(void) 700 { 701 struct test_sockmap_pass_prog *pass; 702 struct bpf_program *prog; 703 struct bpf_link *link; 704 int err, map; 705 706 pass = test_sockmap_pass_prog__open_and_load(); 707 if (!ASSERT_OK_PTR(pass, "open_and_load")) 708 return; 709 prog = pass->progs.prog_skb_verdict; 710 map = bpf_map__fd(pass->maps.sock_map_rx); 711 link = bpf_program__attach_sockmap(prog, map); 712 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 713 goto out; 714 715 err = bpf_link__update_program(link, pass->progs.prog_skb_verdict_clone); 716 if (!ASSERT_OK(err, "bpf_link__update_program")) 717 goto out; 718 719 /* Fail since a prog with different attach type attempts to do update. */ 720 err = bpf_link__update_program(link, pass->progs.prog_skb_parser); 721 if (!ASSERT_ERR(err, "bpf_link__update_program")) 722 goto out; 723 724 test_sockmap_skb_verdict_peek_helper(map); 725 ASSERT_EQ(pass->bss->clone_called, 1, "clone_called"); 726 out: 727 bpf_link__detach(link); 728 test_sockmap_pass_prog__destroy(pass); 729 } 730 731 static void test_sockmap_unconnected_unix(void) 732 { 733 int err, map, stream = 0, dgram = 0, zero = 0; 734 struct test_sockmap_pass_prog *skel; 735 736 skel = test_sockmap_pass_prog__open_and_load(); 737 if (!ASSERT_OK_PTR(skel, "open_and_load")) 738 return; 739 740 map = bpf_map__fd(skel->maps.sock_map_rx); 741 742 stream = xsocket(AF_UNIX, SOCK_STREAM, 0); 743 if (stream < 0) 744 return; 745 746 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 747 if (dgram < 0) { 748 close(stream); 749 return; 750 } 751 752 err = bpf_map_update_elem(map, &zero, &stream, BPF_ANY); 753 ASSERT_ERR(err, "bpf_map_update_elem(stream)"); 754 755 err = bpf_map_update_elem(map, &zero, &dgram, BPF_ANY); 756 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 757 758 close(stream); 759 close(dgram); 760 } 761 762 static void test_sockmap_many_socket(void) 763 { 764 struct test_sockmap_pass_prog *skel; 765 int stream[2], dgram, udp, tcp; 766 int i, err, map, entry = 0; 767 768 skel = test_sockmap_pass_prog__open_and_load(); 769 if (!ASSERT_OK_PTR(skel, "open_and_load")) 770 return; 771 772 map = bpf_map__fd(skel->maps.sock_map_rx); 773 774 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 775 if (dgram < 0) { 776 test_sockmap_pass_prog__destroy(skel); 777 return; 778 } 779 780 tcp = connected_socket_v4(); 781 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) { 782 close(dgram); 783 test_sockmap_pass_prog__destroy(skel); 784 return; 785 } 786 787 udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); 788 if (udp < 0) { 789 close(dgram); 790 close(tcp); 791 test_sockmap_pass_prog__destroy(skel); 792 return; 793 } 794 795 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 796 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 797 if (err) 798 goto out; 799 800 for (i = 0; i < 2; i++, entry++) { 801 err = bpf_map_update_elem(map, &entry, &stream[0], BPF_ANY); 802 ASSERT_OK(err, "bpf_map_update_elem(stream)"); 803 } 804 for (i = 0; i < 2; i++, entry++) { 805 err = bpf_map_update_elem(map, &entry, &dgram, BPF_ANY); 806 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 807 } 808 for (i = 0; i < 2; i++, entry++) { 809 err = bpf_map_update_elem(map, &entry, &udp, BPF_ANY); 810 ASSERT_OK(err, "bpf_map_update_elem(udp)"); 811 } 812 for (i = 0; i < 2; i++, entry++) { 813 err = bpf_map_update_elem(map, &entry, &tcp, BPF_ANY); 814 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 815 } 816 for (entry--; entry >= 0; entry--) { 817 err = bpf_map_delete_elem(map, &entry); 818 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 819 } 820 821 close(stream[0]); 822 close(stream[1]); 823 out: 824 close(dgram); 825 close(tcp); 826 close(udp); 827 test_sockmap_pass_prog__destroy(skel); 828 } 829 830 static void test_sockmap_many_maps(void) 831 { 832 struct test_sockmap_pass_prog *skel; 833 int stream[2], dgram, udp, tcp; 834 int i, err, map[2], entry = 0; 835 836 skel = test_sockmap_pass_prog__open_and_load(); 837 if (!ASSERT_OK_PTR(skel, "open_and_load")) 838 return; 839 840 map[0] = bpf_map__fd(skel->maps.sock_map_rx); 841 map[1] = bpf_map__fd(skel->maps.sock_map_tx); 842 843 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 844 if (dgram < 0) { 845 test_sockmap_pass_prog__destroy(skel); 846 return; 847 } 848 849 tcp = connected_socket_v4(); 850 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) { 851 close(dgram); 852 test_sockmap_pass_prog__destroy(skel); 853 return; 854 } 855 856 udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); 857 if (udp < 0) { 858 close(dgram); 859 close(tcp); 860 test_sockmap_pass_prog__destroy(skel); 861 return; 862 } 863 864 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 865 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 866 if (err) 867 goto out; 868 869 for (i = 0; i < 2; i++, entry++) { 870 err = bpf_map_update_elem(map[i], &entry, &stream[0], BPF_ANY); 871 ASSERT_OK(err, "bpf_map_update_elem(stream)"); 872 } 873 for (i = 0; i < 2; i++, entry++) { 874 err = bpf_map_update_elem(map[i], &entry, &dgram, BPF_ANY); 875 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 876 } 877 for (i = 0; i < 2; i++, entry++) { 878 err = bpf_map_update_elem(map[i], &entry, &udp, BPF_ANY); 879 ASSERT_OK(err, "bpf_map_update_elem(udp)"); 880 } 881 for (i = 0; i < 2; i++, entry++) { 882 err = bpf_map_update_elem(map[i], &entry, &tcp, BPF_ANY); 883 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 884 } 885 for (entry--; entry >= 0; entry--) { 886 err = bpf_map_delete_elem(map[1], &entry); 887 entry--; 888 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 889 err = bpf_map_delete_elem(map[0], &entry); 890 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 891 } 892 893 close(stream[0]); 894 close(stream[1]); 895 out: 896 close(dgram); 897 close(tcp); 898 close(udp); 899 test_sockmap_pass_prog__destroy(skel); 900 } 901 902 static void test_sockmap_same_sock(void) 903 { 904 struct test_sockmap_pass_prog *skel; 905 int stream[2], dgram, udp, tcp; 906 int i, err, map, zero = 0; 907 908 skel = test_sockmap_pass_prog__open_and_load(); 909 if (!ASSERT_OK_PTR(skel, "open_and_load")) 910 return; 911 912 map = bpf_map__fd(skel->maps.sock_map_rx); 913 914 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 915 if (dgram < 0) { 916 test_sockmap_pass_prog__destroy(skel); 917 return; 918 } 919 920 tcp = connected_socket_v4(); 921 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) { 922 close(dgram); 923 test_sockmap_pass_prog__destroy(skel); 924 return; 925 } 926 927 udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); 928 if (udp < 0) { 929 close(dgram); 930 close(tcp); 931 test_sockmap_pass_prog__destroy(skel); 932 return; 933 } 934 935 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 936 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 937 if (err) 938 goto out; 939 940 for (i = 0; i < 2; i++) { 941 err = bpf_map_update_elem(map, &zero, &stream[0], BPF_ANY); 942 ASSERT_OK(err, "bpf_map_update_elem(stream)"); 943 } 944 for (i = 0; i < 2; i++) { 945 err = bpf_map_update_elem(map, &zero, &dgram, BPF_ANY); 946 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 947 } 948 for (i = 0; i < 2; i++) { 949 err = bpf_map_update_elem(map, &zero, &udp, BPF_ANY); 950 ASSERT_OK(err, "bpf_map_update_elem(udp)"); 951 } 952 for (i = 0; i < 2; i++) { 953 err = bpf_map_update_elem(map, &zero, &tcp, BPF_ANY); 954 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 955 } 956 957 err = bpf_map_delete_elem(map, &zero); 958 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 959 960 close(stream[0]); 961 close(stream[1]); 962 out: 963 close(dgram); 964 close(tcp); 965 close(udp); 966 test_sockmap_pass_prog__destroy(skel); 967 } 968 969 static void test_sockmap_skb_verdict_vsock_poll(void) 970 { 971 struct test_sockmap_pass_prog *skel; 972 int err, map, conn, peer; 973 struct bpf_program *prog; 974 struct bpf_link *link; 975 char buf = 'x'; 976 int zero = 0; 977 978 skel = test_sockmap_pass_prog__open_and_load(); 979 if (!ASSERT_OK_PTR(skel, "open_and_load")) 980 return; 981 982 if (create_pair(AF_VSOCK, SOCK_STREAM, &conn, &peer)) 983 goto destroy; 984 985 prog = skel->progs.prog_skb_verdict; 986 map = bpf_map__fd(skel->maps.sock_map_rx); 987 link = bpf_program__attach_sockmap(prog, map); 988 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 989 goto close; 990 991 err = bpf_map_update_elem(map, &zero, &conn, BPF_ANY); 992 if (!ASSERT_OK(err, "bpf_map_update_elem")) 993 goto detach; 994 995 if (xsend(peer, &buf, 1, 0) != 1) 996 goto detach; 997 998 err = poll_read(conn, IO_TIMEOUT_SEC); 999 if (!ASSERT_OK(err, "poll")) 1000 goto detach; 1001 1002 if (xrecv_nonblock(conn, &buf, 1, 0) != 1) 1003 FAIL("xrecv_nonblock"); 1004 detach: 1005 bpf_link__detach(link); 1006 close: 1007 xclose(conn); 1008 xclose(peer); 1009 destroy: 1010 test_sockmap_pass_prog__destroy(skel); 1011 } 1012 1013 void test_sockmap_basic(void) 1014 { 1015 if (test__start_subtest("sockmap create_update_free")) 1016 test_sockmap_create_update_free(BPF_MAP_TYPE_SOCKMAP); 1017 if (test__start_subtest("sockhash create_update_free")) 1018 test_sockmap_create_update_free(BPF_MAP_TYPE_SOCKHASH); 1019 if (test__start_subtest("sockmap vsock delete on close")) 1020 test_sockmap_vsock_delete_on_close(); 1021 if (test__start_subtest("sockmap sk_msg load helpers")) 1022 test_skmsg_helpers(BPF_MAP_TYPE_SOCKMAP); 1023 if (test__start_subtest("sockhash sk_msg load helpers")) 1024 test_skmsg_helpers(BPF_MAP_TYPE_SOCKHASH); 1025 if (test__start_subtest("sockmap update")) 1026 test_sockmap_update(BPF_MAP_TYPE_SOCKMAP); 1027 if (test__start_subtest("sockhash update")) 1028 test_sockmap_update(BPF_MAP_TYPE_SOCKHASH); 1029 if (test__start_subtest("sockmap update in unsafe context")) 1030 test_sockmap_invalid_update(); 1031 if (test__start_subtest("sockmap copy")) 1032 test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP); 1033 if (test__start_subtest("sockhash copy")) 1034 test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH); 1035 if (test__start_subtest("sockmap skb_verdict attach")) { 1036 test_sockmap_skb_verdict_attach(BPF_SK_SKB_VERDICT, 1037 BPF_SK_SKB_STREAM_VERDICT); 1038 test_sockmap_skb_verdict_attach(BPF_SK_SKB_STREAM_VERDICT, 1039 BPF_SK_SKB_VERDICT); 1040 } 1041 if (test__start_subtest("sockmap skb_verdict attach_with_link")) 1042 test_sockmap_skb_verdict_attach_with_link(); 1043 if (test__start_subtest("sockmap msg_verdict progs query")) 1044 test_sockmap_progs_query(BPF_SK_MSG_VERDICT); 1045 if (test__start_subtest("sockmap stream_parser progs query")) 1046 test_sockmap_progs_query(BPF_SK_SKB_STREAM_PARSER); 1047 if (test__start_subtest("sockmap stream_verdict progs query")) 1048 test_sockmap_progs_query(BPF_SK_SKB_STREAM_VERDICT); 1049 if (test__start_subtest("sockmap skb_verdict progs query")) 1050 test_sockmap_progs_query(BPF_SK_SKB_VERDICT); 1051 if (test__start_subtest("sockmap skb_verdict shutdown")) 1052 test_sockmap_skb_verdict_shutdown(); 1053 if (test__start_subtest("sockmap stream parser and verdict pass")) 1054 test_sockmap_stream_pass(); 1055 if (test__start_subtest("sockmap skb_verdict fionread")) 1056 test_sockmap_skb_verdict_fionread(true); 1057 if (test__start_subtest("sockmap skb_verdict fionread on drop")) 1058 test_sockmap_skb_verdict_fionread(false); 1059 if (test__start_subtest("sockmap skb_verdict msg_f_peek")) 1060 test_sockmap_skb_verdict_peek(); 1061 if (test__start_subtest("sockmap skb_verdict msg_f_peek with link")) 1062 test_sockmap_skb_verdict_peek_with_link(); 1063 if (test__start_subtest("sockmap unconnected af_unix")) 1064 test_sockmap_unconnected_unix(); 1065 if (test__start_subtest("sockmap one socket to many map entries")) 1066 test_sockmap_many_socket(); 1067 if (test__start_subtest("sockmap one socket to many maps")) 1068 test_sockmap_many_maps(); 1069 if (test__start_subtest("sockmap same socket replace")) 1070 test_sockmap_same_sock(); 1071 if (test__start_subtest("sockmap sk_msg attach sockmap helpers with link")) 1072 test_skmsg_helpers_with_link(BPF_MAP_TYPE_SOCKMAP); 1073 if (test__start_subtest("sockhash sk_msg attach sockhash helpers with link")) 1074 test_skmsg_helpers_with_link(BPF_MAP_TYPE_SOCKHASH); 1075 if (test__start_subtest("sockmap skb_verdict vsock poll")) 1076 test_sockmap_skb_verdict_vsock_poll(); 1077 } 1078