1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2020 Cloudflare 3 #include <error.h> 4 #include <netinet/tcp.h> 5 #include <sys/epoll.h> 6 7 #include "test_progs.h" 8 #include "test_skmsg_load_helpers.skel.h" 9 #include "test_sockmap_update.skel.h" 10 #include "test_sockmap_invalid_update.skel.h" 11 #include "test_sockmap_skb_verdict_attach.skel.h" 12 #include "test_sockmap_progs_query.skel.h" 13 #include "test_sockmap_pass_prog.skel.h" 14 #include "test_sockmap_drop_prog.skel.h" 15 #include "test_sockmap_change_tail.skel.h" 16 #include "bpf_iter_sockmap.skel.h" 17 18 #include "sockmap_helpers.h" 19 20 #define TCP_REPAIR 19 /* TCP sock is under repair right now */ 21 22 #define TCP_REPAIR_ON 1 23 #define TCP_REPAIR_OFF_NO_WP -1 /* Turn off without window probes */ 24 25 static int connected_socket_v4(void) 26 { 27 struct sockaddr_in addr = { 28 .sin_family = AF_INET, 29 .sin_port = htons(80), 30 .sin_addr = { inet_addr("127.0.0.1") }, 31 }; 32 socklen_t len = sizeof(addr); 33 int s, repair, err; 34 35 s = socket(AF_INET, SOCK_STREAM, 0); 36 if (!ASSERT_GE(s, 0, "socket")) 37 goto error; 38 39 repair = TCP_REPAIR_ON; 40 err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair)); 41 if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)")) 42 goto error; 43 44 err = connect(s, (struct sockaddr *)&addr, len); 45 if (!ASSERT_OK(err, "connect")) 46 goto error; 47 48 repair = TCP_REPAIR_OFF_NO_WP; 49 err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair)); 50 if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)")) 51 goto error; 52 53 return s; 54 error: 55 perror(__func__); 56 close(s); 57 return -1; 58 } 59 60 static void compare_cookies(struct bpf_map *src, struct bpf_map *dst) 61 { 62 __u32 i, max_entries = bpf_map__max_entries(src); 63 int err, src_fd, dst_fd; 64 65 src_fd = bpf_map__fd(src); 66 dst_fd = bpf_map__fd(dst); 67 68 for (i = 0; i < max_entries; i++) { 69 __u64 src_cookie, dst_cookie; 70 71 err = bpf_map_lookup_elem(src_fd, &i, &src_cookie); 72 if (err && errno == ENOENT) { 73 err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); 74 ASSERT_ERR(err, "map_lookup_elem(dst)"); 75 ASSERT_EQ(errno, ENOENT, "map_lookup_elem(dst)"); 76 continue; 77 } 78 if (!ASSERT_OK(err, "lookup_elem(src)")) 79 continue; 80 81 err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); 82 if (!ASSERT_OK(err, "lookup_elem(dst)")) 83 continue; 84 85 ASSERT_EQ(dst_cookie, src_cookie, "cookie mismatch"); 86 } 87 } 88 89 /* Create a map, populate it with one socket, and free the map. */ 90 static void test_sockmap_create_update_free(enum bpf_map_type map_type) 91 { 92 const int zero = 0; 93 int s, map, err; 94 95 s = connected_socket_v4(); 96 if (!ASSERT_GE(s, 0, "connected_socket_v4")) 97 return; 98 99 map = bpf_map_create(map_type, NULL, sizeof(int), sizeof(int), 1, NULL); 100 if (!ASSERT_GE(map, 0, "bpf_map_create")) 101 goto out; 102 103 err = bpf_map_update_elem(map, &zero, &s, BPF_NOEXIST); 104 if (!ASSERT_OK(err, "bpf_map_update")) 105 goto out; 106 107 out: 108 close(map); 109 close(s); 110 } 111 112 static void test_sockmap_vsock_delete_on_close(void) 113 { 114 int map, c, p, err, zero = 0; 115 116 map = bpf_map_create(BPF_MAP_TYPE_SOCKMAP, NULL, sizeof(int), 117 sizeof(int), 1, NULL); 118 if (!ASSERT_OK_FD(map, "bpf_map_create")) 119 return; 120 121 err = create_pair(AF_VSOCK, SOCK_STREAM, &c, &p); 122 if (!ASSERT_OK(err, "create_pair")) 123 goto close_map; 124 125 if (xbpf_map_update_elem(map, &zero, &c, BPF_NOEXIST)) 126 goto close_socks; 127 128 xclose(c); 129 xclose(p); 130 131 err = create_pair(AF_VSOCK, SOCK_STREAM, &c, &p); 132 if (!ASSERT_OK(err, "create_pair")) 133 goto close_map; 134 135 err = bpf_map_update_elem(map, &zero, &c, BPF_NOEXIST); 136 ASSERT_OK(err, "after close(), bpf_map_update"); 137 138 close_socks: 139 xclose(c); 140 xclose(p); 141 close_map: 142 xclose(map); 143 } 144 145 static void test_skmsg_helpers(enum bpf_map_type map_type) 146 { 147 struct test_skmsg_load_helpers *skel; 148 int err, map, verdict; 149 150 skel = test_skmsg_load_helpers__open_and_load(); 151 if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load")) 152 return; 153 154 verdict = bpf_program__fd(skel->progs.prog_msg_verdict); 155 map = bpf_map__fd(skel->maps.sock_map); 156 157 err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0); 158 if (!ASSERT_OK(err, "bpf_prog_attach")) 159 goto out; 160 161 err = bpf_prog_detach2(verdict, map, BPF_SK_MSG_VERDICT); 162 if (!ASSERT_OK(err, "bpf_prog_detach2")) 163 goto out; 164 out: 165 test_skmsg_load_helpers__destroy(skel); 166 } 167 168 static void test_skmsg_helpers_with_link(enum bpf_map_type map_type) 169 { 170 struct bpf_program *prog, *prog_clone, *prog_clone2; 171 DECLARE_LIBBPF_OPTS(bpf_link_update_opts, opts); 172 struct test_skmsg_load_helpers *skel; 173 struct bpf_link *link, *link2; 174 int err, map; 175 176 skel = test_skmsg_load_helpers__open_and_load(); 177 if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load")) 178 return; 179 180 prog = skel->progs.prog_msg_verdict; 181 prog_clone = skel->progs.prog_msg_verdict_clone; 182 prog_clone2 = skel->progs.prog_msg_verdict_clone2; 183 map = bpf_map__fd(skel->maps.sock_map); 184 185 link = bpf_program__attach_sockmap(prog, map); 186 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 187 goto out; 188 189 /* Fail since bpf_link for the same prog has been created. */ 190 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_MSG_VERDICT, 0); 191 if (!ASSERT_ERR(err, "bpf_prog_attach")) 192 goto out; 193 194 /* Fail since bpf_link for the same prog type has been created. */ 195 link2 = bpf_program__attach_sockmap(prog_clone, map); 196 if (!ASSERT_ERR_PTR(link2, "bpf_program__attach_sockmap")) { 197 bpf_link__detach(link2); 198 goto out; 199 } 200 201 err = bpf_link__update_program(link, prog_clone); 202 if (!ASSERT_OK(err, "bpf_link__update_program")) 203 goto out; 204 205 /* Fail since a prog with different type attempts to do update. */ 206 err = bpf_link__update_program(link, skel->progs.prog_skb_verdict); 207 if (!ASSERT_ERR(err, "bpf_link__update_program")) 208 goto out; 209 210 /* Fail since the old prog does not match the one in the kernel. */ 211 opts.old_prog_fd = bpf_program__fd(prog_clone2); 212 opts.flags = BPF_F_REPLACE; 213 err = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), &opts); 214 if (!ASSERT_ERR(err, "bpf_link_update")) 215 goto out; 216 217 opts.old_prog_fd = bpf_program__fd(prog_clone); 218 opts.flags = BPF_F_REPLACE; 219 err = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), &opts); 220 if (!ASSERT_OK(err, "bpf_link_update")) 221 goto out; 222 out: 223 bpf_link__detach(link); 224 test_skmsg_load_helpers__destroy(skel); 225 } 226 227 static void test_sockmap_update(enum bpf_map_type map_type) 228 { 229 int err, prog, src; 230 struct test_sockmap_update *skel; 231 struct bpf_map *dst_map; 232 const __u32 zero = 0; 233 char dummy[14] = {0}; 234 LIBBPF_OPTS(bpf_test_run_opts, topts, 235 .data_in = dummy, 236 .data_size_in = sizeof(dummy), 237 .repeat = 1, 238 ); 239 __s64 sk; 240 241 sk = connected_socket_v4(); 242 if (!ASSERT_NEQ(sk, -1, "connected_socket_v4")) 243 return; 244 245 skel = test_sockmap_update__open_and_load(); 246 if (!ASSERT_OK_PTR(skel, "open_and_load")) 247 goto close_sk; 248 249 prog = bpf_program__fd(skel->progs.copy_sock_map); 250 src = bpf_map__fd(skel->maps.src); 251 if (map_type == BPF_MAP_TYPE_SOCKMAP) 252 dst_map = skel->maps.dst_sock_map; 253 else 254 dst_map = skel->maps.dst_sock_hash; 255 256 err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST); 257 if (!ASSERT_OK(err, "update_elem(src)")) 258 goto out; 259 260 err = bpf_prog_test_run_opts(prog, &topts); 261 if (!ASSERT_OK(err, "test_run")) 262 goto out; 263 if (!ASSERT_NEQ(topts.retval, 0, "test_run retval")) 264 goto out; 265 266 compare_cookies(skel->maps.src, dst_map); 267 268 out: 269 test_sockmap_update__destroy(skel); 270 close_sk: 271 close(sk); 272 } 273 274 static void test_sockmap_invalid_update(void) 275 { 276 struct test_sockmap_invalid_update *skel; 277 278 skel = test_sockmap_invalid_update__open_and_load(); 279 if (!ASSERT_NULL(skel, "open_and_load")) 280 test_sockmap_invalid_update__destroy(skel); 281 } 282 283 static void test_sockmap_copy(enum bpf_map_type map_type) 284 { 285 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 286 int err, len, src_fd, iter_fd; 287 union bpf_iter_link_info linfo = {}; 288 __u32 i, num_sockets, num_elems; 289 struct bpf_iter_sockmap *skel; 290 __s64 *sock_fd = NULL; 291 struct bpf_link *link; 292 struct bpf_map *src; 293 char buf[64]; 294 295 skel = bpf_iter_sockmap__open_and_load(); 296 if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load")) 297 return; 298 299 if (map_type == BPF_MAP_TYPE_SOCKMAP) { 300 src = skel->maps.sockmap; 301 num_elems = bpf_map__max_entries(src); 302 num_sockets = num_elems - 1; 303 } else { 304 src = skel->maps.sockhash; 305 num_elems = bpf_map__max_entries(src) - 1; 306 num_sockets = num_elems; 307 } 308 309 sock_fd = calloc(num_sockets, sizeof(*sock_fd)); 310 if (!ASSERT_OK_PTR(sock_fd, "calloc(sock_fd)")) 311 goto out; 312 313 for (i = 0; i < num_sockets; i++) 314 sock_fd[i] = -1; 315 316 src_fd = bpf_map__fd(src); 317 318 for (i = 0; i < num_sockets; i++) { 319 sock_fd[i] = connected_socket_v4(); 320 if (!ASSERT_NEQ(sock_fd[i], -1, "connected_socket_v4")) 321 goto out; 322 323 err = bpf_map_update_elem(src_fd, &i, &sock_fd[i], BPF_NOEXIST); 324 if (!ASSERT_OK(err, "map_update")) 325 goto out; 326 } 327 328 linfo.map.map_fd = src_fd; 329 opts.link_info = &linfo; 330 opts.link_info_len = sizeof(linfo); 331 link = bpf_program__attach_iter(skel->progs.copy, &opts); 332 if (!ASSERT_OK_PTR(link, "attach_iter")) 333 goto out; 334 335 iter_fd = bpf_iter_create(bpf_link__fd(link)); 336 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 337 goto free_link; 338 339 /* do some tests */ 340 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 341 ; 342 if (!ASSERT_GE(len, 0, "read")) 343 goto close_iter; 344 345 /* test results */ 346 if (!ASSERT_EQ(skel->bss->elems, num_elems, "elems")) 347 goto close_iter; 348 349 if (!ASSERT_EQ(skel->bss->socks, num_sockets, "socks")) 350 goto close_iter; 351 352 compare_cookies(src, skel->maps.dst); 353 354 close_iter: 355 close(iter_fd); 356 free_link: 357 bpf_link__destroy(link); 358 out: 359 for (i = 0; sock_fd && i < num_sockets; i++) 360 if (sock_fd[i] >= 0) 361 close(sock_fd[i]); 362 if (sock_fd) 363 free(sock_fd); 364 bpf_iter_sockmap__destroy(skel); 365 } 366 367 static void test_sockmap_skb_verdict_attach(enum bpf_attach_type first, 368 enum bpf_attach_type second) 369 { 370 struct test_sockmap_skb_verdict_attach *skel; 371 int err, map, verdict; 372 373 skel = test_sockmap_skb_verdict_attach__open_and_load(); 374 if (!ASSERT_OK_PTR(skel, "open_and_load")) 375 return; 376 377 verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 378 map = bpf_map__fd(skel->maps.sock_map); 379 380 err = bpf_prog_attach(verdict, map, first, 0); 381 if (!ASSERT_OK(err, "bpf_prog_attach")) 382 goto out; 383 384 err = bpf_prog_attach(verdict, map, second, 0); 385 ASSERT_EQ(err, -EBUSY, "prog_attach_fail"); 386 387 err = bpf_prog_detach2(verdict, map, first); 388 if (!ASSERT_OK(err, "bpf_prog_detach2")) 389 goto out; 390 out: 391 test_sockmap_skb_verdict_attach__destroy(skel); 392 } 393 394 static void test_sockmap_skb_verdict_attach_with_link(void) 395 { 396 struct test_sockmap_skb_verdict_attach *skel; 397 struct bpf_program *prog; 398 struct bpf_link *link; 399 int err, map; 400 401 skel = test_sockmap_skb_verdict_attach__open_and_load(); 402 if (!ASSERT_OK_PTR(skel, "open_and_load")) 403 return; 404 prog = skel->progs.prog_skb_verdict; 405 map = bpf_map__fd(skel->maps.sock_map); 406 link = bpf_program__attach_sockmap(prog, map); 407 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 408 goto out; 409 410 bpf_link__detach(link); 411 412 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0); 413 if (!ASSERT_OK(err, "bpf_prog_attach")) 414 goto out; 415 416 /* Fail since attaching with the same prog/map has been done. */ 417 link = bpf_program__attach_sockmap(prog, map); 418 if (!ASSERT_ERR_PTR(link, "bpf_program__attach_sockmap")) 419 bpf_link__detach(link); 420 421 err = bpf_prog_detach2(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT); 422 if (!ASSERT_OK(err, "bpf_prog_detach2")) 423 goto out; 424 out: 425 test_sockmap_skb_verdict_attach__destroy(skel); 426 } 427 428 static __u32 query_prog_id(int prog_fd) 429 { 430 struct bpf_prog_info info = {}; 431 __u32 info_len = sizeof(info); 432 int err; 433 434 err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); 435 if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd") || 436 !ASSERT_EQ(info_len, sizeof(info), "bpf_prog_get_info_by_fd")) 437 return 0; 438 439 return info.id; 440 } 441 442 static void test_sockmap_progs_query(enum bpf_attach_type attach_type) 443 { 444 struct test_sockmap_progs_query *skel; 445 int err, map_fd, verdict_fd; 446 __u32 attach_flags = 0; 447 __u32 prog_ids[3] = {}; 448 __u32 prog_cnt = 3; 449 450 skel = test_sockmap_progs_query__open_and_load(); 451 if (!ASSERT_OK_PTR(skel, "test_sockmap_progs_query__open_and_load")) 452 return; 453 454 map_fd = bpf_map__fd(skel->maps.sock_map); 455 456 if (attach_type == BPF_SK_MSG_VERDICT) 457 verdict_fd = bpf_program__fd(skel->progs.prog_skmsg_verdict); 458 else 459 verdict_fd = bpf_program__fd(skel->progs.prog_skb_verdict); 460 461 err = bpf_prog_query(map_fd, attach_type, 0 /* query flags */, 462 &attach_flags, prog_ids, &prog_cnt); 463 ASSERT_OK(err, "bpf_prog_query failed"); 464 ASSERT_EQ(attach_flags, 0, "wrong attach_flags on query"); 465 ASSERT_EQ(prog_cnt, 0, "wrong program count on query"); 466 467 err = bpf_prog_attach(verdict_fd, map_fd, attach_type, 0); 468 if (!ASSERT_OK(err, "bpf_prog_attach failed")) 469 goto out; 470 471 prog_cnt = 1; 472 err = bpf_prog_query(map_fd, attach_type, 0 /* query flags */, 473 &attach_flags, prog_ids, &prog_cnt); 474 ASSERT_OK(err, "bpf_prog_query failed"); 475 ASSERT_EQ(attach_flags, 0, "wrong attach_flags on query"); 476 ASSERT_EQ(prog_cnt, 1, "wrong program count on query"); 477 ASSERT_EQ(prog_ids[0], query_prog_id(verdict_fd), 478 "wrong prog_ids on query"); 479 480 bpf_prog_detach2(verdict_fd, map_fd, attach_type); 481 out: 482 test_sockmap_progs_query__destroy(skel); 483 } 484 485 #define MAX_EVENTS 10 486 static void test_sockmap_skb_verdict_shutdown(void) 487 { 488 int n, err, map, verdict, c1 = -1, p1 = -1; 489 struct epoll_event ev, events[MAX_EVENTS]; 490 struct test_sockmap_pass_prog *skel; 491 int zero = 0; 492 int epollfd; 493 char b; 494 495 skel = test_sockmap_pass_prog__open_and_load(); 496 if (!ASSERT_OK_PTR(skel, "open_and_load")) 497 return; 498 499 verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 500 map = bpf_map__fd(skel->maps.sock_map_rx); 501 502 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 503 if (!ASSERT_OK(err, "bpf_prog_attach")) 504 goto out; 505 506 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); 507 if (err < 0) 508 goto out; 509 510 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 511 if (err < 0) 512 goto out_close; 513 514 shutdown(p1, SHUT_WR); 515 516 ev.events = EPOLLIN; 517 ev.data.fd = c1; 518 519 epollfd = epoll_create1(0); 520 if (!ASSERT_GT(epollfd, -1, "epoll_create(0)")) 521 goto out_close; 522 err = epoll_ctl(epollfd, EPOLL_CTL_ADD, c1, &ev); 523 if (!ASSERT_OK(err, "epoll_ctl(EPOLL_CTL_ADD)")) 524 goto out_close; 525 err = epoll_wait(epollfd, events, MAX_EVENTS, -1); 526 if (!ASSERT_EQ(err, 1, "epoll_wait(fd)")) 527 goto out_close; 528 529 n = recv(c1, &b, 1, SOCK_NONBLOCK); 530 ASSERT_EQ(n, 0, "recv_timeout(fin)"); 531 out_close: 532 close(c1); 533 close(p1); 534 out: 535 test_sockmap_pass_prog__destroy(skel); 536 } 537 538 static void test_sockmap_stream_pass(void) 539 { 540 int zero = 0, sent, recvd; 541 int verdict, parser; 542 int err, map; 543 int c = -1, p = -1; 544 struct test_sockmap_pass_prog *pass = NULL; 545 char snd[256] = "0123456789"; 546 char rcv[256] = "0"; 547 548 pass = test_sockmap_pass_prog__open_and_load(); 549 verdict = bpf_program__fd(pass->progs.prog_skb_verdict); 550 parser = bpf_program__fd(pass->progs.prog_skb_parser); 551 map = bpf_map__fd(pass->maps.sock_map_rx); 552 553 err = bpf_prog_attach(parser, map, BPF_SK_SKB_STREAM_PARSER, 0); 554 if (!ASSERT_OK(err, "bpf_prog_attach stream parser")) 555 goto out; 556 557 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 558 if (!ASSERT_OK(err, "bpf_prog_attach stream verdict")) 559 goto out; 560 561 err = create_pair(AF_INET, SOCK_STREAM, &c, &p); 562 if (err) 563 goto out; 564 565 /* sk_data_ready of 'p' will be replaced by strparser handler */ 566 err = bpf_map_update_elem(map, &zero, &p, BPF_NOEXIST); 567 if (!ASSERT_OK(err, "bpf_map_update_elem(p)")) 568 goto out_close; 569 570 /* 571 * as 'prog_skb_parser' return the original skb len and 572 * 'prog_skb_verdict' return SK_PASS, the kernel will just 573 * pass it through to original socket 'p' 574 */ 575 sent = xsend(c, snd, sizeof(snd), 0); 576 ASSERT_EQ(sent, sizeof(snd), "xsend(c)"); 577 578 recvd = recv_timeout(p, rcv, sizeof(rcv), SOCK_NONBLOCK, 579 IO_TIMEOUT_SEC); 580 ASSERT_EQ(recvd, sizeof(rcv), "recv_timeout(p)"); 581 582 out_close: 583 close(c); 584 close(p); 585 586 out: 587 test_sockmap_pass_prog__destroy(pass); 588 } 589 590 static void test_sockmap_skb_verdict_fionread(bool pass_prog) 591 { 592 int err, map, verdict, c0 = -1, c1 = -1, p0 = -1, p1 = -1; 593 int expected, zero = 0, sent, recvd, avail; 594 struct test_sockmap_pass_prog *pass = NULL; 595 struct test_sockmap_drop_prog *drop = NULL; 596 char buf[256] = "0123456789"; 597 598 if (pass_prog) { 599 pass = test_sockmap_pass_prog__open_and_load(); 600 if (!ASSERT_OK_PTR(pass, "open_and_load")) 601 return; 602 verdict = bpf_program__fd(pass->progs.prog_skb_verdict); 603 map = bpf_map__fd(pass->maps.sock_map_rx); 604 expected = sizeof(buf); 605 } else { 606 drop = test_sockmap_drop_prog__open_and_load(); 607 if (!ASSERT_OK_PTR(drop, "open_and_load")) 608 return; 609 verdict = bpf_program__fd(drop->progs.prog_skb_verdict); 610 map = bpf_map__fd(drop->maps.sock_map_rx); 611 /* On drop data is consumed immediately and copied_seq inc'd */ 612 expected = 0; 613 } 614 615 616 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 617 if (!ASSERT_OK(err, "bpf_prog_attach")) 618 goto out; 619 620 err = create_socket_pairs(AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1); 621 if (!ASSERT_OK(err, "create_socket_pairs()")) 622 goto out; 623 624 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 625 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) 626 goto out_close; 627 628 sent = xsend(p1, &buf, sizeof(buf), 0); 629 ASSERT_EQ(sent, sizeof(buf), "xsend(p0)"); 630 err = ioctl(c1, FIONREAD, &avail); 631 ASSERT_OK(err, "ioctl(FIONREAD) error"); 632 ASSERT_EQ(avail, expected, "ioctl(FIONREAD)"); 633 /* On DROP test there will be no data to read */ 634 if (pass_prog) { 635 recvd = recv_timeout(c1, &buf, sizeof(buf), SOCK_NONBLOCK, IO_TIMEOUT_SEC); 636 ASSERT_EQ(recvd, sizeof(buf), "recv_timeout(c0)"); 637 } 638 639 out_close: 640 close(c0); 641 close(p0); 642 close(c1); 643 close(p1); 644 out: 645 if (pass_prog) 646 test_sockmap_pass_prog__destroy(pass); 647 else 648 test_sockmap_drop_prog__destroy(drop); 649 } 650 651 static void test_sockmap_skb_verdict_change_tail(void) 652 { 653 struct test_sockmap_change_tail *skel; 654 int err, map, verdict; 655 int c1, p1, sent, recvd; 656 int zero = 0; 657 char buf[2]; 658 659 skel = test_sockmap_change_tail__open_and_load(); 660 if (!ASSERT_OK_PTR(skel, "open_and_load")) 661 return; 662 verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 663 map = bpf_map__fd(skel->maps.sock_map_rx); 664 665 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 666 if (!ASSERT_OK(err, "bpf_prog_attach")) 667 goto out; 668 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); 669 if (!ASSERT_OK(err, "create_pair()")) 670 goto out; 671 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 672 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) 673 goto out_close; 674 sent = xsend(p1, "Tr", 2, 0); 675 ASSERT_EQ(sent, 2, "xsend(p1)"); 676 recvd = recv(c1, buf, 2, 0); 677 ASSERT_EQ(recvd, 1, "recv(c1)"); 678 ASSERT_EQ(skel->data->change_tail_ret, 0, "change_tail_ret"); 679 680 sent = xsend(p1, "G", 1, 0); 681 ASSERT_EQ(sent, 1, "xsend(p1)"); 682 recvd = recv(c1, buf, 2, 0); 683 ASSERT_EQ(recvd, 2, "recv(c1)"); 684 ASSERT_EQ(skel->data->change_tail_ret, 0, "change_tail_ret"); 685 686 sent = xsend(p1, "E", 1, 0); 687 ASSERT_EQ(sent, 1, "xsend(p1)"); 688 recvd = recv(c1, buf, 1, 0); 689 ASSERT_EQ(recvd, 1, "recv(c1)"); 690 ASSERT_EQ(skel->data->change_tail_ret, -EINVAL, "change_tail_ret"); 691 692 out_close: 693 close(c1); 694 close(p1); 695 out: 696 test_sockmap_change_tail__destroy(skel); 697 } 698 699 static void test_sockmap_skb_verdict_peek_helper(int map) 700 { 701 int err, c1, p1, zero = 0, sent, recvd, avail; 702 char snd[256] = "0123456789"; 703 char rcv[256] = "0"; 704 705 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); 706 if (!ASSERT_OK(err, "create_pair()")) 707 return; 708 709 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 710 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) 711 goto out_close; 712 713 sent = xsend(p1, snd, sizeof(snd), 0); 714 ASSERT_EQ(sent, sizeof(snd), "xsend(p1)"); 715 recvd = recv(c1, rcv, sizeof(rcv), MSG_PEEK); 716 ASSERT_EQ(recvd, sizeof(rcv), "recv(c1)"); 717 err = ioctl(c1, FIONREAD, &avail); 718 ASSERT_OK(err, "ioctl(FIONREAD) error"); 719 ASSERT_EQ(avail, sizeof(snd), "after peek ioctl(FIONREAD)"); 720 recvd = recv(c1, rcv, sizeof(rcv), 0); 721 ASSERT_EQ(recvd, sizeof(rcv), "recv(p0)"); 722 err = ioctl(c1, FIONREAD, &avail); 723 ASSERT_OK(err, "ioctl(FIONREAD) error"); 724 ASSERT_EQ(avail, 0, "after read ioctl(FIONREAD)"); 725 726 out_close: 727 close(c1); 728 close(p1); 729 } 730 731 static void test_sockmap_skb_verdict_peek(void) 732 { 733 struct test_sockmap_pass_prog *pass; 734 int err, map, verdict; 735 736 pass = test_sockmap_pass_prog__open_and_load(); 737 if (!ASSERT_OK_PTR(pass, "open_and_load")) 738 return; 739 verdict = bpf_program__fd(pass->progs.prog_skb_verdict); 740 map = bpf_map__fd(pass->maps.sock_map_rx); 741 742 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 743 if (!ASSERT_OK(err, "bpf_prog_attach")) 744 goto out; 745 746 test_sockmap_skb_verdict_peek_helper(map); 747 748 out: 749 test_sockmap_pass_prog__destroy(pass); 750 } 751 752 static void test_sockmap_skb_verdict_peek_with_link(void) 753 { 754 struct test_sockmap_pass_prog *pass; 755 struct bpf_program *prog; 756 struct bpf_link *link; 757 int err, map; 758 759 pass = test_sockmap_pass_prog__open_and_load(); 760 if (!ASSERT_OK_PTR(pass, "open_and_load")) 761 return; 762 prog = pass->progs.prog_skb_verdict; 763 map = bpf_map__fd(pass->maps.sock_map_rx); 764 link = bpf_program__attach_sockmap(prog, map); 765 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 766 goto out; 767 768 err = bpf_link__update_program(link, pass->progs.prog_skb_verdict_clone); 769 if (!ASSERT_OK(err, "bpf_link__update_program")) 770 goto out; 771 772 /* Fail since a prog with different attach type attempts to do update. */ 773 err = bpf_link__update_program(link, pass->progs.prog_skb_parser); 774 if (!ASSERT_ERR(err, "bpf_link__update_program")) 775 goto out; 776 777 test_sockmap_skb_verdict_peek_helper(map); 778 ASSERT_EQ(pass->bss->clone_called, 1, "clone_called"); 779 out: 780 bpf_link__detach(link); 781 test_sockmap_pass_prog__destroy(pass); 782 } 783 784 static void test_sockmap_unconnected_unix(void) 785 { 786 int err, map, stream = 0, dgram = 0, zero = 0; 787 struct test_sockmap_pass_prog *skel; 788 789 skel = test_sockmap_pass_prog__open_and_load(); 790 if (!ASSERT_OK_PTR(skel, "open_and_load")) 791 return; 792 793 map = bpf_map__fd(skel->maps.sock_map_rx); 794 795 stream = xsocket(AF_UNIX, SOCK_STREAM, 0); 796 if (stream < 0) 797 return; 798 799 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 800 if (dgram < 0) { 801 close(stream); 802 return; 803 } 804 805 err = bpf_map_update_elem(map, &zero, &stream, BPF_ANY); 806 ASSERT_ERR(err, "bpf_map_update_elem(stream)"); 807 808 err = bpf_map_update_elem(map, &zero, &dgram, BPF_ANY); 809 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 810 811 close(stream); 812 close(dgram); 813 } 814 815 static void test_sockmap_many_socket(void) 816 { 817 struct test_sockmap_pass_prog *skel; 818 int stream[2], dgram, udp, tcp; 819 int i, err, map, entry = 0; 820 821 skel = test_sockmap_pass_prog__open_and_load(); 822 if (!ASSERT_OK_PTR(skel, "open_and_load")) 823 return; 824 825 map = bpf_map__fd(skel->maps.sock_map_rx); 826 827 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 828 if (dgram < 0) { 829 test_sockmap_pass_prog__destroy(skel); 830 return; 831 } 832 833 tcp = connected_socket_v4(); 834 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) { 835 close(dgram); 836 test_sockmap_pass_prog__destroy(skel); 837 return; 838 } 839 840 udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); 841 if (udp < 0) { 842 close(dgram); 843 close(tcp); 844 test_sockmap_pass_prog__destroy(skel); 845 return; 846 } 847 848 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 849 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 850 if (err) 851 goto out; 852 853 for (i = 0; i < 2; i++, entry++) { 854 err = bpf_map_update_elem(map, &entry, &stream[0], BPF_ANY); 855 ASSERT_OK(err, "bpf_map_update_elem(stream)"); 856 } 857 for (i = 0; i < 2; i++, entry++) { 858 err = bpf_map_update_elem(map, &entry, &dgram, BPF_ANY); 859 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 860 } 861 for (i = 0; i < 2; i++, entry++) { 862 err = bpf_map_update_elem(map, &entry, &udp, BPF_ANY); 863 ASSERT_OK(err, "bpf_map_update_elem(udp)"); 864 } 865 for (i = 0; i < 2; i++, entry++) { 866 err = bpf_map_update_elem(map, &entry, &tcp, BPF_ANY); 867 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 868 } 869 for (entry--; entry >= 0; entry--) { 870 err = bpf_map_delete_elem(map, &entry); 871 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 872 } 873 874 close(stream[0]); 875 close(stream[1]); 876 out: 877 close(dgram); 878 close(tcp); 879 close(udp); 880 test_sockmap_pass_prog__destroy(skel); 881 } 882 883 static void test_sockmap_many_maps(void) 884 { 885 struct test_sockmap_pass_prog *skel; 886 int stream[2], dgram, udp, tcp; 887 int i, err, map[2], entry = 0; 888 889 skel = test_sockmap_pass_prog__open_and_load(); 890 if (!ASSERT_OK_PTR(skel, "open_and_load")) 891 return; 892 893 map[0] = bpf_map__fd(skel->maps.sock_map_rx); 894 map[1] = bpf_map__fd(skel->maps.sock_map_tx); 895 896 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 897 if (dgram < 0) { 898 test_sockmap_pass_prog__destroy(skel); 899 return; 900 } 901 902 tcp = connected_socket_v4(); 903 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) { 904 close(dgram); 905 test_sockmap_pass_prog__destroy(skel); 906 return; 907 } 908 909 udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); 910 if (udp < 0) { 911 close(dgram); 912 close(tcp); 913 test_sockmap_pass_prog__destroy(skel); 914 return; 915 } 916 917 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 918 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 919 if (err) 920 goto out; 921 922 for (i = 0; i < 2; i++, entry++) { 923 err = bpf_map_update_elem(map[i], &entry, &stream[0], BPF_ANY); 924 ASSERT_OK(err, "bpf_map_update_elem(stream)"); 925 } 926 for (i = 0; i < 2; i++, entry++) { 927 err = bpf_map_update_elem(map[i], &entry, &dgram, BPF_ANY); 928 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 929 } 930 for (i = 0; i < 2; i++, entry++) { 931 err = bpf_map_update_elem(map[i], &entry, &udp, BPF_ANY); 932 ASSERT_OK(err, "bpf_map_update_elem(udp)"); 933 } 934 for (i = 0; i < 2; i++, entry++) { 935 err = bpf_map_update_elem(map[i], &entry, &tcp, BPF_ANY); 936 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 937 } 938 for (entry--; entry >= 0; entry--) { 939 err = bpf_map_delete_elem(map[1], &entry); 940 entry--; 941 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 942 err = bpf_map_delete_elem(map[0], &entry); 943 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 944 } 945 946 close(stream[0]); 947 close(stream[1]); 948 out: 949 close(dgram); 950 close(tcp); 951 close(udp); 952 test_sockmap_pass_prog__destroy(skel); 953 } 954 955 static void test_sockmap_same_sock(void) 956 { 957 struct test_sockmap_pass_prog *skel; 958 int stream[2], dgram, udp, tcp; 959 int i, err, map, zero = 0; 960 961 skel = test_sockmap_pass_prog__open_and_load(); 962 if (!ASSERT_OK_PTR(skel, "open_and_load")) 963 return; 964 965 map = bpf_map__fd(skel->maps.sock_map_rx); 966 967 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 968 if (dgram < 0) { 969 test_sockmap_pass_prog__destroy(skel); 970 return; 971 } 972 973 tcp = connected_socket_v4(); 974 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) { 975 close(dgram); 976 test_sockmap_pass_prog__destroy(skel); 977 return; 978 } 979 980 udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); 981 if (udp < 0) { 982 close(dgram); 983 close(tcp); 984 test_sockmap_pass_prog__destroy(skel); 985 return; 986 } 987 988 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 989 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 990 if (err) { 991 close(tcp); 992 goto out; 993 } 994 995 for (i = 0; i < 2; i++) { 996 err = bpf_map_update_elem(map, &zero, &stream[0], BPF_ANY); 997 ASSERT_OK(err, "bpf_map_update_elem(stream)"); 998 } 999 for (i = 0; i < 2; i++) { 1000 err = bpf_map_update_elem(map, &zero, &dgram, BPF_ANY); 1001 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 1002 } 1003 for (i = 0; i < 2; i++) { 1004 err = bpf_map_update_elem(map, &zero, &udp, BPF_ANY); 1005 ASSERT_OK(err, "bpf_map_update_elem(udp)"); 1006 } 1007 for (i = 0; i < 2; i++) { 1008 err = bpf_map_update_elem(map, &zero, &tcp, BPF_ANY); 1009 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 1010 } 1011 1012 close(tcp); 1013 err = bpf_map_delete_elem(map, &zero); 1014 ASSERT_ERR(err, "bpf_map_delete_elem(entry)"); 1015 1016 close(stream[0]); 1017 close(stream[1]); 1018 out: 1019 close(dgram); 1020 close(udp); 1021 test_sockmap_pass_prog__destroy(skel); 1022 } 1023 1024 static void test_sockmap_skb_verdict_vsock_poll(void) 1025 { 1026 struct test_sockmap_pass_prog *skel; 1027 int err, map, conn, peer; 1028 struct bpf_program *prog; 1029 struct bpf_link *link; 1030 char buf = 'x'; 1031 int zero = 0; 1032 1033 skel = test_sockmap_pass_prog__open_and_load(); 1034 if (!ASSERT_OK_PTR(skel, "open_and_load")) 1035 return; 1036 1037 if (create_pair(AF_VSOCK, SOCK_STREAM, &conn, &peer)) 1038 goto destroy; 1039 1040 prog = skel->progs.prog_skb_verdict; 1041 map = bpf_map__fd(skel->maps.sock_map_rx); 1042 link = bpf_program__attach_sockmap(prog, map); 1043 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 1044 goto close; 1045 1046 err = bpf_map_update_elem(map, &zero, &conn, BPF_ANY); 1047 if (!ASSERT_OK(err, "bpf_map_update_elem")) 1048 goto detach; 1049 1050 if (xsend(peer, &buf, 1, 0) != 1) 1051 goto detach; 1052 1053 err = poll_read(conn, IO_TIMEOUT_SEC); 1054 if (!ASSERT_OK(err, "poll")) 1055 goto detach; 1056 1057 if (xrecv_nonblock(conn, &buf, 1, 0) != 1) 1058 FAIL("xrecv_nonblock"); 1059 detach: 1060 bpf_link__detach(link); 1061 close: 1062 xclose(conn); 1063 xclose(peer); 1064 destroy: 1065 test_sockmap_pass_prog__destroy(skel); 1066 } 1067 1068 static void test_sockmap_vsock_unconnected(void) 1069 { 1070 struct sockaddr_storage addr; 1071 int map, s, zero = 0; 1072 socklen_t alen; 1073 1074 map = bpf_map_create(BPF_MAP_TYPE_SOCKMAP, NULL, sizeof(int), 1075 sizeof(int), 1, NULL); 1076 if (!ASSERT_OK_FD(map, "bpf_map_create")) 1077 return; 1078 1079 s = xsocket(AF_VSOCK, SOCK_STREAM, 0); 1080 if (s < 0) 1081 goto close_map; 1082 1083 /* Fail connect(), but trigger transport assignment. */ 1084 init_addr_loopback(AF_VSOCK, &addr, &alen); 1085 if (!ASSERT_ERR(connect(s, sockaddr(&addr), alen), "connect")) 1086 goto close_sock; 1087 1088 ASSERT_ERR(bpf_map_update_elem(map, &zero, &s, BPF_ANY), "map_update"); 1089 1090 close_sock: 1091 xclose(s); 1092 close_map: 1093 xclose(map); 1094 } 1095 1096 void test_sockmap_basic(void) 1097 { 1098 if (test__start_subtest("sockmap create_update_free")) 1099 test_sockmap_create_update_free(BPF_MAP_TYPE_SOCKMAP); 1100 if (test__start_subtest("sockhash create_update_free")) 1101 test_sockmap_create_update_free(BPF_MAP_TYPE_SOCKHASH); 1102 if (test__start_subtest("sockmap vsock delete on close")) 1103 test_sockmap_vsock_delete_on_close(); 1104 if (test__start_subtest("sockmap sk_msg load helpers")) 1105 test_skmsg_helpers(BPF_MAP_TYPE_SOCKMAP); 1106 if (test__start_subtest("sockhash sk_msg load helpers")) 1107 test_skmsg_helpers(BPF_MAP_TYPE_SOCKHASH); 1108 if (test__start_subtest("sockmap update")) 1109 test_sockmap_update(BPF_MAP_TYPE_SOCKMAP); 1110 if (test__start_subtest("sockhash update")) 1111 test_sockmap_update(BPF_MAP_TYPE_SOCKHASH); 1112 if (test__start_subtest("sockmap update in unsafe context")) 1113 test_sockmap_invalid_update(); 1114 if (test__start_subtest("sockmap copy")) 1115 test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP); 1116 if (test__start_subtest("sockhash copy")) 1117 test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH); 1118 if (test__start_subtest("sockmap skb_verdict attach")) { 1119 test_sockmap_skb_verdict_attach(BPF_SK_SKB_VERDICT, 1120 BPF_SK_SKB_STREAM_VERDICT); 1121 test_sockmap_skb_verdict_attach(BPF_SK_SKB_STREAM_VERDICT, 1122 BPF_SK_SKB_VERDICT); 1123 } 1124 if (test__start_subtest("sockmap skb_verdict attach_with_link")) 1125 test_sockmap_skb_verdict_attach_with_link(); 1126 if (test__start_subtest("sockmap msg_verdict progs query")) 1127 test_sockmap_progs_query(BPF_SK_MSG_VERDICT); 1128 if (test__start_subtest("sockmap stream_parser progs query")) 1129 test_sockmap_progs_query(BPF_SK_SKB_STREAM_PARSER); 1130 if (test__start_subtest("sockmap stream_verdict progs query")) 1131 test_sockmap_progs_query(BPF_SK_SKB_STREAM_VERDICT); 1132 if (test__start_subtest("sockmap skb_verdict progs query")) 1133 test_sockmap_progs_query(BPF_SK_SKB_VERDICT); 1134 if (test__start_subtest("sockmap skb_verdict shutdown")) 1135 test_sockmap_skb_verdict_shutdown(); 1136 if (test__start_subtest("sockmap stream parser and verdict pass")) 1137 test_sockmap_stream_pass(); 1138 if (test__start_subtest("sockmap skb_verdict fionread")) 1139 test_sockmap_skb_verdict_fionread(true); 1140 if (test__start_subtest("sockmap skb_verdict fionread on drop")) 1141 test_sockmap_skb_verdict_fionread(false); 1142 if (test__start_subtest("sockmap skb_verdict change tail")) 1143 test_sockmap_skb_verdict_change_tail(); 1144 if (test__start_subtest("sockmap skb_verdict msg_f_peek")) 1145 test_sockmap_skb_verdict_peek(); 1146 if (test__start_subtest("sockmap skb_verdict msg_f_peek with link")) 1147 test_sockmap_skb_verdict_peek_with_link(); 1148 if (test__start_subtest("sockmap unconnected af_unix")) 1149 test_sockmap_unconnected_unix(); 1150 if (test__start_subtest("sockmap one socket to many map entries")) 1151 test_sockmap_many_socket(); 1152 if (test__start_subtest("sockmap one socket to many maps")) 1153 test_sockmap_many_maps(); 1154 if (test__start_subtest("sockmap same socket replace")) 1155 test_sockmap_same_sock(); 1156 if (test__start_subtest("sockmap sk_msg attach sockmap helpers with link")) 1157 test_skmsg_helpers_with_link(BPF_MAP_TYPE_SOCKMAP); 1158 if (test__start_subtest("sockhash sk_msg attach sockhash helpers with link")) 1159 test_skmsg_helpers_with_link(BPF_MAP_TYPE_SOCKHASH); 1160 if (test__start_subtest("sockmap skb_verdict vsock poll")) 1161 test_sockmap_skb_verdict_vsock_poll(); 1162 if (test__start_subtest("sockmap vsock unconnected")) 1163 test_sockmap_vsock_unconnected(); 1164 } 1165