1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2020 Cloudflare 3 #include <error.h> 4 #include <linux/tcp.h> 5 #include <linux/socket.h> 6 #include <sys/epoll.h> 7 8 #include "test_progs.h" 9 #include "test_skmsg_load_helpers.skel.h" 10 #include "test_sockmap_update.skel.h" 11 #include "test_sockmap_invalid_update.skel.h" 12 #include "test_sockmap_skb_verdict_attach.skel.h" 13 #include "test_sockmap_progs_query.skel.h" 14 #include "test_sockmap_pass_prog.skel.h" 15 #include "test_sockmap_drop_prog.skel.h" 16 #include "test_sockmap_change_tail.skel.h" 17 #include "bpf_iter_sockmap.skel.h" 18 19 #include "sockmap_helpers.h" 20 21 #define TCP_REPAIR 19 /* TCP sock is under repair right now */ 22 23 #define TCP_REPAIR_ON 1 24 #define TCP_REPAIR_OFF_NO_WP -1 /* Turn off without window probes */ 25 26 /** 27 * SOL_TCP is defined in <netinet/tcp.h> (glibc), but the copybuf_address 28 * field of tcp_zerocopy_receive is not yet included in older versions. 29 * This workaround remains necessary until the glibc update propagates. 30 */ 31 #ifndef SOL_TCP 32 #define SOL_TCP 6 33 #endif 34 35 static int connected_socket_v4(void) 36 { 37 struct sockaddr_in addr = { 38 .sin_family = AF_INET, 39 .sin_port = htons(80), 40 .sin_addr = { inet_addr("127.0.0.1") }, 41 }; 42 socklen_t len = sizeof(addr); 43 int s, repair, err; 44 45 s = socket(AF_INET, SOCK_STREAM, 0); 46 if (!ASSERT_GE(s, 0, "socket")) 47 goto error; 48 49 repair = TCP_REPAIR_ON; 50 err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair)); 51 if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)")) 52 goto error; 53 54 err = connect(s, (struct sockaddr *)&addr, len); 55 if (!ASSERT_OK(err, "connect")) 56 goto error; 57 58 repair = TCP_REPAIR_OFF_NO_WP; 59 err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair)); 60 if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)")) 61 goto error; 62 63 return s; 64 error: 65 perror(__func__); 66 close(s); 67 return -1; 68 } 69 70 static void compare_cookies(struct bpf_map *src, struct bpf_map *dst) 71 { 72 __u32 i, max_entries = bpf_map__max_entries(src); 73 int err, src_fd, dst_fd; 74 75 src_fd = bpf_map__fd(src); 76 dst_fd = bpf_map__fd(dst); 77 78 for (i = 0; i < max_entries; i++) { 79 __u64 src_cookie, dst_cookie; 80 81 err = bpf_map_lookup_elem(src_fd, &i, &src_cookie); 82 if (err && errno == ENOENT) { 83 err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); 84 ASSERT_ERR(err, "map_lookup_elem(dst)"); 85 ASSERT_EQ(errno, ENOENT, "map_lookup_elem(dst)"); 86 continue; 87 } 88 if (!ASSERT_OK(err, "lookup_elem(src)")) 89 continue; 90 91 err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); 92 if (!ASSERT_OK(err, "lookup_elem(dst)")) 93 continue; 94 95 ASSERT_EQ(dst_cookie, src_cookie, "cookie mismatch"); 96 } 97 } 98 99 /* Create a map, populate it with one socket, and free the map. */ 100 static void test_sockmap_create_update_free(enum bpf_map_type map_type) 101 { 102 const int zero = 0; 103 int s, map, err; 104 105 s = connected_socket_v4(); 106 if (!ASSERT_GE(s, 0, "connected_socket_v4")) 107 return; 108 109 map = bpf_map_create(map_type, NULL, sizeof(int), sizeof(int), 1, NULL); 110 if (!ASSERT_GE(map, 0, "bpf_map_create")) 111 goto out; 112 113 err = bpf_map_update_elem(map, &zero, &s, BPF_NOEXIST); 114 if (!ASSERT_OK(err, "bpf_map_update")) 115 goto out; 116 117 out: 118 close(map); 119 close(s); 120 } 121 122 static void test_sockmap_vsock_delete_on_close(void) 123 { 124 int map, c, p, err, zero = 0; 125 126 map = bpf_map_create(BPF_MAP_TYPE_SOCKMAP, NULL, sizeof(int), 127 sizeof(int), 1, NULL); 128 if (!ASSERT_OK_FD(map, "bpf_map_create")) 129 return; 130 131 err = create_pair(AF_VSOCK, SOCK_STREAM, &c, &p); 132 if (!ASSERT_OK(err, "create_pair")) 133 goto close_map; 134 135 if (xbpf_map_update_elem(map, &zero, &c, BPF_NOEXIST)) 136 goto close_socks; 137 138 xclose(c); 139 xclose(p); 140 141 err = create_pair(AF_VSOCK, SOCK_STREAM, &c, &p); 142 if (!ASSERT_OK(err, "create_pair")) 143 goto close_map; 144 145 err = bpf_map_update_elem(map, &zero, &c, BPF_NOEXIST); 146 ASSERT_OK(err, "after close(), bpf_map_update"); 147 148 close_socks: 149 xclose(c); 150 xclose(p); 151 close_map: 152 xclose(map); 153 } 154 155 static void test_skmsg_helpers(enum bpf_map_type map_type) 156 { 157 struct test_skmsg_load_helpers *skel; 158 int err, map, verdict; 159 160 skel = test_skmsg_load_helpers__open_and_load(); 161 if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load")) 162 return; 163 164 verdict = bpf_program__fd(skel->progs.prog_msg_verdict); 165 map = bpf_map__fd(skel->maps.sock_map); 166 167 err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0); 168 if (!ASSERT_OK(err, "bpf_prog_attach")) 169 goto out; 170 171 err = bpf_prog_detach2(verdict, map, BPF_SK_MSG_VERDICT); 172 if (!ASSERT_OK(err, "bpf_prog_detach2")) 173 goto out; 174 out: 175 test_skmsg_load_helpers__destroy(skel); 176 } 177 178 static void test_skmsg_helpers_with_link(enum bpf_map_type map_type) 179 { 180 struct bpf_program *prog, *prog_clone, *prog_clone2; 181 DECLARE_LIBBPF_OPTS(bpf_link_update_opts, opts); 182 struct test_skmsg_load_helpers *skel; 183 struct bpf_link *link, *link2; 184 int err, map; 185 186 skel = test_skmsg_load_helpers__open_and_load(); 187 if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load")) 188 return; 189 190 prog = skel->progs.prog_msg_verdict; 191 prog_clone = skel->progs.prog_msg_verdict_clone; 192 prog_clone2 = skel->progs.prog_msg_verdict_clone2; 193 map = bpf_map__fd(skel->maps.sock_map); 194 195 link = bpf_program__attach_sockmap(prog, map); 196 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 197 goto out; 198 199 /* Fail since bpf_link for the same prog has been created. */ 200 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_MSG_VERDICT, 0); 201 if (!ASSERT_ERR(err, "bpf_prog_attach")) 202 goto out; 203 204 /* Fail since bpf_link for the same prog type has been created. */ 205 link2 = bpf_program__attach_sockmap(prog_clone, map); 206 if (!ASSERT_ERR_PTR(link2, "bpf_program__attach_sockmap")) { 207 bpf_link__detach(link2); 208 goto out; 209 } 210 211 err = bpf_link__update_program(link, prog_clone); 212 if (!ASSERT_OK(err, "bpf_link__update_program")) 213 goto out; 214 215 /* Fail since a prog with different type attempts to do update. */ 216 err = bpf_link__update_program(link, skel->progs.prog_skb_verdict); 217 if (!ASSERT_ERR(err, "bpf_link__update_program")) 218 goto out; 219 220 /* Fail since the old prog does not match the one in the kernel. */ 221 opts.old_prog_fd = bpf_program__fd(prog_clone2); 222 opts.flags = BPF_F_REPLACE; 223 err = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), &opts); 224 if (!ASSERT_ERR(err, "bpf_link_update")) 225 goto out; 226 227 opts.old_prog_fd = bpf_program__fd(prog_clone); 228 opts.flags = BPF_F_REPLACE; 229 err = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), &opts); 230 if (!ASSERT_OK(err, "bpf_link_update")) 231 goto out; 232 out: 233 bpf_link__detach(link); 234 test_skmsg_load_helpers__destroy(skel); 235 } 236 237 static void test_sockmap_update(enum bpf_map_type map_type) 238 { 239 int err, prog, src; 240 struct test_sockmap_update *skel; 241 struct bpf_map *dst_map; 242 const __u32 zero = 0; 243 char dummy[14] = {0}; 244 LIBBPF_OPTS(bpf_test_run_opts, topts, 245 .data_in = dummy, 246 .data_size_in = sizeof(dummy), 247 .repeat = 1, 248 ); 249 __s64 sk; 250 251 sk = connected_socket_v4(); 252 if (!ASSERT_NEQ(sk, -1, "connected_socket_v4")) 253 return; 254 255 skel = test_sockmap_update__open_and_load(); 256 if (!ASSERT_OK_PTR(skel, "open_and_load")) 257 goto close_sk; 258 259 prog = bpf_program__fd(skel->progs.copy_sock_map); 260 src = bpf_map__fd(skel->maps.src); 261 if (map_type == BPF_MAP_TYPE_SOCKMAP) 262 dst_map = skel->maps.dst_sock_map; 263 else 264 dst_map = skel->maps.dst_sock_hash; 265 266 err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST); 267 if (!ASSERT_OK(err, "update_elem(src)")) 268 goto out; 269 270 err = bpf_prog_test_run_opts(prog, &topts); 271 if (!ASSERT_OK(err, "test_run")) 272 goto out; 273 if (!ASSERT_NEQ(topts.retval, 0, "test_run retval")) 274 goto out; 275 276 compare_cookies(skel->maps.src, dst_map); 277 278 out: 279 test_sockmap_update__destroy(skel); 280 close_sk: 281 close(sk); 282 } 283 284 static void test_sockmap_invalid_update(void) 285 { 286 struct test_sockmap_invalid_update *skel; 287 288 skel = test_sockmap_invalid_update__open_and_load(); 289 if (!ASSERT_NULL(skel, "open_and_load")) 290 test_sockmap_invalid_update__destroy(skel); 291 } 292 293 static void test_sockmap_copy(enum bpf_map_type map_type) 294 { 295 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 296 int err, len, src_fd, iter_fd; 297 union bpf_iter_link_info linfo = {}; 298 __u32 i, num_sockets, num_elems; 299 struct bpf_iter_sockmap *skel; 300 __s64 *sock_fd = NULL; 301 struct bpf_link *link; 302 struct bpf_map *src; 303 char buf[64]; 304 305 skel = bpf_iter_sockmap__open_and_load(); 306 if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load")) 307 return; 308 309 if (map_type == BPF_MAP_TYPE_SOCKMAP) { 310 src = skel->maps.sockmap; 311 num_elems = bpf_map__max_entries(src); 312 num_sockets = num_elems - 1; 313 } else { 314 src = skel->maps.sockhash; 315 num_elems = bpf_map__max_entries(src) - 1; 316 num_sockets = num_elems; 317 } 318 319 sock_fd = calloc(num_sockets, sizeof(*sock_fd)); 320 if (!ASSERT_OK_PTR(sock_fd, "calloc(sock_fd)")) 321 goto out; 322 323 for (i = 0; i < num_sockets; i++) 324 sock_fd[i] = -1; 325 326 src_fd = bpf_map__fd(src); 327 328 for (i = 0; i < num_sockets; i++) { 329 sock_fd[i] = connected_socket_v4(); 330 if (!ASSERT_NEQ(sock_fd[i], -1, "connected_socket_v4")) 331 goto out; 332 333 err = bpf_map_update_elem(src_fd, &i, &sock_fd[i], BPF_NOEXIST); 334 if (!ASSERT_OK(err, "map_update")) 335 goto out; 336 } 337 338 linfo.map.map_fd = src_fd; 339 opts.link_info = &linfo; 340 opts.link_info_len = sizeof(linfo); 341 link = bpf_program__attach_iter(skel->progs.copy, &opts); 342 if (!ASSERT_OK_PTR(link, "attach_iter")) 343 goto out; 344 345 iter_fd = bpf_iter_create(bpf_link__fd(link)); 346 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 347 goto free_link; 348 349 /* do some tests */ 350 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 351 ; 352 if (!ASSERT_GE(len, 0, "read")) 353 goto close_iter; 354 355 /* test results */ 356 if (!ASSERT_EQ(skel->bss->elems, num_elems, "elems")) 357 goto close_iter; 358 359 if (!ASSERT_EQ(skel->bss->socks, num_sockets, "socks")) 360 goto close_iter; 361 362 compare_cookies(src, skel->maps.dst); 363 364 close_iter: 365 close(iter_fd); 366 free_link: 367 bpf_link__destroy(link); 368 out: 369 for (i = 0; sock_fd && i < num_sockets; i++) 370 if (sock_fd[i] >= 0) 371 close(sock_fd[i]); 372 if (sock_fd) 373 free(sock_fd); 374 bpf_iter_sockmap__destroy(skel); 375 } 376 377 static void test_sockmap_skb_verdict_attach(enum bpf_attach_type first, 378 enum bpf_attach_type second) 379 { 380 struct test_sockmap_skb_verdict_attach *skel; 381 int err, map, verdict; 382 383 skel = test_sockmap_skb_verdict_attach__open_and_load(); 384 if (!ASSERT_OK_PTR(skel, "open_and_load")) 385 return; 386 387 verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 388 map = bpf_map__fd(skel->maps.sock_map); 389 390 err = bpf_prog_attach(verdict, map, first, 0); 391 if (!ASSERT_OK(err, "bpf_prog_attach")) 392 goto out; 393 394 err = bpf_prog_attach(verdict, map, second, 0); 395 ASSERT_EQ(err, -EBUSY, "prog_attach_fail"); 396 397 err = bpf_prog_detach2(verdict, map, first); 398 if (!ASSERT_OK(err, "bpf_prog_detach2")) 399 goto out; 400 out: 401 test_sockmap_skb_verdict_attach__destroy(skel); 402 } 403 404 static void test_sockmap_skb_verdict_attach_with_link(void) 405 { 406 struct test_sockmap_skb_verdict_attach *skel; 407 struct bpf_program *prog; 408 struct bpf_link *link; 409 int err, map; 410 411 skel = test_sockmap_skb_verdict_attach__open_and_load(); 412 if (!ASSERT_OK_PTR(skel, "open_and_load")) 413 return; 414 prog = skel->progs.prog_skb_verdict; 415 map = bpf_map__fd(skel->maps.sock_map); 416 link = bpf_program__attach_sockmap(prog, map); 417 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 418 goto out; 419 420 bpf_link__detach(link); 421 422 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0); 423 if (!ASSERT_OK(err, "bpf_prog_attach")) 424 goto out; 425 426 /* Fail since attaching with the same prog/map has been done. */ 427 link = bpf_program__attach_sockmap(prog, map); 428 if (!ASSERT_ERR_PTR(link, "bpf_program__attach_sockmap")) 429 bpf_link__detach(link); 430 431 err = bpf_prog_detach2(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT); 432 if (!ASSERT_OK(err, "bpf_prog_detach2")) 433 goto out; 434 out: 435 test_sockmap_skb_verdict_attach__destroy(skel); 436 } 437 438 static __u32 query_prog_id(int prog_fd) 439 { 440 struct bpf_prog_info info = {}; 441 __u32 info_len = sizeof(info); 442 int err; 443 444 err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); 445 if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd") || 446 !ASSERT_EQ(info_len, sizeof(info), "bpf_prog_get_info_by_fd")) 447 return 0; 448 449 return info.id; 450 } 451 452 static void test_sockmap_progs_query(enum bpf_attach_type attach_type) 453 { 454 struct test_sockmap_progs_query *skel; 455 int err, map_fd, verdict_fd; 456 __u32 attach_flags = 0; 457 __u32 prog_ids[3] = {}; 458 __u32 prog_cnt = 3; 459 460 skel = test_sockmap_progs_query__open_and_load(); 461 if (!ASSERT_OK_PTR(skel, "test_sockmap_progs_query__open_and_load")) 462 return; 463 464 map_fd = bpf_map__fd(skel->maps.sock_map); 465 466 if (attach_type == BPF_SK_MSG_VERDICT) 467 verdict_fd = bpf_program__fd(skel->progs.prog_skmsg_verdict); 468 else 469 verdict_fd = bpf_program__fd(skel->progs.prog_skb_verdict); 470 471 err = bpf_prog_query(map_fd, attach_type, 0 /* query flags */, 472 &attach_flags, prog_ids, &prog_cnt); 473 ASSERT_OK(err, "bpf_prog_query failed"); 474 ASSERT_EQ(attach_flags, 0, "wrong attach_flags on query"); 475 ASSERT_EQ(prog_cnt, 0, "wrong program count on query"); 476 477 err = bpf_prog_attach(verdict_fd, map_fd, attach_type, 0); 478 if (!ASSERT_OK(err, "bpf_prog_attach failed")) 479 goto out; 480 481 prog_cnt = 1; 482 err = bpf_prog_query(map_fd, attach_type, 0 /* query flags */, 483 &attach_flags, prog_ids, &prog_cnt); 484 ASSERT_OK(err, "bpf_prog_query failed"); 485 ASSERT_EQ(attach_flags, 0, "wrong attach_flags on query"); 486 ASSERT_EQ(prog_cnt, 1, "wrong program count on query"); 487 ASSERT_EQ(prog_ids[0], query_prog_id(verdict_fd), 488 "wrong prog_ids on query"); 489 490 bpf_prog_detach2(verdict_fd, map_fd, attach_type); 491 out: 492 test_sockmap_progs_query__destroy(skel); 493 } 494 495 #define MAX_EVENTS 10 496 static void test_sockmap_skb_verdict_shutdown(void) 497 { 498 int n, err, map, verdict, c1 = -1, p1 = -1; 499 struct epoll_event ev, events[MAX_EVENTS]; 500 struct test_sockmap_pass_prog *skel; 501 int zero = 0; 502 int epollfd; 503 char b; 504 505 skel = test_sockmap_pass_prog__open_and_load(); 506 if (!ASSERT_OK_PTR(skel, "open_and_load")) 507 return; 508 509 verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 510 map = bpf_map__fd(skel->maps.sock_map_rx); 511 512 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 513 if (!ASSERT_OK(err, "bpf_prog_attach")) 514 goto out; 515 516 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); 517 if (err < 0) 518 goto out; 519 520 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 521 if (err < 0) 522 goto out_close; 523 524 shutdown(p1, SHUT_WR); 525 526 ev.events = EPOLLIN; 527 ev.data.fd = c1; 528 529 epollfd = epoll_create1(0); 530 if (!ASSERT_GT(epollfd, -1, "epoll_create(0)")) 531 goto out_close; 532 err = epoll_ctl(epollfd, EPOLL_CTL_ADD, c1, &ev); 533 if (!ASSERT_OK(err, "epoll_ctl(EPOLL_CTL_ADD)")) 534 goto out_close; 535 err = epoll_wait(epollfd, events, MAX_EVENTS, -1); 536 if (!ASSERT_EQ(err, 1, "epoll_wait(fd)")) 537 goto out_close; 538 539 n = recv(c1, &b, 1, MSG_DONTWAIT); 540 ASSERT_EQ(n, 0, "recv(fin)"); 541 out_close: 542 close(c1); 543 close(p1); 544 out: 545 test_sockmap_pass_prog__destroy(skel); 546 } 547 548 549 static void do_test_sockmap_skb_verdict_fionread(int sotype, bool pass_prog) 550 { 551 int err, map, verdict, c0 = -1, c1 = -1, p0 = -1, p1 = -1; 552 int expected, zero = 0, sent, recvd, avail; 553 struct test_sockmap_pass_prog *pass = NULL; 554 struct test_sockmap_drop_prog *drop = NULL; 555 char buf[256] = "0123456789"; 556 int split_len = sizeof(buf) / 2; 557 558 if (pass_prog) { 559 pass = test_sockmap_pass_prog__open_and_load(); 560 if (!ASSERT_OK_PTR(pass, "open_and_load")) 561 return; 562 verdict = bpf_program__fd(pass->progs.prog_skb_verdict); 563 map = bpf_map__fd(pass->maps.sock_map_rx); 564 if (sotype == SOCK_DGRAM) 565 expected = split_len; /* FIONREAD for UDP is different from TCP */ 566 else 567 expected = sizeof(buf); 568 } else { 569 drop = test_sockmap_drop_prog__open_and_load(); 570 if (!ASSERT_OK_PTR(drop, "open_and_load")) 571 return; 572 verdict = bpf_program__fd(drop->progs.prog_skb_verdict); 573 map = bpf_map__fd(drop->maps.sock_map_rx); 574 /* On drop data is consumed immediately and copied_seq inc'd */ 575 expected = 0; 576 } 577 578 579 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 580 if (!ASSERT_OK(err, "bpf_prog_attach")) 581 goto out; 582 583 err = create_socket_pairs(AF_INET, sotype, &c0, &c1, &p0, &p1); 584 if (!ASSERT_OK(err, "create_socket_pairs()")) 585 goto out; 586 587 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 588 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) 589 goto out_close; 590 591 sent = xsend(p1, &buf, split_len, 0); 592 sent += xsend(p1, &buf, sizeof(buf) - split_len, 0); 593 ASSERT_EQ(sent, sizeof(buf), "xsend(p1)"); 594 err = ioctl(c1, FIONREAD, &avail); 595 ASSERT_OK(err, "ioctl(FIONREAD) error"); 596 ASSERT_EQ(avail, expected, "ioctl(FIONREAD)"); 597 /* On DROP test there will be no data to read */ 598 if (pass_prog) { 599 recvd = recv_timeout(c1, &buf, sizeof(buf), MSG_DONTWAIT, IO_TIMEOUT_SEC); 600 ASSERT_EQ(recvd, sizeof(buf), "recv_timeout(c0)"); 601 } 602 603 out_close: 604 close(c0); 605 close(p0); 606 close(c1); 607 close(p1); 608 out: 609 if (pass_prog) 610 test_sockmap_pass_prog__destroy(pass); 611 else 612 test_sockmap_drop_prog__destroy(drop); 613 } 614 615 static void test_sockmap_skb_verdict_fionread(bool pass_prog) 616 { 617 do_test_sockmap_skb_verdict_fionread(SOCK_STREAM, pass_prog); 618 do_test_sockmap_skb_verdict_fionread(SOCK_DGRAM, pass_prog); 619 } 620 621 static void test_sockmap_skb_verdict_change_tail(void) 622 { 623 struct test_sockmap_change_tail *skel; 624 int err, map, verdict; 625 int c1, p1, sent, recvd; 626 int zero = 0; 627 char buf[2]; 628 629 skel = test_sockmap_change_tail__open_and_load(); 630 if (!ASSERT_OK_PTR(skel, "open_and_load")) 631 return; 632 verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 633 map = bpf_map__fd(skel->maps.sock_map_rx); 634 635 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 636 if (!ASSERT_OK(err, "bpf_prog_attach")) 637 goto out; 638 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); 639 if (!ASSERT_OK(err, "create_pair()")) 640 goto out; 641 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 642 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) 643 goto out_close; 644 sent = xsend(p1, "Tr", 2, 0); 645 ASSERT_EQ(sent, 2, "xsend(p1)"); 646 recvd = recv(c1, buf, 2, 0); 647 ASSERT_EQ(recvd, 1, "recv(c1)"); 648 ASSERT_EQ(skel->data->change_tail_ret, 0, "change_tail_ret"); 649 650 sent = xsend(p1, "G", 1, 0); 651 ASSERT_EQ(sent, 1, "xsend(p1)"); 652 recvd = recv(c1, buf, 2, 0); 653 ASSERT_EQ(recvd, 2, "recv(c1)"); 654 ASSERT_EQ(skel->data->change_tail_ret, 0, "change_tail_ret"); 655 656 sent = xsend(p1, "E", 1, 0); 657 ASSERT_EQ(sent, 1, "xsend(p1)"); 658 recvd = recv(c1, buf, 1, 0); 659 ASSERT_EQ(recvd, 1, "recv(c1)"); 660 ASSERT_EQ(skel->data->change_tail_ret, -EINVAL, "change_tail_ret"); 661 662 out_close: 663 close(c1); 664 close(p1); 665 out: 666 test_sockmap_change_tail__destroy(skel); 667 } 668 669 static void test_sockmap_skb_verdict_peek_helper(int map) 670 { 671 int err, c1, p1, zero = 0, sent, recvd, avail; 672 char snd[256] = "0123456789"; 673 char rcv[256] = "0"; 674 675 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); 676 if (!ASSERT_OK(err, "create_pair()")) 677 return; 678 679 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 680 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) 681 goto out_close; 682 683 sent = xsend(p1, snd, sizeof(snd), 0); 684 ASSERT_EQ(sent, sizeof(snd), "xsend(p1)"); 685 recvd = recv(c1, rcv, sizeof(rcv), MSG_PEEK); 686 ASSERT_EQ(recvd, sizeof(rcv), "recv(c1)"); 687 err = ioctl(c1, FIONREAD, &avail); 688 ASSERT_OK(err, "ioctl(FIONREAD) error"); 689 ASSERT_EQ(avail, sizeof(snd), "after peek ioctl(FIONREAD)"); 690 recvd = recv(c1, rcv, sizeof(rcv), 0); 691 ASSERT_EQ(recvd, sizeof(rcv), "recv(p0)"); 692 err = ioctl(c1, FIONREAD, &avail); 693 ASSERT_OK(err, "ioctl(FIONREAD) error"); 694 ASSERT_EQ(avail, 0, "after read ioctl(FIONREAD)"); 695 696 out_close: 697 close(c1); 698 close(p1); 699 } 700 701 static void test_sockmap_skb_verdict_peek(void) 702 { 703 struct test_sockmap_pass_prog *pass; 704 int err, map, verdict; 705 706 pass = test_sockmap_pass_prog__open_and_load(); 707 if (!ASSERT_OK_PTR(pass, "open_and_load")) 708 return; 709 verdict = bpf_program__fd(pass->progs.prog_skb_verdict); 710 map = bpf_map__fd(pass->maps.sock_map_rx); 711 712 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 713 if (!ASSERT_OK(err, "bpf_prog_attach")) 714 goto out; 715 716 test_sockmap_skb_verdict_peek_helper(map); 717 718 out: 719 test_sockmap_pass_prog__destroy(pass); 720 } 721 722 static void test_sockmap_skb_verdict_peek_with_link(void) 723 { 724 struct test_sockmap_pass_prog *pass; 725 struct bpf_program *prog; 726 struct bpf_link *link; 727 int err, map; 728 729 pass = test_sockmap_pass_prog__open_and_load(); 730 if (!ASSERT_OK_PTR(pass, "open_and_load")) 731 return; 732 prog = pass->progs.prog_skb_verdict; 733 map = bpf_map__fd(pass->maps.sock_map_rx); 734 link = bpf_program__attach_sockmap(prog, map); 735 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 736 goto out; 737 738 err = bpf_link__update_program(link, pass->progs.prog_skb_verdict_clone); 739 if (!ASSERT_OK(err, "bpf_link__update_program")) 740 goto out; 741 742 /* Fail since a prog with different attach type attempts to do update. */ 743 err = bpf_link__update_program(link, pass->progs.prog_skb_parser); 744 if (!ASSERT_ERR(err, "bpf_link__update_program")) 745 goto out; 746 747 test_sockmap_skb_verdict_peek_helper(map); 748 ASSERT_EQ(pass->bss->clone_called, 1, "clone_called"); 749 out: 750 bpf_link__detach(link); 751 test_sockmap_pass_prog__destroy(pass); 752 } 753 754 static void test_sockmap_unconnected_unix(void) 755 { 756 int err, map, stream = 0, dgram = 0, zero = 0; 757 struct test_sockmap_pass_prog *skel; 758 759 skel = test_sockmap_pass_prog__open_and_load(); 760 if (!ASSERT_OK_PTR(skel, "open_and_load")) 761 return; 762 763 map = bpf_map__fd(skel->maps.sock_map_rx); 764 765 stream = xsocket(AF_UNIX, SOCK_STREAM, 0); 766 if (stream < 0) 767 return; 768 769 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 770 if (dgram < 0) { 771 close(stream); 772 return; 773 } 774 775 err = bpf_map_update_elem(map, &zero, &stream, BPF_ANY); 776 ASSERT_ERR(err, "bpf_map_update_elem(stream)"); 777 778 err = bpf_map_update_elem(map, &zero, &dgram, BPF_ANY); 779 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 780 781 close(stream); 782 close(dgram); 783 } 784 785 static void test_sockmap_many_socket(void) 786 { 787 struct test_sockmap_pass_prog *skel; 788 int stream[2], dgram, udp, tcp; 789 int i, err, map, entry = 0; 790 791 skel = test_sockmap_pass_prog__open_and_load(); 792 if (!ASSERT_OK_PTR(skel, "open_and_load")) 793 return; 794 795 map = bpf_map__fd(skel->maps.sock_map_rx); 796 797 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 798 if (dgram < 0) { 799 test_sockmap_pass_prog__destroy(skel); 800 return; 801 } 802 803 tcp = connected_socket_v4(); 804 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) { 805 close(dgram); 806 test_sockmap_pass_prog__destroy(skel); 807 return; 808 } 809 810 udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); 811 if (udp < 0) { 812 close(dgram); 813 close(tcp); 814 test_sockmap_pass_prog__destroy(skel); 815 return; 816 } 817 818 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 819 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 820 if (err) 821 goto out; 822 823 for (i = 0; i < 2; i++, entry++) { 824 err = bpf_map_update_elem(map, &entry, &stream[0], BPF_ANY); 825 ASSERT_OK(err, "bpf_map_update_elem(stream)"); 826 } 827 for (i = 0; i < 2; i++, entry++) { 828 err = bpf_map_update_elem(map, &entry, &dgram, BPF_ANY); 829 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 830 } 831 for (i = 0; i < 2; i++, entry++) { 832 err = bpf_map_update_elem(map, &entry, &udp, BPF_ANY); 833 ASSERT_OK(err, "bpf_map_update_elem(udp)"); 834 } 835 for (i = 0; i < 2; i++, entry++) { 836 err = bpf_map_update_elem(map, &entry, &tcp, BPF_ANY); 837 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 838 } 839 for (entry--; entry >= 0; entry--) { 840 err = bpf_map_delete_elem(map, &entry); 841 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 842 } 843 844 close(stream[0]); 845 close(stream[1]); 846 out: 847 close(dgram); 848 close(tcp); 849 close(udp); 850 test_sockmap_pass_prog__destroy(skel); 851 } 852 853 static void test_sockmap_many_maps(void) 854 { 855 struct test_sockmap_pass_prog *skel; 856 int stream[2], dgram, udp, tcp; 857 int i, err, map[2], entry = 0; 858 859 skel = test_sockmap_pass_prog__open_and_load(); 860 if (!ASSERT_OK_PTR(skel, "open_and_load")) 861 return; 862 863 map[0] = bpf_map__fd(skel->maps.sock_map_rx); 864 map[1] = bpf_map__fd(skel->maps.sock_map_tx); 865 866 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 867 if (dgram < 0) { 868 test_sockmap_pass_prog__destroy(skel); 869 return; 870 } 871 872 tcp = connected_socket_v4(); 873 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) { 874 close(dgram); 875 test_sockmap_pass_prog__destroy(skel); 876 return; 877 } 878 879 udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); 880 if (udp < 0) { 881 close(dgram); 882 close(tcp); 883 test_sockmap_pass_prog__destroy(skel); 884 return; 885 } 886 887 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 888 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 889 if (err) 890 goto out; 891 892 for (i = 0; i < 2; i++, entry++) { 893 err = bpf_map_update_elem(map[i], &entry, &stream[0], BPF_ANY); 894 ASSERT_OK(err, "bpf_map_update_elem(stream)"); 895 } 896 for (i = 0; i < 2; i++, entry++) { 897 err = bpf_map_update_elem(map[i], &entry, &dgram, BPF_ANY); 898 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 899 } 900 for (i = 0; i < 2; i++, entry++) { 901 err = bpf_map_update_elem(map[i], &entry, &udp, BPF_ANY); 902 ASSERT_OK(err, "bpf_map_update_elem(udp)"); 903 } 904 for (i = 0; i < 2; i++, entry++) { 905 err = bpf_map_update_elem(map[i], &entry, &tcp, BPF_ANY); 906 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 907 } 908 for (entry--; entry >= 0; entry--) { 909 err = bpf_map_delete_elem(map[1], &entry); 910 entry--; 911 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 912 err = bpf_map_delete_elem(map[0], &entry); 913 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 914 } 915 916 close(stream[0]); 917 close(stream[1]); 918 out: 919 close(dgram); 920 close(tcp); 921 close(udp); 922 test_sockmap_pass_prog__destroy(skel); 923 } 924 925 static void test_sockmap_same_sock(void) 926 { 927 struct test_sockmap_pass_prog *skel; 928 int stream[2], dgram, udp, tcp; 929 int i, err, map, zero = 0; 930 931 skel = test_sockmap_pass_prog__open_and_load(); 932 if (!ASSERT_OK_PTR(skel, "open_and_load")) 933 return; 934 935 map = bpf_map__fd(skel->maps.sock_map_rx); 936 937 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 938 if (dgram < 0) { 939 test_sockmap_pass_prog__destroy(skel); 940 return; 941 } 942 943 tcp = connected_socket_v4(); 944 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) { 945 close(dgram); 946 test_sockmap_pass_prog__destroy(skel); 947 return; 948 } 949 950 udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); 951 if (udp < 0) { 952 close(dgram); 953 close(tcp); 954 test_sockmap_pass_prog__destroy(skel); 955 return; 956 } 957 958 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 959 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 960 if (err) { 961 close(tcp); 962 goto out; 963 } 964 965 for (i = 0; i < 2; i++) { 966 err = bpf_map_update_elem(map, &zero, &stream[0], BPF_ANY); 967 ASSERT_OK(err, "bpf_map_update_elem(stream)"); 968 } 969 for (i = 0; i < 2; i++) { 970 err = bpf_map_update_elem(map, &zero, &dgram, BPF_ANY); 971 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 972 } 973 for (i = 0; i < 2; i++) { 974 err = bpf_map_update_elem(map, &zero, &udp, BPF_ANY); 975 ASSERT_OK(err, "bpf_map_update_elem(udp)"); 976 } 977 for (i = 0; i < 2; i++) { 978 err = bpf_map_update_elem(map, &zero, &tcp, BPF_ANY); 979 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 980 } 981 982 close(tcp); 983 err = bpf_map_delete_elem(map, &zero); 984 ASSERT_ERR(err, "bpf_map_delete_elem(entry)"); 985 986 close(stream[0]); 987 close(stream[1]); 988 out: 989 close(dgram); 990 close(udp); 991 test_sockmap_pass_prog__destroy(skel); 992 } 993 994 static void test_sockmap_skb_verdict_vsock_poll(void) 995 { 996 struct test_sockmap_pass_prog *skel; 997 int err, map, conn, peer; 998 struct bpf_program *prog; 999 struct bpf_link *link; 1000 char buf = 'x'; 1001 int zero = 0; 1002 1003 skel = test_sockmap_pass_prog__open_and_load(); 1004 if (!ASSERT_OK_PTR(skel, "open_and_load")) 1005 return; 1006 1007 if (create_pair(AF_VSOCK, SOCK_STREAM, &conn, &peer)) 1008 goto destroy; 1009 1010 prog = skel->progs.prog_skb_verdict; 1011 map = bpf_map__fd(skel->maps.sock_map_rx); 1012 link = bpf_program__attach_sockmap(prog, map); 1013 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 1014 goto close; 1015 1016 err = bpf_map_update_elem(map, &zero, &conn, BPF_ANY); 1017 if (!ASSERT_OK(err, "bpf_map_update_elem")) 1018 goto detach; 1019 1020 if (xsend(peer, &buf, 1, 0) != 1) 1021 goto detach; 1022 1023 err = poll_read(conn, IO_TIMEOUT_SEC); 1024 if (!ASSERT_OK(err, "poll")) 1025 goto detach; 1026 1027 if (xrecv_nonblock(conn, &buf, 1, 0) != 1) 1028 FAIL("xrecv_nonblock"); 1029 detach: 1030 bpf_link__detach(link); 1031 close: 1032 xclose(conn); 1033 xclose(peer); 1034 destroy: 1035 test_sockmap_pass_prog__destroy(skel); 1036 } 1037 1038 static void test_sockmap_vsock_unconnected(void) 1039 { 1040 struct sockaddr_storage addr; 1041 int map, s, zero = 0; 1042 socklen_t alen; 1043 1044 map = bpf_map_create(BPF_MAP_TYPE_SOCKMAP, NULL, sizeof(int), 1045 sizeof(int), 1, NULL); 1046 if (!ASSERT_OK_FD(map, "bpf_map_create")) 1047 return; 1048 1049 s = xsocket(AF_VSOCK, SOCK_STREAM, 0); 1050 if (s < 0) 1051 goto close_map; 1052 1053 /* Fail connect(), but trigger transport assignment. */ 1054 init_addr_loopback(AF_VSOCK, &addr, &alen); 1055 if (!ASSERT_ERR(connect(s, sockaddr(&addr), alen), "connect")) 1056 goto close_sock; 1057 1058 ASSERT_ERR(bpf_map_update_elem(map, &zero, &s, BPF_ANY), "map_update"); 1059 1060 close_sock: 1061 xclose(s); 1062 close_map: 1063 xclose(map); 1064 } 1065 1066 /* it is used to reproduce WARNING */ 1067 static void test_sockmap_zc(void) 1068 { 1069 int map, err, sent, recvd, zero = 0, one = 1, on = 1; 1070 char buf[10] = "0123456789", rcv[11], addr[100]; 1071 struct test_sockmap_pass_prog *skel = NULL; 1072 int c0 = -1, p0 = -1, c1 = -1, p1 = -1; 1073 struct tcp_zerocopy_receive zc; 1074 socklen_t zc_len = sizeof(zc); 1075 struct bpf_program *prog; 1076 1077 skel = test_sockmap_pass_prog__open_and_load(); 1078 if (!ASSERT_OK_PTR(skel, "open_and_load")) 1079 return; 1080 1081 if (create_socket_pairs(AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1)) 1082 goto end; 1083 1084 prog = skel->progs.prog_skb_verdict_ingress; 1085 map = bpf_map__fd(skel->maps.sock_map_rx); 1086 1087 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0); 1088 if (!ASSERT_OK(err, "bpf_prog_attach")) 1089 goto end; 1090 1091 err = bpf_map_update_elem(map, &zero, &p0, BPF_ANY); 1092 if (!ASSERT_OK(err, "bpf_map_update_elem")) 1093 goto end; 1094 1095 err = bpf_map_update_elem(map, &one, &p1, BPF_ANY); 1096 if (!ASSERT_OK(err, "bpf_map_update_elem")) 1097 goto end; 1098 1099 sent = xsend(c0, buf, sizeof(buf), 0); 1100 if (!ASSERT_EQ(sent, sizeof(buf), "xsend")) 1101 goto end; 1102 1103 /* trigger tcp_bpf_recvmsg_parser and inc copied_seq of p1 */ 1104 recvd = recv_timeout(p1, rcv, sizeof(rcv), MSG_DONTWAIT, 1); 1105 if (!ASSERT_EQ(recvd, sent, "recv_timeout(p1)")) 1106 goto end; 1107 1108 /* uninstall sockmap of p1 */ 1109 bpf_map_delete_elem(map, &one); 1110 1111 /* trigger tcp stack and the rcv_nxt of p1 is less than copied_seq */ 1112 sent = xsend(c1, buf, sizeof(buf) - 1, 0); 1113 if (!ASSERT_EQ(sent, sizeof(buf) - 1, "xsend")) 1114 goto end; 1115 1116 err = setsockopt(p1, SOL_SOCKET, SO_ZEROCOPY, &on, sizeof(on)); 1117 if (!ASSERT_OK(err, "setsockopt")) 1118 goto end; 1119 1120 memset(&zc, 0, sizeof(zc)); 1121 zc.copybuf_address = (__u64)((unsigned long)addr); 1122 zc.copybuf_len = sizeof(addr); 1123 1124 err = getsockopt(p1, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE, &zc, &zc_len); 1125 if (!ASSERT_OK(err, "getsockopt")) 1126 goto end; 1127 1128 end: 1129 if (c0 >= 0) 1130 close(c0); 1131 if (p0 >= 0) 1132 close(p0); 1133 if (c1 >= 0) 1134 close(c1); 1135 if (p1 >= 0) 1136 close(p1); 1137 test_sockmap_pass_prog__destroy(skel); 1138 } 1139 1140 /* it is used to check whether copied_seq of sk is correct */ 1141 static void test_sockmap_copied_seq(bool strp) 1142 { 1143 int i, map, err, sent, recvd, zero = 0, one = 1; 1144 struct test_sockmap_pass_prog *skel = NULL; 1145 int c0 = -1, p0 = -1, c1 = -1, p1 = -1; 1146 char buf[10] = "0123456789", rcv[11]; 1147 struct bpf_program *prog; 1148 1149 skel = test_sockmap_pass_prog__open_and_load(); 1150 if (!ASSERT_OK_PTR(skel, "open_and_load")) 1151 return; 1152 1153 if (create_socket_pairs(AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1)) 1154 goto end; 1155 1156 prog = skel->progs.prog_skb_verdict_ingress; 1157 map = bpf_map__fd(skel->maps.sock_map_rx); 1158 1159 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0); 1160 if (!ASSERT_OK(err, "bpf_prog_attach verdict")) 1161 goto end; 1162 1163 if (strp) { 1164 prog = skel->progs.prog_skb_verdict_ingress_strp; 1165 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_PARSER, 0); 1166 if (!ASSERT_OK(err, "bpf_prog_attach parser")) 1167 goto end; 1168 } 1169 1170 err = bpf_map_update_elem(map, &zero, &p0, BPF_ANY); 1171 if (!ASSERT_OK(err, "bpf_map_update_elem(p0)")) 1172 goto end; 1173 1174 err = bpf_map_update_elem(map, &one, &p1, BPF_ANY); 1175 if (!ASSERT_OK(err, "bpf_map_update_elem(p1)")) 1176 goto end; 1177 1178 /* just trigger sockamp: data sent by c0 will be received by p1 */ 1179 sent = xsend(c0, buf, sizeof(buf), 0); 1180 if (!ASSERT_EQ(sent, sizeof(buf), "xsend(c0), bpf")) 1181 goto end; 1182 1183 /* do partial read */ 1184 recvd = recv_timeout(p1, rcv, 1, MSG_DONTWAIT, 1); 1185 recvd += recv_timeout(p1, rcv + 1, sizeof(rcv) - 1, MSG_DONTWAIT, 1); 1186 if (!ASSERT_EQ(recvd, sent, "recv_timeout(p1), bpf") || 1187 !ASSERT_OK(memcmp(buf, rcv, recvd), "data mismatch")) 1188 goto end; 1189 1190 /* uninstall sockmap of p1 and p0 */ 1191 err = bpf_map_delete_elem(map, &one); 1192 if (!ASSERT_OK(err, "bpf_map_delete_elem(1)")) 1193 goto end; 1194 1195 err = bpf_map_delete_elem(map, &zero); 1196 if (!ASSERT_OK(err, "bpf_map_delete_elem(0)")) 1197 goto end; 1198 1199 /* now all sockets become plain socket, they should still work */ 1200 for (i = 0; i < 5; i++) { 1201 /* test copied_seq of p1 by running tcp native stack */ 1202 sent = xsend(c1, buf, sizeof(buf), 0); 1203 if (!ASSERT_EQ(sent, sizeof(buf), "xsend(c1), native")) 1204 goto end; 1205 1206 recvd = recv(p1, rcv, sizeof(rcv), MSG_DONTWAIT); 1207 if (!ASSERT_EQ(recvd, sent, "recv_timeout(p1), native")) 1208 goto end; 1209 1210 /* p0 previously redirected skb to p1, we also check copied_seq of p0 */ 1211 sent = xsend(c0, buf, sizeof(buf), 0); 1212 if (!ASSERT_EQ(sent, sizeof(buf), "xsend(c0), native")) 1213 goto end; 1214 1215 recvd = recv(p0, rcv, sizeof(rcv), MSG_DONTWAIT); 1216 if (!ASSERT_EQ(recvd, sent, "recv_timeout(p0), native")) 1217 goto end; 1218 } 1219 1220 end: 1221 if (c0 >= 0) 1222 close(c0); 1223 if (p0 >= 0) 1224 close(p0); 1225 if (c1 >= 0) 1226 close(c1); 1227 if (p1 >= 0) 1228 close(p1); 1229 test_sockmap_pass_prog__destroy(skel); 1230 } 1231 1232 /* Wait until FIONREAD returns the expected value or timeout */ 1233 static int wait_for_fionread(int fd, int expected, unsigned int timeout_ms) 1234 { 1235 unsigned int elapsed = 0; 1236 int avail = 0; 1237 1238 while (elapsed < timeout_ms) { 1239 if (ioctl(fd, FIONREAD, &avail) < 0) 1240 return -errno; 1241 if (avail >= expected) 1242 return avail; 1243 usleep(1000); 1244 elapsed++; 1245 } 1246 return avail; 1247 } 1248 1249 /* it is used to send data to via native stack and BPF redirecting */ 1250 static void test_sockmap_multi_channels(int sotype) 1251 { 1252 int map, err, sent, recvd, zero = 0, one = 1, avail = 0, expected; 1253 struct test_sockmap_pass_prog *skel = NULL; 1254 int c0 = -1, p0 = -1, c1 = -1, p1 = -1; 1255 char buf[10] = "0123456789", rcv[11]; 1256 struct bpf_program *prog; 1257 1258 skel = test_sockmap_pass_prog__open_and_load(); 1259 if (!ASSERT_OK_PTR(skel, "open_and_load")) 1260 return; 1261 1262 err = create_socket_pairs(AF_INET, sotype, &c0, &c1, &p0, &p1); 1263 if (err) 1264 goto end; 1265 1266 prog = skel->progs.prog_skb_verdict_ingress; 1267 map = bpf_map__fd(skel->maps.sock_map_rx); 1268 1269 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0); 1270 if (!ASSERT_OK(err, "bpf_prog_attach verdict")) 1271 goto end; 1272 1273 err = bpf_map_update_elem(map, &zero, &p0, BPF_ANY); 1274 if (!ASSERT_OK(err, "bpf_map_update_elem(p0)")) 1275 goto end; 1276 1277 err = bpf_map_update_elem(map, &one, &p1, BPF_ANY); 1278 if (!ASSERT_OK(err, "bpf_map_update_elem")) 1279 goto end; 1280 1281 /* send data to p1 via native stack */ 1282 sent = xsend(c1, buf, 2, 0); 1283 if (!ASSERT_EQ(sent, 2, "xsend(2)")) 1284 goto end; 1285 1286 avail = wait_for_fionread(p1, 2, IO_TIMEOUT_SEC); 1287 ASSERT_EQ(avail, 2, "ioctl(FIONREAD) partial return"); 1288 1289 /* send data to p1 via bpf redirecting */ 1290 sent = xsend(c0, buf + 2, sizeof(buf) - 2, 0); 1291 if (!ASSERT_EQ(sent, sizeof(buf) - 2, "xsend(remain-data)")) 1292 goto end; 1293 1294 /* Poll FIONREAD until expected bytes arrive, poll_read() is unreliable 1295 * here since it may return immediately if prior data is already queued. 1296 */ 1297 expected = sotype == SOCK_DGRAM ? 2 : sizeof(buf); 1298 avail = wait_for_fionread(p1, expected, IO_TIMEOUT_SEC); 1299 ASSERT_EQ(avail, expected, "ioctl(FIONREAD) full return"); 1300 1301 recvd = recv_timeout(p1, rcv, sizeof(rcv), MSG_DONTWAIT, 1); 1302 if (!ASSERT_EQ(recvd, sizeof(buf), "recv_timeout(p1)") || 1303 !ASSERT_OK(memcmp(buf, rcv, recvd), "data mismatch")) 1304 goto end; 1305 end: 1306 if (c0 >= 0) 1307 close(c0); 1308 if (p0 >= 0) 1309 close(p0); 1310 if (c1 >= 0) 1311 close(c1); 1312 if (p1 >= 0) 1313 close(p1); 1314 test_sockmap_pass_prog__destroy(skel); 1315 } 1316 1317 void test_sockmap_basic(void) 1318 { 1319 if (test__start_subtest("sockmap create_update_free")) 1320 test_sockmap_create_update_free(BPF_MAP_TYPE_SOCKMAP); 1321 if (test__start_subtest("sockhash create_update_free")) 1322 test_sockmap_create_update_free(BPF_MAP_TYPE_SOCKHASH); 1323 if (test__start_subtest("sockmap vsock delete on close")) 1324 test_sockmap_vsock_delete_on_close(); 1325 if (test__start_subtest("sockmap sk_msg load helpers")) 1326 test_skmsg_helpers(BPF_MAP_TYPE_SOCKMAP); 1327 if (test__start_subtest("sockhash sk_msg load helpers")) 1328 test_skmsg_helpers(BPF_MAP_TYPE_SOCKHASH); 1329 if (test__start_subtest("sockmap update")) 1330 test_sockmap_update(BPF_MAP_TYPE_SOCKMAP); 1331 if (test__start_subtest("sockhash update")) 1332 test_sockmap_update(BPF_MAP_TYPE_SOCKHASH); 1333 if (test__start_subtest("sockmap update in unsafe context")) 1334 test_sockmap_invalid_update(); 1335 if (test__start_subtest("sockmap copy")) 1336 test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP); 1337 if (test__start_subtest("sockhash copy")) 1338 test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH); 1339 if (test__start_subtest("sockmap skb_verdict attach")) { 1340 test_sockmap_skb_verdict_attach(BPF_SK_SKB_VERDICT, 1341 BPF_SK_SKB_STREAM_VERDICT); 1342 test_sockmap_skb_verdict_attach(BPF_SK_SKB_STREAM_VERDICT, 1343 BPF_SK_SKB_VERDICT); 1344 } 1345 if (test__start_subtest("sockmap skb_verdict attach_with_link")) 1346 test_sockmap_skb_verdict_attach_with_link(); 1347 if (test__start_subtest("sockmap msg_verdict progs query")) 1348 test_sockmap_progs_query(BPF_SK_MSG_VERDICT); 1349 if (test__start_subtest("sockmap stream_parser progs query")) 1350 test_sockmap_progs_query(BPF_SK_SKB_STREAM_PARSER); 1351 if (test__start_subtest("sockmap stream_verdict progs query")) 1352 test_sockmap_progs_query(BPF_SK_SKB_STREAM_VERDICT); 1353 if (test__start_subtest("sockmap skb_verdict progs query")) 1354 test_sockmap_progs_query(BPF_SK_SKB_VERDICT); 1355 if (test__start_subtest("sockmap skb_verdict shutdown")) 1356 test_sockmap_skb_verdict_shutdown(); 1357 if (test__start_subtest("sockmap skb_verdict fionread")) 1358 test_sockmap_skb_verdict_fionread(true); 1359 if (test__start_subtest("sockmap skb_verdict fionread on drop")) 1360 test_sockmap_skb_verdict_fionread(false); 1361 if (test__start_subtest("sockmap skb_verdict change tail")) 1362 test_sockmap_skb_verdict_change_tail(); 1363 if (test__start_subtest("sockmap skb_verdict msg_f_peek")) 1364 test_sockmap_skb_verdict_peek(); 1365 if (test__start_subtest("sockmap skb_verdict msg_f_peek with link")) 1366 test_sockmap_skb_verdict_peek_with_link(); 1367 if (test__start_subtest("sockmap unconnected af_unix")) 1368 test_sockmap_unconnected_unix(); 1369 if (test__start_subtest("sockmap one socket to many map entries")) 1370 test_sockmap_many_socket(); 1371 if (test__start_subtest("sockmap one socket to many maps")) 1372 test_sockmap_many_maps(); 1373 if (test__start_subtest("sockmap same socket replace")) 1374 test_sockmap_same_sock(); 1375 if (test__start_subtest("sockmap sk_msg attach sockmap helpers with link")) 1376 test_skmsg_helpers_with_link(BPF_MAP_TYPE_SOCKMAP); 1377 if (test__start_subtest("sockhash sk_msg attach sockhash helpers with link")) 1378 test_skmsg_helpers_with_link(BPF_MAP_TYPE_SOCKHASH); 1379 if (test__start_subtest("sockmap skb_verdict vsock poll")) 1380 test_sockmap_skb_verdict_vsock_poll(); 1381 if (test__start_subtest("sockmap vsock unconnected")) 1382 test_sockmap_vsock_unconnected(); 1383 if (test__start_subtest("sockmap with zc")) 1384 test_sockmap_zc(); 1385 if (test__start_subtest("sockmap recover")) 1386 test_sockmap_copied_seq(false); 1387 if (test__start_subtest("sockmap recover with strp")) 1388 test_sockmap_copied_seq(true); 1389 if (test__start_subtest("sockmap tcp multi channels")) 1390 test_sockmap_multi_channels(SOCK_STREAM); 1391 if (test__start_subtest("sockmap udp multi channels")) 1392 test_sockmap_multi_channels(SOCK_DGRAM); 1393 } 1394