1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2020 Cloudflare 3 #include <error.h> 4 #include <linux/tcp.h> 5 #include <linux/socket.h> 6 #include <sys/epoll.h> 7 8 #include "test_progs.h" 9 #include "test_skmsg_load_helpers.skel.h" 10 #include "test_sockmap_invalid_update.skel.h" 11 #include "test_sockmap_skb_verdict_attach.skel.h" 12 #include "test_sockmap_progs_query.skel.h" 13 #include "test_sockmap_pass_prog.skel.h" 14 #include "test_sockmap_drop_prog.skel.h" 15 #include "test_sockmap_change_tail.skel.h" 16 #include "test_sockmap_msg_pop_data.skel.h" 17 #include "bpf_iter_sockmap.skel.h" 18 19 #include "sockmap_helpers.h" 20 21 #define TCP_REPAIR 19 /* TCP sock is under repair right now */ 22 23 #define TCP_REPAIR_ON 1 24 #define TCP_REPAIR_OFF_NO_WP -1 /* Turn off without window probes */ 25 26 /** 27 * SOL_TCP is defined in <netinet/tcp.h> (glibc), but the copybuf_address 28 * field of tcp_zerocopy_receive is not yet included in older versions. 29 * This workaround remains necessary until the glibc update propagates. 30 */ 31 #ifndef SOL_TCP 32 #define SOL_TCP 6 33 #endif 34 35 static int connected_socket_v4(void) 36 { 37 struct sockaddr_in addr = { 38 .sin_family = AF_INET, 39 .sin_port = htons(80), 40 .sin_addr = { inet_addr("127.0.0.1") }, 41 }; 42 socklen_t len = sizeof(addr); 43 int s, repair, err; 44 45 s = socket(AF_INET, SOCK_STREAM, 0); 46 if (!ASSERT_GE(s, 0, "socket")) 47 goto error; 48 49 repair = TCP_REPAIR_ON; 50 err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair)); 51 if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)")) 52 goto error; 53 54 err = connect(s, (struct sockaddr *)&addr, len); 55 if (!ASSERT_OK(err, "connect")) 56 goto error; 57 58 repair = TCP_REPAIR_OFF_NO_WP; 59 err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair)); 60 if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)")) 61 goto error; 62 63 return s; 64 error: 65 perror(__func__); 66 close(s); 67 return -1; 68 } 69 70 static void compare_cookies(struct bpf_map *src, struct bpf_map *dst) 71 { 72 __u32 i, max_entries = bpf_map__max_entries(src); 73 int err, src_fd, dst_fd; 74 75 src_fd = bpf_map__fd(src); 76 dst_fd = bpf_map__fd(dst); 77 78 for (i = 0; i < max_entries; i++) { 79 __u64 src_cookie, dst_cookie; 80 81 err = bpf_map_lookup_elem(src_fd, &i, &src_cookie); 82 if (err && errno == ENOENT) { 83 err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); 84 ASSERT_ERR(err, "map_lookup_elem(dst)"); 85 ASSERT_EQ(errno, ENOENT, "map_lookup_elem(dst)"); 86 continue; 87 } 88 if (!ASSERT_OK(err, "lookup_elem(src)")) 89 continue; 90 91 err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); 92 if (!ASSERT_OK(err, "lookup_elem(dst)")) 93 continue; 94 95 ASSERT_EQ(dst_cookie, src_cookie, "cookie mismatch"); 96 } 97 } 98 99 /* Create a map, populate it with one socket, and free the map. */ 100 static void test_sockmap_create_update_free(enum bpf_map_type map_type) 101 { 102 const int zero = 0; 103 int s, map, err; 104 105 s = connected_socket_v4(); 106 if (!ASSERT_GE(s, 0, "connected_socket_v4")) 107 return; 108 109 map = bpf_map_create(map_type, NULL, sizeof(int), sizeof(int), 1, NULL); 110 if (!ASSERT_GE(map, 0, "bpf_map_create")) 111 goto out; 112 113 err = bpf_map_update_elem(map, &zero, &s, BPF_NOEXIST); 114 if (!ASSERT_OK(err, "bpf_map_update")) 115 goto out; 116 117 out: 118 close(map); 119 close(s); 120 } 121 122 static void test_sockmap_vsock_delete_on_close(void) 123 { 124 int map, c, p, err, zero = 0; 125 126 map = bpf_map_create(BPF_MAP_TYPE_SOCKMAP, NULL, sizeof(int), 127 sizeof(int), 1, NULL); 128 if (!ASSERT_OK_FD(map, "bpf_map_create")) 129 return; 130 131 err = create_pair(AF_VSOCK, SOCK_STREAM, &c, &p); 132 if (!ASSERT_OK(err, "create_pair")) 133 goto close_map; 134 135 if (xbpf_map_update_elem(map, &zero, &c, BPF_NOEXIST)) 136 goto close_socks; 137 138 xclose(c); 139 xclose(p); 140 141 err = create_pair(AF_VSOCK, SOCK_STREAM, &c, &p); 142 if (!ASSERT_OK(err, "create_pair")) 143 goto close_map; 144 145 err = bpf_map_update_elem(map, &zero, &c, BPF_NOEXIST); 146 ASSERT_OK(err, "after close(), bpf_map_update"); 147 148 close_socks: 149 xclose(c); 150 xclose(p); 151 close_map: 152 xclose(map); 153 } 154 155 static void test_skmsg_helpers(enum bpf_map_type map_type) 156 { 157 struct test_skmsg_load_helpers *skel; 158 int err, map, verdict; 159 160 skel = test_skmsg_load_helpers__open_and_load(); 161 if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load")) 162 return; 163 164 verdict = bpf_program__fd(skel->progs.prog_msg_verdict); 165 map = bpf_map__fd(skel->maps.sock_map); 166 167 err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0); 168 if (!ASSERT_OK(err, "bpf_prog_attach")) 169 goto out; 170 171 err = bpf_prog_detach2(verdict, map, BPF_SK_MSG_VERDICT); 172 if (!ASSERT_OK(err, "bpf_prog_detach2")) 173 goto out; 174 out: 175 test_skmsg_load_helpers__destroy(skel); 176 } 177 178 static void test_skmsg_helpers_with_link(enum bpf_map_type map_type) 179 { 180 struct bpf_program *prog, *prog_clone, *prog_clone2; 181 DECLARE_LIBBPF_OPTS(bpf_link_update_opts, opts); 182 struct test_skmsg_load_helpers *skel; 183 struct bpf_link *link, *link2; 184 int err, map; 185 186 skel = test_skmsg_load_helpers__open_and_load(); 187 if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load")) 188 return; 189 190 prog = skel->progs.prog_msg_verdict; 191 prog_clone = skel->progs.prog_msg_verdict_clone; 192 prog_clone2 = skel->progs.prog_msg_verdict_clone2; 193 map = bpf_map__fd(skel->maps.sock_map); 194 195 link = bpf_program__attach_sockmap(prog, map); 196 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 197 goto out; 198 199 /* Fail since bpf_link for the same prog has been created. */ 200 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_MSG_VERDICT, 0); 201 if (!ASSERT_ERR(err, "bpf_prog_attach")) 202 goto out; 203 204 /* Fail since bpf_link for the same prog type has been created. */ 205 link2 = bpf_program__attach_sockmap(prog_clone, map); 206 if (!ASSERT_ERR_PTR(link2, "bpf_program__attach_sockmap")) { 207 bpf_link__destroy(link2); 208 goto out; 209 } 210 211 err = bpf_link__update_program(link, prog_clone); 212 if (!ASSERT_OK(err, "bpf_link__update_program")) 213 goto out; 214 215 /* Fail since a prog with different type attempts to do update. */ 216 err = bpf_link__update_program(link, skel->progs.prog_skb_verdict); 217 if (!ASSERT_ERR(err, "bpf_link__update_program")) 218 goto out; 219 220 /* Fail since the old prog does not match the one in the kernel. */ 221 opts.old_prog_fd = bpf_program__fd(prog_clone2); 222 opts.flags = BPF_F_REPLACE; 223 err = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), &opts); 224 if (!ASSERT_ERR(err, "bpf_link_update")) 225 goto out; 226 227 opts.old_prog_fd = bpf_program__fd(prog_clone); 228 opts.flags = BPF_F_REPLACE; 229 err = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), &opts); 230 if (!ASSERT_OK(err, "bpf_link_update")) 231 goto out; 232 out: 233 bpf_link__destroy(link); 234 test_skmsg_load_helpers__destroy(skel); 235 } 236 237 static void test_sockmap_invalid_update(void) 238 { 239 struct test_sockmap_invalid_update *skel; 240 241 skel = test_sockmap_invalid_update__open_and_load(); 242 if (!ASSERT_NULL(skel, "open_and_load")) 243 test_sockmap_invalid_update__destroy(skel); 244 } 245 246 static void test_sockmap_copy(enum bpf_map_type map_type) 247 { 248 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 249 int err, len, src_fd, iter_fd; 250 union bpf_iter_link_info linfo = {}; 251 __u32 i, num_sockets, num_elems; 252 struct bpf_iter_sockmap *skel; 253 __s64 *sock_fd = NULL; 254 struct bpf_link *link; 255 struct bpf_map *src; 256 char buf[64]; 257 258 skel = bpf_iter_sockmap__open_and_load(); 259 if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load")) 260 return; 261 262 if (map_type == BPF_MAP_TYPE_SOCKMAP) { 263 src = skel->maps.sockmap; 264 num_elems = bpf_map__max_entries(src); 265 num_sockets = num_elems - 1; 266 } else { 267 src = skel->maps.sockhash; 268 num_elems = bpf_map__max_entries(src) - 1; 269 num_sockets = num_elems; 270 } 271 272 sock_fd = calloc(num_sockets, sizeof(*sock_fd)); 273 if (!ASSERT_OK_PTR(sock_fd, "calloc(sock_fd)")) 274 goto out; 275 276 for (i = 0; i < num_sockets; i++) 277 sock_fd[i] = -1; 278 279 src_fd = bpf_map__fd(src); 280 281 for (i = 0; i < num_sockets; i++) { 282 sock_fd[i] = connected_socket_v4(); 283 if (!ASSERT_NEQ(sock_fd[i], -1, "connected_socket_v4")) 284 goto out; 285 286 err = bpf_map_update_elem(src_fd, &i, &sock_fd[i], BPF_NOEXIST); 287 if (!ASSERT_OK(err, "map_update")) 288 goto out; 289 } 290 291 linfo.map.map_fd = src_fd; 292 opts.link_info = &linfo; 293 opts.link_info_len = sizeof(linfo); 294 link = bpf_program__attach_iter(skel->progs.copy, &opts); 295 if (!ASSERT_OK_PTR(link, "attach_iter")) 296 goto out; 297 298 iter_fd = bpf_iter_create(bpf_link__fd(link)); 299 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 300 goto free_link; 301 302 /* do some tests */ 303 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 304 ; 305 if (!ASSERT_GE(len, 0, "read")) 306 goto close_iter; 307 308 /* test results */ 309 if (!ASSERT_EQ(skel->bss->elems, num_elems, "elems")) 310 goto close_iter; 311 312 if (!ASSERT_EQ(skel->bss->socks, num_sockets, "socks")) 313 goto close_iter; 314 315 compare_cookies(src, skel->maps.dst); 316 317 close_iter: 318 close(iter_fd); 319 free_link: 320 bpf_link__destroy(link); 321 out: 322 for (i = 0; sock_fd && i < num_sockets; i++) 323 if (sock_fd[i] >= 0) 324 close(sock_fd[i]); 325 if (sock_fd) 326 free(sock_fd); 327 bpf_iter_sockmap__destroy(skel); 328 } 329 330 static void test_sockmap_skb_verdict_attach(enum bpf_attach_type first, 331 enum bpf_attach_type second) 332 { 333 struct test_sockmap_skb_verdict_attach *skel; 334 int err, map, verdict; 335 336 skel = test_sockmap_skb_verdict_attach__open_and_load(); 337 if (!ASSERT_OK_PTR(skel, "open_and_load")) 338 return; 339 340 verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 341 map = bpf_map__fd(skel->maps.sock_map); 342 343 err = bpf_prog_attach(verdict, map, first, 0); 344 if (!ASSERT_OK(err, "bpf_prog_attach")) 345 goto out; 346 347 err = bpf_prog_attach(verdict, map, second, 0); 348 ASSERT_EQ(err, -EBUSY, "prog_attach_fail"); 349 350 err = bpf_prog_detach2(verdict, map, first); 351 if (!ASSERT_OK(err, "bpf_prog_detach2")) 352 goto out; 353 out: 354 test_sockmap_skb_verdict_attach__destroy(skel); 355 } 356 357 static void test_sockmap_skb_verdict_attach_with_link(void) 358 { 359 struct test_sockmap_skb_verdict_attach *skel; 360 struct bpf_program *prog; 361 struct bpf_link *link; 362 int err, map; 363 364 skel = test_sockmap_skb_verdict_attach__open_and_load(); 365 if (!ASSERT_OK_PTR(skel, "open_and_load")) 366 return; 367 prog = skel->progs.prog_skb_verdict; 368 map = bpf_map__fd(skel->maps.sock_map); 369 link = bpf_program__attach_sockmap(prog, map); 370 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 371 goto out; 372 373 bpf_link__destroy(link); 374 375 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0); 376 if (!ASSERT_OK(err, "bpf_prog_attach")) 377 goto out; 378 379 /* Fail since attaching with the same prog/map has been done. */ 380 link = bpf_program__attach_sockmap(prog, map); 381 if (!ASSERT_ERR_PTR(link, "bpf_program__attach_sockmap")) 382 bpf_link__destroy(link); 383 384 err = bpf_prog_detach2(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT); 385 if (!ASSERT_OK(err, "bpf_prog_detach2")) 386 goto out; 387 out: 388 test_sockmap_skb_verdict_attach__destroy(skel); 389 } 390 391 static __u32 query_prog_id(int prog_fd) 392 { 393 struct bpf_prog_info info = {}; 394 __u32 info_len = sizeof(info); 395 int err; 396 397 err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); 398 if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd") || 399 !ASSERT_EQ(info_len, sizeof(info), "bpf_prog_get_info_by_fd")) 400 return 0; 401 402 return info.id; 403 } 404 405 static void test_sockmap_progs_query(enum bpf_attach_type attach_type) 406 { 407 struct test_sockmap_progs_query *skel; 408 int err, map_fd, verdict_fd; 409 __u32 attach_flags = 0; 410 __u32 prog_ids[3] = {}; 411 __u32 prog_cnt = 3; 412 413 skel = test_sockmap_progs_query__open_and_load(); 414 if (!ASSERT_OK_PTR(skel, "test_sockmap_progs_query__open_and_load")) 415 return; 416 417 map_fd = bpf_map__fd(skel->maps.sock_map); 418 419 if (attach_type == BPF_SK_MSG_VERDICT) 420 verdict_fd = bpf_program__fd(skel->progs.prog_skmsg_verdict); 421 else 422 verdict_fd = bpf_program__fd(skel->progs.prog_skb_verdict); 423 424 err = bpf_prog_query(map_fd, attach_type, 0 /* query flags */, 425 &attach_flags, prog_ids, &prog_cnt); 426 ASSERT_OK(err, "bpf_prog_query failed"); 427 ASSERT_EQ(attach_flags, 0, "wrong attach_flags on query"); 428 ASSERT_EQ(prog_cnt, 0, "wrong program count on query"); 429 430 err = bpf_prog_attach(verdict_fd, map_fd, attach_type, 0); 431 if (!ASSERT_OK(err, "bpf_prog_attach failed")) 432 goto out; 433 434 prog_cnt = 1; 435 err = bpf_prog_query(map_fd, attach_type, 0 /* query flags */, 436 &attach_flags, prog_ids, &prog_cnt); 437 ASSERT_OK(err, "bpf_prog_query failed"); 438 ASSERT_EQ(attach_flags, 0, "wrong attach_flags on query"); 439 ASSERT_EQ(prog_cnt, 1, "wrong program count on query"); 440 ASSERT_EQ(prog_ids[0], query_prog_id(verdict_fd), 441 "wrong prog_ids on query"); 442 443 bpf_prog_detach2(verdict_fd, map_fd, attach_type); 444 out: 445 test_sockmap_progs_query__destroy(skel); 446 } 447 448 #define MAX_EVENTS 10 449 static void test_sockmap_skb_verdict_shutdown(void) 450 { 451 int n, err, map, verdict, c1 = -1, p1 = -1; 452 struct epoll_event ev, events[MAX_EVENTS]; 453 struct test_sockmap_pass_prog *skel; 454 int zero = 0; 455 int epollfd; 456 char b; 457 458 skel = test_sockmap_pass_prog__open_and_load(); 459 if (!ASSERT_OK_PTR(skel, "open_and_load")) 460 return; 461 462 verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 463 map = bpf_map__fd(skel->maps.sock_map_rx); 464 465 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 466 if (!ASSERT_OK(err, "bpf_prog_attach")) 467 goto out; 468 469 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); 470 if (err < 0) 471 goto out; 472 473 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 474 if (err < 0) 475 goto out_close; 476 477 shutdown(p1, SHUT_WR); 478 479 ev.events = EPOLLIN; 480 ev.data.fd = c1; 481 482 epollfd = epoll_create1(0); 483 if (!ASSERT_GT(epollfd, -1, "epoll_create(0)")) 484 goto out_close; 485 err = epoll_ctl(epollfd, EPOLL_CTL_ADD, c1, &ev); 486 if (!ASSERT_OK(err, "epoll_ctl(EPOLL_CTL_ADD)")) 487 goto out_close; 488 err = epoll_wait(epollfd, events, MAX_EVENTS, -1); 489 if (!ASSERT_EQ(err, 1, "epoll_wait(fd)")) 490 goto out_close; 491 492 n = recv(c1, &b, 1, MSG_DONTWAIT); 493 ASSERT_EQ(n, 0, "recv(fin)"); 494 out_close: 495 close(c1); 496 close(p1); 497 out: 498 test_sockmap_pass_prog__destroy(skel); 499 } 500 501 502 static void do_test_sockmap_skb_verdict_fionread(int sotype, bool pass_prog) 503 { 504 int err, map, verdict, c0 = -1, c1 = -1, p0 = -1, p1 = -1; 505 int expected, zero = 0, sent, recvd, avail; 506 struct test_sockmap_pass_prog *pass = NULL; 507 struct test_sockmap_drop_prog *drop = NULL; 508 char buf[256] = "0123456789"; 509 int split_len = sizeof(buf) / 2; 510 511 if (pass_prog) { 512 pass = test_sockmap_pass_prog__open_and_load(); 513 if (!ASSERT_OK_PTR(pass, "open_and_load")) 514 return; 515 verdict = bpf_program__fd(pass->progs.prog_skb_verdict); 516 map = bpf_map__fd(pass->maps.sock_map_rx); 517 if (sotype == SOCK_DGRAM) 518 expected = split_len; /* FIONREAD for UDP is different from TCP */ 519 else 520 expected = sizeof(buf); 521 } else { 522 drop = test_sockmap_drop_prog__open_and_load(); 523 if (!ASSERT_OK_PTR(drop, "open_and_load")) 524 return; 525 verdict = bpf_program__fd(drop->progs.prog_skb_verdict); 526 map = bpf_map__fd(drop->maps.sock_map_rx); 527 /* On drop data is consumed immediately and copied_seq inc'd */ 528 expected = 0; 529 } 530 531 532 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 533 if (!ASSERT_OK(err, "bpf_prog_attach")) 534 goto out; 535 536 err = create_socket_pairs(AF_INET, sotype, &c0, &c1, &p0, &p1); 537 if (!ASSERT_OK(err, "create_socket_pairs()")) 538 goto out; 539 540 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 541 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) 542 goto out_close; 543 544 sent = xsend(p1, &buf, split_len, 0); 545 sent += xsend(p1, &buf, sizeof(buf) - split_len, 0); 546 ASSERT_EQ(sent, sizeof(buf), "xsend(p1)"); 547 err = ioctl(c1, FIONREAD, &avail); 548 ASSERT_OK(err, "ioctl(FIONREAD) error"); 549 ASSERT_EQ(avail, expected, "ioctl(FIONREAD)"); 550 /* On DROP test there will be no data to read */ 551 if (pass_prog) { 552 recvd = recv_timeout(c1, &buf, sizeof(buf), MSG_DONTWAIT, IO_TIMEOUT_SEC); 553 ASSERT_EQ(recvd, sizeof(buf), "recv_timeout(c0)"); 554 } 555 556 out_close: 557 close(c0); 558 close(p0); 559 close(c1); 560 close(p1); 561 out: 562 if (pass_prog) 563 test_sockmap_pass_prog__destroy(pass); 564 else 565 test_sockmap_drop_prog__destroy(drop); 566 } 567 568 static void test_sockmap_skb_verdict_fionread(bool pass_prog) 569 { 570 do_test_sockmap_skb_verdict_fionread(SOCK_STREAM, pass_prog); 571 do_test_sockmap_skb_verdict_fionread(SOCK_DGRAM, pass_prog); 572 } 573 574 static void test_sockmap_skb_verdict_change_tail(void) 575 { 576 struct test_sockmap_change_tail *skel; 577 int err, map, verdict; 578 int c1, p1, sent, recvd; 579 int zero = 0; 580 char buf[2]; 581 582 skel = test_sockmap_change_tail__open_and_load(); 583 if (!ASSERT_OK_PTR(skel, "open_and_load")) 584 return; 585 verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 586 map = bpf_map__fd(skel->maps.sock_map_rx); 587 588 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 589 if (!ASSERT_OK(err, "bpf_prog_attach")) 590 goto out; 591 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); 592 if (!ASSERT_OK(err, "create_pair()")) 593 goto out; 594 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 595 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) 596 goto out_close; 597 sent = xsend(p1, "Tr", 2, 0); 598 ASSERT_EQ(sent, 2, "xsend(p1)"); 599 recvd = recv(c1, buf, 2, 0); 600 ASSERT_EQ(recvd, 1, "recv(c1)"); 601 ASSERT_EQ(skel->data->change_tail_ret, 0, "change_tail_ret"); 602 603 sent = xsend(p1, "G", 1, 0); 604 ASSERT_EQ(sent, 1, "xsend(p1)"); 605 recvd = recv(c1, buf, 2, 0); 606 ASSERT_EQ(recvd, 2, "recv(c1)"); 607 ASSERT_EQ(skel->data->change_tail_ret, 0, "change_tail_ret"); 608 609 sent = xsend(p1, "E", 1, 0); 610 ASSERT_EQ(sent, 1, "xsend(p1)"); 611 recvd = recv(c1, buf, 1, 0); 612 ASSERT_EQ(recvd, 1, "recv(c1)"); 613 ASSERT_EQ(skel->data->change_tail_ret, -EINVAL, "change_tail_ret"); 614 615 out_close: 616 close(c1); 617 close(p1); 618 out: 619 test_sockmap_change_tail__destroy(skel); 620 } 621 622 static void test_sockmap_msg_verdict_pop_data(void) 623 { 624 struct test_sockmap_msg_pop_data *skel; 625 int err, map, verdict; 626 int c1 = -1, p1 = -1, sent; 627 int zero = 0; 628 char *buf; 629 const size_t len = 32 * 1024; 630 631 skel = test_sockmap_msg_pop_data__open_and_load(); 632 if (!ASSERT_OK_PTR(skel, "open_and_load")) 633 return; 634 635 verdict = bpf_program__fd(skel->progs.prog_msg_pop_data); 636 map = bpf_map__fd(skel->maps.sock_map); 637 638 err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0); 639 if (!ASSERT_OK(err, "bpf_prog_attach")) 640 goto out; 641 642 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); 643 if (!ASSERT_OK(err, "create_pair")) 644 goto out; 645 646 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 647 if (!ASSERT_OK(err, "bpf_map_update_elem")) 648 goto out_close; 649 650 buf = calloc(len, 1); 651 if (!ASSERT_OK_PTR(buf, "calloc")) 652 goto out_close; 653 654 sent = xsend(c1, buf, len, 0); 655 ASSERT_EQ(sent, (ssize_t)len, "xsend"); 656 ASSERT_EQ(skel->data->pop_data_ret, -EINVAL, "pop_data_rejects overflow"); 657 658 free(buf); 659 660 out_close: 661 close(c1); 662 close(p1); 663 out: 664 test_sockmap_msg_pop_data__destroy(skel); 665 } 666 667 static void test_sockmap_skb_verdict_peek_helper(int map) 668 { 669 int err, c1, p1, zero = 0, sent, recvd, avail; 670 char snd[256] = "0123456789"; 671 char rcv[256] = "0"; 672 673 err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); 674 if (!ASSERT_OK(err, "create_pair()")) 675 return; 676 677 err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); 678 if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) 679 goto out_close; 680 681 sent = xsend(p1, snd, sizeof(snd), 0); 682 ASSERT_EQ(sent, sizeof(snd), "xsend(p1)"); 683 recvd = recv(c1, rcv, sizeof(rcv), MSG_PEEK); 684 ASSERT_EQ(recvd, sizeof(rcv), "recv(c1)"); 685 err = ioctl(c1, FIONREAD, &avail); 686 ASSERT_OK(err, "ioctl(FIONREAD) error"); 687 ASSERT_EQ(avail, sizeof(snd), "after peek ioctl(FIONREAD)"); 688 recvd = recv(c1, rcv, sizeof(rcv), 0); 689 ASSERT_EQ(recvd, sizeof(rcv), "recv(p0)"); 690 err = ioctl(c1, FIONREAD, &avail); 691 ASSERT_OK(err, "ioctl(FIONREAD) error"); 692 ASSERT_EQ(avail, 0, "after read ioctl(FIONREAD)"); 693 694 out_close: 695 close(c1); 696 close(p1); 697 } 698 699 static void test_sockmap_skb_verdict_peek(void) 700 { 701 struct test_sockmap_pass_prog *pass; 702 int err, map, verdict; 703 704 pass = test_sockmap_pass_prog__open_and_load(); 705 if (!ASSERT_OK_PTR(pass, "open_and_load")) 706 return; 707 verdict = bpf_program__fd(pass->progs.prog_skb_verdict); 708 map = bpf_map__fd(pass->maps.sock_map_rx); 709 710 err = bpf_prog_attach(verdict, map, BPF_SK_SKB_STREAM_VERDICT, 0); 711 if (!ASSERT_OK(err, "bpf_prog_attach")) 712 goto out; 713 714 test_sockmap_skb_verdict_peek_helper(map); 715 716 out: 717 test_sockmap_pass_prog__destroy(pass); 718 } 719 720 static void test_sockmap_skb_verdict_peek_with_link(void) 721 { 722 struct test_sockmap_pass_prog *pass; 723 struct bpf_program *prog; 724 struct bpf_link *link; 725 int err, map; 726 727 pass = test_sockmap_pass_prog__open_and_load(); 728 if (!ASSERT_OK_PTR(pass, "open_and_load")) 729 return; 730 prog = pass->progs.prog_skb_verdict; 731 map = bpf_map__fd(pass->maps.sock_map_rx); 732 link = bpf_program__attach_sockmap(prog, map); 733 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 734 goto out; 735 736 err = bpf_link__update_program(link, pass->progs.prog_skb_verdict_clone); 737 if (!ASSERT_OK(err, "bpf_link__update_program")) 738 goto out; 739 740 /* Fail since a prog with different attach type attempts to do update. */ 741 err = bpf_link__update_program(link, pass->progs.prog_skb_parser); 742 if (!ASSERT_ERR(err, "bpf_link__update_program")) 743 goto out; 744 745 test_sockmap_skb_verdict_peek_helper(map); 746 ASSERT_EQ(pass->bss->clone_called, 1, "clone_called"); 747 out: 748 bpf_link__destroy(link); 749 test_sockmap_pass_prog__destroy(pass); 750 } 751 752 static void test_sockmap_unconnected_unix(void) 753 { 754 int err, map, stream = -1, dgram = -1, zero = 0; 755 struct test_sockmap_pass_prog *skel; 756 757 skel = test_sockmap_pass_prog__open_and_load(); 758 if (!ASSERT_OK_PTR(skel, "open_and_load")) 759 return; 760 761 map = bpf_map__fd(skel->maps.sock_map_rx); 762 763 stream = xsocket(AF_UNIX, SOCK_STREAM, 0); 764 if (stream < 0) 765 goto out; 766 767 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 768 if (dgram < 0) 769 goto out; 770 771 err = bpf_map_update_elem(map, &zero, &stream, BPF_ANY); 772 if (!ASSERT_ERR(err, "bpf_map_update_elem(stream)")) 773 goto out; 774 775 err = bpf_map_update_elem(map, &zero, &dgram, BPF_ANY); 776 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 777 out: 778 close(stream); 779 close(dgram); 780 test_sockmap_pass_prog__destroy(skel); 781 } 782 783 static void test_sockmap_many_socket(void) 784 { 785 struct test_sockmap_pass_prog *skel; 786 int stream[2], dgram, udp, tcp; 787 int i, err, map, entry = 0; 788 789 skel = test_sockmap_pass_prog__open_and_load(); 790 if (!ASSERT_OK_PTR(skel, "open_and_load")) 791 return; 792 793 map = bpf_map__fd(skel->maps.sock_map_rx); 794 795 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 796 if (dgram < 0) { 797 test_sockmap_pass_prog__destroy(skel); 798 return; 799 } 800 801 tcp = connected_socket_v4(); 802 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) { 803 close(dgram); 804 test_sockmap_pass_prog__destroy(skel); 805 return; 806 } 807 808 udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); 809 if (udp < 0) { 810 close(dgram); 811 close(tcp); 812 test_sockmap_pass_prog__destroy(skel); 813 return; 814 } 815 816 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 817 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 818 if (err) 819 goto out; 820 821 for (i = 0; i < 2; i++, entry++) { 822 err = bpf_map_update_elem(map, &entry, &stream[0], BPF_ANY); 823 ASSERT_OK(err, "bpf_map_update_elem(stream)"); 824 } 825 for (i = 0; i < 2; i++, entry++) { 826 err = bpf_map_update_elem(map, &entry, &dgram, BPF_ANY); 827 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 828 } 829 for (i = 0; i < 2; i++, entry++) { 830 err = bpf_map_update_elem(map, &entry, &udp, BPF_ANY); 831 ASSERT_OK(err, "bpf_map_update_elem(udp)"); 832 } 833 for (i = 0; i < 2; i++, entry++) { 834 err = bpf_map_update_elem(map, &entry, &tcp, BPF_ANY); 835 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 836 } 837 for (entry--; entry >= 0; entry--) { 838 err = bpf_map_delete_elem(map, &entry); 839 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 840 } 841 842 close(stream[0]); 843 close(stream[1]); 844 out: 845 close(dgram); 846 close(tcp); 847 close(udp); 848 test_sockmap_pass_prog__destroy(skel); 849 } 850 851 static void test_sockmap_many_maps(void) 852 { 853 struct test_sockmap_pass_prog *skel; 854 int stream[2], dgram, udp, tcp; 855 int i, err, map[2], entry = 0; 856 857 skel = test_sockmap_pass_prog__open_and_load(); 858 if (!ASSERT_OK_PTR(skel, "open_and_load")) 859 return; 860 861 map[0] = bpf_map__fd(skel->maps.sock_map_rx); 862 map[1] = bpf_map__fd(skel->maps.sock_map_tx); 863 864 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 865 if (dgram < 0) { 866 test_sockmap_pass_prog__destroy(skel); 867 return; 868 } 869 870 tcp = connected_socket_v4(); 871 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) { 872 close(dgram); 873 test_sockmap_pass_prog__destroy(skel); 874 return; 875 } 876 877 udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); 878 if (udp < 0) { 879 close(dgram); 880 close(tcp); 881 test_sockmap_pass_prog__destroy(skel); 882 return; 883 } 884 885 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 886 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 887 if (err) 888 goto out; 889 890 for (i = 0; i < 2; i++, entry++) { 891 err = bpf_map_update_elem(map[i], &entry, &stream[0], BPF_ANY); 892 ASSERT_OK(err, "bpf_map_update_elem(stream)"); 893 } 894 for (i = 0; i < 2; i++, entry++) { 895 err = bpf_map_update_elem(map[i], &entry, &dgram, BPF_ANY); 896 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 897 } 898 for (i = 0; i < 2; i++, entry++) { 899 err = bpf_map_update_elem(map[i], &entry, &udp, BPF_ANY); 900 ASSERT_OK(err, "bpf_map_update_elem(udp)"); 901 } 902 for (i = 0; i < 2; i++, entry++) { 903 err = bpf_map_update_elem(map[i], &entry, &tcp, BPF_ANY); 904 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 905 } 906 for (entry--; entry >= 0; entry--) { 907 err = bpf_map_delete_elem(map[1], &entry); 908 entry--; 909 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 910 err = bpf_map_delete_elem(map[0], &entry); 911 ASSERT_OK(err, "bpf_map_delete_elem(entry)"); 912 } 913 914 close(stream[0]); 915 close(stream[1]); 916 out: 917 close(dgram); 918 close(tcp); 919 close(udp); 920 test_sockmap_pass_prog__destroy(skel); 921 } 922 923 static void test_sockmap_same_sock(void) 924 { 925 struct test_sockmap_pass_prog *skel; 926 int stream[2], dgram, udp, tcp; 927 int i, err, map, zero = 0; 928 929 skel = test_sockmap_pass_prog__open_and_load(); 930 if (!ASSERT_OK_PTR(skel, "open_and_load")) 931 return; 932 933 map = bpf_map__fd(skel->maps.sock_map_rx); 934 935 dgram = xsocket(AF_UNIX, SOCK_DGRAM, 0); 936 if (dgram < 0) { 937 test_sockmap_pass_prog__destroy(skel); 938 return; 939 } 940 941 tcp = connected_socket_v4(); 942 if (!ASSERT_GE(tcp, 0, "connected_socket_v4")) { 943 close(dgram); 944 test_sockmap_pass_prog__destroy(skel); 945 return; 946 } 947 948 udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); 949 if (udp < 0) { 950 close(dgram); 951 close(tcp); 952 test_sockmap_pass_prog__destroy(skel); 953 return; 954 } 955 956 err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream); 957 ASSERT_OK(err, "socketpair(af_unix, sock_stream)"); 958 if (err) { 959 close(tcp); 960 goto out; 961 } 962 963 for (i = 0; i < 2; i++) { 964 err = bpf_map_update_elem(map, &zero, &stream[0], BPF_ANY); 965 ASSERT_OK(err, "bpf_map_update_elem(stream)"); 966 } 967 for (i = 0; i < 2; i++) { 968 err = bpf_map_update_elem(map, &zero, &dgram, BPF_ANY); 969 ASSERT_OK(err, "bpf_map_update_elem(dgram)"); 970 } 971 for (i = 0; i < 2; i++) { 972 err = bpf_map_update_elem(map, &zero, &udp, BPF_ANY); 973 ASSERT_OK(err, "bpf_map_update_elem(udp)"); 974 } 975 for (i = 0; i < 2; i++) { 976 err = bpf_map_update_elem(map, &zero, &tcp, BPF_ANY); 977 ASSERT_OK(err, "bpf_map_update_elem(tcp)"); 978 } 979 980 close(tcp); 981 err = bpf_map_delete_elem(map, &zero); 982 ASSERT_ERR(err, "bpf_map_delete_elem(entry)"); 983 984 close(stream[0]); 985 close(stream[1]); 986 out: 987 close(dgram); 988 close(udp); 989 test_sockmap_pass_prog__destroy(skel); 990 } 991 992 static void test_sockmap_skb_verdict_vsock_poll(void) 993 { 994 struct test_sockmap_pass_prog *skel; 995 int err, map, conn, peer; 996 struct bpf_program *prog; 997 struct bpf_link *link; 998 char buf = 'x'; 999 int zero = 0; 1000 1001 skel = test_sockmap_pass_prog__open_and_load(); 1002 if (!ASSERT_OK_PTR(skel, "open_and_load")) 1003 return; 1004 1005 if (create_pair(AF_VSOCK, SOCK_STREAM, &conn, &peer)) 1006 goto destroy; 1007 1008 prog = skel->progs.prog_skb_verdict; 1009 map = bpf_map__fd(skel->maps.sock_map_rx); 1010 link = bpf_program__attach_sockmap(prog, map); 1011 if (!ASSERT_OK_PTR(link, "bpf_program__attach_sockmap")) 1012 goto close; 1013 1014 err = bpf_map_update_elem(map, &zero, &conn, BPF_ANY); 1015 if (!ASSERT_OK(err, "bpf_map_update_elem")) 1016 goto detach; 1017 1018 if (xsend(peer, &buf, 1, 0) != 1) 1019 goto detach; 1020 1021 err = poll_read(conn, IO_TIMEOUT_SEC); 1022 if (!ASSERT_OK(err, "poll")) 1023 goto detach; 1024 1025 if (xrecv_nonblock(conn, &buf, 1, 0) != 1) 1026 FAIL("xrecv_nonblock"); 1027 detach: 1028 bpf_link__destroy(link); 1029 close: 1030 xclose(conn); 1031 xclose(peer); 1032 destroy: 1033 test_sockmap_pass_prog__destroy(skel); 1034 } 1035 1036 static void test_sockmap_vsock_unconnected(void) 1037 { 1038 struct sockaddr_storage addr; 1039 int map, s, zero = 0; 1040 socklen_t alen; 1041 1042 map = bpf_map_create(BPF_MAP_TYPE_SOCKMAP, NULL, sizeof(int), 1043 sizeof(int), 1, NULL); 1044 if (!ASSERT_OK_FD(map, "bpf_map_create")) 1045 return; 1046 1047 s = xsocket(AF_VSOCK, SOCK_STREAM, 0); 1048 if (s < 0) 1049 goto close_map; 1050 1051 /* Fail connect(), but trigger transport assignment. */ 1052 init_addr_loopback(AF_VSOCK, &addr, &alen); 1053 if (!ASSERT_ERR(connect(s, sockaddr(&addr), alen), "connect")) 1054 goto close_sock; 1055 1056 ASSERT_ERR(bpf_map_update_elem(map, &zero, &s, BPF_ANY), "map_update"); 1057 1058 close_sock: 1059 xclose(s); 1060 close_map: 1061 xclose(map); 1062 } 1063 1064 /* it is used to reproduce WARNING */ 1065 static void test_sockmap_zc(void) 1066 { 1067 int map, err, sent, recvd, zero = 0, one = 1, on = 1; 1068 char buf[10] = "0123456789", rcv[11], addr[100]; 1069 struct test_sockmap_pass_prog *skel = NULL; 1070 int c0 = -1, p0 = -1, c1 = -1, p1 = -1; 1071 struct tcp_zerocopy_receive zc; 1072 socklen_t zc_len = sizeof(zc); 1073 struct bpf_program *prog; 1074 1075 skel = test_sockmap_pass_prog__open_and_load(); 1076 if (!ASSERT_OK_PTR(skel, "open_and_load")) 1077 return; 1078 1079 if (create_socket_pairs(AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1)) 1080 goto end; 1081 1082 prog = skel->progs.prog_skb_verdict_ingress; 1083 map = bpf_map__fd(skel->maps.sock_map_rx); 1084 1085 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0); 1086 if (!ASSERT_OK(err, "bpf_prog_attach")) 1087 goto end; 1088 1089 err = bpf_map_update_elem(map, &zero, &p0, BPF_ANY); 1090 if (!ASSERT_OK(err, "bpf_map_update_elem")) 1091 goto end; 1092 1093 err = bpf_map_update_elem(map, &one, &p1, BPF_ANY); 1094 if (!ASSERT_OK(err, "bpf_map_update_elem")) 1095 goto end; 1096 1097 sent = xsend(c0, buf, sizeof(buf), 0); 1098 if (!ASSERT_EQ(sent, sizeof(buf), "xsend")) 1099 goto end; 1100 1101 /* trigger tcp_bpf_recvmsg_parser and inc copied_seq of p1 */ 1102 recvd = recv_timeout(p1, rcv, sizeof(rcv), MSG_DONTWAIT, 1); 1103 if (!ASSERT_EQ(recvd, sent, "recv_timeout(p1)")) 1104 goto end; 1105 1106 /* uninstall sockmap of p1 */ 1107 bpf_map_delete_elem(map, &one); 1108 1109 /* trigger tcp stack and the rcv_nxt of p1 is less than copied_seq */ 1110 sent = xsend(c1, buf, sizeof(buf) - 1, 0); 1111 if (!ASSERT_EQ(sent, sizeof(buf) - 1, "xsend")) 1112 goto end; 1113 1114 err = setsockopt(p1, SOL_SOCKET, SO_ZEROCOPY, &on, sizeof(on)); 1115 if (!ASSERT_OK(err, "setsockopt")) 1116 goto end; 1117 1118 memset(&zc, 0, sizeof(zc)); 1119 zc.copybuf_address = (__u64)((unsigned long)addr); 1120 zc.copybuf_len = sizeof(addr); 1121 1122 err = getsockopt(p1, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE, &zc, &zc_len); 1123 if (!ASSERT_OK(err, "getsockopt")) 1124 goto end; 1125 1126 end: 1127 if (c0 >= 0) 1128 close(c0); 1129 if (p0 >= 0) 1130 close(p0); 1131 if (c1 >= 0) 1132 close(c1); 1133 if (p1 >= 0) 1134 close(p1); 1135 test_sockmap_pass_prog__destroy(skel); 1136 } 1137 1138 /* it is used to check whether copied_seq of sk is correct */ 1139 static void test_sockmap_copied_seq(bool strp) 1140 { 1141 int i, map, err, sent, recvd, zero = 0, one = 1; 1142 struct test_sockmap_pass_prog *skel = NULL; 1143 int c0 = -1, p0 = -1, c1 = -1, p1 = -1; 1144 char buf[10] = "0123456789", rcv[11]; 1145 struct bpf_program *prog; 1146 1147 skel = test_sockmap_pass_prog__open_and_load(); 1148 if (!ASSERT_OK_PTR(skel, "open_and_load")) 1149 return; 1150 1151 if (create_socket_pairs(AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1)) 1152 goto end; 1153 1154 prog = skel->progs.prog_skb_verdict_ingress; 1155 map = bpf_map__fd(skel->maps.sock_map_rx); 1156 1157 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0); 1158 if (!ASSERT_OK(err, "bpf_prog_attach verdict")) 1159 goto end; 1160 1161 if (strp) { 1162 prog = skel->progs.prog_skb_verdict_ingress_strp; 1163 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_PARSER, 0); 1164 if (!ASSERT_OK(err, "bpf_prog_attach parser")) 1165 goto end; 1166 } 1167 1168 err = bpf_map_update_elem(map, &zero, &p0, BPF_ANY); 1169 if (!ASSERT_OK(err, "bpf_map_update_elem(p0)")) 1170 goto end; 1171 1172 err = bpf_map_update_elem(map, &one, &p1, BPF_ANY); 1173 if (!ASSERT_OK(err, "bpf_map_update_elem(p1)")) 1174 goto end; 1175 1176 /* just trigger sockamp: data sent by c0 will be received by p1 */ 1177 sent = xsend(c0, buf, sizeof(buf), 0); 1178 if (!ASSERT_EQ(sent, sizeof(buf), "xsend(c0), bpf")) 1179 goto end; 1180 1181 /* do partial read */ 1182 recvd = recv_timeout(p1, rcv, 1, MSG_DONTWAIT, 1); 1183 recvd += recv_timeout(p1, rcv + 1, sizeof(rcv) - 1, MSG_DONTWAIT, 1); 1184 if (!ASSERT_EQ(recvd, sent, "recv_timeout(p1), bpf") || 1185 !ASSERT_OK(memcmp(buf, rcv, recvd), "data mismatch")) 1186 goto end; 1187 1188 /* uninstall sockmap of p1 and p0 */ 1189 err = bpf_map_delete_elem(map, &one); 1190 if (!ASSERT_OK(err, "bpf_map_delete_elem(1)")) 1191 goto end; 1192 1193 err = bpf_map_delete_elem(map, &zero); 1194 if (!ASSERT_OK(err, "bpf_map_delete_elem(0)")) 1195 goto end; 1196 1197 /* now all sockets become plain socket, they should still work */ 1198 for (i = 0; i < 5; i++) { 1199 /* test copied_seq of p1 by running tcp native stack */ 1200 sent = xsend(c1, buf, sizeof(buf), 0); 1201 if (!ASSERT_EQ(sent, sizeof(buf), "xsend(c1), native")) 1202 goto end; 1203 1204 recvd = recv(p1, rcv, sizeof(rcv), MSG_DONTWAIT); 1205 if (!ASSERT_EQ(recvd, sent, "recv_timeout(p1), native")) 1206 goto end; 1207 1208 /* p0 previously redirected skb to p1, we also check copied_seq of p0 */ 1209 sent = xsend(c0, buf, sizeof(buf), 0); 1210 if (!ASSERT_EQ(sent, sizeof(buf), "xsend(c0), native")) 1211 goto end; 1212 1213 recvd = recv(p0, rcv, sizeof(rcv), MSG_DONTWAIT); 1214 if (!ASSERT_EQ(recvd, sent, "recv_timeout(p0), native")) 1215 goto end; 1216 } 1217 1218 end: 1219 if (c0 >= 0) 1220 close(c0); 1221 if (p0 >= 0) 1222 close(p0); 1223 if (c1 >= 0) 1224 close(c1); 1225 if (p1 >= 0) 1226 close(p1); 1227 test_sockmap_pass_prog__destroy(skel); 1228 } 1229 1230 /* Wait until FIONREAD returns the expected value or timeout */ 1231 static int wait_for_fionread(int fd, int expected, unsigned int timeout_ms) 1232 { 1233 unsigned int elapsed = 0; 1234 int avail = 0; 1235 1236 while (elapsed < timeout_ms) { 1237 if (ioctl(fd, FIONREAD, &avail) < 0) 1238 return -errno; 1239 if (avail >= expected) 1240 return avail; 1241 usleep(1000); 1242 elapsed++; 1243 } 1244 return avail; 1245 } 1246 1247 /* it is used to send data to via native stack and BPF redirecting */ 1248 static void test_sockmap_multi_channels(int sotype) 1249 { 1250 int map, err, sent, recvd, zero = 0, one = 1, avail = 0, expected; 1251 struct test_sockmap_pass_prog *skel = NULL; 1252 int c0 = -1, p0 = -1, c1 = -1, p1 = -1; 1253 char buf[10] = "0123456789", rcv[11]; 1254 struct bpf_program *prog; 1255 1256 skel = test_sockmap_pass_prog__open_and_load(); 1257 if (!ASSERT_OK_PTR(skel, "open_and_load")) 1258 return; 1259 1260 err = create_socket_pairs(AF_INET, sotype, &c0, &c1, &p0, &p1); 1261 if (err) 1262 goto end; 1263 1264 prog = skel->progs.prog_skb_verdict_ingress; 1265 map = bpf_map__fd(skel->maps.sock_map_rx); 1266 1267 err = bpf_prog_attach(bpf_program__fd(prog), map, BPF_SK_SKB_STREAM_VERDICT, 0); 1268 if (!ASSERT_OK(err, "bpf_prog_attach verdict")) 1269 goto end; 1270 1271 err = bpf_map_update_elem(map, &zero, &p0, BPF_ANY); 1272 if (!ASSERT_OK(err, "bpf_map_update_elem(p0)")) 1273 goto end; 1274 1275 err = bpf_map_update_elem(map, &one, &p1, BPF_ANY); 1276 if (!ASSERT_OK(err, "bpf_map_update_elem")) 1277 goto end; 1278 1279 /* send data to p1 via native stack */ 1280 sent = xsend(c1, buf, 2, 0); 1281 if (!ASSERT_EQ(sent, 2, "xsend(2)")) 1282 goto end; 1283 1284 avail = wait_for_fionread(p1, 2, IO_TIMEOUT_SEC); 1285 ASSERT_EQ(avail, 2, "ioctl(FIONREAD) partial return"); 1286 1287 /* send data to p1 via bpf redirecting */ 1288 sent = xsend(c0, buf + 2, sizeof(buf) - 2, 0); 1289 if (!ASSERT_EQ(sent, sizeof(buf) - 2, "xsend(remain-data)")) 1290 goto end; 1291 1292 /* Poll FIONREAD until expected bytes arrive, poll_read() is unreliable 1293 * here since it may return immediately if prior data is already queued. 1294 */ 1295 expected = sotype == SOCK_DGRAM ? 2 : sizeof(buf); 1296 avail = wait_for_fionread(p1, expected, IO_TIMEOUT_SEC); 1297 ASSERT_EQ(avail, expected, "ioctl(FIONREAD) full return"); 1298 1299 recvd = recv_timeout(p1, rcv, expected, MSG_DONTWAIT, 1); 1300 if (!ASSERT_EQ(recvd, expected, "recv_timeout(p1)") || 1301 !ASSERT_OK(memcmp(buf, rcv, recvd), "data mismatch")) 1302 goto end; 1303 1304 /* process remaining data for udp if secondary data is available */ 1305 expected = sizeof(buf) - expected; 1306 if (expected) { 1307 avail = wait_for_fionread(p1, expected, IO_TIMEOUT_SEC); 1308 ASSERT_EQ(avail, expected, "second ioctl(FIONREAD) full return"); 1309 1310 recvd = recv_timeout(p1, rcv, expected, MSG_DONTWAIT, 1); 1311 if (!ASSERT_EQ(recvd, expected, "second recv_timeout(p1)") || 1312 !ASSERT_OK(memcmp(buf + sizeof(buf) - expected, rcv, recvd), 1313 "second data mismatch")) 1314 goto end; 1315 } 1316 end: 1317 if (c0 >= 0) 1318 close(c0); 1319 if (p0 >= 0) 1320 close(p0); 1321 if (c1 >= 0) 1322 close(c1); 1323 if (p1 >= 0) 1324 close(p1); 1325 test_sockmap_pass_prog__destroy(skel); 1326 } 1327 1328 void test_sockmap_basic(void) 1329 { 1330 if (test__start_subtest("sockmap create_update_free")) 1331 test_sockmap_create_update_free(BPF_MAP_TYPE_SOCKMAP); 1332 if (test__start_subtest("sockhash create_update_free")) 1333 test_sockmap_create_update_free(BPF_MAP_TYPE_SOCKHASH); 1334 if (test__start_subtest("sockmap vsock delete on close")) 1335 test_sockmap_vsock_delete_on_close(); 1336 if (test__start_subtest("sockmap sk_msg load helpers")) 1337 test_skmsg_helpers(BPF_MAP_TYPE_SOCKMAP); 1338 if (test__start_subtest("sockhash sk_msg load helpers")) 1339 test_skmsg_helpers(BPF_MAP_TYPE_SOCKHASH); 1340 if (test__start_subtest("sockmap update in unsafe context")) 1341 test_sockmap_invalid_update(); 1342 if (test__start_subtest("sockmap copy")) 1343 test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP); 1344 if (test__start_subtest("sockhash copy")) 1345 test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH); 1346 if (test__start_subtest("sockmap skb_verdict attach")) { 1347 test_sockmap_skb_verdict_attach(BPF_SK_SKB_VERDICT, 1348 BPF_SK_SKB_STREAM_VERDICT); 1349 test_sockmap_skb_verdict_attach(BPF_SK_SKB_STREAM_VERDICT, 1350 BPF_SK_SKB_VERDICT); 1351 } 1352 if (test__start_subtest("sockmap skb_verdict attach_with_link")) 1353 test_sockmap_skb_verdict_attach_with_link(); 1354 if (test__start_subtest("sockmap msg_verdict progs query")) 1355 test_sockmap_progs_query(BPF_SK_MSG_VERDICT); 1356 if (test__start_subtest("sockmap stream_parser progs query")) 1357 test_sockmap_progs_query(BPF_SK_SKB_STREAM_PARSER); 1358 if (test__start_subtest("sockmap stream_verdict progs query")) 1359 test_sockmap_progs_query(BPF_SK_SKB_STREAM_VERDICT); 1360 if (test__start_subtest("sockmap skb_verdict progs query")) 1361 test_sockmap_progs_query(BPF_SK_SKB_VERDICT); 1362 if (test__start_subtest("sockmap skb_verdict shutdown")) 1363 test_sockmap_skb_verdict_shutdown(); 1364 if (test__start_subtest("sockmap skb_verdict fionread")) 1365 test_sockmap_skb_verdict_fionread(true); 1366 if (test__start_subtest("sockmap skb_verdict fionread on drop")) 1367 test_sockmap_skb_verdict_fionread(false); 1368 if (test__start_subtest("sockmap skb_verdict change tail")) 1369 test_sockmap_skb_verdict_change_tail(); 1370 if (test__start_subtest("sockmap msg_verdict pop_data overflow")) 1371 test_sockmap_msg_verdict_pop_data(); 1372 if (test__start_subtest("sockmap skb_verdict msg_f_peek")) 1373 test_sockmap_skb_verdict_peek(); 1374 if (test__start_subtest("sockmap skb_verdict msg_f_peek with link")) 1375 test_sockmap_skb_verdict_peek_with_link(); 1376 if (test__start_subtest("sockmap unconnected af_unix")) 1377 test_sockmap_unconnected_unix(); 1378 if (test__start_subtest("sockmap one socket to many map entries")) 1379 test_sockmap_many_socket(); 1380 if (test__start_subtest("sockmap one socket to many maps")) 1381 test_sockmap_many_maps(); 1382 if (test__start_subtest("sockmap same socket replace")) 1383 test_sockmap_same_sock(); 1384 if (test__start_subtest("sockmap sk_msg attach sockmap helpers with link")) 1385 test_skmsg_helpers_with_link(BPF_MAP_TYPE_SOCKMAP); 1386 if (test__start_subtest("sockhash sk_msg attach sockhash helpers with link")) 1387 test_skmsg_helpers_with_link(BPF_MAP_TYPE_SOCKHASH); 1388 if (test__start_subtest("sockmap skb_verdict vsock poll")) 1389 test_sockmap_skb_verdict_vsock_poll(); 1390 if (test__start_subtest("sockmap vsock unconnected")) 1391 test_sockmap_vsock_unconnected(); 1392 if (test__start_subtest("sockmap with zc")) 1393 test_sockmap_zc(); 1394 if (test__start_subtest("sockmap recover")) 1395 test_sockmap_copied_seq(false); 1396 if (test__start_subtest("sockmap recover with strp")) 1397 test_sockmap_copied_seq(true); 1398 if (test__start_subtest("sockmap tcp multi channels")) 1399 test_sockmap_multi_channels(SOCK_STREAM); 1400 if (test__start_subtest("sockmap udp multi channels")) 1401 test_sockmap_multi_channels(SOCK_DGRAM); 1402 } 1403