1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2022 Facebook */ 3 4 #include <vmlinux.h> 5 #include <string.h> 6 #include <stdbool.h> 7 #include <bpf/bpf_helpers.h> 8 #include <bpf/bpf_tracing.h> 9 #include "bpf_misc.h" 10 #include "errno.h" 11 12 #define PAGE_SIZE_64K 65536 13 14 char _license[] SEC("license") = "GPL"; 15 16 int pid, err, val; 17 18 struct ringbuf_sample { 19 int pid; 20 int seq; 21 long value; 22 char comm[16]; 23 }; 24 25 struct { 26 __uint(type, BPF_MAP_TYPE_RINGBUF); 27 __uint(max_entries, 4096); 28 } ringbuf SEC(".maps"); 29 30 struct { 31 __uint(type, BPF_MAP_TYPE_ARRAY); 32 __uint(max_entries, 1); 33 __type(key, __u32); 34 __type(value, __u32); 35 } array_map SEC(".maps"); 36 37 SEC("?tp/syscalls/sys_enter_nanosleep") 38 int test_read_write(void *ctx) 39 { 40 char write_data[64] = "hello there, world!!"; 41 char read_data[64] = {}; 42 struct bpf_dynptr ptr; 43 int i; 44 45 if (bpf_get_current_pid_tgid() >> 32 != pid) 46 return 0; 47 48 bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(write_data), 0, &ptr); 49 50 /* Write data into the dynptr */ 51 err = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0); 52 53 /* Read the data that was written into the dynptr */ 54 err = err ?: bpf_dynptr_read(read_data, sizeof(read_data), &ptr, 0, 0); 55 56 /* Ensure the data we read matches the data we wrote */ 57 for (i = 0; i < sizeof(read_data); i++) { 58 if (read_data[i] != write_data[i]) { 59 err = 1; 60 break; 61 } 62 } 63 64 bpf_ringbuf_discard_dynptr(&ptr, 0); 65 return 0; 66 } 67 68 SEC("?tp/syscalls/sys_enter_nanosleep") 69 int test_dynptr_data(void *ctx) 70 { 71 __u32 key = 0, val = 235, *map_val; 72 struct bpf_dynptr ptr; 73 __u32 map_val_size; 74 void *data; 75 76 map_val_size = sizeof(*map_val); 77 78 if (bpf_get_current_pid_tgid() >> 32 != pid) 79 return 0; 80 81 bpf_map_update_elem(&array_map, &key, &val, 0); 82 83 map_val = bpf_map_lookup_elem(&array_map, &key); 84 if (!map_val) { 85 err = 1; 86 return 0; 87 } 88 89 bpf_dynptr_from_mem(map_val, map_val_size, 0, &ptr); 90 91 /* Try getting a data slice that is out of range */ 92 data = bpf_dynptr_data(&ptr, map_val_size + 1, 1); 93 if (data) { 94 err = 2; 95 return 0; 96 } 97 98 /* Try getting more bytes than available */ 99 data = bpf_dynptr_data(&ptr, 0, map_val_size + 1); 100 if (data) { 101 err = 3; 102 return 0; 103 } 104 105 data = bpf_dynptr_data(&ptr, 0, sizeof(__u32)); 106 if (!data) { 107 err = 4; 108 return 0; 109 } 110 111 *(__u32 *)data = 999; 112 113 err = bpf_probe_read_kernel(&val, sizeof(val), data); 114 if (err) 115 return 0; 116 117 if (val != *(int *)data) 118 err = 5; 119 120 return 0; 121 } 122 123 static int ringbuf_callback(__u32 index, void *data) 124 { 125 struct ringbuf_sample *sample; 126 127 struct bpf_dynptr *ptr = (struct bpf_dynptr *)data; 128 129 sample = bpf_dynptr_data(ptr, 0, sizeof(*sample)); 130 if (!sample) 131 err = 2; 132 else 133 sample->pid += index; 134 135 return 0; 136 } 137 138 SEC("?tp/syscalls/sys_enter_nanosleep") 139 int test_ringbuf(void *ctx) 140 { 141 struct bpf_dynptr ptr; 142 struct ringbuf_sample *sample; 143 144 if (bpf_get_current_pid_tgid() >> 32 != pid) 145 return 0; 146 147 val = 100; 148 149 /* check that you can reserve a dynamic size reservation */ 150 err = bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); 151 152 sample = err ? NULL : bpf_dynptr_data(&ptr, 0, sizeof(*sample)); 153 if (!sample) { 154 err = 1; 155 goto done; 156 } 157 158 sample->pid = 10; 159 160 /* Can pass dynptr to callback functions */ 161 bpf_loop(10, ringbuf_callback, &ptr, 0); 162 163 if (sample->pid != 55) 164 err = 2; 165 166 done: 167 bpf_ringbuf_discard_dynptr(&ptr, 0); 168 return 0; 169 } 170 171 SEC("?cgroup_skb/egress") 172 int test_skb_readonly(struct __sk_buff *skb) 173 { 174 __u8 write_data[2] = {1, 2}; 175 struct bpf_dynptr ptr; 176 int ret; 177 178 if (bpf_dynptr_from_skb(skb, 0, &ptr)) { 179 err = 1; 180 return 1; 181 } 182 183 /* since cgroup skbs are read only, writes should fail */ 184 ret = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0); 185 if (ret != -EINVAL) { 186 err = 2; 187 return 1; 188 } 189 190 return 1; 191 } 192 193 SEC("?cgroup_skb/egress") 194 int test_dynptr_skb_data(struct __sk_buff *skb) 195 { 196 struct bpf_dynptr ptr; 197 __u64 *data; 198 199 if (bpf_dynptr_from_skb(skb, 0, &ptr)) { 200 err = 1; 201 return 1; 202 } 203 204 /* This should return NULL. Must use bpf_dynptr_slice API */ 205 data = bpf_dynptr_data(&ptr, 0, 1); 206 if (data) { 207 err = 2; 208 return 1; 209 } 210 211 return 1; 212 } 213 214 SEC("?tc") 215 int test_dynptr_skb_meta_data(struct __sk_buff *skb) 216 { 217 struct bpf_dynptr meta; 218 __u8 *md; 219 int ret; 220 221 err = 1; 222 ret = bpf_dynptr_from_skb_meta(skb, 0, &meta); 223 if (ret) 224 return 1; 225 226 /* This should return NULL. Must use bpf_dynptr_slice API */ 227 err = 2; 228 md = bpf_dynptr_data(&meta, 0, sizeof(*md)); 229 if (md) 230 return 1; 231 232 err = 0; 233 return 1; 234 } 235 236 /* Check that skb metadata dynptr ops don't accept any flags. */ 237 SEC("?tc") 238 int test_dynptr_skb_meta_flags(struct __sk_buff *skb) 239 { 240 const __u64 INVALID_FLAGS = ~0ULL; 241 struct bpf_dynptr meta; 242 __u8 buf; 243 int ret; 244 245 err = 1; 246 ret = bpf_dynptr_from_skb_meta(skb, INVALID_FLAGS, &meta); 247 if (ret != -EINVAL) 248 return 1; 249 250 err = 2; 251 ret = bpf_dynptr_from_skb_meta(skb, 0, &meta); 252 if (ret) 253 return 1; 254 255 err = 3; 256 ret = bpf_dynptr_read(&buf, 0, &meta, 0, INVALID_FLAGS); 257 if (ret != -EINVAL) 258 return 1; 259 260 err = 4; 261 ret = bpf_dynptr_write(&meta, 0, &buf, 0, INVALID_FLAGS); 262 if (ret != -EINVAL) 263 return 1; 264 265 err = 0; 266 return 1; 267 } 268 269 SEC("tp/syscalls/sys_enter_nanosleep") 270 int test_adjust(void *ctx) 271 { 272 struct bpf_dynptr ptr; 273 __u32 bytes = 64; 274 __u32 off = 10; 275 __u32 trim = 15; 276 277 if (bpf_get_current_pid_tgid() >> 32 != pid) 278 return 0; 279 280 err = bpf_ringbuf_reserve_dynptr(&ringbuf, bytes, 0, &ptr); 281 if (err) { 282 err = 1; 283 goto done; 284 } 285 286 if (bpf_dynptr_size(&ptr) != bytes) { 287 err = 2; 288 goto done; 289 } 290 291 /* Advance the dynptr by off */ 292 err = bpf_dynptr_adjust(&ptr, off, bpf_dynptr_size(&ptr)); 293 if (err) { 294 err = 3; 295 goto done; 296 } 297 298 if (bpf_dynptr_size(&ptr) != bytes - off) { 299 err = 4; 300 goto done; 301 } 302 303 /* Trim the dynptr */ 304 err = bpf_dynptr_adjust(&ptr, off, 15); 305 if (err) { 306 err = 5; 307 goto done; 308 } 309 310 /* Check that the size was adjusted correctly */ 311 if (bpf_dynptr_size(&ptr) != trim - off) { 312 err = 6; 313 goto done; 314 } 315 316 done: 317 bpf_ringbuf_discard_dynptr(&ptr, 0); 318 return 0; 319 } 320 321 SEC("tp/syscalls/sys_enter_nanosleep") 322 int test_adjust_err(void *ctx) 323 { 324 char write_data[45] = "hello there, world!!"; 325 struct bpf_dynptr ptr; 326 __u32 size = 64; 327 __u32 off = 20; 328 329 if (bpf_get_current_pid_tgid() >> 32 != pid) 330 return 0; 331 332 if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 0, &ptr)) { 333 err = 1; 334 goto done; 335 } 336 337 /* Check that start can't be greater than end */ 338 if (bpf_dynptr_adjust(&ptr, 5, 1) != -EINVAL) { 339 err = 2; 340 goto done; 341 } 342 343 /* Check that start can't be greater than size */ 344 if (bpf_dynptr_adjust(&ptr, size + 1, size + 1) != -ERANGE) { 345 err = 3; 346 goto done; 347 } 348 349 /* Check that end can't be greater than size */ 350 if (bpf_dynptr_adjust(&ptr, 0, size + 1) != -ERANGE) { 351 err = 4; 352 goto done; 353 } 354 355 if (bpf_dynptr_adjust(&ptr, off, size)) { 356 err = 5; 357 goto done; 358 } 359 360 /* Check that you can't write more bytes than available into the dynptr 361 * after you've adjusted it 362 */ 363 if (bpf_dynptr_write(&ptr, 0, &write_data, sizeof(write_data), 0) != -E2BIG) { 364 err = 6; 365 goto done; 366 } 367 368 /* Check that even after adjusting, submitting/discarding 369 * a ringbuf dynptr works 370 */ 371 bpf_ringbuf_submit_dynptr(&ptr, 0); 372 return 0; 373 374 done: 375 bpf_ringbuf_discard_dynptr(&ptr, 0); 376 return 0; 377 } 378 379 SEC("tp/syscalls/sys_enter_nanosleep") 380 int test_zero_size_dynptr(void *ctx) 381 { 382 char write_data = 'x', read_data; 383 struct bpf_dynptr ptr; 384 __u32 size = 64; 385 386 if (bpf_get_current_pid_tgid() >> 32 != pid) 387 return 0; 388 389 if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 0, &ptr)) { 390 err = 1; 391 goto done; 392 } 393 394 /* After this, the dynptr has a size of 0 */ 395 if (bpf_dynptr_adjust(&ptr, size, size)) { 396 err = 2; 397 goto done; 398 } 399 400 /* Test that reading + writing non-zero bytes is not ok */ 401 if (bpf_dynptr_read(&read_data, sizeof(read_data), &ptr, 0, 0) != -E2BIG) { 402 err = 3; 403 goto done; 404 } 405 406 if (bpf_dynptr_write(&ptr, 0, &write_data, sizeof(write_data), 0) != -E2BIG) { 407 err = 4; 408 goto done; 409 } 410 411 /* Test that reading + writing 0 bytes from a 0-size dynptr is ok */ 412 if (bpf_dynptr_read(&read_data, 0, &ptr, 0, 0)) { 413 err = 5; 414 goto done; 415 } 416 417 if (bpf_dynptr_write(&ptr, 0, &write_data, 0, 0)) { 418 err = 6; 419 goto done; 420 } 421 422 err = 0; 423 424 done: 425 bpf_ringbuf_discard_dynptr(&ptr, 0); 426 return 0; 427 } 428 429 SEC("tp/syscalls/sys_enter_nanosleep") 430 int test_dynptr_is_null(void *ctx) 431 { 432 struct bpf_dynptr ptr1; 433 struct bpf_dynptr ptr2; 434 __u64 size = 4; 435 436 if (bpf_get_current_pid_tgid() >> 32 != pid) 437 return 0; 438 439 /* Pass in invalid flags, get back an invalid dynptr */ 440 if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 123, &ptr1) != -EINVAL) { 441 err = 1; 442 goto exit_early; 443 } 444 445 /* Test that the invalid dynptr is null */ 446 if (!bpf_dynptr_is_null(&ptr1)) { 447 err = 2; 448 goto exit_early; 449 } 450 451 /* Get a valid dynptr */ 452 if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 0, &ptr2)) { 453 err = 3; 454 goto exit; 455 } 456 457 /* Test that the valid dynptr is not null */ 458 if (bpf_dynptr_is_null(&ptr2)) { 459 err = 4; 460 goto exit; 461 } 462 463 exit: 464 bpf_ringbuf_discard_dynptr(&ptr2, 0); 465 exit_early: 466 bpf_ringbuf_discard_dynptr(&ptr1, 0); 467 return 0; 468 } 469 470 SEC("cgroup_skb/egress") 471 int test_dynptr_is_rdonly(struct __sk_buff *skb) 472 { 473 struct bpf_dynptr ptr1; 474 struct bpf_dynptr ptr2; 475 struct bpf_dynptr ptr3; 476 477 /* Pass in invalid flags, get back an invalid dynptr */ 478 if (bpf_dynptr_from_skb(skb, 123, &ptr1) != -EINVAL) { 479 err = 1; 480 return 0; 481 } 482 483 /* Test that an invalid dynptr is_rdonly returns false */ 484 if (bpf_dynptr_is_rdonly(&ptr1)) { 485 err = 2; 486 return 0; 487 } 488 489 /* Get a read-only dynptr */ 490 if (bpf_dynptr_from_skb(skb, 0, &ptr2)) { 491 err = 3; 492 return 0; 493 } 494 495 /* Test that the dynptr is read-only */ 496 if (!bpf_dynptr_is_rdonly(&ptr2)) { 497 err = 4; 498 return 0; 499 } 500 501 /* Get a read-writeable dynptr */ 502 if (bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr3)) { 503 err = 5; 504 goto done; 505 } 506 507 /* Test that the dynptr is read-only */ 508 if (bpf_dynptr_is_rdonly(&ptr3)) { 509 err = 6; 510 goto done; 511 } 512 513 done: 514 bpf_ringbuf_discard_dynptr(&ptr3, 0); 515 return 0; 516 } 517 518 SEC("cgroup_skb/egress") 519 int test_dynptr_clone(struct __sk_buff *skb) 520 { 521 struct bpf_dynptr ptr1; 522 struct bpf_dynptr ptr2; 523 __u32 off = 2, size; 524 525 /* Get a dynptr */ 526 if (bpf_dynptr_from_skb(skb, 0, &ptr1)) { 527 err = 1; 528 return 0; 529 } 530 531 if (bpf_dynptr_adjust(&ptr1, off, bpf_dynptr_size(&ptr1))) { 532 err = 2; 533 return 0; 534 } 535 536 /* Clone the dynptr */ 537 if (bpf_dynptr_clone(&ptr1, &ptr2)) { 538 err = 3; 539 return 0; 540 } 541 542 size = bpf_dynptr_size(&ptr1); 543 544 /* Check that the clone has the same size and rd-only */ 545 if (bpf_dynptr_size(&ptr2) != size) { 546 err = 4; 547 return 0; 548 } 549 550 if (bpf_dynptr_is_rdonly(&ptr2) != bpf_dynptr_is_rdonly(&ptr1)) { 551 err = 5; 552 return 0; 553 } 554 555 /* Advance and trim the original dynptr */ 556 bpf_dynptr_adjust(&ptr1, 5, 5); 557 558 /* Check that only original dynptr was affected, and the clone wasn't */ 559 if (bpf_dynptr_size(&ptr2) != size) { 560 err = 6; 561 return 0; 562 } 563 564 return 0; 565 } 566 567 SEC("?cgroup_skb/egress") 568 int test_dynptr_skb_no_buff(struct __sk_buff *skb) 569 { 570 struct bpf_dynptr ptr; 571 __u64 *data; 572 573 if (bpf_dynptr_from_skb(skb, 0, &ptr)) { 574 err = 1; 575 return 1; 576 } 577 578 /* This may return NULL. SKB may require a buffer */ 579 data = bpf_dynptr_slice(&ptr, 0, NULL, 1); 580 581 return !!data; 582 } 583 584 SEC("?cgroup_skb/egress") 585 int test_dynptr_skb_strcmp(struct __sk_buff *skb) 586 { 587 struct bpf_dynptr ptr; 588 char *data; 589 590 if (bpf_dynptr_from_skb(skb, 0, &ptr)) { 591 err = 1; 592 return 1; 593 } 594 595 /* This may return NULL. SKB may require a buffer */ 596 data = bpf_dynptr_slice(&ptr, 0, NULL, 10); 597 if (data) { 598 bpf_strncmp(data, 10, "foo"); 599 return 1; 600 } 601 602 return 1; 603 } 604 605 SEC("tp_btf/kfree_skb") 606 int BPF_PROG(test_dynptr_skb_tp_btf, void *skb, void *location) 607 { 608 __u8 write_data[2] = {1, 2}; 609 struct bpf_dynptr ptr; 610 int ret; 611 612 if (bpf_dynptr_from_skb(skb, 0, &ptr)) { 613 err = 1; 614 return 1; 615 } 616 617 /* since tp_btf skbs are read only, writes should fail */ 618 ret = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0); 619 if (ret != -EINVAL) { 620 err = 2; 621 return 1; 622 } 623 624 return 1; 625 } 626 627 static inline int bpf_memcmp(const char *a, const char *b, u32 size) 628 { 629 int i; 630 631 bpf_for(i, 0, size) { 632 if (a[i] != b[i]) 633 return a[i] < b[i] ? -1 : 1; 634 } 635 return 0; 636 } 637 638 SEC("?tp/syscalls/sys_enter_nanosleep") 639 int test_dynptr_copy(void *ctx) 640 { 641 char data[] = "hello there, world!!"; 642 char buf[32] = {'\0'}; 643 __u32 sz = sizeof(data); 644 struct bpf_dynptr src, dst; 645 646 bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &src); 647 bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &dst); 648 649 /* Test basic case of copying contiguous memory backed dynptrs */ 650 err = bpf_dynptr_write(&src, 0, data, sz, 0); 651 err = err ?: bpf_dynptr_copy(&dst, 0, &src, 0, sz); 652 err = err ?: bpf_dynptr_read(buf, sz, &dst, 0, 0); 653 err = err ?: bpf_memcmp(data, buf, sz); 654 655 /* Test that offsets are handled correctly */ 656 err = err ?: bpf_dynptr_copy(&dst, 3, &src, 5, sz - 5); 657 err = err ?: bpf_dynptr_read(buf, sz - 5, &dst, 3, 0); 658 err = err ?: bpf_memcmp(data + 5, buf, sz - 5); 659 660 bpf_ringbuf_discard_dynptr(&src, 0); 661 bpf_ringbuf_discard_dynptr(&dst, 0); 662 return 0; 663 } 664 665 SEC("xdp") 666 int test_dynptr_copy_xdp(struct xdp_md *xdp) 667 { 668 struct bpf_dynptr ptr_buf, ptr_xdp; 669 char data[] = "qwertyuiopasdfghjkl"; 670 char buf[32] = {'\0'}; 671 __u32 len = sizeof(data), xdp_data_size; 672 int i, chunks = 200; 673 674 /* ptr_xdp is backed by non-contiguous memory */ 675 bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp); 676 xdp_data_size = bpf_dynptr_size(&ptr_xdp); 677 bpf_ringbuf_reserve_dynptr(&ringbuf, len * chunks, 0, &ptr_buf); 678 679 /* Destination dynptr is backed by non-contiguous memory */ 680 bpf_for(i, 0, chunks) { 681 err = bpf_dynptr_write(&ptr_buf, i * len, data, len, 0); 682 if (err) 683 goto out; 684 } 685 686 err = bpf_dynptr_copy(&ptr_xdp, 0, &ptr_buf, 0, len * chunks); 687 if (err) 688 goto out; 689 690 bpf_for(i, 0, chunks) { 691 __builtin_memset(buf, 0, sizeof(buf)); 692 err = bpf_dynptr_read(&buf, len, &ptr_xdp, i * len, 0); 693 if (err) 694 goto out; 695 if (bpf_memcmp(data, buf, len) != 0) 696 goto out; 697 } 698 699 /* Source dynptr is backed by non-contiguous memory */ 700 __builtin_memset(buf, 0, sizeof(buf)); 701 bpf_for(i, 0, chunks) { 702 err = bpf_dynptr_write(&ptr_buf, i * len, buf, len, 0); 703 if (err) 704 goto out; 705 } 706 707 err = bpf_dynptr_copy(&ptr_buf, 0, &ptr_xdp, 0, len * chunks); 708 if (err) 709 goto out; 710 711 bpf_for(i, 0, chunks) { 712 __builtin_memset(buf, 0, sizeof(buf)); 713 err = bpf_dynptr_read(&buf, len, &ptr_buf, i * len, 0); 714 if (err) 715 goto out; 716 if (bpf_memcmp(data, buf, len) != 0) 717 goto out; 718 } 719 720 /* Both source and destination dynptrs are backed by non-contiguous memory */ 721 err = bpf_dynptr_copy(&ptr_xdp, 2, &ptr_xdp, len, len * (chunks - 1)); 722 if (err) 723 goto out; 724 725 bpf_for(i, 0, chunks - 1) { 726 __builtin_memset(buf, 0, sizeof(buf)); 727 err = bpf_dynptr_read(&buf, len, &ptr_xdp, 2 + i * len, 0); 728 if (err) 729 goto out; 730 if (bpf_memcmp(data, buf, len) != 0) 731 goto out; 732 } 733 734 if (bpf_dynptr_copy(&ptr_xdp, xdp_data_size - 3000, &ptr_xdp, 0, len * chunks) != -E2BIG) 735 err = 1; 736 737 out: 738 bpf_ringbuf_discard_dynptr(&ptr_buf, 0); 739 return XDP_DROP; 740 } 741 742 char memset_zero_data[] = "data to be zeroed"; 743 744 SEC("?tp/syscalls/sys_enter_nanosleep") 745 int test_dynptr_memset_zero(void *ctx) 746 { 747 __u32 data_sz = sizeof(memset_zero_data); 748 char zeroes[32] = {'\0'}; 749 struct bpf_dynptr ptr; 750 751 err = bpf_dynptr_from_mem(memset_zero_data, data_sz, 0, &ptr); 752 err = err ?: bpf_dynptr_memset(&ptr, 0, data_sz, 0); 753 err = err ?: bpf_memcmp(zeroes, memset_zero_data, data_sz); 754 755 return 0; 756 } 757 758 #define DYNPTR_MEMSET_VAL 42 759 760 char memset_notzero_data[] = "data to be overwritten"; 761 762 SEC("?tp/syscalls/sys_enter_nanosleep") 763 int test_dynptr_memset_notzero(void *ctx) 764 { 765 u32 data_sz = sizeof(memset_notzero_data); 766 struct bpf_dynptr ptr; 767 char expected[32]; 768 769 __builtin_memset(expected, DYNPTR_MEMSET_VAL, data_sz); 770 771 err = bpf_dynptr_from_mem(memset_notzero_data, data_sz, 0, &ptr); 772 err = err ?: bpf_dynptr_memset(&ptr, 0, data_sz, DYNPTR_MEMSET_VAL); 773 err = err ?: bpf_memcmp(expected, memset_notzero_data, data_sz); 774 775 return 0; 776 } 777 778 char memset_zero_offset_data[] = "data to be zeroed partially"; 779 780 SEC("?tp/syscalls/sys_enter_nanosleep") 781 int test_dynptr_memset_zero_offset(void *ctx) 782 { 783 char expected[] = "data to \0\0\0\0eroed partially"; 784 __u32 data_sz = sizeof(memset_zero_offset_data); 785 struct bpf_dynptr ptr; 786 787 err = bpf_dynptr_from_mem(memset_zero_offset_data, data_sz, 0, &ptr); 788 err = err ?: bpf_dynptr_memset(&ptr, 8, 4, 0); 789 err = err ?: bpf_memcmp(expected, memset_zero_offset_data, data_sz); 790 791 return 0; 792 } 793 794 char memset_zero_adjusted_data[] = "data to be zeroed partially"; 795 796 SEC("?tp/syscalls/sys_enter_nanosleep") 797 int test_dynptr_memset_zero_adjusted(void *ctx) 798 { 799 char expected[] = "data\0\0\0\0be zeroed partially"; 800 __u32 data_sz = sizeof(memset_zero_adjusted_data); 801 struct bpf_dynptr ptr; 802 803 err = bpf_dynptr_from_mem(memset_zero_adjusted_data, data_sz, 0, &ptr); 804 err = err ?: bpf_dynptr_adjust(&ptr, 4, 8); 805 err = err ?: bpf_dynptr_memset(&ptr, 0, bpf_dynptr_size(&ptr), 0); 806 err = err ?: bpf_memcmp(expected, memset_zero_adjusted_data, data_sz); 807 808 return 0; 809 } 810 811 char memset_overflow_data[] = "memset overflow data"; 812 813 SEC("?tp/syscalls/sys_enter_nanosleep") 814 int test_dynptr_memset_overflow(void *ctx) 815 { 816 __u32 data_sz = sizeof(memset_overflow_data); 817 struct bpf_dynptr ptr; 818 int ret; 819 820 err = bpf_dynptr_from_mem(memset_overflow_data, data_sz, 0, &ptr); 821 ret = bpf_dynptr_memset(&ptr, 0, data_sz + 1, 0); 822 if (ret != -E2BIG) 823 err = 1; 824 825 return 0; 826 } 827 828 SEC("?tp/syscalls/sys_enter_nanosleep") 829 int test_dynptr_memset_overflow_offset(void *ctx) 830 { 831 __u32 data_sz = sizeof(memset_overflow_data); 832 struct bpf_dynptr ptr; 833 int ret; 834 835 err = bpf_dynptr_from_mem(memset_overflow_data, data_sz, 0, &ptr); 836 ret = bpf_dynptr_memset(&ptr, 1, data_sz, 0); 837 if (ret != -E2BIG) 838 err = 1; 839 840 return 0; 841 } 842 843 SEC("?cgroup_skb/egress") 844 int test_dynptr_memset_readonly(struct __sk_buff *skb) 845 { 846 struct bpf_dynptr ptr; 847 int ret; 848 849 err = bpf_dynptr_from_skb(skb, 0, &ptr); 850 851 /* cgroup skbs are read only, memset should fail */ 852 ret = bpf_dynptr_memset(&ptr, 0, bpf_dynptr_size(&ptr), 0); 853 if (ret != -EINVAL) 854 err = 1; 855 856 return 0; 857 } 858 859 #define min_t(type, x, y) ({ \ 860 type __x = (x); \ 861 type __y = (y); \ 862 __x < __y ? __x : __y; }) 863 864 SEC("xdp") 865 int test_dynptr_memset_xdp_chunks(struct xdp_md *xdp) 866 { 867 u32 data_sz, chunk_sz, offset = 0; 868 const int max_chunks = 200; 869 struct bpf_dynptr ptr_xdp; 870 char expected_buf[32]; 871 char buf[32]; 872 int i; 873 874 __builtin_memset(expected_buf, DYNPTR_MEMSET_VAL, sizeof(expected_buf)); 875 876 /* ptr_xdp is backed by non-contiguous memory */ 877 bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp); 878 data_sz = bpf_dynptr_size(&ptr_xdp); 879 880 err = bpf_dynptr_memset(&ptr_xdp, 0, data_sz, DYNPTR_MEMSET_VAL); 881 if (err) { 882 /* bpf_dynptr_memset() eventually called bpf_xdp_pointer() 883 * where if data_sz is greater than 0xffff, -EFAULT will be 884 * returned. For 64K page size, data_sz is greater than 885 * 64K, so error is expected and let us zero out error and 886 * return success. 887 */ 888 if (data_sz >= PAGE_SIZE_64K) 889 err = 0; 890 goto out; 891 } 892 893 bpf_for(i, 0, max_chunks) { 894 offset = i * sizeof(buf); 895 if (offset >= data_sz) 896 goto out; 897 chunk_sz = min_t(u32, sizeof(buf), data_sz - offset); 898 err = bpf_dynptr_read(&buf, chunk_sz, &ptr_xdp, offset, 0); 899 if (err) 900 goto out; 901 err = bpf_memcmp(buf, expected_buf, sizeof(buf)); 902 if (err) 903 goto out; 904 } 905 out: 906 return XDP_DROP; 907 } 908 909 void *user_ptr; 910 /* Contains the copy of the data pointed by user_ptr. 911 * Size 384 to make it not fit into a single kernel chunk when copying 912 * but less than the maximum bpf stack size (512). 913 */ 914 char expected_str[384]; 915 __u32 test_len[7] = {0/* placeholder */, 0, 1, 2, 255, 256, 257}; 916 917 typedef int (*bpf_read_dynptr_fn_t)(struct bpf_dynptr *dptr, u32 off, 918 u32 size, const void *unsafe_ptr); 919 920 /* Returns the offset just before the end of the maximum sized xdp fragment. 921 * Any write larger than 32 bytes will be split between 2 fragments. 922 */ 923 __u32 xdp_near_frag_end_offset(void) 924 { 925 const __u32 headroom = 256; 926 const __u32 max_frag_size = __PAGE_SIZE - headroom - sizeof(struct skb_shared_info); 927 928 /* 32 bytes before the approximate end of the fragment */ 929 return max_frag_size - 32; 930 } 931 932 /* Use __always_inline on test_dynptr_probe[_str][_xdp]() and callbacks 933 * of type bpf_read_dynptr_fn_t to prevent compiler from generating 934 * indirect calls that make program fail to load with "unknown opcode" error. 935 */ 936 static __always_inline void test_dynptr_probe(void *ptr, bpf_read_dynptr_fn_t bpf_read_dynptr_fn) 937 { 938 char buf[sizeof(expected_str)]; 939 struct bpf_dynptr ptr_buf; 940 int i; 941 942 if (bpf_get_current_pid_tgid() >> 32 != pid) 943 return; 944 945 err = bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(buf), 0, &ptr_buf); 946 947 bpf_for(i, 0, ARRAY_SIZE(test_len)) { 948 __u32 len = test_len[i]; 949 950 err = err ?: bpf_read_dynptr_fn(&ptr_buf, 0, test_len[i], ptr); 951 if (len > sizeof(buf)) 952 break; 953 err = err ?: bpf_dynptr_read(&buf, len, &ptr_buf, 0, 0); 954 955 if (err || bpf_memcmp(expected_str, buf, len)) 956 err = 1; 957 958 /* Reset buffer and dynptr */ 959 __builtin_memset(buf, 0, sizeof(buf)); 960 err = err ?: bpf_dynptr_write(&ptr_buf, 0, buf, len, 0); 961 } 962 bpf_ringbuf_discard_dynptr(&ptr_buf, 0); 963 } 964 965 static __always_inline void test_dynptr_probe_str(void *ptr, 966 bpf_read_dynptr_fn_t bpf_read_dynptr_fn) 967 { 968 char buf[sizeof(expected_str)]; 969 struct bpf_dynptr ptr_buf; 970 __u32 cnt, i; 971 972 if (bpf_get_current_pid_tgid() >> 32 != pid) 973 return; 974 975 bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(buf), 0, &ptr_buf); 976 977 bpf_for(i, 0, ARRAY_SIZE(test_len)) { 978 __u32 len = test_len[i]; 979 980 cnt = bpf_read_dynptr_fn(&ptr_buf, 0, len, ptr); 981 if (cnt != len) 982 err = 1; 983 984 if (len > sizeof(buf)) 985 continue; 986 err = err ?: bpf_dynptr_read(&buf, len, &ptr_buf, 0, 0); 987 if (!len) 988 continue; 989 if (err || bpf_memcmp(expected_str, buf, len - 1) || buf[len - 1] != '\0') 990 err = 1; 991 } 992 bpf_ringbuf_discard_dynptr(&ptr_buf, 0); 993 } 994 995 static __always_inline void test_dynptr_probe_xdp(struct xdp_md *xdp, void *ptr, 996 bpf_read_dynptr_fn_t bpf_read_dynptr_fn) 997 { 998 struct bpf_dynptr ptr_xdp; 999 char buf[sizeof(expected_str)]; 1000 __u32 off, i; 1001 1002 if (bpf_get_current_pid_tgid() >> 32 != pid) 1003 return; 1004 1005 off = xdp_near_frag_end_offset(); 1006 err = bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp); 1007 1008 bpf_for(i, 0, ARRAY_SIZE(test_len)) { 1009 __u32 len = test_len[i]; 1010 1011 err = err ?: bpf_read_dynptr_fn(&ptr_xdp, off, len, ptr); 1012 if (len > sizeof(buf)) 1013 continue; 1014 err = err ?: bpf_dynptr_read(&buf, len, &ptr_xdp, off, 0); 1015 if (err || bpf_memcmp(expected_str, buf, len)) 1016 err = 1; 1017 /* Reset buffer and dynptr */ 1018 __builtin_memset(buf, 0, sizeof(buf)); 1019 err = err ?: bpf_dynptr_write(&ptr_xdp, off, buf, len, 0); 1020 } 1021 } 1022 1023 static __always_inline void test_dynptr_probe_str_xdp(struct xdp_md *xdp, void *ptr, 1024 bpf_read_dynptr_fn_t bpf_read_dynptr_fn) 1025 { 1026 struct bpf_dynptr ptr_xdp; 1027 char buf[sizeof(expected_str)]; 1028 __u32 cnt, off, i; 1029 1030 if (bpf_get_current_pid_tgid() >> 32 != pid) 1031 return; 1032 1033 off = xdp_near_frag_end_offset(); 1034 err = bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp); 1035 if (err) 1036 return; 1037 1038 bpf_for(i, 0, ARRAY_SIZE(test_len)) { 1039 __u32 len = test_len[i]; 1040 1041 cnt = bpf_read_dynptr_fn(&ptr_xdp, off, len, ptr); 1042 if (cnt != len) 1043 err = 1; 1044 1045 if (len > sizeof(buf)) 1046 continue; 1047 err = err ?: bpf_dynptr_read(&buf, len, &ptr_xdp, off, 0); 1048 1049 if (!len) 1050 continue; 1051 if (err || bpf_memcmp(expected_str, buf, len - 1) || buf[len - 1] != '\0') 1052 err = 1; 1053 1054 __builtin_memset(buf, 0, sizeof(buf)); 1055 err = err ?: bpf_dynptr_write(&ptr_xdp, off, buf, len, 0); 1056 } 1057 } 1058 1059 SEC("xdp") 1060 int test_probe_read_user_dynptr(struct xdp_md *xdp) 1061 { 1062 test_dynptr_probe(user_ptr, bpf_probe_read_user_dynptr); 1063 if (!err) 1064 test_dynptr_probe_xdp(xdp, user_ptr, bpf_probe_read_user_dynptr); 1065 return XDP_PASS; 1066 } 1067 1068 SEC("xdp") 1069 int test_probe_read_kernel_dynptr(struct xdp_md *xdp) 1070 { 1071 test_dynptr_probe(expected_str, bpf_probe_read_kernel_dynptr); 1072 if (!err) 1073 test_dynptr_probe_xdp(xdp, expected_str, bpf_probe_read_kernel_dynptr); 1074 return XDP_PASS; 1075 } 1076 1077 SEC("xdp") 1078 int test_probe_read_user_str_dynptr(struct xdp_md *xdp) 1079 { 1080 test_dynptr_probe_str(user_ptr, bpf_probe_read_user_str_dynptr); 1081 if (!err) 1082 test_dynptr_probe_str_xdp(xdp, user_ptr, bpf_probe_read_user_str_dynptr); 1083 return XDP_PASS; 1084 } 1085 1086 SEC("xdp") 1087 int test_probe_read_kernel_str_dynptr(struct xdp_md *xdp) 1088 { 1089 test_dynptr_probe_str(expected_str, bpf_probe_read_kernel_str_dynptr); 1090 if (!err) 1091 test_dynptr_probe_str_xdp(xdp, expected_str, bpf_probe_read_kernel_str_dynptr); 1092 return XDP_PASS; 1093 } 1094 1095 SEC("fentry.s/" SYS_PREFIX "sys_nanosleep") 1096 int test_copy_from_user_dynptr(void *ctx) 1097 { 1098 test_dynptr_probe(user_ptr, bpf_copy_from_user_dynptr); 1099 return 0; 1100 } 1101 1102 SEC("fentry.s/" SYS_PREFIX "sys_nanosleep") 1103 int test_copy_from_user_str_dynptr(void *ctx) 1104 { 1105 test_dynptr_probe_str(user_ptr, bpf_copy_from_user_str_dynptr); 1106 return 0; 1107 } 1108 1109 static int bpf_copy_data_from_user_task(struct bpf_dynptr *dptr, u32 off, 1110 u32 size, const void *unsafe_ptr) 1111 { 1112 struct task_struct *task = bpf_get_current_task_btf(); 1113 1114 return bpf_copy_from_user_task_dynptr(dptr, off, size, unsafe_ptr, task); 1115 } 1116 1117 static int bpf_copy_data_from_user_task_str(struct bpf_dynptr *dptr, u32 off, 1118 u32 size, const void *unsafe_ptr) 1119 { 1120 struct task_struct *task = bpf_get_current_task_btf(); 1121 1122 return bpf_copy_from_user_task_str_dynptr(dptr, off, size, unsafe_ptr, task); 1123 } 1124 1125 SEC("fentry.s/" SYS_PREFIX "sys_nanosleep") 1126 int test_copy_from_user_task_dynptr(void *ctx) 1127 { 1128 test_dynptr_probe(user_ptr, bpf_copy_data_from_user_task); 1129 return 0; 1130 } 1131 1132 SEC("fentry.s/" SYS_PREFIX "sys_nanosleep") 1133 int test_copy_from_user_task_str_dynptr(void *ctx) 1134 { 1135 test_dynptr_probe_str(user_ptr, bpf_copy_data_from_user_task_str); 1136 return 0; 1137 } 1138