1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2022 Facebook */ 3 4 #include <vmlinux.h> 5 #include <string.h> 6 #include <stdbool.h> 7 #include <bpf/bpf_helpers.h> 8 #include <bpf/bpf_tracing.h> 9 #include "bpf_misc.h" 10 #include "errno.h" 11 12 #define PAGE_SIZE_64K 65536 13 #define TEST_SKB_LINEAR_SIZE (sizeof(struct ethhdr) + sizeof(struct iphdr)) 14 15 char _license[] SEC("license") = "GPL"; 16 17 int pid, err, val; 18 19 struct ringbuf_sample { 20 int pid; 21 int seq; 22 long value; 23 char comm[16]; 24 }; 25 26 struct { 27 __uint(type, BPF_MAP_TYPE_RINGBUF); 28 __uint(max_entries, 4096); 29 } ringbuf SEC(".maps"); 30 31 struct { 32 __uint(type, BPF_MAP_TYPE_ARRAY); 33 __uint(max_entries, 1); 34 __type(key, __u32); 35 __type(value, __u32); 36 } array_map SEC(".maps"); 37 38 SEC("?tp/syscalls/sys_enter_nanosleep") 39 int test_read_write(void *ctx) 40 { 41 char write_data[64] = "hello there, world!!"; 42 char read_data[64] = {}; 43 struct bpf_dynptr ptr; 44 int i; 45 46 if (bpf_get_current_pid_tgid() >> 32 != pid) 47 return 0; 48 49 bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(write_data), 0, &ptr); 50 51 /* Write data into the dynptr */ 52 err = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0); 53 54 /* Read the data that was written into the dynptr */ 55 err = err ?: bpf_dynptr_read(read_data, sizeof(read_data), &ptr, 0, 0); 56 57 /* Ensure the data we read matches the data we wrote */ 58 for (i = 0; i < sizeof(read_data); i++) { 59 if (read_data[i] != write_data[i]) { 60 err = 1; 61 break; 62 } 63 } 64 65 bpf_ringbuf_discard_dynptr(&ptr, 0); 66 return 0; 67 } 68 69 SEC("?tp/syscalls/sys_enter_nanosleep") 70 int test_dynptr_data(void *ctx) 71 { 72 __u32 key = 0, val = 235, *map_val; 73 struct bpf_dynptr ptr; 74 __u32 map_val_size; 75 void *data; 76 77 map_val_size = sizeof(*map_val); 78 79 if (bpf_get_current_pid_tgid() >> 32 != pid) 80 return 0; 81 82 bpf_map_update_elem(&array_map, &key, &val, 0); 83 84 map_val = bpf_map_lookup_elem(&array_map, &key); 85 if (!map_val) { 86 err = 1; 87 return 0; 88 } 89 90 bpf_dynptr_from_mem(map_val, map_val_size, 0, &ptr); 91 92 /* Try getting a data slice that is out of range */ 93 data = bpf_dynptr_data(&ptr, map_val_size + 1, 1); 94 if (data) { 95 err = 2; 96 return 0; 97 } 98 99 /* Try getting more bytes than available */ 100 data = bpf_dynptr_data(&ptr, 0, map_val_size + 1); 101 if (data) { 102 err = 3; 103 return 0; 104 } 105 106 data = bpf_dynptr_data(&ptr, 0, sizeof(__u32)); 107 if (!data) { 108 err = 4; 109 return 0; 110 } 111 112 *(__u32 *)data = 999; 113 114 err = bpf_probe_read_kernel(&val, sizeof(val), data); 115 if (err) 116 return 0; 117 118 if (val != *(int *)data) 119 err = 5; 120 121 return 0; 122 } 123 124 static int ringbuf_callback(__u32 index, void *data) 125 { 126 struct ringbuf_sample *sample; 127 128 struct bpf_dynptr *ptr = (struct bpf_dynptr *)data; 129 130 sample = bpf_dynptr_data(ptr, 0, sizeof(*sample)); 131 if (!sample) 132 err = 2; 133 else 134 sample->pid += index; 135 136 return 0; 137 } 138 139 SEC("?tp/syscalls/sys_enter_nanosleep") 140 int test_ringbuf(void *ctx) 141 { 142 struct bpf_dynptr ptr; 143 struct ringbuf_sample *sample; 144 145 if (bpf_get_current_pid_tgid() >> 32 != pid) 146 return 0; 147 148 val = 100; 149 150 /* check that you can reserve a dynamic size reservation */ 151 err = bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); 152 153 sample = err ? NULL : bpf_dynptr_data(&ptr, 0, sizeof(*sample)); 154 if (!sample) { 155 err = 1; 156 goto done; 157 } 158 159 sample->pid = 10; 160 161 /* Can pass dynptr to callback functions */ 162 bpf_loop(10, ringbuf_callback, &ptr, 0); 163 164 if (sample->pid != 55) 165 err = 2; 166 167 done: 168 bpf_ringbuf_discard_dynptr(&ptr, 0); 169 return 0; 170 } 171 172 SEC("?cgroup_skb/egress") 173 int test_skb_readonly(struct __sk_buff *skb) 174 { 175 __u8 write_data[2] = {1, 2}; 176 struct bpf_dynptr ptr; 177 int ret; 178 179 if (bpf_dynptr_from_skb(skb, 0, &ptr)) { 180 err = 1; 181 return 1; 182 } 183 184 /* since cgroup skbs are read only, writes should fail */ 185 ret = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0); 186 if (ret != -EINVAL) { 187 err = 2; 188 return 1; 189 } 190 191 return 1; 192 } 193 194 SEC("?cgroup_skb/egress") 195 int test_dynptr_skb_data(struct __sk_buff *skb) 196 { 197 struct bpf_dynptr ptr; 198 __u64 *data; 199 200 if (bpf_dynptr_from_skb(skb, 0, &ptr)) { 201 err = 1; 202 return 1; 203 } 204 205 /* This should return NULL. Must use bpf_dynptr_slice API */ 206 data = bpf_dynptr_data(&ptr, 0, 1); 207 if (data) { 208 err = 2; 209 return 1; 210 } 211 212 return 1; 213 } 214 215 SEC("?tc") 216 int test_dynptr_skb_slice_non_linear(struct __sk_buff *skb) 217 { 218 struct bpf_dynptr ptr; 219 void *data; 220 221 if (bpf_dynptr_from_skb(skb, 0, &ptr)) { 222 err = 1; 223 return 1; 224 } 225 226 /* Ensure we cannot read past the end of the buffer. */ 227 data = bpf_dynptr_slice(&ptr, TEST_SKB_LINEAR_SIZE + 1, NULL, 1); 228 if (data) 229 err = 2; 230 231 return 1; 232 } 233 234 SEC("?tc") 235 int test_dynptr_skb_meta_data(struct __sk_buff *skb) 236 { 237 struct bpf_dynptr meta; 238 __u8 *md; 239 int ret; 240 241 err = 1; 242 ret = bpf_dynptr_from_skb_meta(skb, 0, &meta); 243 if (ret) 244 return 1; 245 246 /* This should return NULL. Must use bpf_dynptr_slice API */ 247 err = 2; 248 md = bpf_dynptr_data(&meta, 0, sizeof(*md)); 249 if (md) 250 return 1; 251 252 err = 0; 253 return 1; 254 } 255 256 /* Check that skb metadata dynptr ops don't accept any flags. */ 257 SEC("?tc") 258 int test_dynptr_skb_meta_flags(struct __sk_buff *skb) 259 { 260 const __u64 INVALID_FLAGS = ~0ULL; 261 struct bpf_dynptr meta; 262 __u8 buf; 263 int ret; 264 265 err = 1; 266 ret = bpf_dynptr_from_skb_meta(skb, INVALID_FLAGS, &meta); 267 if (ret != -EINVAL) 268 return 1; 269 270 err = 2; 271 ret = bpf_dynptr_from_skb_meta(skb, 0, &meta); 272 if (ret) 273 return 1; 274 275 err = 3; 276 ret = bpf_dynptr_read(&buf, 0, &meta, 0, INVALID_FLAGS); 277 if (ret != -EINVAL) 278 return 1; 279 280 err = 4; 281 ret = bpf_dynptr_write(&meta, 0, &buf, 0, INVALID_FLAGS); 282 if (ret != -EINVAL) 283 return 1; 284 285 err = 0; 286 return 1; 287 } 288 289 SEC("tp/syscalls/sys_enter_nanosleep") 290 int test_adjust(void *ctx) 291 { 292 struct bpf_dynptr ptr; 293 __u32 bytes = 64; 294 __u32 off = 10; 295 __u32 trim = 15; 296 297 if (bpf_get_current_pid_tgid() >> 32 != pid) 298 return 0; 299 300 err = bpf_ringbuf_reserve_dynptr(&ringbuf, bytes, 0, &ptr); 301 if (err) { 302 err = 1; 303 goto done; 304 } 305 306 if (bpf_dynptr_size(&ptr) != bytes) { 307 err = 2; 308 goto done; 309 } 310 311 /* Advance the dynptr by off */ 312 err = bpf_dynptr_adjust(&ptr, off, bpf_dynptr_size(&ptr)); 313 if (err) { 314 err = 3; 315 goto done; 316 } 317 318 if (bpf_dynptr_size(&ptr) != bytes - off) { 319 err = 4; 320 goto done; 321 } 322 323 /* Trim the dynptr */ 324 err = bpf_dynptr_adjust(&ptr, off, 15); 325 if (err) { 326 err = 5; 327 goto done; 328 } 329 330 /* Check that the size was adjusted correctly */ 331 if (bpf_dynptr_size(&ptr) != trim - off) { 332 err = 6; 333 goto done; 334 } 335 336 done: 337 bpf_ringbuf_discard_dynptr(&ptr, 0); 338 return 0; 339 } 340 341 SEC("tp/syscalls/sys_enter_nanosleep") 342 int test_adjust_err(void *ctx) 343 { 344 char write_data[45] = "hello there, world!!"; 345 struct bpf_dynptr ptr; 346 __u32 size = 64; 347 __u32 off = 20; 348 349 if (bpf_get_current_pid_tgid() >> 32 != pid) 350 return 0; 351 352 if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 0, &ptr)) { 353 err = 1; 354 goto done; 355 } 356 357 /* Check that start can't be greater than end */ 358 if (bpf_dynptr_adjust(&ptr, 5, 1) != -EINVAL) { 359 err = 2; 360 goto done; 361 } 362 363 /* Check that start can't be greater than size */ 364 if (bpf_dynptr_adjust(&ptr, size + 1, size + 1) != -ERANGE) { 365 err = 3; 366 goto done; 367 } 368 369 /* Check that end can't be greater than size */ 370 if (bpf_dynptr_adjust(&ptr, 0, size + 1) != -ERANGE) { 371 err = 4; 372 goto done; 373 } 374 375 if (bpf_dynptr_adjust(&ptr, off, size)) { 376 err = 5; 377 goto done; 378 } 379 380 /* Check that you can't write more bytes than available into the dynptr 381 * after you've adjusted it 382 */ 383 if (bpf_dynptr_write(&ptr, 0, &write_data, sizeof(write_data), 0) != -E2BIG) { 384 err = 6; 385 goto done; 386 } 387 388 /* Check that even after adjusting, submitting/discarding 389 * a ringbuf dynptr works 390 */ 391 bpf_ringbuf_submit_dynptr(&ptr, 0); 392 return 0; 393 394 done: 395 bpf_ringbuf_discard_dynptr(&ptr, 0); 396 return 0; 397 } 398 399 SEC("tp/syscalls/sys_enter_nanosleep") 400 int test_zero_size_dynptr(void *ctx) 401 { 402 char write_data = 'x', read_data; 403 struct bpf_dynptr ptr; 404 __u32 size = 64; 405 406 if (bpf_get_current_pid_tgid() >> 32 != pid) 407 return 0; 408 409 if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 0, &ptr)) { 410 err = 1; 411 goto done; 412 } 413 414 /* After this, the dynptr has a size of 0 */ 415 if (bpf_dynptr_adjust(&ptr, size, size)) { 416 err = 2; 417 goto done; 418 } 419 420 /* Test that reading + writing non-zero bytes is not ok */ 421 if (bpf_dynptr_read(&read_data, sizeof(read_data), &ptr, 0, 0) != -E2BIG) { 422 err = 3; 423 goto done; 424 } 425 426 if (bpf_dynptr_write(&ptr, 0, &write_data, sizeof(write_data), 0) != -E2BIG) { 427 err = 4; 428 goto done; 429 } 430 431 /* Test that reading + writing 0 bytes from a 0-size dynptr is ok */ 432 if (bpf_dynptr_read(&read_data, 0, &ptr, 0, 0)) { 433 err = 5; 434 goto done; 435 } 436 437 if (bpf_dynptr_write(&ptr, 0, &write_data, 0, 0)) { 438 err = 6; 439 goto done; 440 } 441 442 err = 0; 443 444 done: 445 bpf_ringbuf_discard_dynptr(&ptr, 0); 446 return 0; 447 } 448 449 SEC("tp/syscalls/sys_enter_nanosleep") 450 int test_dynptr_is_null(void *ctx) 451 { 452 struct bpf_dynptr ptr1; 453 struct bpf_dynptr ptr2; 454 __u64 size = 4; 455 456 if (bpf_get_current_pid_tgid() >> 32 != pid) 457 return 0; 458 459 /* Pass in invalid flags, get back an invalid dynptr */ 460 if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 123, &ptr1) != -EINVAL) { 461 err = 1; 462 goto exit_early; 463 } 464 465 /* Test that the invalid dynptr is null */ 466 if (!bpf_dynptr_is_null(&ptr1)) { 467 err = 2; 468 goto exit_early; 469 } 470 471 /* Get a valid dynptr */ 472 if (bpf_ringbuf_reserve_dynptr(&ringbuf, size, 0, &ptr2)) { 473 err = 3; 474 goto exit; 475 } 476 477 /* Test that the valid dynptr is not null */ 478 if (bpf_dynptr_is_null(&ptr2)) { 479 err = 4; 480 goto exit; 481 } 482 483 exit: 484 bpf_ringbuf_discard_dynptr(&ptr2, 0); 485 exit_early: 486 bpf_ringbuf_discard_dynptr(&ptr1, 0); 487 return 0; 488 } 489 490 SEC("cgroup_skb/egress") 491 int test_dynptr_is_rdonly(struct __sk_buff *skb) 492 { 493 struct bpf_dynptr ptr1; 494 struct bpf_dynptr ptr2; 495 struct bpf_dynptr ptr3; 496 497 /* Pass in invalid flags, get back an invalid dynptr */ 498 if (bpf_dynptr_from_skb(skb, 123, &ptr1) != -EINVAL) { 499 err = 1; 500 return 0; 501 } 502 503 /* Test that an invalid dynptr is_rdonly returns false */ 504 if (bpf_dynptr_is_rdonly(&ptr1)) { 505 err = 2; 506 return 0; 507 } 508 509 /* Get a read-only dynptr */ 510 if (bpf_dynptr_from_skb(skb, 0, &ptr2)) { 511 err = 3; 512 return 0; 513 } 514 515 /* Test that the dynptr is read-only */ 516 if (!bpf_dynptr_is_rdonly(&ptr2)) { 517 err = 4; 518 return 0; 519 } 520 521 /* Get a read-writeable dynptr */ 522 if (bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr3)) { 523 err = 5; 524 goto done; 525 } 526 527 /* Test that the dynptr is read-only */ 528 if (bpf_dynptr_is_rdonly(&ptr3)) { 529 err = 6; 530 goto done; 531 } 532 533 done: 534 bpf_ringbuf_discard_dynptr(&ptr3, 0); 535 return 0; 536 } 537 538 SEC("cgroup_skb/egress") 539 int test_dynptr_clone(struct __sk_buff *skb) 540 { 541 struct bpf_dynptr ptr1; 542 struct bpf_dynptr ptr2; 543 __u32 off = 2, size; 544 545 /* Get a dynptr */ 546 if (bpf_dynptr_from_skb(skb, 0, &ptr1)) { 547 err = 1; 548 return 0; 549 } 550 551 if (bpf_dynptr_adjust(&ptr1, off, bpf_dynptr_size(&ptr1))) { 552 err = 2; 553 return 0; 554 } 555 556 /* Clone the dynptr */ 557 if (bpf_dynptr_clone(&ptr1, &ptr2)) { 558 err = 3; 559 return 0; 560 } 561 562 size = bpf_dynptr_size(&ptr1); 563 564 /* Check that the clone has the same size and rd-only */ 565 if (bpf_dynptr_size(&ptr2) != size) { 566 err = 4; 567 return 0; 568 } 569 570 if (bpf_dynptr_is_rdonly(&ptr2) != bpf_dynptr_is_rdonly(&ptr1)) { 571 err = 5; 572 return 0; 573 } 574 575 /* Advance and trim the original dynptr */ 576 bpf_dynptr_adjust(&ptr1, 5, 5); 577 578 /* Check that only original dynptr was affected, and the clone wasn't */ 579 if (bpf_dynptr_size(&ptr2) != size) { 580 err = 6; 581 return 0; 582 } 583 584 return 0; 585 } 586 587 SEC("?cgroup_skb/egress") 588 int test_dynptr_skb_no_buff(struct __sk_buff *skb) 589 { 590 struct bpf_dynptr ptr; 591 __u64 *data; 592 593 if (bpf_dynptr_from_skb(skb, 0, &ptr)) { 594 err = 1; 595 return 1; 596 } 597 598 /* This may return NULL. SKB may require a buffer */ 599 data = bpf_dynptr_slice(&ptr, 0, NULL, 1); 600 601 return !!data; 602 } 603 604 SEC("?cgroup_skb/egress") 605 int test_dynptr_skb_strcmp(struct __sk_buff *skb) 606 { 607 struct bpf_dynptr ptr; 608 char *data; 609 610 if (bpf_dynptr_from_skb(skb, 0, &ptr)) { 611 err = 1; 612 return 1; 613 } 614 615 /* This may return NULL. SKB may require a buffer */ 616 data = bpf_dynptr_slice(&ptr, 0, NULL, 10); 617 if (data) { 618 bpf_strncmp(data, 10, "foo"); 619 return 1; 620 } 621 622 return 1; 623 } 624 625 SEC("tp_btf/kfree_skb") 626 int BPF_PROG(test_dynptr_skb_tp_btf, void *skb, void *location) 627 { 628 __u8 write_data[2] = {1, 2}; 629 struct bpf_dynptr ptr; 630 int ret; 631 632 if (bpf_dynptr_from_skb(skb, 0, &ptr)) { 633 err = 1; 634 return 1; 635 } 636 637 /* since tp_btf skbs are read only, writes should fail */ 638 ret = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0); 639 if (ret != -EINVAL) { 640 err = 2; 641 return 1; 642 } 643 644 return 1; 645 } 646 647 static inline int bpf_memcmp(const char *a, const char *b, u32 size) 648 { 649 int i; 650 651 bpf_for(i, 0, size) { 652 if (a[i] != b[i]) 653 return a[i] < b[i] ? -1 : 1; 654 } 655 return 0; 656 } 657 658 SEC("?tp/syscalls/sys_enter_nanosleep") 659 int test_dynptr_copy(void *ctx) 660 { 661 char data[] = "hello there, world!!"; 662 char buf[32] = {'\0'}; 663 __u32 sz = sizeof(data); 664 struct bpf_dynptr src, dst; 665 666 bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &src); 667 bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &dst); 668 669 /* Test basic case of copying contiguous memory backed dynptrs */ 670 err = bpf_dynptr_write(&src, 0, data, sz, 0); 671 err = err ?: bpf_dynptr_copy(&dst, 0, &src, 0, sz); 672 err = err ?: bpf_dynptr_read(buf, sz, &dst, 0, 0); 673 err = err ?: bpf_memcmp(data, buf, sz); 674 675 /* Test that offsets are handled correctly */ 676 err = err ?: bpf_dynptr_copy(&dst, 3, &src, 5, sz - 5); 677 err = err ?: bpf_dynptr_read(buf, sz - 5, &dst, 3, 0); 678 err = err ?: bpf_memcmp(data + 5, buf, sz - 5); 679 680 bpf_ringbuf_discard_dynptr(&src, 0); 681 bpf_ringbuf_discard_dynptr(&dst, 0); 682 return 0; 683 } 684 685 SEC("xdp") 686 int test_dynptr_copy_xdp(struct xdp_md *xdp) 687 { 688 struct bpf_dynptr ptr_buf, ptr_xdp; 689 char data[] = "qwertyuiopasdfghjkl"; 690 char buf[32] = {'\0'}; 691 __u32 len = sizeof(data), xdp_data_size; 692 int i, chunks = 200; 693 694 /* ptr_xdp is backed by non-contiguous memory */ 695 bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp); 696 xdp_data_size = bpf_dynptr_size(&ptr_xdp); 697 bpf_ringbuf_reserve_dynptr(&ringbuf, len * chunks, 0, &ptr_buf); 698 699 /* Destination dynptr is backed by non-contiguous memory */ 700 bpf_for(i, 0, chunks) { 701 err = bpf_dynptr_write(&ptr_buf, i * len, data, len, 0); 702 if (err) 703 goto out; 704 } 705 706 err = bpf_dynptr_copy(&ptr_xdp, 0, &ptr_buf, 0, len * chunks); 707 if (err) 708 goto out; 709 710 bpf_for(i, 0, chunks) { 711 __builtin_memset(buf, 0, sizeof(buf)); 712 err = bpf_dynptr_read(&buf, len, &ptr_xdp, i * len, 0); 713 if (err) 714 goto out; 715 if (bpf_memcmp(data, buf, len) != 0) 716 goto out; 717 } 718 719 /* Source dynptr is backed by non-contiguous memory */ 720 __builtin_memset(buf, 0, sizeof(buf)); 721 bpf_for(i, 0, chunks) { 722 err = bpf_dynptr_write(&ptr_buf, i * len, buf, len, 0); 723 if (err) 724 goto out; 725 } 726 727 err = bpf_dynptr_copy(&ptr_buf, 0, &ptr_xdp, 0, len * chunks); 728 if (err) 729 goto out; 730 731 bpf_for(i, 0, chunks) { 732 __builtin_memset(buf, 0, sizeof(buf)); 733 err = bpf_dynptr_read(&buf, len, &ptr_buf, i * len, 0); 734 if (err) 735 goto out; 736 if (bpf_memcmp(data, buf, len) != 0) 737 goto out; 738 } 739 740 /* Both source and destination dynptrs are backed by non-contiguous memory */ 741 err = bpf_dynptr_copy(&ptr_xdp, 2, &ptr_xdp, len, len * (chunks - 1)); 742 if (err) 743 goto out; 744 745 bpf_for(i, 0, chunks - 1) { 746 __builtin_memset(buf, 0, sizeof(buf)); 747 err = bpf_dynptr_read(&buf, len, &ptr_xdp, 2 + i * len, 0); 748 if (err) 749 goto out; 750 if (bpf_memcmp(data, buf, len) != 0) 751 goto out; 752 } 753 754 if (bpf_dynptr_copy(&ptr_xdp, xdp_data_size - 3000, &ptr_xdp, 0, len * chunks) != -E2BIG) 755 err = 1; 756 757 out: 758 bpf_ringbuf_discard_dynptr(&ptr_buf, 0); 759 return XDP_DROP; 760 } 761 762 char memset_zero_data[] = "data to be zeroed"; 763 764 SEC("?tp/syscalls/sys_enter_nanosleep") 765 int test_dynptr_memset_zero(void *ctx) 766 { 767 __u32 data_sz = sizeof(memset_zero_data); 768 char zeroes[32] = {'\0'}; 769 struct bpf_dynptr ptr; 770 771 err = bpf_dynptr_from_mem(memset_zero_data, data_sz, 0, &ptr); 772 err = err ?: bpf_dynptr_memset(&ptr, 0, data_sz, 0); 773 err = err ?: bpf_memcmp(zeroes, memset_zero_data, data_sz); 774 775 return 0; 776 } 777 778 #define DYNPTR_MEMSET_VAL 42 779 780 char memset_notzero_data[] = "data to be overwritten"; 781 782 SEC("?tp/syscalls/sys_enter_nanosleep") 783 int test_dynptr_memset_notzero(void *ctx) 784 { 785 u32 data_sz = sizeof(memset_notzero_data); 786 struct bpf_dynptr ptr; 787 char expected[32]; 788 789 __builtin_memset(expected, DYNPTR_MEMSET_VAL, data_sz); 790 791 err = bpf_dynptr_from_mem(memset_notzero_data, data_sz, 0, &ptr); 792 err = err ?: bpf_dynptr_memset(&ptr, 0, data_sz, DYNPTR_MEMSET_VAL); 793 err = err ?: bpf_memcmp(expected, memset_notzero_data, data_sz); 794 795 return 0; 796 } 797 798 char memset_zero_offset_data[] = "data to be zeroed partially"; 799 800 SEC("?tp/syscalls/sys_enter_nanosleep") 801 int test_dynptr_memset_zero_offset(void *ctx) 802 { 803 char expected[] = "data to \0\0\0\0eroed partially"; 804 __u32 data_sz = sizeof(memset_zero_offset_data); 805 struct bpf_dynptr ptr; 806 807 err = bpf_dynptr_from_mem(memset_zero_offset_data, data_sz, 0, &ptr); 808 err = err ?: bpf_dynptr_memset(&ptr, 8, 4, 0); 809 err = err ?: bpf_memcmp(expected, memset_zero_offset_data, data_sz); 810 811 return 0; 812 } 813 814 char memset_zero_adjusted_data[] = "data to be zeroed partially"; 815 816 SEC("?tp/syscalls/sys_enter_nanosleep") 817 int test_dynptr_memset_zero_adjusted(void *ctx) 818 { 819 char expected[] = "data\0\0\0\0be zeroed partially"; 820 __u32 data_sz = sizeof(memset_zero_adjusted_data); 821 struct bpf_dynptr ptr; 822 823 err = bpf_dynptr_from_mem(memset_zero_adjusted_data, data_sz, 0, &ptr); 824 err = err ?: bpf_dynptr_adjust(&ptr, 4, 8); 825 err = err ?: bpf_dynptr_memset(&ptr, 0, bpf_dynptr_size(&ptr), 0); 826 err = err ?: bpf_memcmp(expected, memset_zero_adjusted_data, data_sz); 827 828 return 0; 829 } 830 831 char memset_overflow_data[] = "memset overflow data"; 832 833 SEC("?tp/syscalls/sys_enter_nanosleep") 834 int test_dynptr_memset_overflow(void *ctx) 835 { 836 __u32 data_sz = sizeof(memset_overflow_data); 837 struct bpf_dynptr ptr; 838 int ret; 839 840 err = bpf_dynptr_from_mem(memset_overflow_data, data_sz, 0, &ptr); 841 ret = bpf_dynptr_memset(&ptr, 0, data_sz + 1, 0); 842 if (ret != -E2BIG) 843 err = 1; 844 845 return 0; 846 } 847 848 SEC("?tp/syscalls/sys_enter_nanosleep") 849 int test_dynptr_memset_overflow_offset(void *ctx) 850 { 851 __u32 data_sz = sizeof(memset_overflow_data); 852 struct bpf_dynptr ptr; 853 int ret; 854 855 err = bpf_dynptr_from_mem(memset_overflow_data, data_sz, 0, &ptr); 856 ret = bpf_dynptr_memset(&ptr, 1, data_sz, 0); 857 if (ret != -E2BIG) 858 err = 1; 859 860 return 0; 861 } 862 863 SEC("?cgroup_skb/egress") 864 int test_dynptr_memset_readonly(struct __sk_buff *skb) 865 { 866 struct bpf_dynptr ptr; 867 int ret; 868 869 err = bpf_dynptr_from_skb(skb, 0, &ptr); 870 871 /* cgroup skbs are read only, memset should fail */ 872 ret = bpf_dynptr_memset(&ptr, 0, bpf_dynptr_size(&ptr), 0); 873 if (ret != -EINVAL) 874 err = 1; 875 876 return 0; 877 } 878 879 #define min_t(type, x, y) ({ \ 880 type __x = (x); \ 881 type __y = (y); \ 882 __x < __y ? __x : __y; }) 883 884 SEC("xdp") 885 int test_dynptr_memset_xdp_chunks(struct xdp_md *xdp) 886 { 887 u32 data_sz, chunk_sz, offset = 0; 888 const int max_chunks = 200; 889 struct bpf_dynptr ptr_xdp; 890 char expected_buf[32]; 891 char buf[32]; 892 int i; 893 894 __builtin_memset(expected_buf, DYNPTR_MEMSET_VAL, sizeof(expected_buf)); 895 896 /* ptr_xdp is backed by non-contiguous memory */ 897 bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp); 898 data_sz = bpf_dynptr_size(&ptr_xdp); 899 900 err = bpf_dynptr_memset(&ptr_xdp, 0, data_sz, DYNPTR_MEMSET_VAL); 901 if (err) { 902 /* bpf_dynptr_memset() eventually called bpf_xdp_pointer() 903 * where if data_sz is greater than 0xffff, -EFAULT will be 904 * returned. For 64K page size, data_sz is greater than 905 * 64K, so error is expected and let us zero out error and 906 * return success. 907 */ 908 if (data_sz >= PAGE_SIZE_64K) 909 err = 0; 910 goto out; 911 } 912 913 bpf_for(i, 0, max_chunks) { 914 offset = i * sizeof(buf); 915 if (offset >= data_sz) 916 goto out; 917 chunk_sz = min_t(u32, sizeof(buf), data_sz - offset); 918 err = bpf_dynptr_read(&buf, chunk_sz, &ptr_xdp, offset, 0); 919 if (err) 920 goto out; 921 err = bpf_memcmp(buf, expected_buf, sizeof(buf)); 922 if (err) 923 goto out; 924 } 925 out: 926 return XDP_DROP; 927 } 928 929 void *user_ptr; 930 /* Contains the copy of the data pointed by user_ptr. 931 * Size 384 to make it not fit into a single kernel chunk when copying 932 * but less than the maximum bpf stack size (512). 933 */ 934 char expected_str[384]; 935 __u32 test_len[7] = {0/* placeholder */, 0, 1, 2, 255, 256, 257}; 936 937 typedef int (*bpf_read_dynptr_fn_t)(const struct bpf_dynptr *dptr, u64 off, 938 u64 size, const void *unsafe_ptr); 939 940 /* Returns the offset just before the end of the maximum sized xdp fragment. 941 * Any write larger than 32 bytes will be split between 2 fragments. 942 */ 943 __u32 xdp_near_frag_end_offset(void) 944 { 945 const __u32 headroom = 256; 946 const __u32 max_frag_size = __PAGE_SIZE - headroom - sizeof(struct skb_shared_info); 947 948 /* 32 bytes before the approximate end of the fragment */ 949 return max_frag_size - 32; 950 } 951 952 /* Use __always_inline on test_dynptr_probe[_str][_xdp]() and callbacks 953 * of type bpf_read_dynptr_fn_t to prevent compiler from generating 954 * indirect calls that make program fail to load with "unknown opcode" error. 955 */ 956 static __always_inline void test_dynptr_probe(void *ptr, bpf_read_dynptr_fn_t bpf_read_dynptr_fn) 957 { 958 char buf[sizeof(expected_str)]; 959 struct bpf_dynptr ptr_buf; 960 int i; 961 962 if (bpf_get_current_pid_tgid() >> 32 != pid) 963 return; 964 965 err = bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(buf), 0, &ptr_buf); 966 967 bpf_for(i, 0, ARRAY_SIZE(test_len)) { 968 __u32 len = test_len[i]; 969 970 err = err ?: bpf_read_dynptr_fn(&ptr_buf, 0, test_len[i], ptr); 971 if (len > sizeof(buf)) 972 break; 973 err = err ?: bpf_dynptr_read(&buf, len, &ptr_buf, 0, 0); 974 975 if (err || bpf_memcmp(expected_str, buf, len)) 976 err = 1; 977 978 /* Reset buffer and dynptr */ 979 __builtin_memset(buf, 0, sizeof(buf)); 980 err = err ?: bpf_dynptr_write(&ptr_buf, 0, buf, len, 0); 981 } 982 bpf_ringbuf_discard_dynptr(&ptr_buf, 0); 983 } 984 985 static __always_inline void test_dynptr_probe_str(void *ptr, 986 bpf_read_dynptr_fn_t bpf_read_dynptr_fn) 987 { 988 char buf[sizeof(expected_str)]; 989 struct bpf_dynptr ptr_buf; 990 __u32 cnt, i; 991 992 if (bpf_get_current_pid_tgid() >> 32 != pid) 993 return; 994 995 bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(buf), 0, &ptr_buf); 996 997 bpf_for(i, 0, ARRAY_SIZE(test_len)) { 998 __u32 len = test_len[i]; 999 1000 cnt = bpf_read_dynptr_fn(&ptr_buf, 0, len, ptr); 1001 if (cnt != len) 1002 err = 1; 1003 1004 if (len > sizeof(buf)) 1005 continue; 1006 err = err ?: bpf_dynptr_read(&buf, len, &ptr_buf, 0, 0); 1007 if (!len) 1008 continue; 1009 if (err || bpf_memcmp(expected_str, buf, len - 1) || buf[len - 1] != '\0') 1010 err = 1; 1011 } 1012 bpf_ringbuf_discard_dynptr(&ptr_buf, 0); 1013 } 1014 1015 static __always_inline void test_dynptr_probe_xdp(struct xdp_md *xdp, void *ptr, 1016 bpf_read_dynptr_fn_t bpf_read_dynptr_fn) 1017 { 1018 struct bpf_dynptr ptr_xdp; 1019 char buf[sizeof(expected_str)]; 1020 __u32 off, i; 1021 1022 if (bpf_get_current_pid_tgid() >> 32 != pid) 1023 return; 1024 1025 off = xdp_near_frag_end_offset(); 1026 err = bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp); 1027 1028 bpf_for(i, 0, ARRAY_SIZE(test_len)) { 1029 __u32 len = test_len[i]; 1030 1031 err = err ?: bpf_read_dynptr_fn(&ptr_xdp, off, len, ptr); 1032 if (len > sizeof(buf)) 1033 continue; 1034 err = err ?: bpf_dynptr_read(&buf, len, &ptr_xdp, off, 0); 1035 if (err || bpf_memcmp(expected_str, buf, len)) 1036 err = 1; 1037 /* Reset buffer and dynptr */ 1038 __builtin_memset(buf, 0, sizeof(buf)); 1039 err = err ?: bpf_dynptr_write(&ptr_xdp, off, buf, len, 0); 1040 } 1041 } 1042 1043 static __always_inline void test_dynptr_probe_str_xdp(struct xdp_md *xdp, void *ptr, 1044 bpf_read_dynptr_fn_t bpf_read_dynptr_fn) 1045 { 1046 struct bpf_dynptr ptr_xdp; 1047 char buf[sizeof(expected_str)]; 1048 __u32 cnt, off, i; 1049 1050 if (bpf_get_current_pid_tgid() >> 32 != pid) 1051 return; 1052 1053 off = xdp_near_frag_end_offset(); 1054 err = bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp); 1055 if (err) 1056 return; 1057 1058 bpf_for(i, 0, ARRAY_SIZE(test_len)) { 1059 __u32 len = test_len[i]; 1060 1061 cnt = bpf_read_dynptr_fn(&ptr_xdp, off, len, ptr); 1062 if (cnt != len) 1063 err = 1; 1064 1065 if (len > sizeof(buf)) 1066 continue; 1067 err = err ?: bpf_dynptr_read(&buf, len, &ptr_xdp, off, 0); 1068 1069 if (!len) 1070 continue; 1071 if (err || bpf_memcmp(expected_str, buf, len - 1) || buf[len - 1] != '\0') 1072 err = 1; 1073 1074 __builtin_memset(buf, 0, sizeof(buf)); 1075 err = err ?: bpf_dynptr_write(&ptr_xdp, off, buf, len, 0); 1076 } 1077 } 1078 1079 SEC("xdp") 1080 int test_probe_read_user_dynptr(struct xdp_md *xdp) 1081 { 1082 test_dynptr_probe(user_ptr, bpf_probe_read_user_dynptr); 1083 if (!err) 1084 test_dynptr_probe_xdp(xdp, user_ptr, bpf_probe_read_user_dynptr); 1085 return XDP_PASS; 1086 } 1087 1088 SEC("xdp") 1089 int test_probe_read_kernel_dynptr(struct xdp_md *xdp) 1090 { 1091 test_dynptr_probe(expected_str, bpf_probe_read_kernel_dynptr); 1092 if (!err) 1093 test_dynptr_probe_xdp(xdp, expected_str, bpf_probe_read_kernel_dynptr); 1094 return XDP_PASS; 1095 } 1096 1097 SEC("xdp") 1098 int test_probe_read_user_str_dynptr(struct xdp_md *xdp) 1099 { 1100 test_dynptr_probe_str(user_ptr, bpf_probe_read_user_str_dynptr); 1101 if (!err) 1102 test_dynptr_probe_str_xdp(xdp, user_ptr, bpf_probe_read_user_str_dynptr); 1103 return XDP_PASS; 1104 } 1105 1106 SEC("xdp") 1107 int test_probe_read_kernel_str_dynptr(struct xdp_md *xdp) 1108 { 1109 test_dynptr_probe_str(expected_str, bpf_probe_read_kernel_str_dynptr); 1110 if (!err) 1111 test_dynptr_probe_str_xdp(xdp, expected_str, bpf_probe_read_kernel_str_dynptr); 1112 return XDP_PASS; 1113 } 1114 1115 SEC("fentry.s/" SYS_PREFIX "sys_nanosleep") 1116 int test_copy_from_user_dynptr(void *ctx) 1117 { 1118 test_dynptr_probe(user_ptr, bpf_copy_from_user_dynptr); 1119 return 0; 1120 } 1121 1122 SEC("fentry.s/" SYS_PREFIX "sys_nanosleep") 1123 int test_copy_from_user_str_dynptr(void *ctx) 1124 { 1125 test_dynptr_probe_str(user_ptr, bpf_copy_from_user_str_dynptr); 1126 return 0; 1127 } 1128 1129 static int bpf_copy_data_from_user_task(const struct bpf_dynptr *dptr, u64 off, 1130 u64 size, const void *unsafe_ptr) 1131 { 1132 struct task_struct *task = bpf_get_current_task_btf(); 1133 1134 return bpf_copy_from_user_task_dynptr(dptr, off, size, unsafe_ptr, task); 1135 } 1136 1137 static int bpf_copy_data_from_user_task_str(const struct bpf_dynptr *dptr, u64 off, 1138 u64 size, const void *unsafe_ptr) 1139 { 1140 struct task_struct *task = bpf_get_current_task_btf(); 1141 1142 return bpf_copy_from_user_task_str_dynptr(dptr, off, size, unsafe_ptr, task); 1143 } 1144 1145 SEC("fentry.s/" SYS_PREFIX "sys_nanosleep") 1146 int test_copy_from_user_task_dynptr(void *ctx) 1147 { 1148 test_dynptr_probe(user_ptr, bpf_copy_data_from_user_task); 1149 return 0; 1150 } 1151 1152 SEC("fentry.s/" SYS_PREFIX "sys_nanosleep") 1153 int test_copy_from_user_task_str_dynptr(void *ctx) 1154 { 1155 test_dynptr_probe_str(user_ptr, bpf_copy_data_from_user_task_str); 1156 return 0; 1157 } 1158