1 // SPDX-License-Identifier: GPL-2.0 2 #include <vmlinux.h> 3 4 #include <bpf/bpf_endian.h> 5 #include <bpf/bpf_helpers.h> 6 #include <errno.h> 7 8 #include "bpf_kfuncs.h" 9 #include "bpf_tracing_net.h" 10 11 #define META_SIZE 32 12 13 #define ctx_ptr(ctx, mem) (void *)(unsigned long)ctx->mem 14 15 /* Demonstrate passing metadata from XDP to TC using bpf_xdp_adjust_meta. 16 * 17 * The XDP program extracts a fixed-size payload following the Ethernet header 18 * and stores it as packet metadata to test the driver's metadata support. The 19 * TC program then verifies if the passed metadata is correct. 20 */ 21 22 bool test_pass; 23 24 static const __u8 meta_want[META_SIZE] = { 25 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 26 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 27 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 28 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 29 }; 30 31 static bool check_metadata(const char *file, int line, __u8 *meta_have) 32 { 33 if (!__builtin_memcmp(meta_have, meta_want, META_SIZE)) 34 return true; 35 36 bpf_stream_printk(BPF_STDERR, 37 "FAIL:%s:%d: metadata mismatch\n" 38 " have:\n %pI6\n %pI6\n" 39 " want:\n %pI6\n %pI6\n", 40 file, line, 41 &meta_have[0x00], &meta_have[0x10], 42 &meta_want[0x00], &meta_want[0x10]); 43 return false; 44 } 45 46 #define check_metadata(meta_have) check_metadata(__FILE__, __LINE__, meta_have) 47 48 static bool check_skb_metadata(const char *file, int line, struct __sk_buff *skb) 49 { 50 __u8 *data_meta = ctx_ptr(skb, data_meta); 51 __u8 *data = ctx_ptr(skb, data); 52 53 return data_meta + META_SIZE <= data && (check_metadata)(file, line, data_meta); 54 } 55 56 #define check_skb_metadata(skb) check_skb_metadata(__FILE__, __LINE__, skb) 57 58 SEC("tc") 59 int ing_cls(struct __sk_buff *ctx) 60 { 61 __u8 *meta_have = ctx_ptr(ctx, data_meta); 62 __u8 *data = ctx_ptr(ctx, data); 63 64 if (meta_have + META_SIZE > data) 65 goto out; 66 67 if (!check_metadata(meta_have)) 68 goto out; 69 70 test_pass = true; 71 out: 72 return TC_ACT_SHOT; 73 } 74 75 /* Read from metadata using bpf_dynptr_read helper */ 76 SEC("tc") 77 int ing_cls_dynptr_read(struct __sk_buff *ctx) 78 { 79 __u8 meta_have[META_SIZE]; 80 struct bpf_dynptr meta; 81 82 bpf_dynptr_from_skb_meta(ctx, 0, &meta); 83 bpf_dynptr_read(meta_have, META_SIZE, &meta, 0, 0); 84 85 if (!check_metadata(meta_have)) 86 goto out; 87 88 test_pass = true; 89 out: 90 return TC_ACT_SHOT; 91 } 92 93 /* Write to metadata using bpf_dynptr_write helper */ 94 SEC("tc") 95 int ing_cls_dynptr_write(struct __sk_buff *ctx) 96 { 97 struct bpf_dynptr data, meta; 98 __u8 *src; 99 100 bpf_dynptr_from_skb(ctx, 0, &data); 101 src = bpf_dynptr_slice(&data, sizeof(struct ethhdr), NULL, META_SIZE); 102 if (!src) 103 return TC_ACT_SHOT; 104 105 bpf_dynptr_from_skb_meta(ctx, 0, &meta); 106 bpf_dynptr_write(&meta, 0, src, META_SIZE, 0); 107 108 return TC_ACT_UNSPEC; /* pass */ 109 } 110 111 /* Read from metadata using read-only dynptr slice */ 112 SEC("tc") 113 int ing_cls_dynptr_slice(struct __sk_buff *ctx) 114 { 115 struct bpf_dynptr meta; 116 __u8 *meta_have; 117 118 bpf_dynptr_from_skb_meta(ctx, 0, &meta); 119 meta_have = bpf_dynptr_slice(&meta, 0, NULL, META_SIZE); 120 if (!meta_have) 121 goto out; 122 123 if (!check_metadata(meta_have)) 124 goto out; 125 126 test_pass = true; 127 out: 128 return TC_ACT_SHOT; 129 } 130 131 /* Write to metadata using writeable dynptr slice */ 132 SEC("tc") 133 int ing_cls_dynptr_slice_rdwr(struct __sk_buff *ctx) 134 { 135 struct bpf_dynptr data, meta; 136 __u8 *src, *dst; 137 138 bpf_dynptr_from_skb(ctx, 0, &data); 139 src = bpf_dynptr_slice(&data, sizeof(struct ethhdr), NULL, META_SIZE); 140 if (!src) 141 return TC_ACT_SHOT; 142 143 bpf_dynptr_from_skb_meta(ctx, 0, &meta); 144 dst = bpf_dynptr_slice_rdwr(&meta, 0, NULL, META_SIZE); 145 if (!dst) 146 return TC_ACT_SHOT; 147 148 __builtin_memcpy(dst, src, META_SIZE); 149 150 return TC_ACT_UNSPEC; /* pass */ 151 } 152 153 /* Read skb metadata in chunks from various offsets in different ways. */ 154 SEC("tc") 155 int ing_cls_dynptr_offset_rd(struct __sk_buff *ctx) 156 { 157 const __u32 chunk_len = META_SIZE / 4; 158 __u8 meta_have[META_SIZE]; 159 struct bpf_dynptr meta; 160 __u8 *dst, *src; 161 162 dst = meta_have; 163 164 /* 1. Regular read */ 165 bpf_dynptr_from_skb_meta(ctx, 0, &meta); 166 bpf_dynptr_read(dst, chunk_len, &meta, 0, 0); 167 dst += chunk_len; 168 169 /* 2. Read from an offset-adjusted dynptr */ 170 bpf_dynptr_adjust(&meta, chunk_len, bpf_dynptr_size(&meta)); 171 bpf_dynptr_read(dst, chunk_len, &meta, 0, 0); 172 dst += chunk_len; 173 174 /* 3. Read at an offset */ 175 bpf_dynptr_read(dst, chunk_len, &meta, chunk_len, 0); 176 dst += chunk_len; 177 178 /* 4. Read from a slice starting at an offset */ 179 src = bpf_dynptr_slice(&meta, 2 * chunk_len, NULL, chunk_len); 180 if (!src) 181 goto out; 182 __builtin_memcpy(dst, src, chunk_len); 183 184 if (!check_metadata(meta_have)) 185 goto out; 186 187 test_pass = true; 188 out: 189 return TC_ACT_SHOT; 190 } 191 192 /* Write skb metadata in chunks at various offsets in different ways. */ 193 SEC("tc") 194 int ing_cls_dynptr_offset_wr(struct __sk_buff *ctx) 195 { 196 const __u32 chunk_len = META_SIZE / 4; 197 __u8 payload[META_SIZE]; 198 struct bpf_dynptr meta; 199 __u8 *dst, *src; 200 201 bpf_skb_load_bytes(ctx, sizeof(struct ethhdr), payload, sizeof(payload)); 202 src = payload; 203 204 /* 1. Regular write */ 205 bpf_dynptr_from_skb_meta(ctx, 0, &meta); 206 bpf_dynptr_write(&meta, 0, src, chunk_len, 0); 207 src += chunk_len; 208 209 /* 2. Write to an offset-adjusted dynptr */ 210 bpf_dynptr_adjust(&meta, chunk_len, bpf_dynptr_size(&meta)); 211 bpf_dynptr_write(&meta, 0, src, chunk_len, 0); 212 src += chunk_len; 213 214 /* 3. Write at an offset */ 215 bpf_dynptr_write(&meta, chunk_len, src, chunk_len, 0); 216 src += chunk_len; 217 218 /* 4. Write to a slice starting at an offset */ 219 dst = bpf_dynptr_slice_rdwr(&meta, 2 * chunk_len, NULL, chunk_len); 220 if (!dst) 221 return TC_ACT_SHOT; 222 __builtin_memcpy(dst, src, chunk_len); 223 224 return TC_ACT_UNSPEC; /* pass */ 225 } 226 227 /* Pass an OOB offset to dynptr read, write, adjust, slice. */ 228 SEC("tc") 229 int ing_cls_dynptr_offset_oob(struct __sk_buff *ctx) 230 { 231 struct bpf_dynptr meta; 232 __u8 md, *p; 233 int err; 234 235 err = bpf_dynptr_from_skb_meta(ctx, 0, &meta); 236 if (err) 237 goto fail; 238 239 /* read offset OOB */ 240 err = bpf_dynptr_read(&md, sizeof(md), &meta, META_SIZE, 0); 241 if (err != -E2BIG) 242 goto fail; 243 244 /* write offset OOB */ 245 err = bpf_dynptr_write(&meta, META_SIZE, &md, sizeof(md), 0); 246 if (err != -E2BIG) 247 goto fail; 248 249 /* adjust end offset OOB */ 250 err = bpf_dynptr_adjust(&meta, 0, META_SIZE + 1); 251 if (err != -ERANGE) 252 goto fail; 253 254 /* adjust start offset OOB */ 255 err = bpf_dynptr_adjust(&meta, META_SIZE + 1, META_SIZE + 1); 256 if (err != -ERANGE) 257 goto fail; 258 259 /* slice offset OOB */ 260 p = bpf_dynptr_slice(&meta, META_SIZE, NULL, sizeof(*p)); 261 if (p) 262 goto fail; 263 264 /* slice rdwr offset OOB */ 265 p = bpf_dynptr_slice_rdwr(&meta, META_SIZE, NULL, sizeof(*p)); 266 if (p) 267 goto fail; 268 269 return TC_ACT_UNSPEC; 270 fail: 271 return TC_ACT_SHOT; 272 } 273 274 /* Test packets carry test metadata pattern as payload. */ 275 static bool is_test_packet_xdp(struct xdp_md *ctx) 276 { 277 __u8 meta_have[META_SIZE]; 278 __u32 len; 279 280 len = bpf_xdp_get_buff_len(ctx); 281 if (len < META_SIZE) 282 return false; 283 if (bpf_xdp_load_bytes(ctx, len - META_SIZE, meta_have, META_SIZE)) 284 return false; 285 if (__builtin_memcmp(meta_have, meta_want, META_SIZE)) 286 return false; 287 288 return true; 289 } 290 291 /* Test packets carry test metadata pattern as payload. */ 292 static bool is_test_packet_tc(struct __sk_buff *ctx) 293 { 294 __u8 meta_have[META_SIZE]; 295 296 if (ctx->len < META_SIZE) 297 return false; 298 if (bpf_skb_load_bytes(ctx, ctx->len - META_SIZE, meta_have, META_SIZE)) 299 return false; 300 if (__builtin_memcmp(meta_have, meta_want, META_SIZE)) 301 return false; 302 303 return true; 304 } 305 306 /* Reserve and clear space for metadata but don't populate it */ 307 SEC("xdp") 308 int ing_xdp_zalloc_meta(struct xdp_md *ctx) 309 { 310 __u8 *meta; 311 int ret; 312 313 /* Drop any non-test packets */ 314 if (!is_test_packet_xdp(ctx)) 315 return XDP_DROP; 316 317 ret = bpf_xdp_adjust_meta(ctx, -META_SIZE); 318 if (ret < 0) 319 return XDP_DROP; 320 321 meta = ctx_ptr(ctx, data_meta); 322 if (meta + META_SIZE > ctx_ptr(ctx, data)) 323 return XDP_DROP; 324 325 __builtin_memset(meta, 0, META_SIZE); 326 327 return XDP_PASS; 328 } 329 330 SEC("xdp") 331 int ing_xdp(struct xdp_md *ctx) 332 { 333 __u8 *data, *data_meta; 334 int ret; 335 336 /* Drop any non-test packets */ 337 if (!is_test_packet_xdp(ctx)) 338 return XDP_DROP; 339 340 ret = bpf_xdp_adjust_meta(ctx, -META_SIZE); 341 if (ret < 0) 342 return XDP_DROP; 343 344 data_meta = ctx_ptr(ctx, data_meta); 345 data = ctx_ptr(ctx, data); 346 347 if (data_meta + META_SIZE > data) 348 return XDP_DROP; 349 350 __builtin_memcpy(data_meta, meta_want, META_SIZE); 351 return XDP_PASS; 352 } 353 354 /* 355 * Check that, when operating on a cloned packet, skb->data_meta..skb->data is 356 * kept intact if prog writes to packet _payload_ using packet pointers. 357 */ 358 SEC("tc") 359 int clone_data_meta_survives_data_write(struct __sk_buff *ctx) 360 { 361 __u8 *meta_have = ctx_ptr(ctx, data_meta); 362 struct ethhdr *eth = ctx_ptr(ctx, data); 363 364 if (eth + 1 > ctx_ptr(ctx, data_end)) 365 goto out; 366 /* Ignore non-test packets */ 367 if (!is_test_packet_tc(ctx)) 368 goto out; 369 370 if (meta_have + META_SIZE > eth) 371 goto out; 372 373 if (!check_metadata(meta_have)) 374 goto out; 375 376 /* Packet write to trigger unclone in prologue */ 377 eth->h_proto = 42; 378 379 test_pass = true; 380 out: 381 return TC_ACT_SHOT; 382 } 383 384 /* 385 * Check that, when operating on a cloned packet, skb->data_meta..skb->data is 386 * kept intact if prog writes to packet _metadata_ using packet pointers. 387 */ 388 SEC("tc") 389 int clone_data_meta_survives_meta_write(struct __sk_buff *ctx) 390 { 391 __u8 *meta_have = ctx_ptr(ctx, data_meta); 392 struct ethhdr *eth = ctx_ptr(ctx, data); 393 394 if (eth + 1 > ctx_ptr(ctx, data_end)) 395 goto out; 396 /* Ignore non-test packets */ 397 if (!is_test_packet_tc(ctx)) 398 goto out; 399 400 if (meta_have + META_SIZE > eth) 401 goto out; 402 403 if (!check_metadata(meta_have)) 404 goto out; 405 406 /* Metadata write to trigger unclone in prologue */ 407 *meta_have = 42; 408 409 test_pass = true; 410 out: 411 return TC_ACT_SHOT; 412 } 413 414 /* 415 * Check that, when operating on a cloned packet, metadata remains intact if 416 * prog creates a r/w slice to packet _payload_. 417 */ 418 SEC("tc") 419 int clone_meta_dynptr_survives_data_slice_write(struct __sk_buff *ctx) 420 { 421 struct bpf_dynptr data, meta; 422 __u8 meta_have[META_SIZE]; 423 struct ethhdr *eth; 424 425 bpf_dynptr_from_skb(ctx, 0, &data); 426 eth = bpf_dynptr_slice_rdwr(&data, 0, NULL, sizeof(*eth)); 427 if (!eth) 428 goto out; 429 /* Ignore non-test packets */ 430 if (!is_test_packet_tc(ctx)) 431 goto out; 432 433 bpf_dynptr_from_skb_meta(ctx, 0, &meta); 434 bpf_dynptr_read(meta_have, META_SIZE, &meta, 0, 0); 435 if (!check_metadata(meta_have)) 436 goto out; 437 438 test_pass = true; 439 out: 440 return TC_ACT_SHOT; 441 } 442 443 /* 444 * Check that, when operating on a cloned packet, metadata remains intact if 445 * prog creates an r/w slice to packet _metadata_. 446 */ 447 SEC("tc") 448 int clone_meta_dynptr_survives_meta_slice_write(struct __sk_buff *ctx) 449 { 450 struct bpf_dynptr meta; 451 __u8 *meta_have; 452 453 /* Ignore non-test packets */ 454 if (!is_test_packet_tc(ctx)) 455 goto out; 456 457 bpf_dynptr_from_skb_meta(ctx, 0, &meta); 458 meta_have = bpf_dynptr_slice_rdwr(&meta, 0, NULL, META_SIZE); 459 if (!meta_have) 460 goto out; 461 462 if (!check_metadata(meta_have)) 463 goto out; 464 465 test_pass = true; 466 out: 467 return TC_ACT_SHOT; 468 } 469 470 /* 471 * Check that, when operating on a cloned packet, skb_meta dynptr is read-write 472 * before prog writes to packet _payload_ using dynptr_write helper and metadata 473 * remains intact before and after the write. 474 */ 475 SEC("tc") 476 int clone_meta_dynptr_rw_before_data_dynptr_write(struct __sk_buff *ctx) 477 { 478 struct bpf_dynptr data, meta; 479 __u8 meta_have[META_SIZE]; 480 int err; 481 482 /* Ignore non-test packets */ 483 if (!is_test_packet_tc(ctx)) 484 goto out; 485 486 /* Expect read-write metadata before unclone */ 487 bpf_dynptr_from_skb_meta(ctx, 0, &meta); 488 if (bpf_dynptr_is_rdonly(&meta)) 489 goto out; 490 491 err = bpf_dynptr_read(meta_have, META_SIZE, &meta, 0, 0); 492 if (err || !check_metadata(meta_have)) 493 goto out; 494 495 /* Helper write to payload will unclone the packet */ 496 bpf_dynptr_from_skb(ctx, 0, &data); 497 bpf_dynptr_write(&data, offsetof(struct ethhdr, h_proto), "x", 1, 0); 498 499 err = bpf_dynptr_read(meta_have, META_SIZE, &meta, 0, 0); 500 if (err || !check_metadata(meta_have)) 501 goto out; 502 503 test_pass = true; 504 out: 505 return TC_ACT_SHOT; 506 } 507 508 /* 509 * Check that, when operating on a cloned packet, skb_meta dynptr is read-write 510 * before prog writes to packet _metadata_ using dynptr_write helper and 511 * metadata remains intact before and after the write. 512 */ 513 SEC("tc") 514 int clone_meta_dynptr_rw_before_meta_dynptr_write(struct __sk_buff *ctx) 515 { 516 struct bpf_dynptr meta; 517 __u8 meta_have[META_SIZE]; 518 int err; 519 520 /* Ignore non-test packets */ 521 if (!is_test_packet_tc(ctx)) 522 goto out; 523 524 /* Expect read-write metadata before unclone */ 525 bpf_dynptr_from_skb_meta(ctx, 0, &meta); 526 if (bpf_dynptr_is_rdonly(&meta)) 527 goto out; 528 529 err = bpf_dynptr_read(meta_have, META_SIZE, &meta, 0, 0); 530 if (err || !check_metadata(meta_have)) 531 goto out; 532 533 /* Helper write to metadata will unclone the packet */ 534 bpf_dynptr_write(&meta, 0, &meta_have[0], 1, 0); 535 536 err = bpf_dynptr_read(meta_have, META_SIZE, &meta, 0, 0); 537 if (err || !check_metadata(meta_have)) 538 goto out; 539 540 test_pass = true; 541 out: 542 return TC_ACT_SHOT; 543 } 544 545 SEC("lwt_xmit") 546 int dummy_lwt_xmit(struct __sk_buff *ctx) 547 { 548 if (bpf_skb_change_head(ctx, sizeof(struct ipv6hdr), 0)) 549 return BPF_DROP; 550 551 return BPF_OK; 552 } 553 554 SEC("tc") 555 int tc_is_meta_empty(struct __sk_buff *ctx) 556 { 557 if (!is_test_packet_tc(ctx)) 558 return TC_ACT_OK; 559 560 if (ctx->data_meta != ctx->data) 561 return TC_ACT_OK; 562 563 test_pass = true; 564 return TC_ACT_OK; 565 } 566 567 SEC("tc") 568 int helper_skb_vlan_push_pop(struct __sk_buff *ctx) 569 { 570 int err; 571 572 /* bpf_skb_vlan_push assumes HW offload for primary VLAN tag. Only 573 * secondary tag push triggers an actual MAC header modification. 574 */ 575 err = bpf_skb_vlan_push(ctx, 0, 42); 576 if (err) 577 goto out; 578 err = bpf_skb_vlan_push(ctx, 0, 207); 579 if (err) 580 goto out; 581 582 if (!check_skb_metadata(ctx)) 583 goto out; 584 585 err = bpf_skb_vlan_pop(ctx); 586 if (err) 587 goto out; 588 err = bpf_skb_vlan_pop(ctx); 589 if (err) 590 goto out; 591 592 if (!check_skb_metadata(ctx)) 593 goto out; 594 595 test_pass = true; 596 out: 597 return TC_ACT_SHOT; 598 } 599 600 SEC("tc") 601 int helper_skb_adjust_room(struct __sk_buff *ctx) 602 { 603 int err; 604 605 /* Grow a 1 byte hole after the MAC header */ 606 err = bpf_skb_adjust_room(ctx, 1, BPF_ADJ_ROOM_MAC, 0); 607 if (err) 608 goto out; 609 610 if (!check_skb_metadata(ctx)) 611 goto out; 612 613 /* Shrink a 1 byte hole after the MAC header */ 614 err = bpf_skb_adjust_room(ctx, -1, BPF_ADJ_ROOM_MAC, 0); 615 if (err) 616 goto out; 617 618 if (!check_skb_metadata(ctx)) 619 goto out; 620 621 /* Grow a 256 byte hole to trigger head reallocation */ 622 err = bpf_skb_adjust_room(ctx, 256, BPF_ADJ_ROOM_MAC, 0); 623 if (err) 624 goto out; 625 626 if (!check_skb_metadata(ctx)) 627 goto out; 628 629 test_pass = true; 630 out: 631 return TC_ACT_SHOT; 632 } 633 634 SEC("tc") 635 int helper_skb_change_head_tail(struct __sk_buff *ctx) 636 { 637 int err; 638 639 /* Reserve 1 extra in the front for packet data */ 640 err = bpf_skb_change_head(ctx, 1, 0); 641 if (err) 642 goto out; 643 644 if (!check_skb_metadata(ctx)) 645 goto out; 646 647 /* Reserve 256 extra bytes in the front to trigger head reallocation */ 648 err = bpf_skb_change_head(ctx, 256, 0); 649 if (err) 650 goto out; 651 652 if (!check_skb_metadata(ctx)) 653 goto out; 654 655 /* Reserve 4k extra bytes in the back to trigger head reallocation */ 656 err = bpf_skb_change_tail(ctx, ctx->len + 4096, 0); 657 if (err) 658 goto out; 659 660 if (!check_skb_metadata(ctx)) 661 goto out; 662 663 test_pass = true; 664 out: 665 return TC_ACT_SHOT; 666 } 667 668 SEC("tc") 669 int helper_skb_change_proto(struct __sk_buff *ctx) 670 { 671 int err; 672 673 err = bpf_skb_change_proto(ctx, bpf_htons(ETH_P_IPV6), 0); 674 if (err) 675 goto out; 676 677 if (!check_skb_metadata(ctx)) 678 goto out; 679 680 err = bpf_skb_change_proto(ctx, bpf_htons(ETH_P_IP), 0); 681 if (err) 682 goto out; 683 684 if (!check_skb_metadata(ctx)) 685 goto out; 686 687 test_pass = true; 688 out: 689 return TC_ACT_SHOT; 690 } 691 692 char _license[] SEC("license") = "GPL"; 693