1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2017 Facebook 3 */ 4 #include <linux/bpf.h> 5 #include <linux/btf.h> 6 #include <linux/btf_ids.h> 7 #include <linux/slab.h> 8 #include <linux/init.h> 9 #include <linux/vmalloc.h> 10 #include <linux/etherdevice.h> 11 #include <linux/filter.h> 12 #include <linux/rcupdate_trace.h> 13 #include <linux/sched/signal.h> 14 #include <net/bpf_sk_storage.h> 15 #include <net/sock.h> 16 #include <net/tcp.h> 17 #include <net/net_namespace.h> 18 #include <net/page_pool.h> 19 #include <linux/error-injection.h> 20 #include <linux/smp.h> 21 #include <linux/sock_diag.h> 22 #include <net/xdp.h> 23 24 #define CREATE_TRACE_POINTS 25 #include <trace/events/bpf_test_run.h> 26 27 struct bpf_test_timer { 28 enum { NO_PREEMPT, NO_MIGRATE } mode; 29 u32 i; 30 u64 time_start, time_spent; 31 }; 32 33 static void bpf_test_timer_enter(struct bpf_test_timer *t) 34 __acquires(rcu) 35 { 36 rcu_read_lock(); 37 if (t->mode == NO_PREEMPT) 38 preempt_disable(); 39 else 40 migrate_disable(); 41 42 t->time_start = ktime_get_ns(); 43 } 44 45 static void bpf_test_timer_leave(struct bpf_test_timer *t) 46 __releases(rcu) 47 { 48 t->time_start = 0; 49 50 if (t->mode == NO_PREEMPT) 51 preempt_enable(); 52 else 53 migrate_enable(); 54 rcu_read_unlock(); 55 } 56 57 static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations, 58 u32 repeat, int *err, u32 *duration) 59 __must_hold(rcu) 60 { 61 t->i += iterations; 62 if (t->i >= repeat) { 63 /* We're done. */ 64 t->time_spent += ktime_get_ns() - t->time_start; 65 do_div(t->time_spent, t->i); 66 *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent; 67 *err = 0; 68 goto reset; 69 } 70 71 if (signal_pending(current)) { 72 /* During iteration: we've been cancelled, abort. */ 73 *err = -EINTR; 74 goto reset; 75 } 76 77 if (need_resched()) { 78 /* During iteration: we need to reschedule between runs. */ 79 t->time_spent += ktime_get_ns() - t->time_start; 80 bpf_test_timer_leave(t); 81 cond_resched(); 82 bpf_test_timer_enter(t); 83 } 84 85 /* Do another round. */ 86 return true; 87 88 reset: 89 t->i = 0; 90 return false; 91 } 92 93 /* We put this struct at the head of each page with a context and frame 94 * initialised when the page is allocated, so we don't have to do this on each 95 * repetition of the test run. 96 */ 97 struct xdp_page_head { 98 struct xdp_buff orig_ctx; 99 struct xdp_buff ctx; 100 struct xdp_frame frm; 101 u8 data[]; 102 }; 103 104 struct xdp_test_data { 105 struct xdp_buff *orig_ctx; 106 struct xdp_rxq_info rxq; 107 struct net_device *dev; 108 struct page_pool *pp; 109 struct xdp_frame **frames; 110 struct sk_buff **skbs; 111 struct xdp_mem_info mem; 112 u32 batch_size; 113 u32 frame_cnt; 114 }; 115 116 #define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head)) 117 #define TEST_XDP_MAX_BATCH 256 118 119 static void xdp_test_run_init_page(struct page *page, void *arg) 120 { 121 struct xdp_page_head *head = phys_to_virt(page_to_phys(page)); 122 struct xdp_buff *new_ctx, *orig_ctx; 123 u32 headroom = XDP_PACKET_HEADROOM; 124 struct xdp_test_data *xdp = arg; 125 size_t frm_len, meta_len; 126 struct xdp_frame *frm; 127 void *data; 128 129 orig_ctx = xdp->orig_ctx; 130 frm_len = orig_ctx->data_end - orig_ctx->data_meta; 131 meta_len = orig_ctx->data - orig_ctx->data_meta; 132 headroom -= meta_len; 133 134 new_ctx = &head->ctx; 135 frm = &head->frm; 136 data = &head->data; 137 memcpy(data + headroom, orig_ctx->data_meta, frm_len); 138 139 xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq); 140 xdp_prepare_buff(new_ctx, data, headroom, frm_len, true); 141 new_ctx->data = new_ctx->data_meta + meta_len; 142 143 xdp_update_frame_from_buff(new_ctx, frm); 144 frm->mem = new_ctx->rxq->mem; 145 146 memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx)); 147 } 148 149 static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx) 150 { 151 struct page_pool *pp; 152 int err = -ENOMEM; 153 struct page_pool_params pp_params = { 154 .order = 0, 155 .flags = 0, 156 .pool_size = xdp->batch_size, 157 .nid = NUMA_NO_NODE, 158 .init_callback = xdp_test_run_init_page, 159 .init_arg = xdp, 160 }; 161 162 xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); 163 if (!xdp->frames) 164 return -ENOMEM; 165 166 xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); 167 if (!xdp->skbs) 168 goto err_skbs; 169 170 pp = page_pool_create(&pp_params); 171 if (IS_ERR(pp)) { 172 err = PTR_ERR(pp); 173 goto err_pp; 174 } 175 176 /* will copy 'mem.id' into pp->xdp_mem_id */ 177 err = xdp_reg_mem_model(&xdp->mem, MEM_TYPE_PAGE_POOL, pp); 178 if (err) 179 goto err_mmodel; 180 181 xdp->pp = pp; 182 183 /* We create a 'fake' RXQ referencing the original dev, but with an 184 * xdp_mem_info pointing to our page_pool 185 */ 186 xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0); 187 xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL; 188 xdp->rxq.mem.id = pp->xdp_mem_id; 189 xdp->dev = orig_ctx->rxq->dev; 190 xdp->orig_ctx = orig_ctx; 191 192 return 0; 193 194 err_mmodel: 195 page_pool_destroy(pp); 196 err_pp: 197 kvfree(xdp->skbs); 198 err_skbs: 199 kvfree(xdp->frames); 200 return err; 201 } 202 203 static void xdp_test_run_teardown(struct xdp_test_data *xdp) 204 { 205 xdp_unreg_mem_model(&xdp->mem); 206 page_pool_destroy(xdp->pp); 207 kfree(xdp->frames); 208 kfree(xdp->skbs); 209 } 210 211 static bool frame_was_changed(const struct xdp_page_head *head) 212 { 213 /* xdp_scrub_frame() zeroes the data pointer, flags is the last field, 214 * i.e. has the highest chances to be overwritten. If those two are 215 * untouched, it's most likely safe to skip the context reset. 216 */ 217 return head->frm.data != head->orig_ctx.data || 218 head->frm.flags != head->orig_ctx.flags; 219 } 220 221 static bool ctx_was_changed(struct xdp_page_head *head) 222 { 223 return head->orig_ctx.data != head->ctx.data || 224 head->orig_ctx.data_meta != head->ctx.data_meta || 225 head->orig_ctx.data_end != head->ctx.data_end; 226 } 227 228 static void reset_ctx(struct xdp_page_head *head) 229 { 230 if (likely(!frame_was_changed(head) && !ctx_was_changed(head))) 231 return; 232 233 head->ctx.data = head->orig_ctx.data; 234 head->ctx.data_meta = head->orig_ctx.data_meta; 235 head->ctx.data_end = head->orig_ctx.data_end; 236 xdp_update_frame_from_buff(&head->ctx, &head->frm); 237 } 238 239 static int xdp_recv_frames(struct xdp_frame **frames, int nframes, 240 struct sk_buff **skbs, 241 struct net_device *dev) 242 { 243 gfp_t gfp = __GFP_ZERO | GFP_ATOMIC; 244 int i, n; 245 LIST_HEAD(list); 246 247 n = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, (void **)skbs); 248 if (unlikely(n == 0)) { 249 for (i = 0; i < nframes; i++) 250 xdp_return_frame(frames[i]); 251 return -ENOMEM; 252 } 253 254 for (i = 0; i < nframes; i++) { 255 struct xdp_frame *xdpf = frames[i]; 256 struct sk_buff *skb = skbs[i]; 257 258 skb = __xdp_build_skb_from_frame(xdpf, skb, dev); 259 if (!skb) { 260 xdp_return_frame(xdpf); 261 continue; 262 } 263 264 list_add_tail(&skb->list, &list); 265 } 266 netif_receive_skb_list(&list); 267 268 return 0; 269 } 270 271 static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog, 272 u32 repeat) 273 { 274 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); 275 int err = 0, act, ret, i, nframes = 0, batch_sz; 276 struct xdp_frame **frames = xdp->frames; 277 struct xdp_page_head *head; 278 struct xdp_frame *frm; 279 bool redirect = false; 280 struct xdp_buff *ctx; 281 struct page *page; 282 283 batch_sz = min_t(u32, repeat, xdp->batch_size); 284 285 local_bh_disable(); 286 xdp_set_return_frame_no_direct(); 287 288 for (i = 0; i < batch_sz; i++) { 289 page = page_pool_dev_alloc_pages(xdp->pp); 290 if (!page) { 291 err = -ENOMEM; 292 goto out; 293 } 294 295 head = phys_to_virt(page_to_phys(page)); 296 reset_ctx(head); 297 ctx = &head->ctx; 298 frm = &head->frm; 299 xdp->frame_cnt++; 300 301 act = bpf_prog_run_xdp(prog, ctx); 302 303 /* if program changed pkt bounds we need to update the xdp_frame */ 304 if (unlikely(ctx_was_changed(head))) { 305 ret = xdp_update_frame_from_buff(ctx, frm); 306 if (ret) { 307 xdp_return_buff(ctx); 308 continue; 309 } 310 } 311 312 switch (act) { 313 case XDP_TX: 314 /* we can't do a real XDP_TX since we're not in the 315 * driver, so turn it into a REDIRECT back to the same 316 * index 317 */ 318 ri->tgt_index = xdp->dev->ifindex; 319 ri->map_id = INT_MAX; 320 ri->map_type = BPF_MAP_TYPE_UNSPEC; 321 fallthrough; 322 case XDP_REDIRECT: 323 redirect = true; 324 ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog); 325 if (ret) 326 xdp_return_buff(ctx); 327 break; 328 case XDP_PASS: 329 frames[nframes++] = frm; 330 break; 331 default: 332 bpf_warn_invalid_xdp_action(NULL, prog, act); 333 fallthrough; 334 case XDP_DROP: 335 xdp_return_buff(ctx); 336 break; 337 } 338 } 339 340 out: 341 if (redirect) 342 xdp_do_flush(); 343 if (nframes) { 344 ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev); 345 if (ret) 346 err = ret; 347 } 348 349 xdp_clear_return_frame_no_direct(); 350 local_bh_enable(); 351 return err; 352 } 353 354 static int bpf_test_run_xdp_live(struct bpf_prog *prog, struct xdp_buff *ctx, 355 u32 repeat, u32 batch_size, u32 *time) 356 357 { 358 struct xdp_test_data xdp = { .batch_size = batch_size }; 359 struct bpf_test_timer t = { .mode = NO_MIGRATE }; 360 int ret; 361 362 if (!repeat) 363 repeat = 1; 364 365 ret = xdp_test_run_setup(&xdp, ctx); 366 if (ret) 367 return ret; 368 369 bpf_test_timer_enter(&t); 370 do { 371 xdp.frame_cnt = 0; 372 ret = xdp_test_run_batch(&xdp, prog, repeat - t.i); 373 if (unlikely(ret < 0)) 374 break; 375 } while (bpf_test_timer_continue(&t, xdp.frame_cnt, repeat, &ret, time)); 376 bpf_test_timer_leave(&t); 377 378 xdp_test_run_teardown(&xdp); 379 return ret; 380 } 381 382 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 383 u32 *retval, u32 *time, bool xdp) 384 { 385 struct bpf_prog_array_item item = {.prog = prog}; 386 struct bpf_run_ctx *old_ctx; 387 struct bpf_cg_run_ctx run_ctx; 388 struct bpf_test_timer t = { NO_MIGRATE }; 389 enum bpf_cgroup_storage_type stype; 390 int ret; 391 392 for_each_cgroup_storage_type(stype) { 393 item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 394 if (IS_ERR(item.cgroup_storage[stype])) { 395 item.cgroup_storage[stype] = NULL; 396 for_each_cgroup_storage_type(stype) 397 bpf_cgroup_storage_free(item.cgroup_storage[stype]); 398 return -ENOMEM; 399 } 400 } 401 402 if (!repeat) 403 repeat = 1; 404 405 bpf_test_timer_enter(&t); 406 old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 407 do { 408 run_ctx.prog_item = &item; 409 local_bh_disable(); 410 if (xdp) 411 *retval = bpf_prog_run_xdp(prog, ctx); 412 else 413 *retval = bpf_prog_run(prog, ctx); 414 local_bh_enable(); 415 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, time)); 416 bpf_reset_run_ctx(old_ctx); 417 bpf_test_timer_leave(&t); 418 419 for_each_cgroup_storage_type(stype) 420 bpf_cgroup_storage_free(item.cgroup_storage[stype]); 421 422 return ret; 423 } 424 425 static int bpf_test_finish(const union bpf_attr *kattr, 426 union bpf_attr __user *uattr, const void *data, 427 struct skb_shared_info *sinfo, u32 size, 428 u32 retval, u32 duration) 429 { 430 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 431 int err = -EFAULT; 432 u32 copy_size = size; 433 434 /* Clamp copy if the user has provided a size hint, but copy the full 435 * buffer if not to retain old behaviour. 436 */ 437 if (kattr->test.data_size_out && 438 copy_size > kattr->test.data_size_out) { 439 copy_size = kattr->test.data_size_out; 440 err = -ENOSPC; 441 } 442 443 if (data_out) { 444 int len = sinfo ? copy_size - sinfo->xdp_frags_size : copy_size; 445 446 if (len < 0) { 447 err = -ENOSPC; 448 goto out; 449 } 450 451 if (copy_to_user(data_out, data, len)) 452 goto out; 453 454 if (sinfo) { 455 int i, offset = len; 456 u32 data_len; 457 458 for (i = 0; i < sinfo->nr_frags; i++) { 459 skb_frag_t *frag = &sinfo->frags[i]; 460 461 if (offset >= copy_size) { 462 err = -ENOSPC; 463 break; 464 } 465 466 data_len = min_t(u32, copy_size - offset, 467 skb_frag_size(frag)); 468 469 if (copy_to_user(data_out + offset, 470 skb_frag_address(frag), 471 data_len)) 472 goto out; 473 474 offset += data_len; 475 } 476 } 477 } 478 479 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 480 goto out; 481 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 482 goto out; 483 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 484 goto out; 485 if (err != -ENOSPC) 486 err = 0; 487 out: 488 trace_bpf_test_finish(&err); 489 return err; 490 } 491 492 /* Integer types of various sizes and pointer combinations cover variety of 493 * architecture dependent calling conventions. 7+ can be supported in the 494 * future. 495 */ 496 __diag_push(); 497 __diag_ignore_all("-Wmissing-prototypes", 498 "Global functions as their definitions will be in vmlinux BTF"); 499 __bpf_kfunc int bpf_fentry_test1(int a) 500 { 501 return a + 1; 502 } 503 EXPORT_SYMBOL_GPL(bpf_fentry_test1); 504 505 int noinline bpf_fentry_test2(int a, u64 b) 506 { 507 return a + b; 508 } 509 510 int noinline bpf_fentry_test3(char a, int b, u64 c) 511 { 512 return a + b + c; 513 } 514 515 int noinline bpf_fentry_test4(void *a, char b, int c, u64 d) 516 { 517 return (long)a + b + c + d; 518 } 519 520 int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e) 521 { 522 return a + (long)b + c + d + e; 523 } 524 525 int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f) 526 { 527 return a + (long)b + c + d + (long)e + f; 528 } 529 530 struct bpf_fentry_test_t { 531 struct bpf_fentry_test_t *a; 532 }; 533 534 int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg) 535 { 536 return (long)arg; 537 } 538 539 int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg) 540 { 541 return (long)arg->a; 542 } 543 544 __bpf_kfunc int bpf_modify_return_test(int a, int *b) 545 { 546 *b += 1; 547 return a + *b; 548 } 549 550 __bpf_kfunc u64 bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d) 551 { 552 return a + b + c + d; 553 } 554 555 __bpf_kfunc int bpf_kfunc_call_test2(struct sock *sk, u32 a, u32 b) 556 { 557 return a + b; 558 } 559 560 __bpf_kfunc struct sock *bpf_kfunc_call_test3(struct sock *sk) 561 { 562 return sk; 563 } 564 565 long noinline bpf_kfunc_call_test4(signed char a, short b, int c, long d) 566 { 567 /* Provoke the compiler to assume that the caller has sign-extended a, 568 * b and c on platforms where this is required (e.g. s390x). 569 */ 570 return (long)a + (long)b + (long)c + d; 571 } 572 573 int noinline bpf_fentry_shadow_test(int a) 574 { 575 return a + 1; 576 } 577 578 struct prog_test_member1 { 579 int a; 580 }; 581 582 struct prog_test_member { 583 struct prog_test_member1 m; 584 int c; 585 }; 586 587 struct prog_test_ref_kfunc { 588 int a; 589 int b; 590 struct prog_test_member memb; 591 struct prog_test_ref_kfunc *next; 592 refcount_t cnt; 593 }; 594 595 static struct prog_test_ref_kfunc prog_test_struct = { 596 .a = 42, 597 .b = 108, 598 .next = &prog_test_struct, 599 .cnt = REFCOUNT_INIT(1), 600 }; 601 602 __bpf_kfunc struct prog_test_ref_kfunc * 603 bpf_kfunc_call_test_acquire(unsigned long *scalar_ptr) 604 { 605 refcount_inc(&prog_test_struct.cnt); 606 return &prog_test_struct; 607 } 608 609 __bpf_kfunc void bpf_kfunc_call_test_offset(struct prog_test_ref_kfunc *p) 610 { 611 WARN_ON_ONCE(1); 612 } 613 614 __bpf_kfunc struct prog_test_member * 615 bpf_kfunc_call_memb_acquire(void) 616 { 617 WARN_ON_ONCE(1); 618 return NULL; 619 } 620 621 __bpf_kfunc void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) 622 { 623 refcount_dec(&p->cnt); 624 } 625 626 __bpf_kfunc void bpf_kfunc_call_memb_release(struct prog_test_member *p) 627 { 628 } 629 630 __bpf_kfunc void bpf_kfunc_call_memb1_release(struct prog_test_member1 *p) 631 { 632 WARN_ON_ONCE(1); 633 } 634 635 static int *__bpf_kfunc_call_test_get_mem(struct prog_test_ref_kfunc *p, const int size) 636 { 637 if (size > 2 * sizeof(int)) 638 return NULL; 639 640 return (int *)p; 641 } 642 643 __bpf_kfunc int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p, 644 const int rdwr_buf_size) 645 { 646 return __bpf_kfunc_call_test_get_mem(p, rdwr_buf_size); 647 } 648 649 __bpf_kfunc int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p, 650 const int rdonly_buf_size) 651 { 652 return __bpf_kfunc_call_test_get_mem(p, rdonly_buf_size); 653 } 654 655 /* the next 2 ones can't be really used for testing expect to ensure 656 * that the verifier rejects the call. 657 * Acquire functions must return struct pointers, so these ones are 658 * failing. 659 */ 660 __bpf_kfunc int *bpf_kfunc_call_test_acq_rdonly_mem(struct prog_test_ref_kfunc *p, 661 const int rdonly_buf_size) 662 { 663 return __bpf_kfunc_call_test_get_mem(p, rdonly_buf_size); 664 } 665 666 __bpf_kfunc void bpf_kfunc_call_int_mem_release(int *p) 667 { 668 } 669 670 __bpf_kfunc struct prog_test_ref_kfunc * 671 bpf_kfunc_call_test_kptr_get(struct prog_test_ref_kfunc **pp, int a, int b) 672 { 673 struct prog_test_ref_kfunc *p = READ_ONCE(*pp); 674 675 if (!p) 676 return NULL; 677 refcount_inc(&p->cnt); 678 return p; 679 } 680 681 struct prog_test_pass1 { 682 int x0; 683 struct { 684 int x1; 685 struct { 686 int x2; 687 struct { 688 int x3; 689 }; 690 }; 691 }; 692 }; 693 694 struct prog_test_pass2 { 695 int len; 696 short arr1[4]; 697 struct { 698 char arr2[4]; 699 unsigned long arr3[8]; 700 } x; 701 }; 702 703 struct prog_test_fail1 { 704 void *p; 705 int x; 706 }; 707 708 struct prog_test_fail2 { 709 int x8; 710 struct prog_test_pass1 x; 711 }; 712 713 struct prog_test_fail3 { 714 int len; 715 char arr1[2]; 716 char arr2[]; 717 }; 718 719 __bpf_kfunc void bpf_kfunc_call_test_pass_ctx(struct __sk_buff *skb) 720 { 721 } 722 723 __bpf_kfunc void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p) 724 { 725 } 726 727 __bpf_kfunc void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) 728 { 729 } 730 731 __bpf_kfunc void bpf_kfunc_call_test_fail1(struct prog_test_fail1 *p) 732 { 733 } 734 735 __bpf_kfunc void bpf_kfunc_call_test_fail2(struct prog_test_fail2 *p) 736 { 737 } 738 739 __bpf_kfunc void bpf_kfunc_call_test_fail3(struct prog_test_fail3 *p) 740 { 741 } 742 743 __bpf_kfunc void bpf_kfunc_call_test_mem_len_pass1(void *mem, int mem__sz) 744 { 745 } 746 747 __bpf_kfunc void bpf_kfunc_call_test_mem_len_fail1(void *mem, int len) 748 { 749 } 750 751 __bpf_kfunc void bpf_kfunc_call_test_mem_len_fail2(u64 *mem, int len) 752 { 753 } 754 755 __bpf_kfunc void bpf_kfunc_call_test_ref(struct prog_test_ref_kfunc *p) 756 { 757 /* p != NULL, but p->cnt could be 0 */ 758 } 759 760 __bpf_kfunc void bpf_kfunc_call_test_destructive(void) 761 { 762 } 763 764 __bpf_kfunc static u32 bpf_kfunc_call_test_static_unused_arg(u32 arg, u32 unused) 765 { 766 return arg; 767 } 768 769 __diag_pop(); 770 771 BTF_SET8_START(bpf_test_modify_return_ids) 772 BTF_ID_FLAGS(func, bpf_modify_return_test) 773 BTF_ID_FLAGS(func, bpf_fentry_test1, KF_SLEEPABLE) 774 BTF_SET8_END(bpf_test_modify_return_ids) 775 776 static const struct btf_kfunc_id_set bpf_test_modify_return_set = { 777 .owner = THIS_MODULE, 778 .set = &bpf_test_modify_return_ids, 779 }; 780 781 BTF_SET8_START(test_sk_check_kfunc_ids) 782 BTF_ID_FLAGS(func, bpf_kfunc_call_test1) 783 BTF_ID_FLAGS(func, bpf_kfunc_call_test2) 784 BTF_ID_FLAGS(func, bpf_kfunc_call_test3) 785 BTF_ID_FLAGS(func, bpf_kfunc_call_test4) 786 BTF_ID_FLAGS(func, bpf_kfunc_call_test_acquire, KF_ACQUIRE | KF_RET_NULL) 787 BTF_ID_FLAGS(func, bpf_kfunc_call_memb_acquire, KF_ACQUIRE | KF_RET_NULL) 788 BTF_ID_FLAGS(func, bpf_kfunc_call_test_release, KF_RELEASE) 789 BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE) 790 BTF_ID_FLAGS(func, bpf_kfunc_call_memb1_release, KF_RELEASE) 791 BTF_ID_FLAGS(func, bpf_kfunc_call_test_get_rdwr_mem, KF_RET_NULL) 792 BTF_ID_FLAGS(func, bpf_kfunc_call_test_get_rdonly_mem, KF_RET_NULL) 793 BTF_ID_FLAGS(func, bpf_kfunc_call_test_acq_rdonly_mem, KF_ACQUIRE | KF_RET_NULL) 794 BTF_ID_FLAGS(func, bpf_kfunc_call_int_mem_release, KF_RELEASE) 795 BTF_ID_FLAGS(func, bpf_kfunc_call_test_kptr_get, KF_ACQUIRE | KF_RET_NULL | KF_KPTR_GET) 796 BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass_ctx) 797 BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass1) 798 BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass2) 799 BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail1) 800 BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail2) 801 BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail3) 802 BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1) 803 BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail1) 804 BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail2) 805 BTF_ID_FLAGS(func, bpf_kfunc_call_test_ref, KF_TRUSTED_ARGS | KF_RCU) 806 BTF_ID_FLAGS(func, bpf_kfunc_call_test_destructive, KF_DESTRUCTIVE) 807 BTF_ID_FLAGS(func, bpf_kfunc_call_test_static_unused_arg) 808 BTF_ID_FLAGS(func, bpf_kfunc_call_test_offset) 809 BTF_SET8_END(test_sk_check_kfunc_ids) 810 811 static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size, 812 u32 size, u32 headroom, u32 tailroom) 813 { 814 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 815 void *data; 816 817 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 818 return ERR_PTR(-EINVAL); 819 820 if (user_size > size) 821 return ERR_PTR(-EMSGSIZE); 822 823 size = SKB_DATA_ALIGN(size); 824 data = kzalloc(size + headroom + tailroom, GFP_USER); 825 if (!data) 826 return ERR_PTR(-ENOMEM); 827 828 if (copy_from_user(data + headroom, data_in, user_size)) { 829 kfree(data); 830 return ERR_PTR(-EFAULT); 831 } 832 833 return data; 834 } 835 836 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 837 const union bpf_attr *kattr, 838 union bpf_attr __user *uattr) 839 { 840 struct bpf_fentry_test_t arg = {}; 841 u16 side_effect = 0, ret = 0; 842 int b = 2, err = -EFAULT; 843 u32 retval = 0; 844 845 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 846 return -EINVAL; 847 848 switch (prog->expected_attach_type) { 849 case BPF_TRACE_FENTRY: 850 case BPF_TRACE_FEXIT: 851 if (bpf_fentry_test1(1) != 2 || 852 bpf_fentry_test2(2, 3) != 5 || 853 bpf_fentry_test3(4, 5, 6) != 15 || 854 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 || 855 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 || 856 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 || 857 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 || 858 bpf_fentry_test8(&arg) != 0) 859 goto out; 860 break; 861 case BPF_MODIFY_RETURN: 862 ret = bpf_modify_return_test(1, &b); 863 if (b != 2) 864 side_effect = 1; 865 break; 866 default: 867 goto out; 868 } 869 870 retval = ((u32)side_effect << 16) | ret; 871 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 872 goto out; 873 874 err = 0; 875 out: 876 trace_bpf_test_finish(&err); 877 return err; 878 } 879 880 struct bpf_raw_tp_test_run_info { 881 struct bpf_prog *prog; 882 void *ctx; 883 u32 retval; 884 }; 885 886 static void 887 __bpf_prog_test_run_raw_tp(void *data) 888 { 889 struct bpf_raw_tp_test_run_info *info = data; 890 891 rcu_read_lock(); 892 info->retval = bpf_prog_run(info->prog, info->ctx); 893 rcu_read_unlock(); 894 } 895 896 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, 897 const union bpf_attr *kattr, 898 union bpf_attr __user *uattr) 899 { 900 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); 901 __u32 ctx_size_in = kattr->test.ctx_size_in; 902 struct bpf_raw_tp_test_run_info info; 903 int cpu = kattr->test.cpu, err = 0; 904 int current_cpu; 905 906 /* doesn't support data_in/out, ctx_out, duration, or repeat */ 907 if (kattr->test.data_in || kattr->test.data_out || 908 kattr->test.ctx_out || kattr->test.duration || 909 kattr->test.repeat || kattr->test.batch_size) 910 return -EINVAL; 911 912 if (ctx_size_in < prog->aux->max_ctx_offset || 913 ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64)) 914 return -EINVAL; 915 916 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) 917 return -EINVAL; 918 919 if (ctx_size_in) { 920 info.ctx = memdup_user(ctx_in, ctx_size_in); 921 if (IS_ERR(info.ctx)) 922 return PTR_ERR(info.ctx); 923 } else { 924 info.ctx = NULL; 925 } 926 927 info.prog = prog; 928 929 current_cpu = get_cpu(); 930 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || 931 cpu == current_cpu) { 932 __bpf_prog_test_run_raw_tp(&info); 933 } else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) { 934 /* smp_call_function_single() also checks cpu_online() 935 * after csd_lock(). However, since cpu is from user 936 * space, let's do an extra quick check to filter out 937 * invalid value before smp_call_function_single(). 938 */ 939 err = -ENXIO; 940 } else { 941 err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp, 942 &info, 1); 943 } 944 put_cpu(); 945 946 if (!err && 947 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) 948 err = -EFAULT; 949 950 kfree(info.ctx); 951 return err; 952 } 953 954 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) 955 { 956 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); 957 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 958 u32 size = kattr->test.ctx_size_in; 959 void *data; 960 int err; 961 962 if (!data_in && !data_out) 963 return NULL; 964 965 data = kzalloc(max_size, GFP_USER); 966 if (!data) 967 return ERR_PTR(-ENOMEM); 968 969 if (data_in) { 970 err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size); 971 if (err) { 972 kfree(data); 973 return ERR_PTR(err); 974 } 975 976 size = min_t(u32, max_size, size); 977 if (copy_from_user(data, data_in, size)) { 978 kfree(data); 979 return ERR_PTR(-EFAULT); 980 } 981 } 982 return data; 983 } 984 985 static int bpf_ctx_finish(const union bpf_attr *kattr, 986 union bpf_attr __user *uattr, const void *data, 987 u32 size) 988 { 989 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 990 int err = -EFAULT; 991 u32 copy_size = size; 992 993 if (!data || !data_out) 994 return 0; 995 996 if (copy_size > kattr->test.ctx_size_out) { 997 copy_size = kattr->test.ctx_size_out; 998 err = -ENOSPC; 999 } 1000 1001 if (copy_to_user(data_out, data, copy_size)) 1002 goto out; 1003 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) 1004 goto out; 1005 if (err != -ENOSPC) 1006 err = 0; 1007 out: 1008 return err; 1009 } 1010 1011 /** 1012 * range_is_zero - test whether buffer is initialized 1013 * @buf: buffer to check 1014 * @from: check from this position 1015 * @to: check up until (excluding) this position 1016 * 1017 * This function returns true if the there is a non-zero byte 1018 * in the buf in the range [from,to). 1019 */ 1020 static inline bool range_is_zero(void *buf, size_t from, size_t to) 1021 { 1022 return !memchr_inv((u8 *)buf + from, 0, to - from); 1023 } 1024 1025 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) 1026 { 1027 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 1028 1029 if (!__skb) 1030 return 0; 1031 1032 /* make sure the fields we don't use are zeroed */ 1033 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark))) 1034 return -EINVAL; 1035 1036 /* mark is allowed */ 1037 1038 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark), 1039 offsetof(struct __sk_buff, priority))) 1040 return -EINVAL; 1041 1042 /* priority is allowed */ 1043 /* ingress_ifindex is allowed */ 1044 /* ifindex is allowed */ 1045 1046 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex), 1047 offsetof(struct __sk_buff, cb))) 1048 return -EINVAL; 1049 1050 /* cb is allowed */ 1051 1052 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb), 1053 offsetof(struct __sk_buff, tstamp))) 1054 return -EINVAL; 1055 1056 /* tstamp is allowed */ 1057 /* wire_len is allowed */ 1058 /* gso_segs is allowed */ 1059 1060 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs), 1061 offsetof(struct __sk_buff, gso_size))) 1062 return -EINVAL; 1063 1064 /* gso_size is allowed */ 1065 1066 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size), 1067 offsetof(struct __sk_buff, hwtstamp))) 1068 return -EINVAL; 1069 1070 /* hwtstamp is allowed */ 1071 1072 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp), 1073 sizeof(struct __sk_buff))) 1074 return -EINVAL; 1075 1076 skb->mark = __skb->mark; 1077 skb->priority = __skb->priority; 1078 skb->skb_iif = __skb->ingress_ifindex; 1079 skb->tstamp = __skb->tstamp; 1080 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); 1081 1082 if (__skb->wire_len == 0) { 1083 cb->pkt_len = skb->len; 1084 } else { 1085 if (__skb->wire_len < skb->len || 1086 __skb->wire_len > GSO_LEGACY_MAX_SIZE) 1087 return -EINVAL; 1088 cb->pkt_len = __skb->wire_len; 1089 } 1090 1091 if (__skb->gso_segs > GSO_MAX_SEGS) 1092 return -EINVAL; 1093 skb_shinfo(skb)->gso_segs = __skb->gso_segs; 1094 skb_shinfo(skb)->gso_size = __skb->gso_size; 1095 skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp; 1096 1097 return 0; 1098 } 1099 1100 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) 1101 { 1102 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 1103 1104 if (!__skb) 1105 return; 1106 1107 __skb->mark = skb->mark; 1108 __skb->priority = skb->priority; 1109 __skb->ingress_ifindex = skb->skb_iif; 1110 __skb->ifindex = skb->dev->ifindex; 1111 __skb->tstamp = skb->tstamp; 1112 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); 1113 __skb->wire_len = cb->pkt_len; 1114 __skb->gso_segs = skb_shinfo(skb)->gso_segs; 1115 __skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp; 1116 } 1117 1118 static struct proto bpf_dummy_proto = { 1119 .name = "bpf_dummy", 1120 .owner = THIS_MODULE, 1121 .obj_size = sizeof(struct sock), 1122 }; 1123 1124 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 1125 union bpf_attr __user *uattr) 1126 { 1127 bool is_l2 = false, is_direct_pkt_access = false; 1128 struct net *net = current->nsproxy->net_ns; 1129 struct net_device *dev = net->loopback_dev; 1130 u32 size = kattr->test.data_size_in; 1131 u32 repeat = kattr->test.repeat; 1132 struct __sk_buff *ctx = NULL; 1133 u32 retval, duration; 1134 int hh_len = ETH_HLEN; 1135 struct sk_buff *skb; 1136 struct sock *sk; 1137 void *data; 1138 int ret; 1139 1140 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 1141 return -EINVAL; 1142 1143 data = bpf_test_init(kattr, kattr->test.data_size_in, 1144 size, NET_SKB_PAD + NET_IP_ALIGN, 1145 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 1146 if (IS_ERR(data)) 1147 return PTR_ERR(data); 1148 1149 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); 1150 if (IS_ERR(ctx)) { 1151 kfree(data); 1152 return PTR_ERR(ctx); 1153 } 1154 1155 switch (prog->type) { 1156 case BPF_PROG_TYPE_SCHED_CLS: 1157 case BPF_PROG_TYPE_SCHED_ACT: 1158 is_l2 = true; 1159 fallthrough; 1160 case BPF_PROG_TYPE_LWT_IN: 1161 case BPF_PROG_TYPE_LWT_OUT: 1162 case BPF_PROG_TYPE_LWT_XMIT: 1163 is_direct_pkt_access = true; 1164 break; 1165 default: 1166 break; 1167 } 1168 1169 sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1); 1170 if (!sk) { 1171 kfree(data); 1172 kfree(ctx); 1173 return -ENOMEM; 1174 } 1175 sock_init_data(NULL, sk); 1176 1177 skb = slab_build_skb(data); 1178 if (!skb) { 1179 kfree(data); 1180 kfree(ctx); 1181 sk_free(sk); 1182 return -ENOMEM; 1183 } 1184 skb->sk = sk; 1185 1186 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 1187 __skb_put(skb, size); 1188 if (ctx && ctx->ifindex > 1) { 1189 dev = dev_get_by_index(net, ctx->ifindex); 1190 if (!dev) { 1191 ret = -ENODEV; 1192 goto out; 1193 } 1194 } 1195 skb->protocol = eth_type_trans(skb, dev); 1196 skb_reset_network_header(skb); 1197 1198 switch (skb->protocol) { 1199 case htons(ETH_P_IP): 1200 sk->sk_family = AF_INET; 1201 if (sizeof(struct iphdr) <= skb_headlen(skb)) { 1202 sk->sk_rcv_saddr = ip_hdr(skb)->saddr; 1203 sk->sk_daddr = ip_hdr(skb)->daddr; 1204 } 1205 break; 1206 #if IS_ENABLED(CONFIG_IPV6) 1207 case htons(ETH_P_IPV6): 1208 sk->sk_family = AF_INET6; 1209 if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) { 1210 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; 1211 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; 1212 } 1213 break; 1214 #endif 1215 default: 1216 break; 1217 } 1218 1219 if (is_l2) 1220 __skb_push(skb, hh_len); 1221 if (is_direct_pkt_access) 1222 bpf_compute_data_pointers(skb); 1223 ret = convert___skb_to_skb(skb, ctx); 1224 if (ret) 1225 goto out; 1226 ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false); 1227 if (ret) 1228 goto out; 1229 if (!is_l2) { 1230 if (skb_headroom(skb) < hh_len) { 1231 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 1232 1233 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 1234 ret = -ENOMEM; 1235 goto out; 1236 } 1237 } 1238 memset(__skb_push(skb, hh_len), 0, hh_len); 1239 } 1240 convert_skb_to___skb(skb, ctx); 1241 1242 size = skb->len; 1243 /* bpf program can never convert linear skb to non-linear */ 1244 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 1245 size = skb_headlen(skb); 1246 ret = bpf_test_finish(kattr, uattr, skb->data, NULL, size, retval, 1247 duration); 1248 if (!ret) 1249 ret = bpf_ctx_finish(kattr, uattr, ctx, 1250 sizeof(struct __sk_buff)); 1251 out: 1252 if (dev && dev != net->loopback_dev) 1253 dev_put(dev); 1254 kfree_skb(skb); 1255 sk_free(sk); 1256 kfree(ctx); 1257 return ret; 1258 } 1259 1260 static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp) 1261 { 1262 unsigned int ingress_ifindex, rx_queue_index; 1263 struct netdev_rx_queue *rxqueue; 1264 struct net_device *device; 1265 1266 if (!xdp_md) 1267 return 0; 1268 1269 if (xdp_md->egress_ifindex != 0) 1270 return -EINVAL; 1271 1272 ingress_ifindex = xdp_md->ingress_ifindex; 1273 rx_queue_index = xdp_md->rx_queue_index; 1274 1275 if (!ingress_ifindex && rx_queue_index) 1276 return -EINVAL; 1277 1278 if (ingress_ifindex) { 1279 device = dev_get_by_index(current->nsproxy->net_ns, 1280 ingress_ifindex); 1281 if (!device) 1282 return -ENODEV; 1283 1284 if (rx_queue_index >= device->real_num_rx_queues) 1285 goto free_dev; 1286 1287 rxqueue = __netif_get_rx_queue(device, rx_queue_index); 1288 1289 if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq)) 1290 goto free_dev; 1291 1292 xdp->rxq = &rxqueue->xdp_rxq; 1293 /* The device is now tracked in the xdp->rxq for later 1294 * dev_put() 1295 */ 1296 } 1297 1298 xdp->data = xdp->data_meta + xdp_md->data; 1299 return 0; 1300 1301 free_dev: 1302 dev_put(device); 1303 return -EINVAL; 1304 } 1305 1306 static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md) 1307 { 1308 if (!xdp_md) 1309 return; 1310 1311 xdp_md->data = xdp->data - xdp->data_meta; 1312 xdp_md->data_end = xdp->data_end - xdp->data_meta; 1313 1314 if (xdp_md->ingress_ifindex) 1315 dev_put(xdp->rxq->dev); 1316 } 1317 1318 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 1319 union bpf_attr __user *uattr) 1320 { 1321 bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES); 1322 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1323 u32 batch_size = kattr->test.batch_size; 1324 u32 retval = 0, duration, max_data_sz; 1325 u32 size = kattr->test.data_size_in; 1326 u32 headroom = XDP_PACKET_HEADROOM; 1327 u32 repeat = kattr->test.repeat; 1328 struct netdev_rx_queue *rxqueue; 1329 struct skb_shared_info *sinfo; 1330 struct xdp_buff xdp = {}; 1331 int i, ret = -EINVAL; 1332 struct xdp_md *ctx; 1333 void *data; 1334 1335 if (prog->expected_attach_type == BPF_XDP_DEVMAP || 1336 prog->expected_attach_type == BPF_XDP_CPUMAP) 1337 return -EINVAL; 1338 1339 if (kattr->test.flags & ~BPF_F_TEST_XDP_LIVE_FRAMES) 1340 return -EINVAL; 1341 1342 if (bpf_prog_is_dev_bound(prog->aux)) 1343 return -EINVAL; 1344 1345 if (do_live) { 1346 if (!batch_size) 1347 batch_size = NAPI_POLL_WEIGHT; 1348 else if (batch_size > TEST_XDP_MAX_BATCH) 1349 return -E2BIG; 1350 1351 headroom += sizeof(struct xdp_page_head); 1352 } else if (batch_size) { 1353 return -EINVAL; 1354 } 1355 1356 ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md)); 1357 if (IS_ERR(ctx)) 1358 return PTR_ERR(ctx); 1359 1360 if (ctx) { 1361 /* There can't be user provided data before the meta data */ 1362 if (ctx->data_meta || ctx->data_end != size || 1363 ctx->data > ctx->data_end || 1364 unlikely(xdp_metalen_invalid(ctx->data)) || 1365 (do_live && (kattr->test.data_out || kattr->test.ctx_out))) 1366 goto free_ctx; 1367 /* Meta data is allocated from the headroom */ 1368 headroom -= ctx->data; 1369 } 1370 1371 max_data_sz = 4096 - headroom - tailroom; 1372 if (size > max_data_sz) { 1373 /* disallow live data mode for jumbo frames */ 1374 if (do_live) 1375 goto free_ctx; 1376 size = max_data_sz; 1377 } 1378 1379 data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom); 1380 if (IS_ERR(data)) { 1381 ret = PTR_ERR(data); 1382 goto free_ctx; 1383 } 1384 1385 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 1386 rxqueue->xdp_rxq.frag_size = headroom + max_data_sz + tailroom; 1387 xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq); 1388 xdp_prepare_buff(&xdp, data, headroom, size, true); 1389 sinfo = xdp_get_shared_info_from_buff(&xdp); 1390 1391 ret = xdp_convert_md_to_buff(ctx, &xdp); 1392 if (ret) 1393 goto free_data; 1394 1395 if (unlikely(kattr->test.data_size_in > size)) { 1396 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 1397 1398 while (size < kattr->test.data_size_in) { 1399 struct page *page; 1400 skb_frag_t *frag; 1401 u32 data_len; 1402 1403 if (sinfo->nr_frags == MAX_SKB_FRAGS) { 1404 ret = -ENOMEM; 1405 goto out; 1406 } 1407 1408 page = alloc_page(GFP_KERNEL); 1409 if (!page) { 1410 ret = -ENOMEM; 1411 goto out; 1412 } 1413 1414 frag = &sinfo->frags[sinfo->nr_frags++]; 1415 __skb_frag_set_page(frag, page); 1416 1417 data_len = min_t(u32, kattr->test.data_size_in - size, 1418 PAGE_SIZE); 1419 skb_frag_size_set(frag, data_len); 1420 1421 if (copy_from_user(page_address(page), data_in + size, 1422 data_len)) { 1423 ret = -EFAULT; 1424 goto out; 1425 } 1426 sinfo->xdp_frags_size += data_len; 1427 size += data_len; 1428 } 1429 xdp_buff_set_frags_flag(&xdp); 1430 } 1431 1432 if (repeat > 1) 1433 bpf_prog_change_xdp(NULL, prog); 1434 1435 if (do_live) 1436 ret = bpf_test_run_xdp_live(prog, &xdp, repeat, batch_size, &duration); 1437 else 1438 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); 1439 /* We convert the xdp_buff back to an xdp_md before checking the return 1440 * code so the reference count of any held netdevice will be decremented 1441 * even if the test run failed. 1442 */ 1443 xdp_convert_buff_to_md(&xdp, ctx); 1444 if (ret) 1445 goto out; 1446 1447 size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size; 1448 ret = bpf_test_finish(kattr, uattr, xdp.data_meta, sinfo, size, 1449 retval, duration); 1450 if (!ret) 1451 ret = bpf_ctx_finish(kattr, uattr, ctx, 1452 sizeof(struct xdp_md)); 1453 1454 out: 1455 if (repeat > 1) 1456 bpf_prog_change_xdp(prog, NULL); 1457 free_data: 1458 for (i = 0; i < sinfo->nr_frags; i++) 1459 __free_page(skb_frag_page(&sinfo->frags[i])); 1460 kfree(data); 1461 free_ctx: 1462 kfree(ctx); 1463 return ret; 1464 } 1465 1466 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) 1467 { 1468 /* make sure the fields we don't use are zeroed */ 1469 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) 1470 return -EINVAL; 1471 1472 /* flags is allowed */ 1473 1474 if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags), 1475 sizeof(struct bpf_flow_keys))) 1476 return -EINVAL; 1477 1478 return 0; 1479 } 1480 1481 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 1482 const union bpf_attr *kattr, 1483 union bpf_attr __user *uattr) 1484 { 1485 struct bpf_test_timer t = { NO_PREEMPT }; 1486 u32 size = kattr->test.data_size_in; 1487 struct bpf_flow_dissector ctx = {}; 1488 u32 repeat = kattr->test.repeat; 1489 struct bpf_flow_keys *user_ctx; 1490 struct bpf_flow_keys flow_keys; 1491 const struct ethhdr *eth; 1492 unsigned int flags = 0; 1493 u32 retval, duration; 1494 void *data; 1495 int ret; 1496 1497 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 1498 return -EINVAL; 1499 1500 if (size < ETH_HLEN) 1501 return -EINVAL; 1502 1503 data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0); 1504 if (IS_ERR(data)) 1505 return PTR_ERR(data); 1506 1507 eth = (struct ethhdr *)data; 1508 1509 if (!repeat) 1510 repeat = 1; 1511 1512 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); 1513 if (IS_ERR(user_ctx)) { 1514 kfree(data); 1515 return PTR_ERR(user_ctx); 1516 } 1517 if (user_ctx) { 1518 ret = verify_user_bpf_flow_keys(user_ctx); 1519 if (ret) 1520 goto out; 1521 flags = user_ctx->flags; 1522 } 1523 1524 ctx.flow_keys = &flow_keys; 1525 ctx.data = data; 1526 ctx.data_end = (__u8 *)data + size; 1527 1528 bpf_test_timer_enter(&t); 1529 do { 1530 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, 1531 size, flags); 1532 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); 1533 bpf_test_timer_leave(&t); 1534 1535 if (ret < 0) 1536 goto out; 1537 1538 ret = bpf_test_finish(kattr, uattr, &flow_keys, NULL, 1539 sizeof(flow_keys), retval, duration); 1540 if (!ret) 1541 ret = bpf_ctx_finish(kattr, uattr, user_ctx, 1542 sizeof(struct bpf_flow_keys)); 1543 1544 out: 1545 kfree(user_ctx); 1546 kfree(data); 1547 return ret; 1548 } 1549 1550 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr, 1551 union bpf_attr __user *uattr) 1552 { 1553 struct bpf_test_timer t = { NO_PREEMPT }; 1554 struct bpf_prog_array *progs = NULL; 1555 struct bpf_sk_lookup_kern ctx = {}; 1556 u32 repeat = kattr->test.repeat; 1557 struct bpf_sk_lookup *user_ctx; 1558 u32 retval, duration; 1559 int ret = -EINVAL; 1560 1561 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 1562 return -EINVAL; 1563 1564 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out || 1565 kattr->test.data_size_out) 1566 return -EINVAL; 1567 1568 if (!repeat) 1569 repeat = 1; 1570 1571 user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx)); 1572 if (IS_ERR(user_ctx)) 1573 return PTR_ERR(user_ctx); 1574 1575 if (!user_ctx) 1576 return -EINVAL; 1577 1578 if (user_ctx->sk) 1579 goto out; 1580 1581 if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx))) 1582 goto out; 1583 1584 if (user_ctx->local_port > U16_MAX) { 1585 ret = -ERANGE; 1586 goto out; 1587 } 1588 1589 ctx.family = (u16)user_ctx->family; 1590 ctx.protocol = (u16)user_ctx->protocol; 1591 ctx.dport = (u16)user_ctx->local_port; 1592 ctx.sport = user_ctx->remote_port; 1593 1594 switch (ctx.family) { 1595 case AF_INET: 1596 ctx.v4.daddr = (__force __be32)user_ctx->local_ip4; 1597 ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4; 1598 break; 1599 1600 #if IS_ENABLED(CONFIG_IPV6) 1601 case AF_INET6: 1602 ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6; 1603 ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6; 1604 break; 1605 #endif 1606 1607 default: 1608 ret = -EAFNOSUPPORT; 1609 goto out; 1610 } 1611 1612 progs = bpf_prog_array_alloc(1, GFP_KERNEL); 1613 if (!progs) { 1614 ret = -ENOMEM; 1615 goto out; 1616 } 1617 1618 progs->items[0].prog = prog; 1619 1620 bpf_test_timer_enter(&t); 1621 do { 1622 ctx.selected_sk = NULL; 1623 retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run); 1624 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); 1625 bpf_test_timer_leave(&t); 1626 1627 if (ret < 0) 1628 goto out; 1629 1630 user_ctx->cookie = 0; 1631 if (ctx.selected_sk) { 1632 if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { 1633 ret = -EOPNOTSUPP; 1634 goto out; 1635 } 1636 1637 user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); 1638 } 1639 1640 ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, retval, duration); 1641 if (!ret) 1642 ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx)); 1643 1644 out: 1645 bpf_prog_array_free(progs); 1646 kfree(user_ctx); 1647 return ret; 1648 } 1649 1650 int bpf_prog_test_run_syscall(struct bpf_prog *prog, 1651 const union bpf_attr *kattr, 1652 union bpf_attr __user *uattr) 1653 { 1654 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); 1655 __u32 ctx_size_in = kattr->test.ctx_size_in; 1656 void *ctx = NULL; 1657 u32 retval; 1658 int err = 0; 1659 1660 /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */ 1661 if (kattr->test.data_in || kattr->test.data_out || 1662 kattr->test.ctx_out || kattr->test.duration || 1663 kattr->test.repeat || kattr->test.flags || 1664 kattr->test.batch_size) 1665 return -EINVAL; 1666 1667 if (ctx_size_in < prog->aux->max_ctx_offset || 1668 ctx_size_in > U16_MAX) 1669 return -EINVAL; 1670 1671 if (ctx_size_in) { 1672 ctx = memdup_user(ctx_in, ctx_size_in); 1673 if (IS_ERR(ctx)) 1674 return PTR_ERR(ctx); 1675 } 1676 1677 rcu_read_lock_trace(); 1678 retval = bpf_prog_run_pin_on_cpu(prog, ctx); 1679 rcu_read_unlock_trace(); 1680 1681 if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) { 1682 err = -EFAULT; 1683 goto out; 1684 } 1685 if (ctx_size_in) 1686 if (copy_to_user(ctx_in, ctx, ctx_size_in)) 1687 err = -EFAULT; 1688 out: 1689 kfree(ctx); 1690 return err; 1691 } 1692 1693 static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = { 1694 .owner = THIS_MODULE, 1695 .set = &test_sk_check_kfunc_ids, 1696 }; 1697 1698 BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids) 1699 BTF_ID(struct, prog_test_ref_kfunc) 1700 BTF_ID(func, bpf_kfunc_call_test_release) 1701 BTF_ID(struct, prog_test_member) 1702 BTF_ID(func, bpf_kfunc_call_memb_release) 1703 1704 static int __init bpf_prog_test_run_init(void) 1705 { 1706 const struct btf_id_dtor_kfunc bpf_prog_test_dtor_kfunc[] = { 1707 { 1708 .btf_id = bpf_prog_test_dtor_kfunc_ids[0], 1709 .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[1] 1710 }, 1711 { 1712 .btf_id = bpf_prog_test_dtor_kfunc_ids[2], 1713 .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[3], 1714 }, 1715 }; 1716 int ret; 1717 1718 ret = register_btf_fmodret_id_set(&bpf_test_modify_return_set); 1719 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_prog_test_kfunc_set); 1720 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_prog_test_kfunc_set); 1721 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &bpf_prog_test_kfunc_set); 1722 return ret ?: register_btf_id_dtor_kfuncs(bpf_prog_test_dtor_kfunc, 1723 ARRAY_SIZE(bpf_prog_test_dtor_kfunc), 1724 THIS_MODULE); 1725 } 1726 late_initcall(bpf_prog_test_run_init); 1727