1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2017 Facebook 3 */ 4 #include <linux/bpf.h> 5 #include <linux/btf.h> 6 #include <linux/btf_ids.h> 7 #include <linux/slab.h> 8 #include <linux/init.h> 9 #include <linux/vmalloc.h> 10 #include <linux/etherdevice.h> 11 #include <linux/filter.h> 12 #include <linux/rcupdate_trace.h> 13 #include <linux/sched/signal.h> 14 #include <net/bpf_sk_storage.h> 15 #include <net/sock.h> 16 #include <net/tcp.h> 17 #include <net/net_namespace.h> 18 #include <net/page_pool/helpers.h> 19 #include <linux/error-injection.h> 20 #include <linux/smp.h> 21 #include <linux/sock_diag.h> 22 #include <linux/netfilter.h> 23 #include <net/netdev_rx_queue.h> 24 #include <net/xdp.h> 25 #include <net/netfilter/nf_bpf_link.h> 26 27 #define CREATE_TRACE_POINTS 28 #include <trace/events/bpf_test_run.h> 29 30 struct bpf_test_timer { 31 enum { NO_PREEMPT, NO_MIGRATE } mode; 32 u32 i; 33 u64 time_start, time_spent; 34 }; 35 36 static void bpf_test_timer_enter(struct bpf_test_timer *t) 37 __acquires(rcu) 38 { 39 rcu_read_lock(); 40 if (t->mode == NO_PREEMPT) 41 preempt_disable(); 42 else 43 migrate_disable(); 44 45 t->time_start = ktime_get_ns(); 46 } 47 48 static void bpf_test_timer_leave(struct bpf_test_timer *t) 49 __releases(rcu) 50 { 51 t->time_start = 0; 52 53 if (t->mode == NO_PREEMPT) 54 preempt_enable(); 55 else 56 migrate_enable(); 57 rcu_read_unlock(); 58 } 59 60 static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations, 61 u32 repeat, int *err, u32 *duration) 62 __must_hold(rcu) 63 { 64 t->i += iterations; 65 if (t->i >= repeat) { 66 /* We're done. */ 67 t->time_spent += ktime_get_ns() - t->time_start; 68 do_div(t->time_spent, t->i); 69 *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent; 70 *err = 0; 71 goto reset; 72 } 73 74 if (signal_pending(current)) { 75 /* During iteration: we've been cancelled, abort. */ 76 *err = -EINTR; 77 goto reset; 78 } 79 80 if (need_resched()) { 81 /* During iteration: we need to reschedule between runs. */ 82 t->time_spent += ktime_get_ns() - t->time_start; 83 bpf_test_timer_leave(t); 84 cond_resched(); 85 bpf_test_timer_enter(t); 86 } 87 88 /* Do another round. */ 89 return true; 90 91 reset: 92 t->i = 0; 93 return false; 94 } 95 96 /* We put this struct at the head of each page with a context and frame 97 * initialised when the page is allocated, so we don't have to do this on each 98 * repetition of the test run. 99 */ 100 struct xdp_page_head { 101 struct xdp_buff orig_ctx; 102 struct xdp_buff ctx; 103 union { 104 /* ::data_hard_start starts here */ 105 DECLARE_FLEX_ARRAY(struct xdp_frame, frame); 106 DECLARE_FLEX_ARRAY(u8, data); 107 }; 108 }; 109 110 struct xdp_test_data { 111 struct xdp_buff *orig_ctx; 112 struct xdp_rxq_info rxq; 113 struct net_device *dev; 114 struct page_pool *pp; 115 struct xdp_frame **frames; 116 struct sk_buff **skbs; 117 struct xdp_mem_info mem; 118 u32 batch_size; 119 u32 frame_cnt; 120 }; 121 122 /* tools/testing/selftests/bpf/prog_tests/xdp_do_redirect.c:%MAX_PKT_SIZE 123 * must be updated accordingly this gets changed, otherwise BPF selftests 124 * will fail. 125 */ 126 #define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head)) 127 #define TEST_XDP_MAX_BATCH 256 128 129 static void xdp_test_run_init_page(struct page *page, void *arg) 130 { 131 struct xdp_page_head *head = phys_to_virt(page_to_phys(page)); 132 struct xdp_buff *new_ctx, *orig_ctx; 133 u32 headroom = XDP_PACKET_HEADROOM; 134 struct xdp_test_data *xdp = arg; 135 size_t frm_len, meta_len; 136 struct xdp_frame *frm; 137 void *data; 138 139 orig_ctx = xdp->orig_ctx; 140 frm_len = orig_ctx->data_end - orig_ctx->data_meta; 141 meta_len = orig_ctx->data - orig_ctx->data_meta; 142 headroom -= meta_len; 143 144 new_ctx = &head->ctx; 145 frm = head->frame; 146 data = head->data; 147 memcpy(data + headroom, orig_ctx->data_meta, frm_len); 148 149 xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq); 150 xdp_prepare_buff(new_ctx, data, headroom, frm_len, true); 151 new_ctx->data = new_ctx->data_meta + meta_len; 152 153 xdp_update_frame_from_buff(new_ctx, frm); 154 frm->mem = new_ctx->rxq->mem; 155 156 memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx)); 157 } 158 159 static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx) 160 { 161 struct page_pool *pp; 162 int err = -ENOMEM; 163 struct page_pool_params pp_params = { 164 .order = 0, 165 .flags = 0, 166 .pool_size = xdp->batch_size, 167 .nid = NUMA_NO_NODE, 168 .init_callback = xdp_test_run_init_page, 169 .init_arg = xdp, 170 }; 171 172 xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); 173 if (!xdp->frames) 174 return -ENOMEM; 175 176 xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); 177 if (!xdp->skbs) 178 goto err_skbs; 179 180 pp = page_pool_create(&pp_params); 181 if (IS_ERR(pp)) { 182 err = PTR_ERR(pp); 183 goto err_pp; 184 } 185 186 /* will copy 'mem.id' into pp->xdp_mem_id */ 187 err = xdp_reg_mem_model(&xdp->mem, MEM_TYPE_PAGE_POOL, pp); 188 if (err) 189 goto err_mmodel; 190 191 xdp->pp = pp; 192 193 /* We create a 'fake' RXQ referencing the original dev, but with an 194 * xdp_mem_info pointing to our page_pool 195 */ 196 xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0); 197 xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL; 198 xdp->rxq.mem.id = pp->xdp_mem_id; 199 xdp->dev = orig_ctx->rxq->dev; 200 xdp->orig_ctx = orig_ctx; 201 202 return 0; 203 204 err_mmodel: 205 page_pool_destroy(pp); 206 err_pp: 207 kvfree(xdp->skbs); 208 err_skbs: 209 kvfree(xdp->frames); 210 return err; 211 } 212 213 static void xdp_test_run_teardown(struct xdp_test_data *xdp) 214 { 215 xdp_unreg_mem_model(&xdp->mem); 216 page_pool_destroy(xdp->pp); 217 kfree(xdp->frames); 218 kfree(xdp->skbs); 219 } 220 221 static bool frame_was_changed(const struct xdp_page_head *head) 222 { 223 /* xdp_scrub_frame() zeroes the data pointer, flags is the last field, 224 * i.e. has the highest chances to be overwritten. If those two are 225 * untouched, it's most likely safe to skip the context reset. 226 */ 227 return head->frame->data != head->orig_ctx.data || 228 head->frame->flags != head->orig_ctx.flags; 229 } 230 231 static bool ctx_was_changed(struct xdp_page_head *head) 232 { 233 return head->orig_ctx.data != head->ctx.data || 234 head->orig_ctx.data_meta != head->ctx.data_meta || 235 head->orig_ctx.data_end != head->ctx.data_end; 236 } 237 238 static void reset_ctx(struct xdp_page_head *head) 239 { 240 if (likely(!frame_was_changed(head) && !ctx_was_changed(head))) 241 return; 242 243 head->ctx.data = head->orig_ctx.data; 244 head->ctx.data_meta = head->orig_ctx.data_meta; 245 head->ctx.data_end = head->orig_ctx.data_end; 246 xdp_update_frame_from_buff(&head->ctx, head->frame); 247 } 248 249 static int xdp_recv_frames(struct xdp_frame **frames, int nframes, 250 struct sk_buff **skbs, 251 struct net_device *dev) 252 { 253 gfp_t gfp = __GFP_ZERO | GFP_ATOMIC; 254 int i, n; 255 LIST_HEAD(list); 256 257 n = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, (void **)skbs); 258 if (unlikely(n == 0)) { 259 for (i = 0; i < nframes; i++) 260 xdp_return_frame(frames[i]); 261 return -ENOMEM; 262 } 263 264 for (i = 0; i < nframes; i++) { 265 struct xdp_frame *xdpf = frames[i]; 266 struct sk_buff *skb = skbs[i]; 267 268 skb = __xdp_build_skb_from_frame(xdpf, skb, dev); 269 if (!skb) { 270 xdp_return_frame(xdpf); 271 continue; 272 } 273 274 list_add_tail(&skb->list, &list); 275 } 276 netif_receive_skb_list(&list); 277 278 return 0; 279 } 280 281 static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog, 282 u32 repeat) 283 { 284 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); 285 int err = 0, act, ret, i, nframes = 0, batch_sz; 286 struct xdp_frame **frames = xdp->frames; 287 struct xdp_page_head *head; 288 struct xdp_frame *frm; 289 bool redirect = false; 290 struct xdp_buff *ctx; 291 struct page *page; 292 293 batch_sz = min_t(u32, repeat, xdp->batch_size); 294 295 local_bh_disable(); 296 xdp_set_return_frame_no_direct(); 297 298 for (i = 0; i < batch_sz; i++) { 299 page = page_pool_dev_alloc_pages(xdp->pp); 300 if (!page) { 301 err = -ENOMEM; 302 goto out; 303 } 304 305 head = phys_to_virt(page_to_phys(page)); 306 reset_ctx(head); 307 ctx = &head->ctx; 308 frm = head->frame; 309 xdp->frame_cnt++; 310 311 act = bpf_prog_run_xdp(prog, ctx); 312 313 /* if program changed pkt bounds we need to update the xdp_frame */ 314 if (unlikely(ctx_was_changed(head))) { 315 ret = xdp_update_frame_from_buff(ctx, frm); 316 if (ret) { 317 xdp_return_buff(ctx); 318 continue; 319 } 320 } 321 322 switch (act) { 323 case XDP_TX: 324 /* we can't do a real XDP_TX since we're not in the 325 * driver, so turn it into a REDIRECT back to the same 326 * index 327 */ 328 ri->tgt_index = xdp->dev->ifindex; 329 ri->map_id = INT_MAX; 330 ri->map_type = BPF_MAP_TYPE_UNSPEC; 331 fallthrough; 332 case XDP_REDIRECT: 333 redirect = true; 334 ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog); 335 if (ret) 336 xdp_return_buff(ctx); 337 break; 338 case XDP_PASS: 339 frames[nframes++] = frm; 340 break; 341 default: 342 bpf_warn_invalid_xdp_action(NULL, prog, act); 343 fallthrough; 344 case XDP_DROP: 345 xdp_return_buff(ctx); 346 break; 347 } 348 } 349 350 out: 351 if (redirect) 352 xdp_do_flush(); 353 if (nframes) { 354 ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev); 355 if (ret) 356 err = ret; 357 } 358 359 xdp_clear_return_frame_no_direct(); 360 local_bh_enable(); 361 return err; 362 } 363 364 static int bpf_test_run_xdp_live(struct bpf_prog *prog, struct xdp_buff *ctx, 365 u32 repeat, u32 batch_size, u32 *time) 366 367 { 368 struct xdp_test_data xdp = { .batch_size = batch_size }; 369 struct bpf_test_timer t = { .mode = NO_MIGRATE }; 370 int ret; 371 372 if (!repeat) 373 repeat = 1; 374 375 ret = xdp_test_run_setup(&xdp, ctx); 376 if (ret) 377 return ret; 378 379 bpf_test_timer_enter(&t); 380 do { 381 xdp.frame_cnt = 0; 382 ret = xdp_test_run_batch(&xdp, prog, repeat - t.i); 383 if (unlikely(ret < 0)) 384 break; 385 } while (bpf_test_timer_continue(&t, xdp.frame_cnt, repeat, &ret, time)); 386 bpf_test_timer_leave(&t); 387 388 xdp_test_run_teardown(&xdp); 389 return ret; 390 } 391 392 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 393 u32 *retval, u32 *time, bool xdp) 394 { 395 struct bpf_prog_array_item item = {.prog = prog}; 396 struct bpf_run_ctx *old_ctx; 397 struct bpf_cg_run_ctx run_ctx; 398 struct bpf_test_timer t = { NO_MIGRATE }; 399 enum bpf_cgroup_storage_type stype; 400 int ret; 401 402 for_each_cgroup_storage_type(stype) { 403 item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 404 if (IS_ERR(item.cgroup_storage[stype])) { 405 item.cgroup_storage[stype] = NULL; 406 for_each_cgroup_storage_type(stype) 407 bpf_cgroup_storage_free(item.cgroup_storage[stype]); 408 return -ENOMEM; 409 } 410 } 411 412 if (!repeat) 413 repeat = 1; 414 415 bpf_test_timer_enter(&t); 416 old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 417 do { 418 run_ctx.prog_item = &item; 419 local_bh_disable(); 420 if (xdp) 421 *retval = bpf_prog_run_xdp(prog, ctx); 422 else 423 *retval = bpf_prog_run(prog, ctx); 424 local_bh_enable(); 425 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, time)); 426 bpf_reset_run_ctx(old_ctx); 427 bpf_test_timer_leave(&t); 428 429 for_each_cgroup_storage_type(stype) 430 bpf_cgroup_storage_free(item.cgroup_storage[stype]); 431 432 return ret; 433 } 434 435 static int bpf_test_finish(const union bpf_attr *kattr, 436 union bpf_attr __user *uattr, const void *data, 437 struct skb_shared_info *sinfo, u32 size, 438 u32 retval, u32 duration) 439 { 440 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 441 int err = -EFAULT; 442 u32 copy_size = size; 443 444 /* Clamp copy if the user has provided a size hint, but copy the full 445 * buffer if not to retain old behaviour. 446 */ 447 if (kattr->test.data_size_out && 448 copy_size > kattr->test.data_size_out) { 449 copy_size = kattr->test.data_size_out; 450 err = -ENOSPC; 451 } 452 453 if (data_out) { 454 int len = sinfo ? copy_size - sinfo->xdp_frags_size : copy_size; 455 456 if (len < 0) { 457 err = -ENOSPC; 458 goto out; 459 } 460 461 if (copy_to_user(data_out, data, len)) 462 goto out; 463 464 if (sinfo) { 465 int i, offset = len; 466 u32 data_len; 467 468 for (i = 0; i < sinfo->nr_frags; i++) { 469 skb_frag_t *frag = &sinfo->frags[i]; 470 471 if (offset >= copy_size) { 472 err = -ENOSPC; 473 break; 474 } 475 476 data_len = min_t(u32, copy_size - offset, 477 skb_frag_size(frag)); 478 479 if (copy_to_user(data_out + offset, 480 skb_frag_address(frag), 481 data_len)) 482 goto out; 483 484 offset += data_len; 485 } 486 } 487 } 488 489 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 490 goto out; 491 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 492 goto out; 493 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 494 goto out; 495 if (err != -ENOSPC) 496 err = 0; 497 out: 498 trace_bpf_test_finish(&err); 499 return err; 500 } 501 502 /* Integer types of various sizes and pointer combinations cover variety of 503 * architecture dependent calling conventions. 7+ can be supported in the 504 * future. 505 */ 506 __bpf_kfunc_start_defs(); 507 508 __bpf_kfunc int bpf_fentry_test1(int a) 509 { 510 return a + 1; 511 } 512 EXPORT_SYMBOL_GPL(bpf_fentry_test1); 513 514 int noinline bpf_fentry_test2(int a, u64 b) 515 { 516 return a + b; 517 } 518 519 int noinline bpf_fentry_test3(char a, int b, u64 c) 520 { 521 return a + b + c; 522 } 523 524 int noinline bpf_fentry_test4(void *a, char b, int c, u64 d) 525 { 526 return (long)a + b + c + d; 527 } 528 529 int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e) 530 { 531 return a + (long)b + c + d + e; 532 } 533 534 int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f) 535 { 536 return a + (long)b + c + d + (long)e + f; 537 } 538 539 struct bpf_fentry_test_t { 540 struct bpf_fentry_test_t *a; 541 }; 542 543 int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg) 544 { 545 asm volatile ("": "+r"(arg)); 546 return (long)arg; 547 } 548 549 int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg) 550 { 551 return (long)arg->a; 552 } 553 554 __bpf_kfunc u32 bpf_fentry_test9(u32 *a) 555 { 556 return *a; 557 } 558 559 void noinline bpf_fentry_test_sinfo(struct skb_shared_info *sinfo) 560 { 561 } 562 563 __bpf_kfunc int bpf_modify_return_test(int a, int *b) 564 { 565 *b += 1; 566 return a + *b; 567 } 568 569 __bpf_kfunc int bpf_modify_return_test2(int a, int *b, short c, int d, 570 void *e, char f, int g) 571 { 572 *b += 1; 573 return a + *b + c + d + (long)e + f + g; 574 } 575 576 int noinline bpf_fentry_shadow_test(int a) 577 { 578 return a + 1; 579 } 580 581 struct prog_test_member1 { 582 int a; 583 }; 584 585 struct prog_test_member { 586 struct prog_test_member1 m; 587 int c; 588 }; 589 590 struct prog_test_ref_kfunc { 591 int a; 592 int b; 593 struct prog_test_member memb; 594 struct prog_test_ref_kfunc *next; 595 refcount_t cnt; 596 }; 597 598 __bpf_kfunc void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) 599 { 600 refcount_dec(&p->cnt); 601 } 602 603 __bpf_kfunc void bpf_kfunc_call_test_release_dtor(void *p) 604 { 605 bpf_kfunc_call_test_release(p); 606 } 607 CFI_NOSEAL(bpf_kfunc_call_test_release_dtor); 608 609 __bpf_kfunc void bpf_kfunc_call_memb_release(struct prog_test_member *p) 610 { 611 } 612 613 __bpf_kfunc void bpf_kfunc_call_memb_release_dtor(void *p) 614 { 615 } 616 CFI_NOSEAL(bpf_kfunc_call_memb_release_dtor); 617 618 __bpf_kfunc_end_defs(); 619 620 BTF_SET8_START(bpf_test_modify_return_ids) 621 BTF_ID_FLAGS(func, bpf_modify_return_test) 622 BTF_ID_FLAGS(func, bpf_modify_return_test2) 623 BTF_ID_FLAGS(func, bpf_fentry_test1, KF_SLEEPABLE) 624 BTF_SET8_END(bpf_test_modify_return_ids) 625 626 static const struct btf_kfunc_id_set bpf_test_modify_return_set = { 627 .owner = THIS_MODULE, 628 .set = &bpf_test_modify_return_ids, 629 }; 630 631 BTF_SET8_START(test_sk_check_kfunc_ids) 632 BTF_ID_FLAGS(func, bpf_kfunc_call_test_release, KF_RELEASE) 633 BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE) 634 BTF_SET8_END(test_sk_check_kfunc_ids) 635 636 static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size, 637 u32 size, u32 headroom, u32 tailroom) 638 { 639 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 640 void *data; 641 642 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 643 return ERR_PTR(-EINVAL); 644 645 if (user_size > size) 646 return ERR_PTR(-EMSGSIZE); 647 648 size = SKB_DATA_ALIGN(size); 649 data = kzalloc(size + headroom + tailroom, GFP_USER); 650 if (!data) 651 return ERR_PTR(-ENOMEM); 652 653 if (copy_from_user(data + headroom, data_in, user_size)) { 654 kfree(data); 655 return ERR_PTR(-EFAULT); 656 } 657 658 return data; 659 } 660 661 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 662 const union bpf_attr *kattr, 663 union bpf_attr __user *uattr) 664 { 665 struct bpf_fentry_test_t arg = {}; 666 u16 side_effect = 0, ret = 0; 667 int b = 2, err = -EFAULT; 668 u32 retval = 0; 669 670 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 671 return -EINVAL; 672 673 switch (prog->expected_attach_type) { 674 case BPF_TRACE_FENTRY: 675 case BPF_TRACE_FEXIT: 676 if (bpf_fentry_test1(1) != 2 || 677 bpf_fentry_test2(2, 3) != 5 || 678 bpf_fentry_test3(4, 5, 6) != 15 || 679 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 || 680 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 || 681 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 || 682 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 || 683 bpf_fentry_test8(&arg) != 0 || 684 bpf_fentry_test9(&retval) != 0) 685 goto out; 686 break; 687 case BPF_MODIFY_RETURN: 688 ret = bpf_modify_return_test(1, &b); 689 if (b != 2) 690 side_effect++; 691 b = 2; 692 ret += bpf_modify_return_test2(1, &b, 3, 4, (void *)5, 6, 7); 693 if (b != 2) 694 side_effect++; 695 break; 696 default: 697 goto out; 698 } 699 700 retval = ((u32)side_effect << 16) | ret; 701 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 702 goto out; 703 704 err = 0; 705 out: 706 trace_bpf_test_finish(&err); 707 return err; 708 } 709 710 struct bpf_raw_tp_test_run_info { 711 struct bpf_prog *prog; 712 void *ctx; 713 u32 retval; 714 }; 715 716 static void 717 __bpf_prog_test_run_raw_tp(void *data) 718 { 719 struct bpf_raw_tp_test_run_info *info = data; 720 721 rcu_read_lock(); 722 info->retval = bpf_prog_run(info->prog, info->ctx); 723 rcu_read_unlock(); 724 } 725 726 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, 727 const union bpf_attr *kattr, 728 union bpf_attr __user *uattr) 729 { 730 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); 731 __u32 ctx_size_in = kattr->test.ctx_size_in; 732 struct bpf_raw_tp_test_run_info info; 733 int cpu = kattr->test.cpu, err = 0; 734 int current_cpu; 735 736 /* doesn't support data_in/out, ctx_out, duration, or repeat */ 737 if (kattr->test.data_in || kattr->test.data_out || 738 kattr->test.ctx_out || kattr->test.duration || 739 kattr->test.repeat || kattr->test.batch_size) 740 return -EINVAL; 741 742 if (ctx_size_in < prog->aux->max_ctx_offset || 743 ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64)) 744 return -EINVAL; 745 746 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) 747 return -EINVAL; 748 749 if (ctx_size_in) { 750 info.ctx = memdup_user(ctx_in, ctx_size_in); 751 if (IS_ERR(info.ctx)) 752 return PTR_ERR(info.ctx); 753 } else { 754 info.ctx = NULL; 755 } 756 757 info.prog = prog; 758 759 current_cpu = get_cpu(); 760 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || 761 cpu == current_cpu) { 762 __bpf_prog_test_run_raw_tp(&info); 763 } else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) { 764 /* smp_call_function_single() also checks cpu_online() 765 * after csd_lock(). However, since cpu is from user 766 * space, let's do an extra quick check to filter out 767 * invalid value before smp_call_function_single(). 768 */ 769 err = -ENXIO; 770 } else { 771 err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp, 772 &info, 1); 773 } 774 put_cpu(); 775 776 if (!err && 777 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) 778 err = -EFAULT; 779 780 kfree(info.ctx); 781 return err; 782 } 783 784 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) 785 { 786 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); 787 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 788 u32 size = kattr->test.ctx_size_in; 789 void *data; 790 int err; 791 792 if (!data_in && !data_out) 793 return NULL; 794 795 data = kzalloc(max_size, GFP_USER); 796 if (!data) 797 return ERR_PTR(-ENOMEM); 798 799 if (data_in) { 800 err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size); 801 if (err) { 802 kfree(data); 803 return ERR_PTR(err); 804 } 805 806 size = min_t(u32, max_size, size); 807 if (copy_from_user(data, data_in, size)) { 808 kfree(data); 809 return ERR_PTR(-EFAULT); 810 } 811 } 812 return data; 813 } 814 815 static int bpf_ctx_finish(const union bpf_attr *kattr, 816 union bpf_attr __user *uattr, const void *data, 817 u32 size) 818 { 819 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 820 int err = -EFAULT; 821 u32 copy_size = size; 822 823 if (!data || !data_out) 824 return 0; 825 826 if (copy_size > kattr->test.ctx_size_out) { 827 copy_size = kattr->test.ctx_size_out; 828 err = -ENOSPC; 829 } 830 831 if (copy_to_user(data_out, data, copy_size)) 832 goto out; 833 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) 834 goto out; 835 if (err != -ENOSPC) 836 err = 0; 837 out: 838 return err; 839 } 840 841 /** 842 * range_is_zero - test whether buffer is initialized 843 * @buf: buffer to check 844 * @from: check from this position 845 * @to: check up until (excluding) this position 846 * 847 * This function returns true if the there is a non-zero byte 848 * in the buf in the range [from,to). 849 */ 850 static inline bool range_is_zero(void *buf, size_t from, size_t to) 851 { 852 return !memchr_inv((u8 *)buf + from, 0, to - from); 853 } 854 855 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) 856 { 857 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 858 859 if (!__skb) 860 return 0; 861 862 /* make sure the fields we don't use are zeroed */ 863 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark))) 864 return -EINVAL; 865 866 /* mark is allowed */ 867 868 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark), 869 offsetof(struct __sk_buff, priority))) 870 return -EINVAL; 871 872 /* priority is allowed */ 873 /* ingress_ifindex is allowed */ 874 /* ifindex is allowed */ 875 876 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex), 877 offsetof(struct __sk_buff, cb))) 878 return -EINVAL; 879 880 /* cb is allowed */ 881 882 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb), 883 offsetof(struct __sk_buff, tstamp))) 884 return -EINVAL; 885 886 /* tstamp is allowed */ 887 /* wire_len is allowed */ 888 /* gso_segs is allowed */ 889 890 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs), 891 offsetof(struct __sk_buff, gso_size))) 892 return -EINVAL; 893 894 /* gso_size is allowed */ 895 896 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size), 897 offsetof(struct __sk_buff, hwtstamp))) 898 return -EINVAL; 899 900 /* hwtstamp is allowed */ 901 902 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp), 903 sizeof(struct __sk_buff))) 904 return -EINVAL; 905 906 skb->mark = __skb->mark; 907 skb->priority = __skb->priority; 908 skb->skb_iif = __skb->ingress_ifindex; 909 skb->tstamp = __skb->tstamp; 910 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); 911 912 if (__skb->wire_len == 0) { 913 cb->pkt_len = skb->len; 914 } else { 915 if (__skb->wire_len < skb->len || 916 __skb->wire_len > GSO_LEGACY_MAX_SIZE) 917 return -EINVAL; 918 cb->pkt_len = __skb->wire_len; 919 } 920 921 if (__skb->gso_segs > GSO_MAX_SEGS) 922 return -EINVAL; 923 skb_shinfo(skb)->gso_segs = __skb->gso_segs; 924 skb_shinfo(skb)->gso_size = __skb->gso_size; 925 skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp; 926 927 return 0; 928 } 929 930 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) 931 { 932 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 933 934 if (!__skb) 935 return; 936 937 __skb->mark = skb->mark; 938 __skb->priority = skb->priority; 939 __skb->ingress_ifindex = skb->skb_iif; 940 __skb->ifindex = skb->dev->ifindex; 941 __skb->tstamp = skb->tstamp; 942 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); 943 __skb->wire_len = cb->pkt_len; 944 __skb->gso_segs = skb_shinfo(skb)->gso_segs; 945 __skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp; 946 } 947 948 static struct proto bpf_dummy_proto = { 949 .name = "bpf_dummy", 950 .owner = THIS_MODULE, 951 .obj_size = sizeof(struct sock), 952 }; 953 954 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 955 union bpf_attr __user *uattr) 956 { 957 bool is_l2 = false, is_direct_pkt_access = false; 958 struct net *net = current->nsproxy->net_ns; 959 struct net_device *dev = net->loopback_dev; 960 u32 size = kattr->test.data_size_in; 961 u32 repeat = kattr->test.repeat; 962 struct __sk_buff *ctx = NULL; 963 u32 retval, duration; 964 int hh_len = ETH_HLEN; 965 struct sk_buff *skb; 966 struct sock *sk; 967 void *data; 968 int ret; 969 970 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 971 return -EINVAL; 972 973 data = bpf_test_init(kattr, kattr->test.data_size_in, 974 size, NET_SKB_PAD + NET_IP_ALIGN, 975 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 976 if (IS_ERR(data)) 977 return PTR_ERR(data); 978 979 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); 980 if (IS_ERR(ctx)) { 981 kfree(data); 982 return PTR_ERR(ctx); 983 } 984 985 switch (prog->type) { 986 case BPF_PROG_TYPE_SCHED_CLS: 987 case BPF_PROG_TYPE_SCHED_ACT: 988 is_l2 = true; 989 fallthrough; 990 case BPF_PROG_TYPE_LWT_IN: 991 case BPF_PROG_TYPE_LWT_OUT: 992 case BPF_PROG_TYPE_LWT_XMIT: 993 is_direct_pkt_access = true; 994 break; 995 default: 996 break; 997 } 998 999 sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1); 1000 if (!sk) { 1001 kfree(data); 1002 kfree(ctx); 1003 return -ENOMEM; 1004 } 1005 sock_init_data(NULL, sk); 1006 1007 skb = slab_build_skb(data); 1008 if (!skb) { 1009 kfree(data); 1010 kfree(ctx); 1011 sk_free(sk); 1012 return -ENOMEM; 1013 } 1014 skb->sk = sk; 1015 1016 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 1017 __skb_put(skb, size); 1018 if (ctx && ctx->ifindex > 1) { 1019 dev = dev_get_by_index(net, ctx->ifindex); 1020 if (!dev) { 1021 ret = -ENODEV; 1022 goto out; 1023 } 1024 } 1025 skb->protocol = eth_type_trans(skb, dev); 1026 skb_reset_network_header(skb); 1027 1028 switch (skb->protocol) { 1029 case htons(ETH_P_IP): 1030 sk->sk_family = AF_INET; 1031 if (sizeof(struct iphdr) <= skb_headlen(skb)) { 1032 sk->sk_rcv_saddr = ip_hdr(skb)->saddr; 1033 sk->sk_daddr = ip_hdr(skb)->daddr; 1034 } 1035 break; 1036 #if IS_ENABLED(CONFIG_IPV6) 1037 case htons(ETH_P_IPV6): 1038 sk->sk_family = AF_INET6; 1039 if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) { 1040 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; 1041 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; 1042 } 1043 break; 1044 #endif 1045 default: 1046 break; 1047 } 1048 1049 if (is_l2) 1050 __skb_push(skb, hh_len); 1051 if (is_direct_pkt_access) 1052 bpf_compute_data_pointers(skb); 1053 ret = convert___skb_to_skb(skb, ctx); 1054 if (ret) 1055 goto out; 1056 ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false); 1057 if (ret) 1058 goto out; 1059 if (!is_l2) { 1060 if (skb_headroom(skb) < hh_len) { 1061 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 1062 1063 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 1064 ret = -ENOMEM; 1065 goto out; 1066 } 1067 } 1068 memset(__skb_push(skb, hh_len), 0, hh_len); 1069 } 1070 convert_skb_to___skb(skb, ctx); 1071 1072 size = skb->len; 1073 /* bpf program can never convert linear skb to non-linear */ 1074 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 1075 size = skb_headlen(skb); 1076 ret = bpf_test_finish(kattr, uattr, skb->data, NULL, size, retval, 1077 duration); 1078 if (!ret) 1079 ret = bpf_ctx_finish(kattr, uattr, ctx, 1080 sizeof(struct __sk_buff)); 1081 out: 1082 if (dev && dev != net->loopback_dev) 1083 dev_put(dev); 1084 kfree_skb(skb); 1085 sk_free(sk); 1086 kfree(ctx); 1087 return ret; 1088 } 1089 1090 static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp) 1091 { 1092 unsigned int ingress_ifindex, rx_queue_index; 1093 struct netdev_rx_queue *rxqueue; 1094 struct net_device *device; 1095 1096 if (!xdp_md) 1097 return 0; 1098 1099 if (xdp_md->egress_ifindex != 0) 1100 return -EINVAL; 1101 1102 ingress_ifindex = xdp_md->ingress_ifindex; 1103 rx_queue_index = xdp_md->rx_queue_index; 1104 1105 if (!ingress_ifindex && rx_queue_index) 1106 return -EINVAL; 1107 1108 if (ingress_ifindex) { 1109 device = dev_get_by_index(current->nsproxy->net_ns, 1110 ingress_ifindex); 1111 if (!device) 1112 return -ENODEV; 1113 1114 if (rx_queue_index >= device->real_num_rx_queues) 1115 goto free_dev; 1116 1117 rxqueue = __netif_get_rx_queue(device, rx_queue_index); 1118 1119 if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq)) 1120 goto free_dev; 1121 1122 xdp->rxq = &rxqueue->xdp_rxq; 1123 /* The device is now tracked in the xdp->rxq for later 1124 * dev_put() 1125 */ 1126 } 1127 1128 xdp->data = xdp->data_meta + xdp_md->data; 1129 return 0; 1130 1131 free_dev: 1132 dev_put(device); 1133 return -EINVAL; 1134 } 1135 1136 static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md) 1137 { 1138 if (!xdp_md) 1139 return; 1140 1141 xdp_md->data = xdp->data - xdp->data_meta; 1142 xdp_md->data_end = xdp->data_end - xdp->data_meta; 1143 1144 if (xdp_md->ingress_ifindex) 1145 dev_put(xdp->rxq->dev); 1146 } 1147 1148 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 1149 union bpf_attr __user *uattr) 1150 { 1151 bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES); 1152 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1153 u32 batch_size = kattr->test.batch_size; 1154 u32 retval = 0, duration, max_data_sz; 1155 u32 size = kattr->test.data_size_in; 1156 u32 headroom = XDP_PACKET_HEADROOM; 1157 u32 repeat = kattr->test.repeat; 1158 struct netdev_rx_queue *rxqueue; 1159 struct skb_shared_info *sinfo; 1160 struct xdp_buff xdp = {}; 1161 int i, ret = -EINVAL; 1162 struct xdp_md *ctx; 1163 void *data; 1164 1165 if (prog->expected_attach_type == BPF_XDP_DEVMAP || 1166 prog->expected_attach_type == BPF_XDP_CPUMAP) 1167 return -EINVAL; 1168 1169 if (kattr->test.flags & ~BPF_F_TEST_XDP_LIVE_FRAMES) 1170 return -EINVAL; 1171 1172 if (bpf_prog_is_dev_bound(prog->aux)) 1173 return -EINVAL; 1174 1175 if (do_live) { 1176 if (!batch_size) 1177 batch_size = NAPI_POLL_WEIGHT; 1178 else if (batch_size > TEST_XDP_MAX_BATCH) 1179 return -E2BIG; 1180 1181 headroom += sizeof(struct xdp_page_head); 1182 } else if (batch_size) { 1183 return -EINVAL; 1184 } 1185 1186 ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md)); 1187 if (IS_ERR(ctx)) 1188 return PTR_ERR(ctx); 1189 1190 if (ctx) { 1191 /* There can't be user provided data before the meta data */ 1192 if (ctx->data_meta || ctx->data_end != size || 1193 ctx->data > ctx->data_end || 1194 unlikely(xdp_metalen_invalid(ctx->data)) || 1195 (do_live && (kattr->test.data_out || kattr->test.ctx_out))) 1196 goto free_ctx; 1197 /* Meta data is allocated from the headroom */ 1198 headroom -= ctx->data; 1199 } 1200 1201 max_data_sz = 4096 - headroom - tailroom; 1202 if (size > max_data_sz) { 1203 /* disallow live data mode for jumbo frames */ 1204 if (do_live) 1205 goto free_ctx; 1206 size = max_data_sz; 1207 } 1208 1209 data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom); 1210 if (IS_ERR(data)) { 1211 ret = PTR_ERR(data); 1212 goto free_ctx; 1213 } 1214 1215 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 1216 rxqueue->xdp_rxq.frag_size = headroom + max_data_sz + tailroom; 1217 xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq); 1218 xdp_prepare_buff(&xdp, data, headroom, size, true); 1219 sinfo = xdp_get_shared_info_from_buff(&xdp); 1220 1221 ret = xdp_convert_md_to_buff(ctx, &xdp); 1222 if (ret) 1223 goto free_data; 1224 1225 if (unlikely(kattr->test.data_size_in > size)) { 1226 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 1227 1228 while (size < kattr->test.data_size_in) { 1229 struct page *page; 1230 skb_frag_t *frag; 1231 u32 data_len; 1232 1233 if (sinfo->nr_frags == MAX_SKB_FRAGS) { 1234 ret = -ENOMEM; 1235 goto out; 1236 } 1237 1238 page = alloc_page(GFP_KERNEL); 1239 if (!page) { 1240 ret = -ENOMEM; 1241 goto out; 1242 } 1243 1244 frag = &sinfo->frags[sinfo->nr_frags++]; 1245 1246 data_len = min_t(u32, kattr->test.data_size_in - size, 1247 PAGE_SIZE); 1248 skb_frag_fill_page_desc(frag, page, 0, data_len); 1249 1250 if (copy_from_user(page_address(page), data_in + size, 1251 data_len)) { 1252 ret = -EFAULT; 1253 goto out; 1254 } 1255 sinfo->xdp_frags_size += data_len; 1256 size += data_len; 1257 } 1258 xdp_buff_set_frags_flag(&xdp); 1259 } 1260 1261 if (repeat > 1) 1262 bpf_prog_change_xdp(NULL, prog); 1263 1264 if (do_live) 1265 ret = bpf_test_run_xdp_live(prog, &xdp, repeat, batch_size, &duration); 1266 else 1267 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); 1268 /* We convert the xdp_buff back to an xdp_md before checking the return 1269 * code so the reference count of any held netdevice will be decremented 1270 * even if the test run failed. 1271 */ 1272 xdp_convert_buff_to_md(&xdp, ctx); 1273 if (ret) 1274 goto out; 1275 1276 size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size; 1277 ret = bpf_test_finish(kattr, uattr, xdp.data_meta, sinfo, size, 1278 retval, duration); 1279 if (!ret) 1280 ret = bpf_ctx_finish(kattr, uattr, ctx, 1281 sizeof(struct xdp_md)); 1282 1283 out: 1284 if (repeat > 1) 1285 bpf_prog_change_xdp(prog, NULL); 1286 free_data: 1287 for (i = 0; i < sinfo->nr_frags; i++) 1288 __free_page(skb_frag_page(&sinfo->frags[i])); 1289 kfree(data); 1290 free_ctx: 1291 kfree(ctx); 1292 return ret; 1293 } 1294 1295 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) 1296 { 1297 /* make sure the fields we don't use are zeroed */ 1298 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) 1299 return -EINVAL; 1300 1301 /* flags is allowed */ 1302 1303 if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags), 1304 sizeof(struct bpf_flow_keys))) 1305 return -EINVAL; 1306 1307 return 0; 1308 } 1309 1310 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 1311 const union bpf_attr *kattr, 1312 union bpf_attr __user *uattr) 1313 { 1314 struct bpf_test_timer t = { NO_PREEMPT }; 1315 u32 size = kattr->test.data_size_in; 1316 struct bpf_flow_dissector ctx = {}; 1317 u32 repeat = kattr->test.repeat; 1318 struct bpf_flow_keys *user_ctx; 1319 struct bpf_flow_keys flow_keys; 1320 const struct ethhdr *eth; 1321 unsigned int flags = 0; 1322 u32 retval, duration; 1323 void *data; 1324 int ret; 1325 1326 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 1327 return -EINVAL; 1328 1329 if (size < ETH_HLEN) 1330 return -EINVAL; 1331 1332 data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0); 1333 if (IS_ERR(data)) 1334 return PTR_ERR(data); 1335 1336 eth = (struct ethhdr *)data; 1337 1338 if (!repeat) 1339 repeat = 1; 1340 1341 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); 1342 if (IS_ERR(user_ctx)) { 1343 kfree(data); 1344 return PTR_ERR(user_ctx); 1345 } 1346 if (user_ctx) { 1347 ret = verify_user_bpf_flow_keys(user_ctx); 1348 if (ret) 1349 goto out; 1350 flags = user_ctx->flags; 1351 } 1352 1353 ctx.flow_keys = &flow_keys; 1354 ctx.data = data; 1355 ctx.data_end = (__u8 *)data + size; 1356 1357 bpf_test_timer_enter(&t); 1358 do { 1359 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, 1360 size, flags); 1361 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); 1362 bpf_test_timer_leave(&t); 1363 1364 if (ret < 0) 1365 goto out; 1366 1367 ret = bpf_test_finish(kattr, uattr, &flow_keys, NULL, 1368 sizeof(flow_keys), retval, duration); 1369 if (!ret) 1370 ret = bpf_ctx_finish(kattr, uattr, user_ctx, 1371 sizeof(struct bpf_flow_keys)); 1372 1373 out: 1374 kfree(user_ctx); 1375 kfree(data); 1376 return ret; 1377 } 1378 1379 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr, 1380 union bpf_attr __user *uattr) 1381 { 1382 struct bpf_test_timer t = { NO_PREEMPT }; 1383 struct bpf_prog_array *progs = NULL; 1384 struct bpf_sk_lookup_kern ctx = {}; 1385 u32 repeat = kattr->test.repeat; 1386 struct bpf_sk_lookup *user_ctx; 1387 u32 retval, duration; 1388 int ret = -EINVAL; 1389 1390 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 1391 return -EINVAL; 1392 1393 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out || 1394 kattr->test.data_size_out) 1395 return -EINVAL; 1396 1397 if (!repeat) 1398 repeat = 1; 1399 1400 user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx)); 1401 if (IS_ERR(user_ctx)) 1402 return PTR_ERR(user_ctx); 1403 1404 if (!user_ctx) 1405 return -EINVAL; 1406 1407 if (user_ctx->sk) 1408 goto out; 1409 1410 if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx))) 1411 goto out; 1412 1413 if (user_ctx->local_port > U16_MAX) { 1414 ret = -ERANGE; 1415 goto out; 1416 } 1417 1418 ctx.family = (u16)user_ctx->family; 1419 ctx.protocol = (u16)user_ctx->protocol; 1420 ctx.dport = (u16)user_ctx->local_port; 1421 ctx.sport = user_ctx->remote_port; 1422 1423 switch (ctx.family) { 1424 case AF_INET: 1425 ctx.v4.daddr = (__force __be32)user_ctx->local_ip4; 1426 ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4; 1427 break; 1428 1429 #if IS_ENABLED(CONFIG_IPV6) 1430 case AF_INET6: 1431 ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6; 1432 ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6; 1433 break; 1434 #endif 1435 1436 default: 1437 ret = -EAFNOSUPPORT; 1438 goto out; 1439 } 1440 1441 progs = bpf_prog_array_alloc(1, GFP_KERNEL); 1442 if (!progs) { 1443 ret = -ENOMEM; 1444 goto out; 1445 } 1446 1447 progs->items[0].prog = prog; 1448 1449 bpf_test_timer_enter(&t); 1450 do { 1451 ctx.selected_sk = NULL; 1452 retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run); 1453 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); 1454 bpf_test_timer_leave(&t); 1455 1456 if (ret < 0) 1457 goto out; 1458 1459 user_ctx->cookie = 0; 1460 if (ctx.selected_sk) { 1461 if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { 1462 ret = -EOPNOTSUPP; 1463 goto out; 1464 } 1465 1466 user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); 1467 } 1468 1469 ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, retval, duration); 1470 if (!ret) 1471 ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx)); 1472 1473 out: 1474 bpf_prog_array_free(progs); 1475 kfree(user_ctx); 1476 return ret; 1477 } 1478 1479 int bpf_prog_test_run_syscall(struct bpf_prog *prog, 1480 const union bpf_attr *kattr, 1481 union bpf_attr __user *uattr) 1482 { 1483 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); 1484 __u32 ctx_size_in = kattr->test.ctx_size_in; 1485 void *ctx = NULL; 1486 u32 retval; 1487 int err = 0; 1488 1489 /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */ 1490 if (kattr->test.data_in || kattr->test.data_out || 1491 kattr->test.ctx_out || kattr->test.duration || 1492 kattr->test.repeat || kattr->test.flags || 1493 kattr->test.batch_size) 1494 return -EINVAL; 1495 1496 if (ctx_size_in < prog->aux->max_ctx_offset || 1497 ctx_size_in > U16_MAX) 1498 return -EINVAL; 1499 1500 if (ctx_size_in) { 1501 ctx = memdup_user(ctx_in, ctx_size_in); 1502 if (IS_ERR(ctx)) 1503 return PTR_ERR(ctx); 1504 } 1505 1506 rcu_read_lock_trace(); 1507 retval = bpf_prog_run_pin_on_cpu(prog, ctx); 1508 rcu_read_unlock_trace(); 1509 1510 if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) { 1511 err = -EFAULT; 1512 goto out; 1513 } 1514 if (ctx_size_in) 1515 if (copy_to_user(ctx_in, ctx, ctx_size_in)) 1516 err = -EFAULT; 1517 out: 1518 kfree(ctx); 1519 return err; 1520 } 1521 1522 static int verify_and_copy_hook_state(struct nf_hook_state *state, 1523 const struct nf_hook_state *user, 1524 struct net_device *dev) 1525 { 1526 if (user->in || user->out) 1527 return -EINVAL; 1528 1529 if (user->net || user->sk || user->okfn) 1530 return -EINVAL; 1531 1532 switch (user->pf) { 1533 case NFPROTO_IPV4: 1534 case NFPROTO_IPV6: 1535 switch (state->hook) { 1536 case NF_INET_PRE_ROUTING: 1537 state->in = dev; 1538 break; 1539 case NF_INET_LOCAL_IN: 1540 state->in = dev; 1541 break; 1542 case NF_INET_FORWARD: 1543 state->in = dev; 1544 state->out = dev; 1545 break; 1546 case NF_INET_LOCAL_OUT: 1547 state->out = dev; 1548 break; 1549 case NF_INET_POST_ROUTING: 1550 state->out = dev; 1551 break; 1552 } 1553 1554 break; 1555 default: 1556 return -EINVAL; 1557 } 1558 1559 state->pf = user->pf; 1560 state->hook = user->hook; 1561 1562 return 0; 1563 } 1564 1565 static __be16 nfproto_eth(int nfproto) 1566 { 1567 switch (nfproto) { 1568 case NFPROTO_IPV4: 1569 return htons(ETH_P_IP); 1570 case NFPROTO_IPV6: 1571 break; 1572 } 1573 1574 return htons(ETH_P_IPV6); 1575 } 1576 1577 int bpf_prog_test_run_nf(struct bpf_prog *prog, 1578 const union bpf_attr *kattr, 1579 union bpf_attr __user *uattr) 1580 { 1581 struct net *net = current->nsproxy->net_ns; 1582 struct net_device *dev = net->loopback_dev; 1583 struct nf_hook_state *user_ctx, hook_state = { 1584 .pf = NFPROTO_IPV4, 1585 .hook = NF_INET_LOCAL_OUT, 1586 }; 1587 u32 size = kattr->test.data_size_in; 1588 u32 repeat = kattr->test.repeat; 1589 struct bpf_nf_ctx ctx = { 1590 .state = &hook_state, 1591 }; 1592 struct sk_buff *skb = NULL; 1593 u32 retval, duration; 1594 void *data; 1595 int ret; 1596 1597 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) 1598 return -EINVAL; 1599 1600 if (size < sizeof(struct iphdr)) 1601 return -EINVAL; 1602 1603 data = bpf_test_init(kattr, kattr->test.data_size_in, size, 1604 NET_SKB_PAD + NET_IP_ALIGN, 1605 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 1606 if (IS_ERR(data)) 1607 return PTR_ERR(data); 1608 1609 if (!repeat) 1610 repeat = 1; 1611 1612 user_ctx = bpf_ctx_init(kattr, sizeof(struct nf_hook_state)); 1613 if (IS_ERR(user_ctx)) { 1614 kfree(data); 1615 return PTR_ERR(user_ctx); 1616 } 1617 1618 if (user_ctx) { 1619 ret = verify_and_copy_hook_state(&hook_state, user_ctx, dev); 1620 if (ret) 1621 goto out; 1622 } 1623 1624 skb = slab_build_skb(data); 1625 if (!skb) { 1626 ret = -ENOMEM; 1627 goto out; 1628 } 1629 1630 data = NULL; /* data released via kfree_skb */ 1631 1632 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 1633 __skb_put(skb, size); 1634 1635 ret = -EINVAL; 1636 1637 if (hook_state.hook != NF_INET_LOCAL_OUT) { 1638 if (size < ETH_HLEN + sizeof(struct iphdr)) 1639 goto out; 1640 1641 skb->protocol = eth_type_trans(skb, dev); 1642 switch (skb->protocol) { 1643 case htons(ETH_P_IP): 1644 if (hook_state.pf == NFPROTO_IPV4) 1645 break; 1646 goto out; 1647 case htons(ETH_P_IPV6): 1648 if (size < ETH_HLEN + sizeof(struct ipv6hdr)) 1649 goto out; 1650 if (hook_state.pf == NFPROTO_IPV6) 1651 break; 1652 goto out; 1653 default: 1654 ret = -EPROTO; 1655 goto out; 1656 } 1657 1658 skb_reset_network_header(skb); 1659 } else { 1660 skb->protocol = nfproto_eth(hook_state.pf); 1661 } 1662 1663 ctx.skb = skb; 1664 1665 ret = bpf_test_run(prog, &ctx, repeat, &retval, &duration, false); 1666 if (ret) 1667 goto out; 1668 1669 ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, retval, duration); 1670 1671 out: 1672 kfree(user_ctx); 1673 kfree_skb(skb); 1674 kfree(data); 1675 return ret; 1676 } 1677 1678 static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = { 1679 .owner = THIS_MODULE, 1680 .set = &test_sk_check_kfunc_ids, 1681 }; 1682 1683 BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids) 1684 BTF_ID(struct, prog_test_ref_kfunc) 1685 BTF_ID(func, bpf_kfunc_call_test_release_dtor) 1686 BTF_ID(struct, prog_test_member) 1687 BTF_ID(func, bpf_kfunc_call_memb_release_dtor) 1688 1689 static int __init bpf_prog_test_run_init(void) 1690 { 1691 const struct btf_id_dtor_kfunc bpf_prog_test_dtor_kfunc[] = { 1692 { 1693 .btf_id = bpf_prog_test_dtor_kfunc_ids[0], 1694 .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[1] 1695 }, 1696 { 1697 .btf_id = bpf_prog_test_dtor_kfunc_ids[2], 1698 .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[3], 1699 }, 1700 }; 1701 int ret; 1702 1703 ret = register_btf_fmodret_id_set(&bpf_test_modify_return_set); 1704 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_prog_test_kfunc_set); 1705 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_prog_test_kfunc_set); 1706 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &bpf_prog_test_kfunc_set); 1707 return ret ?: register_btf_id_dtor_kfuncs(bpf_prog_test_dtor_kfunc, 1708 ARRAY_SIZE(bpf_prog_test_dtor_kfunc), 1709 THIS_MODULE); 1710 } 1711 late_initcall(bpf_prog_test_run_init); 1712