1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2017 Facebook 3 */ 4 #include <linux/bpf.h> 5 #include <linux/btf.h> 6 #include <linux/btf_ids.h> 7 #include <linux/slab.h> 8 #include <linux/init.h> 9 #include <linux/vmalloc.h> 10 #include <linux/etherdevice.h> 11 #include <linux/filter.h> 12 #include <linux/rcupdate_trace.h> 13 #include <linux/sched/signal.h> 14 #include <net/bpf_sk_storage.h> 15 #include <net/sock.h> 16 #include <net/tcp.h> 17 #include <net/net_namespace.h> 18 #include <linux/error-injection.h> 19 #include <linux/smp.h> 20 #include <linux/sock_diag.h> 21 #include <net/xdp.h> 22 23 #define CREATE_TRACE_POINTS 24 #include <trace/events/bpf_test_run.h> 25 26 struct bpf_test_timer { 27 enum { NO_PREEMPT, NO_MIGRATE } mode; 28 u32 i; 29 u64 time_start, time_spent; 30 }; 31 32 static void bpf_test_timer_enter(struct bpf_test_timer *t) 33 __acquires(rcu) 34 { 35 rcu_read_lock(); 36 if (t->mode == NO_PREEMPT) 37 preempt_disable(); 38 else 39 migrate_disable(); 40 41 t->time_start = ktime_get_ns(); 42 } 43 44 static void bpf_test_timer_leave(struct bpf_test_timer *t) 45 __releases(rcu) 46 { 47 t->time_start = 0; 48 49 if (t->mode == NO_PREEMPT) 50 preempt_enable(); 51 else 52 migrate_enable(); 53 rcu_read_unlock(); 54 } 55 56 static bool bpf_test_timer_continue(struct bpf_test_timer *t, u32 repeat, int *err, u32 *duration) 57 __must_hold(rcu) 58 { 59 t->i++; 60 if (t->i >= repeat) { 61 /* We're done. */ 62 t->time_spent += ktime_get_ns() - t->time_start; 63 do_div(t->time_spent, t->i); 64 *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent; 65 *err = 0; 66 goto reset; 67 } 68 69 if (signal_pending(current)) { 70 /* During iteration: we've been cancelled, abort. */ 71 *err = -EINTR; 72 goto reset; 73 } 74 75 if (need_resched()) { 76 /* During iteration: we need to reschedule between runs. */ 77 t->time_spent += ktime_get_ns() - t->time_start; 78 bpf_test_timer_leave(t); 79 cond_resched(); 80 bpf_test_timer_enter(t); 81 } 82 83 /* Do another round. */ 84 return true; 85 86 reset: 87 t->i = 0; 88 return false; 89 } 90 91 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 92 u32 *retval, u32 *time, bool xdp) 93 { 94 struct bpf_prog_array_item item = {.prog = prog}; 95 struct bpf_run_ctx *old_ctx; 96 struct bpf_cg_run_ctx run_ctx; 97 struct bpf_test_timer t = { NO_MIGRATE }; 98 enum bpf_cgroup_storage_type stype; 99 int ret; 100 101 for_each_cgroup_storage_type(stype) { 102 item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 103 if (IS_ERR(item.cgroup_storage[stype])) { 104 item.cgroup_storage[stype] = NULL; 105 for_each_cgroup_storage_type(stype) 106 bpf_cgroup_storage_free(item.cgroup_storage[stype]); 107 return -ENOMEM; 108 } 109 } 110 111 if (!repeat) 112 repeat = 1; 113 114 bpf_test_timer_enter(&t); 115 old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 116 do { 117 run_ctx.prog_item = &item; 118 if (xdp) 119 *retval = bpf_prog_run_xdp(prog, ctx); 120 else 121 *retval = bpf_prog_run(prog, ctx); 122 } while (bpf_test_timer_continue(&t, repeat, &ret, time)); 123 bpf_reset_run_ctx(old_ctx); 124 bpf_test_timer_leave(&t); 125 126 for_each_cgroup_storage_type(stype) 127 bpf_cgroup_storage_free(item.cgroup_storage[stype]); 128 129 return ret; 130 } 131 132 static int bpf_test_finish(const union bpf_attr *kattr, 133 union bpf_attr __user *uattr, const void *data, 134 u32 size, u32 retval, u32 duration) 135 { 136 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 137 int err = -EFAULT; 138 u32 copy_size = size; 139 140 /* Clamp copy if the user has provided a size hint, but copy the full 141 * buffer if not to retain old behaviour. 142 */ 143 if (kattr->test.data_size_out && 144 copy_size > kattr->test.data_size_out) { 145 copy_size = kattr->test.data_size_out; 146 err = -ENOSPC; 147 } 148 149 if (data_out && copy_to_user(data_out, data, copy_size)) 150 goto out; 151 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 152 goto out; 153 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 154 goto out; 155 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 156 goto out; 157 if (err != -ENOSPC) 158 err = 0; 159 out: 160 trace_bpf_test_finish(&err); 161 return err; 162 } 163 164 /* Integer types of various sizes and pointer combinations cover variety of 165 * architecture dependent calling conventions. 7+ can be supported in the 166 * future. 167 */ 168 __diag_push(); 169 __diag_ignore(GCC, 8, "-Wmissing-prototypes", 170 "Global functions as their definitions will be in vmlinux BTF"); 171 int noinline bpf_fentry_test1(int a) 172 { 173 return a + 1; 174 } 175 176 int noinline bpf_fentry_test2(int a, u64 b) 177 { 178 return a + b; 179 } 180 181 int noinline bpf_fentry_test3(char a, int b, u64 c) 182 { 183 return a + b + c; 184 } 185 186 int noinline bpf_fentry_test4(void *a, char b, int c, u64 d) 187 { 188 return (long)a + b + c + d; 189 } 190 191 int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e) 192 { 193 return a + (long)b + c + d + e; 194 } 195 196 int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f) 197 { 198 return a + (long)b + c + d + (long)e + f; 199 } 200 201 struct bpf_fentry_test_t { 202 struct bpf_fentry_test_t *a; 203 }; 204 205 int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg) 206 { 207 return (long)arg; 208 } 209 210 int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg) 211 { 212 return (long)arg->a; 213 } 214 215 int noinline bpf_modify_return_test(int a, int *b) 216 { 217 *b += 1; 218 return a + *b; 219 } 220 221 u64 noinline bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d) 222 { 223 return a + b + c + d; 224 } 225 226 int noinline bpf_kfunc_call_test2(struct sock *sk, u32 a, u32 b) 227 { 228 return a + b; 229 } 230 231 struct sock * noinline bpf_kfunc_call_test3(struct sock *sk) 232 { 233 return sk; 234 } 235 236 struct prog_test_ref_kfunc { 237 int a; 238 int b; 239 struct prog_test_ref_kfunc *next; 240 }; 241 242 static struct prog_test_ref_kfunc prog_test_struct = { 243 .a = 42, 244 .b = 108, 245 .next = &prog_test_struct, 246 }; 247 248 noinline struct prog_test_ref_kfunc * 249 bpf_kfunc_call_test_acquire(unsigned long *scalar_ptr) 250 { 251 /* randomly return NULL */ 252 if (get_jiffies_64() % 2) 253 return NULL; 254 return &prog_test_struct; 255 } 256 257 noinline void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) 258 { 259 } 260 261 struct prog_test_pass1 { 262 int x0; 263 struct { 264 int x1; 265 struct { 266 int x2; 267 struct { 268 int x3; 269 }; 270 }; 271 }; 272 }; 273 274 struct prog_test_pass2 { 275 int len; 276 short arr1[4]; 277 struct { 278 char arr2[4]; 279 unsigned long arr3[8]; 280 } x; 281 }; 282 283 struct prog_test_fail1 { 284 void *p; 285 int x; 286 }; 287 288 struct prog_test_fail2 { 289 int x8; 290 struct prog_test_pass1 x; 291 }; 292 293 struct prog_test_fail3 { 294 int len; 295 char arr1[2]; 296 char arr2[0]; 297 }; 298 299 noinline void bpf_kfunc_call_test_pass_ctx(struct __sk_buff *skb) 300 { 301 } 302 303 noinline void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p) 304 { 305 } 306 307 noinline void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) 308 { 309 } 310 311 noinline void bpf_kfunc_call_test_fail1(struct prog_test_fail1 *p) 312 { 313 } 314 315 noinline void bpf_kfunc_call_test_fail2(struct prog_test_fail2 *p) 316 { 317 } 318 319 noinline void bpf_kfunc_call_test_fail3(struct prog_test_fail3 *p) 320 { 321 } 322 323 noinline void bpf_kfunc_call_test_mem_len_pass1(void *mem, int mem__sz) 324 { 325 } 326 327 noinline void bpf_kfunc_call_test_mem_len_fail1(void *mem, int len) 328 { 329 } 330 331 noinline void bpf_kfunc_call_test_mem_len_fail2(u64 *mem, int len) 332 { 333 } 334 335 __diag_pop(); 336 337 ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO); 338 339 BTF_SET_START(test_sk_check_kfunc_ids) 340 BTF_ID(func, bpf_kfunc_call_test1) 341 BTF_ID(func, bpf_kfunc_call_test2) 342 BTF_ID(func, bpf_kfunc_call_test3) 343 BTF_ID(func, bpf_kfunc_call_test_acquire) 344 BTF_ID(func, bpf_kfunc_call_test_release) 345 BTF_ID(func, bpf_kfunc_call_test_pass_ctx) 346 BTF_ID(func, bpf_kfunc_call_test_pass1) 347 BTF_ID(func, bpf_kfunc_call_test_pass2) 348 BTF_ID(func, bpf_kfunc_call_test_fail1) 349 BTF_ID(func, bpf_kfunc_call_test_fail2) 350 BTF_ID(func, bpf_kfunc_call_test_fail3) 351 BTF_ID(func, bpf_kfunc_call_test_mem_len_pass1) 352 BTF_ID(func, bpf_kfunc_call_test_mem_len_fail1) 353 BTF_ID(func, bpf_kfunc_call_test_mem_len_fail2) 354 BTF_SET_END(test_sk_check_kfunc_ids) 355 356 BTF_SET_START(test_sk_acquire_kfunc_ids) 357 BTF_ID(func, bpf_kfunc_call_test_acquire) 358 BTF_SET_END(test_sk_acquire_kfunc_ids) 359 360 BTF_SET_START(test_sk_release_kfunc_ids) 361 BTF_ID(func, bpf_kfunc_call_test_release) 362 BTF_SET_END(test_sk_release_kfunc_ids) 363 364 BTF_SET_START(test_sk_ret_null_kfunc_ids) 365 BTF_ID(func, bpf_kfunc_call_test_acquire) 366 BTF_SET_END(test_sk_ret_null_kfunc_ids) 367 368 static void *bpf_test_init(const union bpf_attr *kattr, u32 size, 369 u32 headroom, u32 tailroom) 370 { 371 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 372 u32 user_size = kattr->test.data_size_in; 373 void *data; 374 375 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 376 return ERR_PTR(-EINVAL); 377 378 if (user_size > size) 379 return ERR_PTR(-EMSGSIZE); 380 381 data = kzalloc(size + headroom + tailroom, GFP_USER); 382 if (!data) 383 return ERR_PTR(-ENOMEM); 384 385 if (copy_from_user(data + headroom, data_in, user_size)) { 386 kfree(data); 387 return ERR_PTR(-EFAULT); 388 } 389 390 return data; 391 } 392 393 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 394 const union bpf_attr *kattr, 395 union bpf_attr __user *uattr) 396 { 397 struct bpf_fentry_test_t arg = {}; 398 u16 side_effect = 0, ret = 0; 399 int b = 2, err = -EFAULT; 400 u32 retval = 0; 401 402 if (kattr->test.flags || kattr->test.cpu) 403 return -EINVAL; 404 405 switch (prog->expected_attach_type) { 406 case BPF_TRACE_FENTRY: 407 case BPF_TRACE_FEXIT: 408 if (bpf_fentry_test1(1) != 2 || 409 bpf_fentry_test2(2, 3) != 5 || 410 bpf_fentry_test3(4, 5, 6) != 15 || 411 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 || 412 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 || 413 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 || 414 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 || 415 bpf_fentry_test8(&arg) != 0) 416 goto out; 417 break; 418 case BPF_MODIFY_RETURN: 419 ret = bpf_modify_return_test(1, &b); 420 if (b != 2) 421 side_effect = 1; 422 break; 423 default: 424 goto out; 425 } 426 427 retval = ((u32)side_effect << 16) | ret; 428 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 429 goto out; 430 431 err = 0; 432 out: 433 trace_bpf_test_finish(&err); 434 return err; 435 } 436 437 struct bpf_raw_tp_test_run_info { 438 struct bpf_prog *prog; 439 void *ctx; 440 u32 retval; 441 }; 442 443 static void 444 __bpf_prog_test_run_raw_tp(void *data) 445 { 446 struct bpf_raw_tp_test_run_info *info = data; 447 448 rcu_read_lock(); 449 info->retval = bpf_prog_run(info->prog, info->ctx); 450 rcu_read_unlock(); 451 } 452 453 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, 454 const union bpf_attr *kattr, 455 union bpf_attr __user *uattr) 456 { 457 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); 458 __u32 ctx_size_in = kattr->test.ctx_size_in; 459 struct bpf_raw_tp_test_run_info info; 460 int cpu = kattr->test.cpu, err = 0; 461 int current_cpu; 462 463 /* doesn't support data_in/out, ctx_out, duration, or repeat */ 464 if (kattr->test.data_in || kattr->test.data_out || 465 kattr->test.ctx_out || kattr->test.duration || 466 kattr->test.repeat) 467 return -EINVAL; 468 469 if (ctx_size_in < prog->aux->max_ctx_offset || 470 ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64)) 471 return -EINVAL; 472 473 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) 474 return -EINVAL; 475 476 if (ctx_size_in) { 477 info.ctx = memdup_user(ctx_in, ctx_size_in); 478 if (IS_ERR(info.ctx)) 479 return PTR_ERR(info.ctx); 480 } else { 481 info.ctx = NULL; 482 } 483 484 info.prog = prog; 485 486 current_cpu = get_cpu(); 487 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || 488 cpu == current_cpu) { 489 __bpf_prog_test_run_raw_tp(&info); 490 } else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) { 491 /* smp_call_function_single() also checks cpu_online() 492 * after csd_lock(). However, since cpu is from user 493 * space, let's do an extra quick check to filter out 494 * invalid value before smp_call_function_single(). 495 */ 496 err = -ENXIO; 497 } else { 498 err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp, 499 &info, 1); 500 } 501 put_cpu(); 502 503 if (!err && 504 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) 505 err = -EFAULT; 506 507 kfree(info.ctx); 508 return err; 509 } 510 511 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) 512 { 513 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); 514 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 515 u32 size = kattr->test.ctx_size_in; 516 void *data; 517 int err; 518 519 if (!data_in && !data_out) 520 return NULL; 521 522 data = kzalloc(max_size, GFP_USER); 523 if (!data) 524 return ERR_PTR(-ENOMEM); 525 526 if (data_in) { 527 err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size); 528 if (err) { 529 kfree(data); 530 return ERR_PTR(err); 531 } 532 533 size = min_t(u32, max_size, size); 534 if (copy_from_user(data, data_in, size)) { 535 kfree(data); 536 return ERR_PTR(-EFAULT); 537 } 538 } 539 return data; 540 } 541 542 static int bpf_ctx_finish(const union bpf_attr *kattr, 543 union bpf_attr __user *uattr, const void *data, 544 u32 size) 545 { 546 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 547 int err = -EFAULT; 548 u32 copy_size = size; 549 550 if (!data || !data_out) 551 return 0; 552 553 if (copy_size > kattr->test.ctx_size_out) { 554 copy_size = kattr->test.ctx_size_out; 555 err = -ENOSPC; 556 } 557 558 if (copy_to_user(data_out, data, copy_size)) 559 goto out; 560 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) 561 goto out; 562 if (err != -ENOSPC) 563 err = 0; 564 out: 565 return err; 566 } 567 568 /** 569 * range_is_zero - test whether buffer is initialized 570 * @buf: buffer to check 571 * @from: check from this position 572 * @to: check up until (excluding) this position 573 * 574 * This function returns true if the there is a non-zero byte 575 * in the buf in the range [from,to). 576 */ 577 static inline bool range_is_zero(void *buf, size_t from, size_t to) 578 { 579 return !memchr_inv((u8 *)buf + from, 0, to - from); 580 } 581 582 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) 583 { 584 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 585 586 if (!__skb) 587 return 0; 588 589 /* make sure the fields we don't use are zeroed */ 590 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark))) 591 return -EINVAL; 592 593 /* mark is allowed */ 594 595 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark), 596 offsetof(struct __sk_buff, priority))) 597 return -EINVAL; 598 599 /* priority is allowed */ 600 /* ingress_ifindex is allowed */ 601 /* ifindex is allowed */ 602 603 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex), 604 offsetof(struct __sk_buff, cb))) 605 return -EINVAL; 606 607 /* cb is allowed */ 608 609 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb), 610 offsetof(struct __sk_buff, tstamp))) 611 return -EINVAL; 612 613 /* tstamp is allowed */ 614 /* wire_len is allowed */ 615 /* gso_segs is allowed */ 616 617 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs), 618 offsetof(struct __sk_buff, gso_size))) 619 return -EINVAL; 620 621 /* gso_size is allowed */ 622 623 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size), 624 offsetof(struct __sk_buff, hwtstamp))) 625 return -EINVAL; 626 627 /* hwtstamp is allowed */ 628 629 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp), 630 sizeof(struct __sk_buff))) 631 return -EINVAL; 632 633 skb->mark = __skb->mark; 634 skb->priority = __skb->priority; 635 skb->skb_iif = __skb->ingress_ifindex; 636 skb->tstamp = __skb->tstamp; 637 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); 638 639 if (__skb->wire_len == 0) { 640 cb->pkt_len = skb->len; 641 } else { 642 if (__skb->wire_len < skb->len || 643 __skb->wire_len > GSO_MAX_SIZE) 644 return -EINVAL; 645 cb->pkt_len = __skb->wire_len; 646 } 647 648 if (__skb->gso_segs > GSO_MAX_SEGS) 649 return -EINVAL; 650 skb_shinfo(skb)->gso_segs = __skb->gso_segs; 651 skb_shinfo(skb)->gso_size = __skb->gso_size; 652 skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp; 653 654 return 0; 655 } 656 657 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) 658 { 659 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 660 661 if (!__skb) 662 return; 663 664 __skb->mark = skb->mark; 665 __skb->priority = skb->priority; 666 __skb->ingress_ifindex = skb->skb_iif; 667 __skb->ifindex = skb->dev->ifindex; 668 __skb->tstamp = skb->tstamp; 669 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); 670 __skb->wire_len = cb->pkt_len; 671 __skb->gso_segs = skb_shinfo(skb)->gso_segs; 672 __skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp; 673 } 674 675 static struct proto bpf_dummy_proto = { 676 .name = "bpf_dummy", 677 .owner = THIS_MODULE, 678 .obj_size = sizeof(struct sock), 679 }; 680 681 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 682 union bpf_attr __user *uattr) 683 { 684 bool is_l2 = false, is_direct_pkt_access = false; 685 struct net *net = current->nsproxy->net_ns; 686 struct net_device *dev = net->loopback_dev; 687 u32 size = kattr->test.data_size_in; 688 u32 repeat = kattr->test.repeat; 689 struct __sk_buff *ctx = NULL; 690 u32 retval, duration; 691 int hh_len = ETH_HLEN; 692 struct sk_buff *skb; 693 struct sock *sk; 694 void *data; 695 int ret; 696 697 if (kattr->test.flags || kattr->test.cpu) 698 return -EINVAL; 699 700 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 701 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 702 if (IS_ERR(data)) 703 return PTR_ERR(data); 704 705 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); 706 if (IS_ERR(ctx)) { 707 kfree(data); 708 return PTR_ERR(ctx); 709 } 710 711 switch (prog->type) { 712 case BPF_PROG_TYPE_SCHED_CLS: 713 case BPF_PROG_TYPE_SCHED_ACT: 714 is_l2 = true; 715 fallthrough; 716 case BPF_PROG_TYPE_LWT_IN: 717 case BPF_PROG_TYPE_LWT_OUT: 718 case BPF_PROG_TYPE_LWT_XMIT: 719 is_direct_pkt_access = true; 720 break; 721 default: 722 break; 723 } 724 725 sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1); 726 if (!sk) { 727 kfree(data); 728 kfree(ctx); 729 return -ENOMEM; 730 } 731 sock_init_data(NULL, sk); 732 733 skb = build_skb(data, 0); 734 if (!skb) { 735 kfree(data); 736 kfree(ctx); 737 sk_free(sk); 738 return -ENOMEM; 739 } 740 skb->sk = sk; 741 742 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 743 __skb_put(skb, size); 744 if (ctx && ctx->ifindex > 1) { 745 dev = dev_get_by_index(net, ctx->ifindex); 746 if (!dev) { 747 ret = -ENODEV; 748 goto out; 749 } 750 } 751 skb->protocol = eth_type_trans(skb, dev); 752 skb_reset_network_header(skb); 753 754 switch (skb->protocol) { 755 case htons(ETH_P_IP): 756 sk->sk_family = AF_INET; 757 if (sizeof(struct iphdr) <= skb_headlen(skb)) { 758 sk->sk_rcv_saddr = ip_hdr(skb)->saddr; 759 sk->sk_daddr = ip_hdr(skb)->daddr; 760 } 761 break; 762 #if IS_ENABLED(CONFIG_IPV6) 763 case htons(ETH_P_IPV6): 764 sk->sk_family = AF_INET6; 765 if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) { 766 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; 767 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; 768 } 769 break; 770 #endif 771 default: 772 break; 773 } 774 775 if (is_l2) 776 __skb_push(skb, hh_len); 777 if (is_direct_pkt_access) 778 bpf_compute_data_pointers(skb); 779 ret = convert___skb_to_skb(skb, ctx); 780 if (ret) 781 goto out; 782 ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false); 783 if (ret) 784 goto out; 785 if (!is_l2) { 786 if (skb_headroom(skb) < hh_len) { 787 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 788 789 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 790 ret = -ENOMEM; 791 goto out; 792 } 793 } 794 memset(__skb_push(skb, hh_len), 0, hh_len); 795 } 796 convert_skb_to___skb(skb, ctx); 797 798 size = skb->len; 799 /* bpf program can never convert linear skb to non-linear */ 800 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 801 size = skb_headlen(skb); 802 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); 803 if (!ret) 804 ret = bpf_ctx_finish(kattr, uattr, ctx, 805 sizeof(struct __sk_buff)); 806 out: 807 if (dev && dev != net->loopback_dev) 808 dev_put(dev); 809 kfree_skb(skb); 810 sk_free(sk); 811 kfree(ctx); 812 return ret; 813 } 814 815 static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp) 816 { 817 unsigned int ingress_ifindex, rx_queue_index; 818 struct netdev_rx_queue *rxqueue; 819 struct net_device *device; 820 821 if (!xdp_md) 822 return 0; 823 824 if (xdp_md->egress_ifindex != 0) 825 return -EINVAL; 826 827 ingress_ifindex = xdp_md->ingress_ifindex; 828 rx_queue_index = xdp_md->rx_queue_index; 829 830 if (!ingress_ifindex && rx_queue_index) 831 return -EINVAL; 832 833 if (ingress_ifindex) { 834 device = dev_get_by_index(current->nsproxy->net_ns, 835 ingress_ifindex); 836 if (!device) 837 return -ENODEV; 838 839 if (rx_queue_index >= device->real_num_rx_queues) 840 goto free_dev; 841 842 rxqueue = __netif_get_rx_queue(device, rx_queue_index); 843 844 if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq)) 845 goto free_dev; 846 847 xdp->rxq = &rxqueue->xdp_rxq; 848 /* The device is now tracked in the xdp->rxq for later 849 * dev_put() 850 */ 851 } 852 853 xdp->data = xdp->data_meta + xdp_md->data; 854 return 0; 855 856 free_dev: 857 dev_put(device); 858 return -EINVAL; 859 } 860 861 static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md) 862 { 863 if (!xdp_md) 864 return; 865 866 xdp_md->data = xdp->data - xdp->data_meta; 867 xdp_md->data_end = xdp->data_end - xdp->data_meta; 868 869 if (xdp_md->ingress_ifindex) 870 dev_put(xdp->rxq->dev); 871 } 872 873 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 874 union bpf_attr __user *uattr) 875 { 876 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 877 u32 headroom = XDP_PACKET_HEADROOM; 878 u32 size = kattr->test.data_size_in; 879 u32 repeat = kattr->test.repeat; 880 struct netdev_rx_queue *rxqueue; 881 struct xdp_buff xdp = {}; 882 u32 retval, duration; 883 struct xdp_md *ctx; 884 u32 max_data_sz; 885 void *data; 886 int ret = -EINVAL; 887 888 if (prog->expected_attach_type == BPF_XDP_DEVMAP || 889 prog->expected_attach_type == BPF_XDP_CPUMAP) 890 return -EINVAL; 891 892 ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md)); 893 if (IS_ERR(ctx)) 894 return PTR_ERR(ctx); 895 896 if (ctx) { 897 /* There can't be user provided data before the meta data */ 898 if (ctx->data_meta || ctx->data_end != size || 899 ctx->data > ctx->data_end || 900 unlikely(xdp_metalen_invalid(ctx->data))) 901 goto free_ctx; 902 /* Meta data is allocated from the headroom */ 903 headroom -= ctx->data; 904 } 905 906 /* XDP have extra tailroom as (most) drivers use full page */ 907 max_data_sz = 4096 - headroom - tailroom; 908 909 data = bpf_test_init(kattr, max_data_sz, headroom, tailroom); 910 if (IS_ERR(data)) { 911 ret = PTR_ERR(data); 912 goto free_ctx; 913 } 914 915 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 916 xdp_init_buff(&xdp, headroom + max_data_sz + tailroom, 917 &rxqueue->xdp_rxq); 918 xdp_prepare_buff(&xdp, data, headroom, size, true); 919 920 ret = xdp_convert_md_to_buff(ctx, &xdp); 921 if (ret) 922 goto free_data; 923 924 if (repeat > 1) 925 bpf_prog_change_xdp(NULL, prog); 926 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); 927 /* We convert the xdp_buff back to an xdp_md before checking the return 928 * code so the reference count of any held netdevice will be decremented 929 * even if the test run failed. 930 */ 931 xdp_convert_buff_to_md(&xdp, ctx); 932 if (ret) 933 goto out; 934 935 if (xdp.data_meta != data + headroom || 936 xdp.data_end != xdp.data_meta + size) 937 size = xdp.data_end - xdp.data_meta; 938 939 ret = bpf_test_finish(kattr, uattr, xdp.data_meta, size, retval, 940 duration); 941 if (!ret) 942 ret = bpf_ctx_finish(kattr, uattr, ctx, 943 sizeof(struct xdp_md)); 944 945 out: 946 if (repeat > 1) 947 bpf_prog_change_xdp(prog, NULL); 948 free_data: 949 kfree(data); 950 free_ctx: 951 kfree(ctx); 952 return ret; 953 } 954 955 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) 956 { 957 /* make sure the fields we don't use are zeroed */ 958 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) 959 return -EINVAL; 960 961 /* flags is allowed */ 962 963 if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags), 964 sizeof(struct bpf_flow_keys))) 965 return -EINVAL; 966 967 return 0; 968 } 969 970 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 971 const union bpf_attr *kattr, 972 union bpf_attr __user *uattr) 973 { 974 struct bpf_test_timer t = { NO_PREEMPT }; 975 u32 size = kattr->test.data_size_in; 976 struct bpf_flow_dissector ctx = {}; 977 u32 repeat = kattr->test.repeat; 978 struct bpf_flow_keys *user_ctx; 979 struct bpf_flow_keys flow_keys; 980 const struct ethhdr *eth; 981 unsigned int flags = 0; 982 u32 retval, duration; 983 void *data; 984 int ret; 985 986 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) 987 return -EINVAL; 988 989 if (kattr->test.flags || kattr->test.cpu) 990 return -EINVAL; 991 992 if (size < ETH_HLEN) 993 return -EINVAL; 994 995 data = bpf_test_init(kattr, size, 0, 0); 996 if (IS_ERR(data)) 997 return PTR_ERR(data); 998 999 eth = (struct ethhdr *)data; 1000 1001 if (!repeat) 1002 repeat = 1; 1003 1004 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); 1005 if (IS_ERR(user_ctx)) { 1006 kfree(data); 1007 return PTR_ERR(user_ctx); 1008 } 1009 if (user_ctx) { 1010 ret = verify_user_bpf_flow_keys(user_ctx); 1011 if (ret) 1012 goto out; 1013 flags = user_ctx->flags; 1014 } 1015 1016 ctx.flow_keys = &flow_keys; 1017 ctx.data = data; 1018 ctx.data_end = (__u8 *)data + size; 1019 1020 bpf_test_timer_enter(&t); 1021 do { 1022 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, 1023 size, flags); 1024 } while (bpf_test_timer_continue(&t, repeat, &ret, &duration)); 1025 bpf_test_timer_leave(&t); 1026 1027 if (ret < 0) 1028 goto out; 1029 1030 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), 1031 retval, duration); 1032 if (!ret) 1033 ret = bpf_ctx_finish(kattr, uattr, user_ctx, 1034 sizeof(struct bpf_flow_keys)); 1035 1036 out: 1037 kfree(user_ctx); 1038 kfree(data); 1039 return ret; 1040 } 1041 1042 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr, 1043 union bpf_attr __user *uattr) 1044 { 1045 struct bpf_test_timer t = { NO_PREEMPT }; 1046 struct bpf_prog_array *progs = NULL; 1047 struct bpf_sk_lookup_kern ctx = {}; 1048 u32 repeat = kattr->test.repeat; 1049 struct bpf_sk_lookup *user_ctx; 1050 u32 retval, duration; 1051 int ret = -EINVAL; 1052 1053 if (prog->type != BPF_PROG_TYPE_SK_LOOKUP) 1054 return -EINVAL; 1055 1056 if (kattr->test.flags || kattr->test.cpu) 1057 return -EINVAL; 1058 1059 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out || 1060 kattr->test.data_size_out) 1061 return -EINVAL; 1062 1063 if (!repeat) 1064 repeat = 1; 1065 1066 user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx)); 1067 if (IS_ERR(user_ctx)) 1068 return PTR_ERR(user_ctx); 1069 1070 if (!user_ctx) 1071 return -EINVAL; 1072 1073 if (user_ctx->sk) 1074 goto out; 1075 1076 if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx))) 1077 goto out; 1078 1079 if (user_ctx->local_port > U16_MAX || user_ctx->remote_port > U16_MAX) { 1080 ret = -ERANGE; 1081 goto out; 1082 } 1083 1084 ctx.family = (u16)user_ctx->family; 1085 ctx.protocol = (u16)user_ctx->protocol; 1086 ctx.dport = (u16)user_ctx->local_port; 1087 ctx.sport = (__force __be16)user_ctx->remote_port; 1088 1089 switch (ctx.family) { 1090 case AF_INET: 1091 ctx.v4.daddr = (__force __be32)user_ctx->local_ip4; 1092 ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4; 1093 break; 1094 1095 #if IS_ENABLED(CONFIG_IPV6) 1096 case AF_INET6: 1097 ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6; 1098 ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6; 1099 break; 1100 #endif 1101 1102 default: 1103 ret = -EAFNOSUPPORT; 1104 goto out; 1105 } 1106 1107 progs = bpf_prog_array_alloc(1, GFP_KERNEL); 1108 if (!progs) { 1109 ret = -ENOMEM; 1110 goto out; 1111 } 1112 1113 progs->items[0].prog = prog; 1114 1115 bpf_test_timer_enter(&t); 1116 do { 1117 ctx.selected_sk = NULL; 1118 retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run); 1119 } while (bpf_test_timer_continue(&t, repeat, &ret, &duration)); 1120 bpf_test_timer_leave(&t); 1121 1122 if (ret < 0) 1123 goto out; 1124 1125 user_ctx->cookie = 0; 1126 if (ctx.selected_sk) { 1127 if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { 1128 ret = -EOPNOTSUPP; 1129 goto out; 1130 } 1131 1132 user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); 1133 } 1134 1135 ret = bpf_test_finish(kattr, uattr, NULL, 0, retval, duration); 1136 if (!ret) 1137 ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx)); 1138 1139 out: 1140 bpf_prog_array_free(progs); 1141 kfree(user_ctx); 1142 return ret; 1143 } 1144 1145 int bpf_prog_test_run_syscall(struct bpf_prog *prog, 1146 const union bpf_attr *kattr, 1147 union bpf_attr __user *uattr) 1148 { 1149 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); 1150 __u32 ctx_size_in = kattr->test.ctx_size_in; 1151 void *ctx = NULL; 1152 u32 retval; 1153 int err = 0; 1154 1155 /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */ 1156 if (kattr->test.data_in || kattr->test.data_out || 1157 kattr->test.ctx_out || kattr->test.duration || 1158 kattr->test.repeat || kattr->test.flags) 1159 return -EINVAL; 1160 1161 if (ctx_size_in < prog->aux->max_ctx_offset || 1162 ctx_size_in > U16_MAX) 1163 return -EINVAL; 1164 1165 if (ctx_size_in) { 1166 ctx = memdup_user(ctx_in, ctx_size_in); 1167 if (IS_ERR(ctx)) 1168 return PTR_ERR(ctx); 1169 } 1170 1171 rcu_read_lock_trace(); 1172 retval = bpf_prog_run_pin_on_cpu(prog, ctx); 1173 rcu_read_unlock_trace(); 1174 1175 if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) { 1176 err = -EFAULT; 1177 goto out; 1178 } 1179 if (ctx_size_in) 1180 if (copy_to_user(ctx_in, ctx, ctx_size_in)) 1181 err = -EFAULT; 1182 out: 1183 kfree(ctx); 1184 return err; 1185 } 1186 1187 static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = { 1188 .owner = THIS_MODULE, 1189 .check_set = &test_sk_check_kfunc_ids, 1190 .acquire_set = &test_sk_acquire_kfunc_ids, 1191 .release_set = &test_sk_release_kfunc_ids, 1192 .ret_null_set = &test_sk_ret_null_kfunc_ids, 1193 }; 1194 1195 static int __init bpf_prog_test_run_init(void) 1196 { 1197 return register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_prog_test_kfunc_set); 1198 } 1199 late_initcall(bpf_prog_test_run_init); 1200