1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2017 Facebook 3 */ 4 #include <linux/bpf.h> 5 #include <linux/slab.h> 6 #include <linux/vmalloc.h> 7 #include <linux/etherdevice.h> 8 #include <linux/filter.h> 9 #include <linux/sched/signal.h> 10 #include <net/bpf_sk_storage.h> 11 #include <net/sock.h> 12 #include <net/tcp.h> 13 #include <linux/error-injection.h> 14 15 #define CREATE_TRACE_POINTS 16 #include <trace/events/bpf_test_run.h> 17 18 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 19 u32 *retval, u32 *time, bool xdp) 20 { 21 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL }; 22 enum bpf_cgroup_storage_type stype; 23 u64 time_start, time_spent = 0; 24 int ret = 0; 25 u32 i; 26 27 for_each_cgroup_storage_type(stype) { 28 storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 29 if (IS_ERR(storage[stype])) { 30 storage[stype] = NULL; 31 for_each_cgroup_storage_type(stype) 32 bpf_cgroup_storage_free(storage[stype]); 33 return -ENOMEM; 34 } 35 } 36 37 if (!repeat) 38 repeat = 1; 39 40 rcu_read_lock(); 41 migrate_disable(); 42 time_start = ktime_get_ns(); 43 for (i = 0; i < repeat; i++) { 44 bpf_cgroup_storage_set(storage); 45 46 if (xdp) 47 *retval = bpf_prog_run_xdp(prog, ctx); 48 else 49 *retval = BPF_PROG_RUN(prog, ctx); 50 51 if (signal_pending(current)) { 52 ret = -EINTR; 53 break; 54 } 55 56 if (need_resched()) { 57 time_spent += ktime_get_ns() - time_start; 58 migrate_enable(); 59 rcu_read_unlock(); 60 61 cond_resched(); 62 63 rcu_read_lock(); 64 migrate_disable(); 65 time_start = ktime_get_ns(); 66 } 67 } 68 time_spent += ktime_get_ns() - time_start; 69 migrate_enable(); 70 rcu_read_unlock(); 71 72 do_div(time_spent, repeat); 73 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 74 75 for_each_cgroup_storage_type(stype) 76 bpf_cgroup_storage_free(storage[stype]); 77 78 return ret; 79 } 80 81 static int bpf_test_finish(const union bpf_attr *kattr, 82 union bpf_attr __user *uattr, const void *data, 83 u32 size, u32 retval, u32 duration) 84 { 85 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 86 int err = -EFAULT; 87 u32 copy_size = size; 88 89 /* Clamp copy if the user has provided a size hint, but copy the full 90 * buffer if not to retain old behaviour. 91 */ 92 if (kattr->test.data_size_out && 93 copy_size > kattr->test.data_size_out) { 94 copy_size = kattr->test.data_size_out; 95 err = -ENOSPC; 96 } 97 98 if (data_out && copy_to_user(data_out, data, copy_size)) 99 goto out; 100 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 101 goto out; 102 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 103 goto out; 104 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 105 goto out; 106 if (err != -ENOSPC) 107 err = 0; 108 out: 109 trace_bpf_test_finish(&err); 110 return err; 111 } 112 113 /* Integer types of various sizes and pointer combinations cover variety of 114 * architecture dependent calling conventions. 7+ can be supported in the 115 * future. 116 */ 117 int noinline bpf_fentry_test1(int a) 118 { 119 return a + 1; 120 } 121 122 int noinline bpf_fentry_test2(int a, u64 b) 123 { 124 return a + b; 125 } 126 127 int noinline bpf_fentry_test3(char a, int b, u64 c) 128 { 129 return a + b + c; 130 } 131 132 int noinline bpf_fentry_test4(void *a, char b, int c, u64 d) 133 { 134 return (long)a + b + c + d; 135 } 136 137 int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e) 138 { 139 return a + (long)b + c + d + e; 140 } 141 142 int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f) 143 { 144 return a + (long)b + c + d + (long)e + f; 145 } 146 147 int noinline bpf_modify_return_test(int a, int *b) 148 { 149 *b += 1; 150 return a + *b; 151 } 152 153 ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO); 154 155 static void *bpf_test_init(const union bpf_attr *kattr, u32 size, 156 u32 headroom, u32 tailroom) 157 { 158 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 159 void *data; 160 161 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 162 return ERR_PTR(-EINVAL); 163 164 data = kzalloc(size + headroom + tailroom, GFP_USER); 165 if (!data) 166 return ERR_PTR(-ENOMEM); 167 168 if (copy_from_user(data + headroom, data_in, size)) { 169 kfree(data); 170 return ERR_PTR(-EFAULT); 171 } 172 173 return data; 174 } 175 176 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 177 const union bpf_attr *kattr, 178 union bpf_attr __user *uattr) 179 { 180 u16 side_effect = 0, ret = 0; 181 int b = 2, err = -EFAULT; 182 u32 retval = 0; 183 184 switch (prog->expected_attach_type) { 185 case BPF_TRACE_FENTRY: 186 case BPF_TRACE_FEXIT: 187 if (bpf_fentry_test1(1) != 2 || 188 bpf_fentry_test2(2, 3) != 5 || 189 bpf_fentry_test3(4, 5, 6) != 15 || 190 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 || 191 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 || 192 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111) 193 goto out; 194 break; 195 case BPF_MODIFY_RETURN: 196 ret = bpf_modify_return_test(1, &b); 197 if (b != 2) 198 side_effect = 1; 199 break; 200 default: 201 goto out; 202 } 203 204 retval = ((u32)side_effect << 16) | ret; 205 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 206 goto out; 207 208 err = 0; 209 out: 210 trace_bpf_test_finish(&err); 211 return err; 212 } 213 214 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) 215 { 216 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); 217 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 218 u32 size = kattr->test.ctx_size_in; 219 void *data; 220 int err; 221 222 if (!data_in && !data_out) 223 return NULL; 224 225 data = kzalloc(max_size, GFP_USER); 226 if (!data) 227 return ERR_PTR(-ENOMEM); 228 229 if (data_in) { 230 err = bpf_check_uarg_tail_zero(data_in, max_size, size); 231 if (err) { 232 kfree(data); 233 return ERR_PTR(err); 234 } 235 236 size = min_t(u32, max_size, size); 237 if (copy_from_user(data, data_in, size)) { 238 kfree(data); 239 return ERR_PTR(-EFAULT); 240 } 241 } 242 return data; 243 } 244 245 static int bpf_ctx_finish(const union bpf_attr *kattr, 246 union bpf_attr __user *uattr, const void *data, 247 u32 size) 248 { 249 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 250 int err = -EFAULT; 251 u32 copy_size = size; 252 253 if (!data || !data_out) 254 return 0; 255 256 if (copy_size > kattr->test.ctx_size_out) { 257 copy_size = kattr->test.ctx_size_out; 258 err = -ENOSPC; 259 } 260 261 if (copy_to_user(data_out, data, copy_size)) 262 goto out; 263 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) 264 goto out; 265 if (err != -ENOSPC) 266 err = 0; 267 out: 268 return err; 269 } 270 271 /** 272 * range_is_zero - test whether buffer is initialized 273 * @buf: buffer to check 274 * @from: check from this position 275 * @to: check up until (excluding) this position 276 * 277 * This function returns true if the there is a non-zero byte 278 * in the buf in the range [from,to). 279 */ 280 static inline bool range_is_zero(void *buf, size_t from, size_t to) 281 { 282 return !memchr_inv((u8 *)buf + from, 0, to - from); 283 } 284 285 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) 286 { 287 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 288 289 if (!__skb) 290 return 0; 291 292 /* make sure the fields we don't use are zeroed */ 293 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark))) 294 return -EINVAL; 295 296 /* mark is allowed */ 297 298 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark), 299 offsetof(struct __sk_buff, priority))) 300 return -EINVAL; 301 302 /* priority is allowed */ 303 304 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, priority), 305 offsetof(struct __sk_buff, cb))) 306 return -EINVAL; 307 308 /* cb is allowed */ 309 310 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb), 311 offsetof(struct __sk_buff, tstamp))) 312 return -EINVAL; 313 314 /* tstamp is allowed */ 315 /* wire_len is allowed */ 316 /* gso_segs is allowed */ 317 318 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs), 319 offsetof(struct __sk_buff, gso_size))) 320 return -EINVAL; 321 322 /* gso_size is allowed */ 323 324 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size), 325 sizeof(struct __sk_buff))) 326 return -EINVAL; 327 328 skb->mark = __skb->mark; 329 skb->priority = __skb->priority; 330 skb->tstamp = __skb->tstamp; 331 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); 332 333 if (__skb->wire_len == 0) { 334 cb->pkt_len = skb->len; 335 } else { 336 if (__skb->wire_len < skb->len || 337 __skb->wire_len > GSO_MAX_SIZE) 338 return -EINVAL; 339 cb->pkt_len = __skb->wire_len; 340 } 341 342 if (__skb->gso_segs > GSO_MAX_SEGS) 343 return -EINVAL; 344 skb_shinfo(skb)->gso_segs = __skb->gso_segs; 345 skb_shinfo(skb)->gso_size = __skb->gso_size; 346 347 return 0; 348 } 349 350 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) 351 { 352 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 353 354 if (!__skb) 355 return; 356 357 __skb->mark = skb->mark; 358 __skb->priority = skb->priority; 359 __skb->tstamp = skb->tstamp; 360 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); 361 __skb->wire_len = cb->pkt_len; 362 __skb->gso_segs = skb_shinfo(skb)->gso_segs; 363 } 364 365 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 366 union bpf_attr __user *uattr) 367 { 368 bool is_l2 = false, is_direct_pkt_access = false; 369 u32 size = kattr->test.data_size_in; 370 u32 repeat = kattr->test.repeat; 371 struct __sk_buff *ctx = NULL; 372 u32 retval, duration; 373 int hh_len = ETH_HLEN; 374 struct sk_buff *skb; 375 struct sock *sk; 376 void *data; 377 int ret; 378 379 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 380 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 381 if (IS_ERR(data)) 382 return PTR_ERR(data); 383 384 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); 385 if (IS_ERR(ctx)) { 386 kfree(data); 387 return PTR_ERR(ctx); 388 } 389 390 switch (prog->type) { 391 case BPF_PROG_TYPE_SCHED_CLS: 392 case BPF_PROG_TYPE_SCHED_ACT: 393 is_l2 = true; 394 /* fall through */ 395 case BPF_PROG_TYPE_LWT_IN: 396 case BPF_PROG_TYPE_LWT_OUT: 397 case BPF_PROG_TYPE_LWT_XMIT: 398 is_direct_pkt_access = true; 399 break; 400 default: 401 break; 402 } 403 404 sk = kzalloc(sizeof(struct sock), GFP_USER); 405 if (!sk) { 406 kfree(data); 407 kfree(ctx); 408 return -ENOMEM; 409 } 410 sock_net_set(sk, current->nsproxy->net_ns); 411 sock_init_data(NULL, sk); 412 413 skb = build_skb(data, 0); 414 if (!skb) { 415 kfree(data); 416 kfree(ctx); 417 kfree(sk); 418 return -ENOMEM; 419 } 420 skb->sk = sk; 421 422 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 423 __skb_put(skb, size); 424 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev); 425 skb_reset_network_header(skb); 426 427 if (is_l2) 428 __skb_push(skb, hh_len); 429 if (is_direct_pkt_access) 430 bpf_compute_data_pointers(skb); 431 ret = convert___skb_to_skb(skb, ctx); 432 if (ret) 433 goto out; 434 ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false); 435 if (ret) 436 goto out; 437 if (!is_l2) { 438 if (skb_headroom(skb) < hh_len) { 439 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 440 441 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 442 ret = -ENOMEM; 443 goto out; 444 } 445 } 446 memset(__skb_push(skb, hh_len), 0, hh_len); 447 } 448 convert_skb_to___skb(skb, ctx); 449 450 size = skb->len; 451 /* bpf program can never convert linear skb to non-linear */ 452 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 453 size = skb_headlen(skb); 454 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); 455 if (!ret) 456 ret = bpf_ctx_finish(kattr, uattr, ctx, 457 sizeof(struct __sk_buff)); 458 out: 459 kfree_skb(skb); 460 bpf_sk_storage_free(sk); 461 kfree(sk); 462 kfree(ctx); 463 return ret; 464 } 465 466 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 467 union bpf_attr __user *uattr) 468 { 469 u32 size = kattr->test.data_size_in; 470 u32 repeat = kattr->test.repeat; 471 struct netdev_rx_queue *rxqueue; 472 struct xdp_buff xdp = {}; 473 u32 retval, duration; 474 void *data; 475 int ret; 476 477 if (kattr->test.ctx_in || kattr->test.ctx_out) 478 return -EINVAL; 479 480 data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0); 481 if (IS_ERR(data)) 482 return PTR_ERR(data); 483 484 xdp.data_hard_start = data; 485 xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN; 486 xdp.data_meta = xdp.data; 487 xdp.data_end = xdp.data + size; 488 489 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 490 xdp.rxq = &rxqueue->xdp_rxq; 491 bpf_prog_change_xdp(NULL, prog); 492 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true); 493 if (ret) 494 goto out; 495 if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN || 496 xdp.data_end != xdp.data + size) 497 size = xdp.data_end - xdp.data; 498 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); 499 out: 500 bpf_prog_change_xdp(prog, NULL); 501 kfree(data); 502 return ret; 503 } 504 505 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) 506 { 507 /* make sure the fields we don't use are zeroed */ 508 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) 509 return -EINVAL; 510 511 /* flags is allowed */ 512 513 if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags), 514 sizeof(struct bpf_flow_keys))) 515 return -EINVAL; 516 517 return 0; 518 } 519 520 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 521 const union bpf_attr *kattr, 522 union bpf_attr __user *uattr) 523 { 524 u32 size = kattr->test.data_size_in; 525 struct bpf_flow_dissector ctx = {}; 526 u32 repeat = kattr->test.repeat; 527 struct bpf_flow_keys *user_ctx; 528 struct bpf_flow_keys flow_keys; 529 u64 time_start, time_spent = 0; 530 const struct ethhdr *eth; 531 unsigned int flags = 0; 532 u32 retval, duration; 533 void *data; 534 int ret; 535 u32 i; 536 537 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) 538 return -EINVAL; 539 540 if (size < ETH_HLEN) 541 return -EINVAL; 542 543 data = bpf_test_init(kattr, size, 0, 0); 544 if (IS_ERR(data)) 545 return PTR_ERR(data); 546 547 eth = (struct ethhdr *)data; 548 549 if (!repeat) 550 repeat = 1; 551 552 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); 553 if (IS_ERR(user_ctx)) { 554 kfree(data); 555 return PTR_ERR(user_ctx); 556 } 557 if (user_ctx) { 558 ret = verify_user_bpf_flow_keys(user_ctx); 559 if (ret) 560 goto out; 561 flags = user_ctx->flags; 562 } 563 564 ctx.flow_keys = &flow_keys; 565 ctx.data = data; 566 ctx.data_end = (__u8 *)data + size; 567 568 rcu_read_lock(); 569 preempt_disable(); 570 time_start = ktime_get_ns(); 571 for (i = 0; i < repeat; i++) { 572 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, 573 size, flags); 574 575 if (signal_pending(current)) { 576 preempt_enable(); 577 rcu_read_unlock(); 578 579 ret = -EINTR; 580 goto out; 581 } 582 583 if (need_resched()) { 584 time_spent += ktime_get_ns() - time_start; 585 preempt_enable(); 586 rcu_read_unlock(); 587 588 cond_resched(); 589 590 rcu_read_lock(); 591 preempt_disable(); 592 time_start = ktime_get_ns(); 593 } 594 } 595 time_spent += ktime_get_ns() - time_start; 596 preempt_enable(); 597 rcu_read_unlock(); 598 599 do_div(time_spent, repeat); 600 duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 601 602 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), 603 retval, duration); 604 if (!ret) 605 ret = bpf_ctx_finish(kattr, uattr, user_ctx, 606 sizeof(struct bpf_flow_keys)); 607 608 out: 609 kfree(user_ctx); 610 kfree(data); 611 return ret; 612 } 613