1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2017 Facebook 3 */ 4 #include <linux/bpf.h> 5 #include <linux/slab.h> 6 #include <linux/vmalloc.h> 7 #include <linux/etherdevice.h> 8 #include <linux/filter.h> 9 #include <linux/sched/signal.h> 10 #include <net/bpf_sk_storage.h> 11 #include <net/sock.h> 12 #include <net/tcp.h> 13 14 #define CREATE_TRACE_POINTS 15 #include <trace/events/bpf_test_run.h> 16 17 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 18 u32 *retval, u32 *time) 19 { 20 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL }; 21 enum bpf_cgroup_storage_type stype; 22 u64 time_start, time_spent = 0; 23 int ret = 0; 24 u32 i; 25 26 for_each_cgroup_storage_type(stype) { 27 storage[stype] = bpf_cgroup_storage_alloc(prog, stype); 28 if (IS_ERR(storage[stype])) { 29 storage[stype] = NULL; 30 for_each_cgroup_storage_type(stype) 31 bpf_cgroup_storage_free(storage[stype]); 32 return -ENOMEM; 33 } 34 } 35 36 if (!repeat) 37 repeat = 1; 38 39 rcu_read_lock(); 40 preempt_disable(); 41 time_start = ktime_get_ns(); 42 for (i = 0; i < repeat; i++) { 43 bpf_cgroup_storage_set(storage); 44 *retval = BPF_PROG_RUN(prog, ctx); 45 46 if (signal_pending(current)) { 47 ret = -EINTR; 48 break; 49 } 50 51 if (need_resched()) { 52 time_spent += ktime_get_ns() - time_start; 53 preempt_enable(); 54 rcu_read_unlock(); 55 56 cond_resched(); 57 58 rcu_read_lock(); 59 preempt_disable(); 60 time_start = ktime_get_ns(); 61 } 62 } 63 time_spent += ktime_get_ns() - time_start; 64 preempt_enable(); 65 rcu_read_unlock(); 66 67 do_div(time_spent, repeat); 68 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 69 70 for_each_cgroup_storage_type(stype) 71 bpf_cgroup_storage_free(storage[stype]); 72 73 return ret; 74 } 75 76 static int bpf_test_finish(const union bpf_attr *kattr, 77 union bpf_attr __user *uattr, const void *data, 78 u32 size, u32 retval, u32 duration) 79 { 80 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); 81 int err = -EFAULT; 82 u32 copy_size = size; 83 84 /* Clamp copy if the user has provided a size hint, but copy the full 85 * buffer if not to retain old behaviour. 86 */ 87 if (kattr->test.data_size_out && 88 copy_size > kattr->test.data_size_out) { 89 copy_size = kattr->test.data_size_out; 90 err = -ENOSPC; 91 } 92 93 if (data_out && copy_to_user(data_out, data, copy_size)) 94 goto out; 95 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 96 goto out; 97 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 98 goto out; 99 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 100 goto out; 101 if (err != -ENOSPC) 102 err = 0; 103 out: 104 trace_bpf_test_finish(&err); 105 return err; 106 } 107 108 static void *bpf_test_init(const union bpf_attr *kattr, u32 size, 109 u32 headroom, u32 tailroom) 110 { 111 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 112 void *data; 113 114 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 115 return ERR_PTR(-EINVAL); 116 117 data = kzalloc(size + headroom + tailroom, GFP_USER); 118 if (!data) 119 return ERR_PTR(-ENOMEM); 120 121 if (copy_from_user(data + headroom, data_in, size)) { 122 kfree(data); 123 return ERR_PTR(-EFAULT); 124 } 125 return data; 126 } 127 128 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) 129 { 130 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); 131 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 132 u32 size = kattr->test.ctx_size_in; 133 void *data; 134 int err; 135 136 if (!data_in && !data_out) 137 return NULL; 138 139 data = kzalloc(max_size, GFP_USER); 140 if (!data) 141 return ERR_PTR(-ENOMEM); 142 143 if (data_in) { 144 err = bpf_check_uarg_tail_zero(data_in, max_size, size); 145 if (err) { 146 kfree(data); 147 return ERR_PTR(err); 148 } 149 150 size = min_t(u32, max_size, size); 151 if (copy_from_user(data, data_in, size)) { 152 kfree(data); 153 return ERR_PTR(-EFAULT); 154 } 155 } 156 return data; 157 } 158 159 static int bpf_ctx_finish(const union bpf_attr *kattr, 160 union bpf_attr __user *uattr, const void *data, 161 u32 size) 162 { 163 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); 164 int err = -EFAULT; 165 u32 copy_size = size; 166 167 if (!data || !data_out) 168 return 0; 169 170 if (copy_size > kattr->test.ctx_size_out) { 171 copy_size = kattr->test.ctx_size_out; 172 err = -ENOSPC; 173 } 174 175 if (copy_to_user(data_out, data, copy_size)) 176 goto out; 177 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) 178 goto out; 179 if (err != -ENOSPC) 180 err = 0; 181 out: 182 return err; 183 } 184 185 /** 186 * range_is_zero - test whether buffer is initialized 187 * @buf: buffer to check 188 * @from: check from this position 189 * @to: check up until (excluding) this position 190 * 191 * This function returns true if the there is a non-zero byte 192 * in the buf in the range [from,to). 193 */ 194 static inline bool range_is_zero(void *buf, size_t from, size_t to) 195 { 196 return !memchr_inv((u8 *)buf + from, 0, to - from); 197 } 198 199 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) 200 { 201 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 202 203 if (!__skb) 204 return 0; 205 206 /* make sure the fields we don't use are zeroed */ 207 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, priority))) 208 return -EINVAL; 209 210 /* priority is allowed */ 211 212 if (!range_is_zero(__skb, offsetof(struct __sk_buff, priority) + 213 FIELD_SIZEOF(struct __sk_buff, priority), 214 offsetof(struct __sk_buff, cb))) 215 return -EINVAL; 216 217 /* cb is allowed */ 218 219 if (!range_is_zero(__skb, offsetof(struct __sk_buff, cb) + 220 FIELD_SIZEOF(struct __sk_buff, cb), 221 offsetof(struct __sk_buff, tstamp))) 222 return -EINVAL; 223 224 /* tstamp is allowed */ 225 226 if (!range_is_zero(__skb, offsetof(struct __sk_buff, tstamp) + 227 FIELD_SIZEOF(struct __sk_buff, tstamp), 228 sizeof(struct __sk_buff))) 229 return -EINVAL; 230 231 skb->priority = __skb->priority; 232 skb->tstamp = __skb->tstamp; 233 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); 234 235 return 0; 236 } 237 238 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) 239 { 240 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; 241 242 if (!__skb) 243 return; 244 245 __skb->priority = skb->priority; 246 __skb->tstamp = skb->tstamp; 247 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); 248 } 249 250 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 251 union bpf_attr __user *uattr) 252 { 253 bool is_l2 = false, is_direct_pkt_access = false; 254 u32 size = kattr->test.data_size_in; 255 u32 repeat = kattr->test.repeat; 256 struct __sk_buff *ctx = NULL; 257 u32 retval, duration; 258 int hh_len = ETH_HLEN; 259 struct sk_buff *skb; 260 struct sock *sk; 261 void *data; 262 int ret; 263 264 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, 265 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 266 if (IS_ERR(data)) 267 return PTR_ERR(data); 268 269 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff)); 270 if (IS_ERR(ctx)) { 271 kfree(data); 272 return PTR_ERR(ctx); 273 } 274 275 switch (prog->type) { 276 case BPF_PROG_TYPE_SCHED_CLS: 277 case BPF_PROG_TYPE_SCHED_ACT: 278 is_l2 = true; 279 /* fall through */ 280 case BPF_PROG_TYPE_LWT_IN: 281 case BPF_PROG_TYPE_LWT_OUT: 282 case BPF_PROG_TYPE_LWT_XMIT: 283 is_direct_pkt_access = true; 284 break; 285 default: 286 break; 287 } 288 289 sk = kzalloc(sizeof(struct sock), GFP_USER); 290 if (!sk) { 291 kfree(data); 292 kfree(ctx); 293 return -ENOMEM; 294 } 295 sock_net_set(sk, current->nsproxy->net_ns); 296 sock_init_data(NULL, sk); 297 298 skb = build_skb(data, 0); 299 if (!skb) { 300 kfree(data); 301 kfree(ctx); 302 kfree(sk); 303 return -ENOMEM; 304 } 305 skb->sk = sk; 306 307 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 308 __skb_put(skb, size); 309 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev); 310 skb_reset_network_header(skb); 311 312 if (is_l2) 313 __skb_push(skb, hh_len); 314 if (is_direct_pkt_access) 315 bpf_compute_data_pointers(skb); 316 ret = convert___skb_to_skb(skb, ctx); 317 if (ret) 318 goto out; 319 ret = bpf_test_run(prog, skb, repeat, &retval, &duration); 320 if (ret) 321 goto out; 322 if (!is_l2) { 323 if (skb_headroom(skb) < hh_len) { 324 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); 325 326 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) { 327 ret = -ENOMEM; 328 goto out; 329 } 330 } 331 memset(__skb_push(skb, hh_len), 0, hh_len); 332 } 333 convert_skb_to___skb(skb, ctx); 334 335 size = skb->len; 336 /* bpf program can never convert linear skb to non-linear */ 337 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 338 size = skb_headlen(skb); 339 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); 340 if (!ret) 341 ret = bpf_ctx_finish(kattr, uattr, ctx, 342 sizeof(struct __sk_buff)); 343 out: 344 kfree_skb(skb); 345 bpf_sk_storage_free(sk); 346 kfree(sk); 347 kfree(ctx); 348 return ret; 349 } 350 351 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 352 union bpf_attr __user *uattr) 353 { 354 u32 size = kattr->test.data_size_in; 355 u32 repeat = kattr->test.repeat; 356 struct netdev_rx_queue *rxqueue; 357 struct xdp_buff xdp = {}; 358 u32 retval, duration; 359 void *data; 360 int ret; 361 362 if (kattr->test.ctx_in || kattr->test.ctx_out) 363 return -EINVAL; 364 365 data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0); 366 if (IS_ERR(data)) 367 return PTR_ERR(data); 368 369 xdp.data_hard_start = data; 370 xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN; 371 xdp.data_meta = xdp.data; 372 xdp.data_end = xdp.data + size; 373 374 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); 375 xdp.rxq = &rxqueue->xdp_rxq; 376 377 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration); 378 if (ret) 379 goto out; 380 if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN || 381 xdp.data_end != xdp.data + size) 382 size = xdp.data_end - xdp.data; 383 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); 384 out: 385 kfree(data); 386 return ret; 387 } 388 389 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx) 390 { 391 /* make sure the fields we don't use are zeroed */ 392 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags))) 393 return -EINVAL; 394 395 /* flags is allowed */ 396 397 if (!range_is_zero(ctx, offsetof(struct bpf_flow_keys, flags) + 398 FIELD_SIZEOF(struct bpf_flow_keys, flags), 399 sizeof(struct bpf_flow_keys))) 400 return -EINVAL; 401 402 return 0; 403 } 404 405 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 406 const union bpf_attr *kattr, 407 union bpf_attr __user *uattr) 408 { 409 u32 size = kattr->test.data_size_in; 410 struct bpf_flow_dissector ctx = {}; 411 u32 repeat = kattr->test.repeat; 412 struct bpf_flow_keys *user_ctx; 413 struct bpf_flow_keys flow_keys; 414 u64 time_start, time_spent = 0; 415 const struct ethhdr *eth; 416 unsigned int flags = 0; 417 u32 retval, duration; 418 void *data; 419 int ret; 420 u32 i; 421 422 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) 423 return -EINVAL; 424 425 if (size < ETH_HLEN) 426 return -EINVAL; 427 428 data = bpf_test_init(kattr, size, 0, 0); 429 if (IS_ERR(data)) 430 return PTR_ERR(data); 431 432 eth = (struct ethhdr *)data; 433 434 if (!repeat) 435 repeat = 1; 436 437 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys)); 438 if (IS_ERR(user_ctx)) { 439 kfree(data); 440 return PTR_ERR(user_ctx); 441 } 442 if (user_ctx) { 443 ret = verify_user_bpf_flow_keys(user_ctx); 444 if (ret) 445 goto out; 446 flags = user_ctx->flags; 447 } 448 449 ctx.flow_keys = &flow_keys; 450 ctx.data = data; 451 ctx.data_end = (__u8 *)data + size; 452 453 rcu_read_lock(); 454 preempt_disable(); 455 time_start = ktime_get_ns(); 456 for (i = 0; i < repeat; i++) { 457 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, 458 size, flags); 459 460 if (signal_pending(current)) { 461 preempt_enable(); 462 rcu_read_unlock(); 463 464 ret = -EINTR; 465 goto out; 466 } 467 468 if (need_resched()) { 469 time_spent += ktime_get_ns() - time_start; 470 preempt_enable(); 471 rcu_read_unlock(); 472 473 cond_resched(); 474 475 rcu_read_lock(); 476 preempt_disable(); 477 time_start = ktime_get_ns(); 478 } 479 } 480 time_spent += ktime_get_ns() - time_start; 481 preempt_enable(); 482 rcu_read_unlock(); 483 484 do_div(time_spent, repeat); 485 duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 486 487 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys), 488 retval, duration); 489 if (!ret) 490 ret = bpf_ctx_finish(kattr, uattr, user_ctx, 491 sizeof(struct bpf_flow_keys)); 492 493 out: 494 kfree(user_ctx); 495 kfree(data); 496 return ret; 497 } 498