xref: /linux/net/bpf/test_run.c (revision 6015fb905d89063231ed33bc15be19ef0fc339b8) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2017 Facebook
3  */
4 #include <linux/bpf.h>
5 #include <linux/btf.h>
6 #include <linux/btf_ids.h>
7 #include <linux/slab.h>
8 #include <linux/init.h>
9 #include <linux/vmalloc.h>
10 #include <linux/etherdevice.h>
11 #include <linux/filter.h>
12 #include <linux/rcupdate_trace.h>
13 #include <linux/sched/signal.h>
14 #include <net/bpf_sk_storage.h>
15 #include <net/sock.h>
16 #include <net/tcp.h>
17 #include <net/net_namespace.h>
18 #include <linux/error-injection.h>
19 #include <linux/smp.h>
20 #include <linux/sock_diag.h>
21 #include <net/xdp.h>
22 
23 #define CREATE_TRACE_POINTS
24 #include <trace/events/bpf_test_run.h>
25 
26 struct bpf_test_timer {
27 	enum { NO_PREEMPT, NO_MIGRATE } mode;
28 	u32 i;
29 	u64 time_start, time_spent;
30 };
31 
32 static void bpf_test_timer_enter(struct bpf_test_timer *t)
33 	__acquires(rcu)
34 {
35 	rcu_read_lock();
36 	if (t->mode == NO_PREEMPT)
37 		preempt_disable();
38 	else
39 		migrate_disable();
40 
41 	t->time_start = ktime_get_ns();
42 }
43 
44 static void bpf_test_timer_leave(struct bpf_test_timer *t)
45 	__releases(rcu)
46 {
47 	t->time_start = 0;
48 
49 	if (t->mode == NO_PREEMPT)
50 		preempt_enable();
51 	else
52 		migrate_enable();
53 	rcu_read_unlock();
54 }
55 
56 static bool bpf_test_timer_continue(struct bpf_test_timer *t, u32 repeat, int *err, u32 *duration)
57 	__must_hold(rcu)
58 {
59 	t->i++;
60 	if (t->i >= repeat) {
61 		/* We're done. */
62 		t->time_spent += ktime_get_ns() - t->time_start;
63 		do_div(t->time_spent, t->i);
64 		*duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent;
65 		*err = 0;
66 		goto reset;
67 	}
68 
69 	if (signal_pending(current)) {
70 		/* During iteration: we've been cancelled, abort. */
71 		*err = -EINTR;
72 		goto reset;
73 	}
74 
75 	if (need_resched()) {
76 		/* During iteration: we need to reschedule between runs. */
77 		t->time_spent += ktime_get_ns() - t->time_start;
78 		bpf_test_timer_leave(t);
79 		cond_resched();
80 		bpf_test_timer_enter(t);
81 	}
82 
83 	/* Do another round. */
84 	return true;
85 
86 reset:
87 	t->i = 0;
88 	return false;
89 }
90 
91 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
92 			u32 *retval, u32 *time, bool xdp)
93 {
94 	struct bpf_prog_array_item item = {.prog = prog};
95 	struct bpf_run_ctx *old_ctx;
96 	struct bpf_cg_run_ctx run_ctx;
97 	struct bpf_test_timer t = { NO_MIGRATE };
98 	enum bpf_cgroup_storage_type stype;
99 	int ret;
100 
101 	for_each_cgroup_storage_type(stype) {
102 		item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
103 		if (IS_ERR(item.cgroup_storage[stype])) {
104 			item.cgroup_storage[stype] = NULL;
105 			for_each_cgroup_storage_type(stype)
106 				bpf_cgroup_storage_free(item.cgroup_storage[stype]);
107 			return -ENOMEM;
108 		}
109 	}
110 
111 	if (!repeat)
112 		repeat = 1;
113 
114 	bpf_test_timer_enter(&t);
115 	old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
116 	do {
117 		run_ctx.prog_item = &item;
118 		if (xdp)
119 			*retval = bpf_prog_run_xdp(prog, ctx);
120 		else
121 			*retval = bpf_prog_run(prog, ctx);
122 	} while (bpf_test_timer_continue(&t, repeat, &ret, time));
123 	bpf_reset_run_ctx(old_ctx);
124 	bpf_test_timer_leave(&t);
125 
126 	for_each_cgroup_storage_type(stype)
127 		bpf_cgroup_storage_free(item.cgroup_storage[stype]);
128 
129 	return ret;
130 }
131 
132 static int bpf_test_finish(const union bpf_attr *kattr,
133 			   union bpf_attr __user *uattr, const void *data,
134 			   struct skb_shared_info *sinfo, u32 size,
135 			   u32 retval, u32 duration)
136 {
137 	void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
138 	int err = -EFAULT;
139 	u32 copy_size = size;
140 
141 	/* Clamp copy if the user has provided a size hint, but copy the full
142 	 * buffer if not to retain old behaviour.
143 	 */
144 	if (kattr->test.data_size_out &&
145 	    copy_size > kattr->test.data_size_out) {
146 		copy_size = kattr->test.data_size_out;
147 		err = -ENOSPC;
148 	}
149 
150 	if (data_out) {
151 		int len = sinfo ? copy_size - sinfo->xdp_frags_size : copy_size;
152 
153 		if (copy_to_user(data_out, data, len))
154 			goto out;
155 
156 		if (sinfo) {
157 			int i, offset = len;
158 			u32 data_len;
159 
160 			for (i = 0; i < sinfo->nr_frags; i++) {
161 				skb_frag_t *frag = &sinfo->frags[i];
162 
163 				if (offset >= copy_size) {
164 					err = -ENOSPC;
165 					break;
166 				}
167 
168 				data_len = min_t(u32, copy_size - offset,
169 						 skb_frag_size(frag));
170 
171 				if (copy_to_user(data_out + offset,
172 						 skb_frag_address(frag),
173 						 data_len))
174 					goto out;
175 
176 				offset += data_len;
177 			}
178 		}
179 	}
180 
181 	if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
182 		goto out;
183 	if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
184 		goto out;
185 	if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
186 		goto out;
187 	if (err != -ENOSPC)
188 		err = 0;
189 out:
190 	trace_bpf_test_finish(&err);
191 	return err;
192 }
193 
194 /* Integer types of various sizes and pointer combinations cover variety of
195  * architecture dependent calling conventions. 7+ can be supported in the
196  * future.
197  */
198 __diag_push();
199 __diag_ignore(GCC, 8, "-Wmissing-prototypes",
200 	      "Global functions as their definitions will be in vmlinux BTF");
201 int noinline bpf_fentry_test1(int a)
202 {
203 	return a + 1;
204 }
205 EXPORT_SYMBOL_GPL(bpf_fentry_test1);
206 ALLOW_ERROR_INJECTION(bpf_fentry_test1, ERRNO);
207 
208 int noinline bpf_fentry_test2(int a, u64 b)
209 {
210 	return a + b;
211 }
212 
213 int noinline bpf_fentry_test3(char a, int b, u64 c)
214 {
215 	return a + b + c;
216 }
217 
218 int noinline bpf_fentry_test4(void *a, char b, int c, u64 d)
219 {
220 	return (long)a + b + c + d;
221 }
222 
223 int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e)
224 {
225 	return a + (long)b + c + d + e;
226 }
227 
228 int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f)
229 {
230 	return a + (long)b + c + d + (long)e + f;
231 }
232 
233 struct bpf_fentry_test_t {
234 	struct bpf_fentry_test_t *a;
235 };
236 
237 int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg)
238 {
239 	return (long)arg;
240 }
241 
242 int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg)
243 {
244 	return (long)arg->a;
245 }
246 
247 int noinline bpf_modify_return_test(int a, int *b)
248 {
249 	*b += 1;
250 	return a + *b;
251 }
252 
253 u64 noinline bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d)
254 {
255 	return a + b + c + d;
256 }
257 
258 int noinline bpf_kfunc_call_test2(struct sock *sk, u32 a, u32 b)
259 {
260 	return a + b;
261 }
262 
263 struct sock * noinline bpf_kfunc_call_test3(struct sock *sk)
264 {
265 	return sk;
266 }
267 
268 struct prog_test_ref_kfunc {
269 	int a;
270 	int b;
271 	struct prog_test_ref_kfunc *next;
272 };
273 
274 static struct prog_test_ref_kfunc prog_test_struct = {
275 	.a = 42,
276 	.b = 108,
277 	.next = &prog_test_struct,
278 };
279 
280 noinline struct prog_test_ref_kfunc *
281 bpf_kfunc_call_test_acquire(unsigned long *scalar_ptr)
282 {
283 	/* randomly return NULL */
284 	if (get_jiffies_64() % 2)
285 		return NULL;
286 	return &prog_test_struct;
287 }
288 
289 noinline void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p)
290 {
291 }
292 
293 struct prog_test_pass1 {
294 	int x0;
295 	struct {
296 		int x1;
297 		struct {
298 			int x2;
299 			struct {
300 				int x3;
301 			};
302 		};
303 	};
304 };
305 
306 struct prog_test_pass2 {
307 	int len;
308 	short arr1[4];
309 	struct {
310 		char arr2[4];
311 		unsigned long arr3[8];
312 	} x;
313 };
314 
315 struct prog_test_fail1 {
316 	void *p;
317 	int x;
318 };
319 
320 struct prog_test_fail2 {
321 	int x8;
322 	struct prog_test_pass1 x;
323 };
324 
325 struct prog_test_fail3 {
326 	int len;
327 	char arr1[2];
328 	char arr2[];
329 };
330 
331 noinline void bpf_kfunc_call_test_pass_ctx(struct __sk_buff *skb)
332 {
333 }
334 
335 noinline void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p)
336 {
337 }
338 
339 noinline void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p)
340 {
341 }
342 
343 noinline void bpf_kfunc_call_test_fail1(struct prog_test_fail1 *p)
344 {
345 }
346 
347 noinline void bpf_kfunc_call_test_fail2(struct prog_test_fail2 *p)
348 {
349 }
350 
351 noinline void bpf_kfunc_call_test_fail3(struct prog_test_fail3 *p)
352 {
353 }
354 
355 noinline void bpf_kfunc_call_test_mem_len_pass1(void *mem, int mem__sz)
356 {
357 }
358 
359 noinline void bpf_kfunc_call_test_mem_len_fail1(void *mem, int len)
360 {
361 }
362 
363 noinline void bpf_kfunc_call_test_mem_len_fail2(u64 *mem, int len)
364 {
365 }
366 
367 __diag_pop();
368 
369 ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO);
370 
371 BTF_SET_START(test_sk_check_kfunc_ids)
372 BTF_ID(func, bpf_kfunc_call_test1)
373 BTF_ID(func, bpf_kfunc_call_test2)
374 BTF_ID(func, bpf_kfunc_call_test3)
375 BTF_ID(func, bpf_kfunc_call_test_acquire)
376 BTF_ID(func, bpf_kfunc_call_test_release)
377 BTF_ID(func, bpf_kfunc_call_test_pass_ctx)
378 BTF_ID(func, bpf_kfunc_call_test_pass1)
379 BTF_ID(func, bpf_kfunc_call_test_pass2)
380 BTF_ID(func, bpf_kfunc_call_test_fail1)
381 BTF_ID(func, bpf_kfunc_call_test_fail2)
382 BTF_ID(func, bpf_kfunc_call_test_fail3)
383 BTF_ID(func, bpf_kfunc_call_test_mem_len_pass1)
384 BTF_ID(func, bpf_kfunc_call_test_mem_len_fail1)
385 BTF_ID(func, bpf_kfunc_call_test_mem_len_fail2)
386 BTF_SET_END(test_sk_check_kfunc_ids)
387 
388 BTF_SET_START(test_sk_acquire_kfunc_ids)
389 BTF_ID(func, bpf_kfunc_call_test_acquire)
390 BTF_SET_END(test_sk_acquire_kfunc_ids)
391 
392 BTF_SET_START(test_sk_release_kfunc_ids)
393 BTF_ID(func, bpf_kfunc_call_test_release)
394 BTF_SET_END(test_sk_release_kfunc_ids)
395 
396 BTF_SET_START(test_sk_ret_null_kfunc_ids)
397 BTF_ID(func, bpf_kfunc_call_test_acquire)
398 BTF_SET_END(test_sk_ret_null_kfunc_ids)
399 
400 static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size,
401 			   u32 size, u32 headroom, u32 tailroom)
402 {
403 	void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
404 	void *data;
405 
406 	if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
407 		return ERR_PTR(-EINVAL);
408 
409 	if (user_size > size)
410 		return ERR_PTR(-EMSGSIZE);
411 
412 	data = kzalloc(size + headroom + tailroom, GFP_USER);
413 	if (!data)
414 		return ERR_PTR(-ENOMEM);
415 
416 	if (copy_from_user(data + headroom, data_in, user_size)) {
417 		kfree(data);
418 		return ERR_PTR(-EFAULT);
419 	}
420 
421 	return data;
422 }
423 
424 int bpf_prog_test_run_tracing(struct bpf_prog *prog,
425 			      const union bpf_attr *kattr,
426 			      union bpf_attr __user *uattr)
427 {
428 	struct bpf_fentry_test_t arg = {};
429 	u16 side_effect = 0, ret = 0;
430 	int b = 2, err = -EFAULT;
431 	u32 retval = 0;
432 
433 	if (kattr->test.flags || kattr->test.cpu)
434 		return -EINVAL;
435 
436 	switch (prog->expected_attach_type) {
437 	case BPF_TRACE_FENTRY:
438 	case BPF_TRACE_FEXIT:
439 		if (bpf_fentry_test1(1) != 2 ||
440 		    bpf_fentry_test2(2, 3) != 5 ||
441 		    bpf_fentry_test3(4, 5, 6) != 15 ||
442 		    bpf_fentry_test4((void *)7, 8, 9, 10) != 34 ||
443 		    bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 ||
444 		    bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 ||
445 		    bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 ||
446 		    bpf_fentry_test8(&arg) != 0)
447 			goto out;
448 		break;
449 	case BPF_MODIFY_RETURN:
450 		ret = bpf_modify_return_test(1, &b);
451 		if (b != 2)
452 			side_effect = 1;
453 		break;
454 	default:
455 		goto out;
456 	}
457 
458 	retval = ((u32)side_effect << 16) | ret;
459 	if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
460 		goto out;
461 
462 	err = 0;
463 out:
464 	trace_bpf_test_finish(&err);
465 	return err;
466 }
467 
468 struct bpf_raw_tp_test_run_info {
469 	struct bpf_prog *prog;
470 	void *ctx;
471 	u32 retval;
472 };
473 
474 static void
475 __bpf_prog_test_run_raw_tp(void *data)
476 {
477 	struct bpf_raw_tp_test_run_info *info = data;
478 
479 	rcu_read_lock();
480 	info->retval = bpf_prog_run(info->prog, info->ctx);
481 	rcu_read_unlock();
482 }
483 
484 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
485 			     const union bpf_attr *kattr,
486 			     union bpf_attr __user *uattr)
487 {
488 	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
489 	__u32 ctx_size_in = kattr->test.ctx_size_in;
490 	struct bpf_raw_tp_test_run_info info;
491 	int cpu = kattr->test.cpu, err = 0;
492 	int current_cpu;
493 
494 	/* doesn't support data_in/out, ctx_out, duration, or repeat */
495 	if (kattr->test.data_in || kattr->test.data_out ||
496 	    kattr->test.ctx_out || kattr->test.duration ||
497 	    kattr->test.repeat)
498 		return -EINVAL;
499 
500 	if (ctx_size_in < prog->aux->max_ctx_offset ||
501 	    ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64))
502 		return -EINVAL;
503 
504 	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)
505 		return -EINVAL;
506 
507 	if (ctx_size_in) {
508 		info.ctx = memdup_user(ctx_in, ctx_size_in);
509 		if (IS_ERR(info.ctx))
510 			return PTR_ERR(info.ctx);
511 	} else {
512 		info.ctx = NULL;
513 	}
514 
515 	info.prog = prog;
516 
517 	current_cpu = get_cpu();
518 	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||
519 	    cpu == current_cpu) {
520 		__bpf_prog_test_run_raw_tp(&info);
521 	} else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
522 		/* smp_call_function_single() also checks cpu_online()
523 		 * after csd_lock(). However, since cpu is from user
524 		 * space, let's do an extra quick check to filter out
525 		 * invalid value before smp_call_function_single().
526 		 */
527 		err = -ENXIO;
528 	} else {
529 		err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp,
530 					       &info, 1);
531 	}
532 	put_cpu();
533 
534 	if (!err &&
535 	    copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32)))
536 		err = -EFAULT;
537 
538 	kfree(info.ctx);
539 	return err;
540 }
541 
542 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
543 {
544 	void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
545 	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
546 	u32 size = kattr->test.ctx_size_in;
547 	void *data;
548 	int err;
549 
550 	if (!data_in && !data_out)
551 		return NULL;
552 
553 	data = kzalloc(max_size, GFP_USER);
554 	if (!data)
555 		return ERR_PTR(-ENOMEM);
556 
557 	if (data_in) {
558 		err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size);
559 		if (err) {
560 			kfree(data);
561 			return ERR_PTR(err);
562 		}
563 
564 		size = min_t(u32, max_size, size);
565 		if (copy_from_user(data, data_in, size)) {
566 			kfree(data);
567 			return ERR_PTR(-EFAULT);
568 		}
569 	}
570 	return data;
571 }
572 
573 static int bpf_ctx_finish(const union bpf_attr *kattr,
574 			  union bpf_attr __user *uattr, const void *data,
575 			  u32 size)
576 {
577 	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
578 	int err = -EFAULT;
579 	u32 copy_size = size;
580 
581 	if (!data || !data_out)
582 		return 0;
583 
584 	if (copy_size > kattr->test.ctx_size_out) {
585 		copy_size = kattr->test.ctx_size_out;
586 		err = -ENOSPC;
587 	}
588 
589 	if (copy_to_user(data_out, data, copy_size))
590 		goto out;
591 	if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
592 		goto out;
593 	if (err != -ENOSPC)
594 		err = 0;
595 out:
596 	return err;
597 }
598 
599 /**
600  * range_is_zero - test whether buffer is initialized
601  * @buf: buffer to check
602  * @from: check from this position
603  * @to: check up until (excluding) this position
604  *
605  * This function returns true if the there is a non-zero byte
606  * in the buf in the range [from,to).
607  */
608 static inline bool range_is_zero(void *buf, size_t from, size_t to)
609 {
610 	return !memchr_inv((u8 *)buf + from, 0, to - from);
611 }
612 
613 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
614 {
615 	struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
616 
617 	if (!__skb)
618 		return 0;
619 
620 	/* make sure the fields we don't use are zeroed */
621 	if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark)))
622 		return -EINVAL;
623 
624 	/* mark is allowed */
625 
626 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark),
627 			   offsetof(struct __sk_buff, priority)))
628 		return -EINVAL;
629 
630 	/* priority is allowed */
631 	/* ingress_ifindex is allowed */
632 	/* ifindex is allowed */
633 
634 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex),
635 			   offsetof(struct __sk_buff, cb)))
636 		return -EINVAL;
637 
638 	/* cb is allowed */
639 
640 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb),
641 			   offsetof(struct __sk_buff, tstamp)))
642 		return -EINVAL;
643 
644 	/* tstamp is allowed */
645 	/* wire_len is allowed */
646 	/* gso_segs is allowed */
647 
648 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs),
649 			   offsetof(struct __sk_buff, gso_size)))
650 		return -EINVAL;
651 
652 	/* gso_size is allowed */
653 
654 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size),
655 			   offsetof(struct __sk_buff, hwtstamp)))
656 		return -EINVAL;
657 
658 	/* hwtstamp is allowed */
659 
660 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp),
661 			   sizeof(struct __sk_buff)))
662 		return -EINVAL;
663 
664 	skb->mark = __skb->mark;
665 	skb->priority = __skb->priority;
666 	skb->skb_iif = __skb->ingress_ifindex;
667 	skb->tstamp = __skb->tstamp;
668 	memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
669 
670 	if (__skb->wire_len == 0) {
671 		cb->pkt_len = skb->len;
672 	} else {
673 		if (__skb->wire_len < skb->len ||
674 		    __skb->wire_len > GSO_MAX_SIZE)
675 			return -EINVAL;
676 		cb->pkt_len = __skb->wire_len;
677 	}
678 
679 	if (__skb->gso_segs > GSO_MAX_SEGS)
680 		return -EINVAL;
681 	skb_shinfo(skb)->gso_segs = __skb->gso_segs;
682 	skb_shinfo(skb)->gso_size = __skb->gso_size;
683 	skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp;
684 
685 	return 0;
686 }
687 
688 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
689 {
690 	struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
691 
692 	if (!__skb)
693 		return;
694 
695 	__skb->mark = skb->mark;
696 	__skb->priority = skb->priority;
697 	__skb->ingress_ifindex = skb->skb_iif;
698 	__skb->ifindex = skb->dev->ifindex;
699 	__skb->tstamp = skb->tstamp;
700 	memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
701 	__skb->wire_len = cb->pkt_len;
702 	__skb->gso_segs = skb_shinfo(skb)->gso_segs;
703 	__skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp;
704 }
705 
706 static struct proto bpf_dummy_proto = {
707 	.name   = "bpf_dummy",
708 	.owner  = THIS_MODULE,
709 	.obj_size = sizeof(struct sock),
710 };
711 
712 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
713 			  union bpf_attr __user *uattr)
714 {
715 	bool is_l2 = false, is_direct_pkt_access = false;
716 	struct net *net = current->nsproxy->net_ns;
717 	struct net_device *dev = net->loopback_dev;
718 	u32 size = kattr->test.data_size_in;
719 	u32 repeat = kattr->test.repeat;
720 	struct __sk_buff *ctx = NULL;
721 	u32 retval, duration;
722 	int hh_len = ETH_HLEN;
723 	struct sk_buff *skb;
724 	struct sock *sk;
725 	void *data;
726 	int ret;
727 
728 	if (kattr->test.flags || kattr->test.cpu)
729 		return -EINVAL;
730 
731 	data = bpf_test_init(kattr, kattr->test.data_size_in,
732 			     size, NET_SKB_PAD + NET_IP_ALIGN,
733 			     SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
734 	if (IS_ERR(data))
735 		return PTR_ERR(data);
736 
737 	ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
738 	if (IS_ERR(ctx)) {
739 		kfree(data);
740 		return PTR_ERR(ctx);
741 	}
742 
743 	switch (prog->type) {
744 	case BPF_PROG_TYPE_SCHED_CLS:
745 	case BPF_PROG_TYPE_SCHED_ACT:
746 		is_l2 = true;
747 		fallthrough;
748 	case BPF_PROG_TYPE_LWT_IN:
749 	case BPF_PROG_TYPE_LWT_OUT:
750 	case BPF_PROG_TYPE_LWT_XMIT:
751 		is_direct_pkt_access = true;
752 		break;
753 	default:
754 		break;
755 	}
756 
757 	sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1);
758 	if (!sk) {
759 		kfree(data);
760 		kfree(ctx);
761 		return -ENOMEM;
762 	}
763 	sock_init_data(NULL, sk);
764 
765 	skb = build_skb(data, 0);
766 	if (!skb) {
767 		kfree(data);
768 		kfree(ctx);
769 		sk_free(sk);
770 		return -ENOMEM;
771 	}
772 	skb->sk = sk;
773 
774 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
775 	__skb_put(skb, size);
776 	if (ctx && ctx->ifindex > 1) {
777 		dev = dev_get_by_index(net, ctx->ifindex);
778 		if (!dev) {
779 			ret = -ENODEV;
780 			goto out;
781 		}
782 	}
783 	skb->protocol = eth_type_trans(skb, dev);
784 	skb_reset_network_header(skb);
785 
786 	switch (skb->protocol) {
787 	case htons(ETH_P_IP):
788 		sk->sk_family = AF_INET;
789 		if (sizeof(struct iphdr) <= skb_headlen(skb)) {
790 			sk->sk_rcv_saddr = ip_hdr(skb)->saddr;
791 			sk->sk_daddr = ip_hdr(skb)->daddr;
792 		}
793 		break;
794 #if IS_ENABLED(CONFIG_IPV6)
795 	case htons(ETH_P_IPV6):
796 		sk->sk_family = AF_INET6;
797 		if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) {
798 			sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr;
799 			sk->sk_v6_daddr = ipv6_hdr(skb)->daddr;
800 		}
801 		break;
802 #endif
803 	default:
804 		break;
805 	}
806 
807 	if (is_l2)
808 		__skb_push(skb, hh_len);
809 	if (is_direct_pkt_access)
810 		bpf_compute_data_pointers(skb);
811 	ret = convert___skb_to_skb(skb, ctx);
812 	if (ret)
813 		goto out;
814 	ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false);
815 	if (ret)
816 		goto out;
817 	if (!is_l2) {
818 		if (skb_headroom(skb) < hh_len) {
819 			int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
820 
821 			if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
822 				ret = -ENOMEM;
823 				goto out;
824 			}
825 		}
826 		memset(__skb_push(skb, hh_len), 0, hh_len);
827 	}
828 	convert_skb_to___skb(skb, ctx);
829 
830 	size = skb->len;
831 	/* bpf program can never convert linear skb to non-linear */
832 	if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
833 		size = skb_headlen(skb);
834 	ret = bpf_test_finish(kattr, uattr, skb->data, NULL, size, retval,
835 			      duration);
836 	if (!ret)
837 		ret = bpf_ctx_finish(kattr, uattr, ctx,
838 				     sizeof(struct __sk_buff));
839 out:
840 	if (dev && dev != net->loopback_dev)
841 		dev_put(dev);
842 	kfree_skb(skb);
843 	sk_free(sk);
844 	kfree(ctx);
845 	return ret;
846 }
847 
848 static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp)
849 {
850 	unsigned int ingress_ifindex, rx_queue_index;
851 	struct netdev_rx_queue *rxqueue;
852 	struct net_device *device;
853 
854 	if (!xdp_md)
855 		return 0;
856 
857 	if (xdp_md->egress_ifindex != 0)
858 		return -EINVAL;
859 
860 	ingress_ifindex = xdp_md->ingress_ifindex;
861 	rx_queue_index = xdp_md->rx_queue_index;
862 
863 	if (!ingress_ifindex && rx_queue_index)
864 		return -EINVAL;
865 
866 	if (ingress_ifindex) {
867 		device = dev_get_by_index(current->nsproxy->net_ns,
868 					  ingress_ifindex);
869 		if (!device)
870 			return -ENODEV;
871 
872 		if (rx_queue_index >= device->real_num_rx_queues)
873 			goto free_dev;
874 
875 		rxqueue = __netif_get_rx_queue(device, rx_queue_index);
876 
877 		if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq))
878 			goto free_dev;
879 
880 		xdp->rxq = &rxqueue->xdp_rxq;
881 		/* The device is now tracked in the xdp->rxq for later
882 		 * dev_put()
883 		 */
884 	}
885 
886 	xdp->data = xdp->data_meta + xdp_md->data;
887 	return 0;
888 
889 free_dev:
890 	dev_put(device);
891 	return -EINVAL;
892 }
893 
894 static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md)
895 {
896 	if (!xdp_md)
897 		return;
898 
899 	xdp_md->data = xdp->data - xdp->data_meta;
900 	xdp_md->data_end = xdp->data_end - xdp->data_meta;
901 
902 	if (xdp_md->ingress_ifindex)
903 		dev_put(xdp->rxq->dev);
904 }
905 
906 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
907 			  union bpf_attr __user *uattr)
908 {
909 	u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
910 	u32 size = kattr->test.data_size_in;
911 	u32 headroom = XDP_PACKET_HEADROOM;
912 	u32 retval, duration, max_data_sz;
913 	u32 repeat = kattr->test.repeat;
914 	struct netdev_rx_queue *rxqueue;
915 	struct skb_shared_info *sinfo;
916 	struct xdp_buff xdp = {};
917 	int i, ret = -EINVAL;
918 	struct xdp_md *ctx;
919 	void *data;
920 
921 	if (prog->expected_attach_type == BPF_XDP_DEVMAP ||
922 	    prog->expected_attach_type == BPF_XDP_CPUMAP)
923 		return -EINVAL;
924 
925 	ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md));
926 	if (IS_ERR(ctx))
927 		return PTR_ERR(ctx);
928 
929 	if (ctx) {
930 		/* There can't be user provided data before the meta data */
931 		if (ctx->data_meta || ctx->data_end != size ||
932 		    ctx->data > ctx->data_end ||
933 		    unlikely(xdp_metalen_invalid(ctx->data)))
934 			goto free_ctx;
935 		/* Meta data is allocated from the headroom */
936 		headroom -= ctx->data;
937 	}
938 
939 	max_data_sz = 4096 - headroom - tailroom;
940 	size = min_t(u32, size, max_data_sz);
941 
942 	data = bpf_test_init(kattr, size, max_data_sz, headroom, tailroom);
943 	if (IS_ERR(data)) {
944 		ret = PTR_ERR(data);
945 		goto free_ctx;
946 	}
947 
948 	rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
949 	rxqueue->xdp_rxq.frag_size = headroom + max_data_sz + tailroom;
950 	xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq);
951 	xdp_prepare_buff(&xdp, data, headroom, size, true);
952 	sinfo = xdp_get_shared_info_from_buff(&xdp);
953 
954 	ret = xdp_convert_md_to_buff(ctx, &xdp);
955 	if (ret)
956 		goto free_data;
957 
958 	if (unlikely(kattr->test.data_size_in > size)) {
959 		void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
960 
961 		while (size < kattr->test.data_size_in) {
962 			struct page *page;
963 			skb_frag_t *frag;
964 			u32 data_len;
965 
966 			if (sinfo->nr_frags == MAX_SKB_FRAGS) {
967 				ret = -ENOMEM;
968 				goto out;
969 			}
970 
971 			page = alloc_page(GFP_KERNEL);
972 			if (!page) {
973 				ret = -ENOMEM;
974 				goto out;
975 			}
976 
977 			frag = &sinfo->frags[sinfo->nr_frags++];
978 			__skb_frag_set_page(frag, page);
979 
980 			data_len = min_t(u32, kattr->test.data_size_in - size,
981 					 PAGE_SIZE);
982 			skb_frag_size_set(frag, data_len);
983 
984 			if (copy_from_user(page_address(page), data_in + size,
985 					   data_len)) {
986 				ret = -EFAULT;
987 				goto out;
988 			}
989 			sinfo->xdp_frags_size += data_len;
990 			size += data_len;
991 		}
992 		xdp_buff_set_frags_flag(&xdp);
993 	}
994 
995 	if (repeat > 1)
996 		bpf_prog_change_xdp(NULL, prog);
997 
998 	ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true);
999 	/* We convert the xdp_buff back to an xdp_md before checking the return
1000 	 * code so the reference count of any held netdevice will be decremented
1001 	 * even if the test run failed.
1002 	 */
1003 	xdp_convert_buff_to_md(&xdp, ctx);
1004 	if (ret)
1005 		goto out;
1006 
1007 	size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size;
1008 	ret = bpf_test_finish(kattr, uattr, xdp.data_meta, sinfo, size,
1009 			      retval, duration);
1010 	if (!ret)
1011 		ret = bpf_ctx_finish(kattr, uattr, ctx,
1012 				     sizeof(struct xdp_md));
1013 
1014 out:
1015 	if (repeat > 1)
1016 		bpf_prog_change_xdp(prog, NULL);
1017 free_data:
1018 	for (i = 0; i < sinfo->nr_frags; i++)
1019 		__free_page(skb_frag_page(&sinfo->frags[i]));
1020 	kfree(data);
1021 free_ctx:
1022 	kfree(ctx);
1023 	return ret;
1024 }
1025 
1026 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
1027 {
1028 	/* make sure the fields we don't use are zeroed */
1029 	if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags)))
1030 		return -EINVAL;
1031 
1032 	/* flags is allowed */
1033 
1034 	if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags),
1035 			   sizeof(struct bpf_flow_keys)))
1036 		return -EINVAL;
1037 
1038 	return 0;
1039 }
1040 
1041 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
1042 				     const union bpf_attr *kattr,
1043 				     union bpf_attr __user *uattr)
1044 {
1045 	struct bpf_test_timer t = { NO_PREEMPT };
1046 	u32 size = kattr->test.data_size_in;
1047 	struct bpf_flow_dissector ctx = {};
1048 	u32 repeat = kattr->test.repeat;
1049 	struct bpf_flow_keys *user_ctx;
1050 	struct bpf_flow_keys flow_keys;
1051 	const struct ethhdr *eth;
1052 	unsigned int flags = 0;
1053 	u32 retval, duration;
1054 	void *data;
1055 	int ret;
1056 
1057 	if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
1058 		return -EINVAL;
1059 
1060 	if (kattr->test.flags || kattr->test.cpu)
1061 		return -EINVAL;
1062 
1063 	if (size < ETH_HLEN)
1064 		return -EINVAL;
1065 
1066 	data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0);
1067 	if (IS_ERR(data))
1068 		return PTR_ERR(data);
1069 
1070 	eth = (struct ethhdr *)data;
1071 
1072 	if (!repeat)
1073 		repeat = 1;
1074 
1075 	user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys));
1076 	if (IS_ERR(user_ctx)) {
1077 		kfree(data);
1078 		return PTR_ERR(user_ctx);
1079 	}
1080 	if (user_ctx) {
1081 		ret = verify_user_bpf_flow_keys(user_ctx);
1082 		if (ret)
1083 			goto out;
1084 		flags = user_ctx->flags;
1085 	}
1086 
1087 	ctx.flow_keys = &flow_keys;
1088 	ctx.data = data;
1089 	ctx.data_end = (__u8 *)data + size;
1090 
1091 	bpf_test_timer_enter(&t);
1092 	do {
1093 		retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
1094 					  size, flags);
1095 	} while (bpf_test_timer_continue(&t, repeat, &ret, &duration));
1096 	bpf_test_timer_leave(&t);
1097 
1098 	if (ret < 0)
1099 		goto out;
1100 
1101 	ret = bpf_test_finish(kattr, uattr, &flow_keys, NULL,
1102 			      sizeof(flow_keys), retval, duration);
1103 	if (!ret)
1104 		ret = bpf_ctx_finish(kattr, uattr, user_ctx,
1105 				     sizeof(struct bpf_flow_keys));
1106 
1107 out:
1108 	kfree(user_ctx);
1109 	kfree(data);
1110 	return ret;
1111 }
1112 
1113 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,
1114 				union bpf_attr __user *uattr)
1115 {
1116 	struct bpf_test_timer t = { NO_PREEMPT };
1117 	struct bpf_prog_array *progs = NULL;
1118 	struct bpf_sk_lookup_kern ctx = {};
1119 	u32 repeat = kattr->test.repeat;
1120 	struct bpf_sk_lookup *user_ctx;
1121 	u32 retval, duration;
1122 	int ret = -EINVAL;
1123 
1124 	if (prog->type != BPF_PROG_TYPE_SK_LOOKUP)
1125 		return -EINVAL;
1126 
1127 	if (kattr->test.flags || kattr->test.cpu)
1128 		return -EINVAL;
1129 
1130 	if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||
1131 	    kattr->test.data_size_out)
1132 		return -EINVAL;
1133 
1134 	if (!repeat)
1135 		repeat = 1;
1136 
1137 	user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx));
1138 	if (IS_ERR(user_ctx))
1139 		return PTR_ERR(user_ctx);
1140 
1141 	if (!user_ctx)
1142 		return -EINVAL;
1143 
1144 	if (user_ctx->sk)
1145 		goto out;
1146 
1147 	if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx)))
1148 		goto out;
1149 
1150 	if (user_ctx->local_port > U16_MAX) {
1151 		ret = -ERANGE;
1152 		goto out;
1153 	}
1154 
1155 	ctx.family = (u16)user_ctx->family;
1156 	ctx.protocol = (u16)user_ctx->protocol;
1157 	ctx.dport = (u16)user_ctx->local_port;
1158 	ctx.sport = user_ctx->remote_port;
1159 
1160 	switch (ctx.family) {
1161 	case AF_INET:
1162 		ctx.v4.daddr = (__force __be32)user_ctx->local_ip4;
1163 		ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4;
1164 		break;
1165 
1166 #if IS_ENABLED(CONFIG_IPV6)
1167 	case AF_INET6:
1168 		ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6;
1169 		ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6;
1170 		break;
1171 #endif
1172 
1173 	default:
1174 		ret = -EAFNOSUPPORT;
1175 		goto out;
1176 	}
1177 
1178 	progs = bpf_prog_array_alloc(1, GFP_KERNEL);
1179 	if (!progs) {
1180 		ret = -ENOMEM;
1181 		goto out;
1182 	}
1183 
1184 	progs->items[0].prog = prog;
1185 
1186 	bpf_test_timer_enter(&t);
1187 	do {
1188 		ctx.selected_sk = NULL;
1189 		retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run);
1190 	} while (bpf_test_timer_continue(&t, repeat, &ret, &duration));
1191 	bpf_test_timer_leave(&t);
1192 
1193 	if (ret < 0)
1194 		goto out;
1195 
1196 	user_ctx->cookie = 0;
1197 	if (ctx.selected_sk) {
1198 		if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) {
1199 			ret = -EOPNOTSUPP;
1200 			goto out;
1201 		}
1202 
1203 		user_ctx->cookie = sock_gen_cookie(ctx.selected_sk);
1204 	}
1205 
1206 	ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, retval, duration);
1207 	if (!ret)
1208 		ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx));
1209 
1210 out:
1211 	bpf_prog_array_free(progs);
1212 	kfree(user_ctx);
1213 	return ret;
1214 }
1215 
1216 int bpf_prog_test_run_syscall(struct bpf_prog *prog,
1217 			      const union bpf_attr *kattr,
1218 			      union bpf_attr __user *uattr)
1219 {
1220 	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
1221 	__u32 ctx_size_in = kattr->test.ctx_size_in;
1222 	void *ctx = NULL;
1223 	u32 retval;
1224 	int err = 0;
1225 
1226 	/* doesn't support data_in/out, ctx_out, duration, or repeat or flags */
1227 	if (kattr->test.data_in || kattr->test.data_out ||
1228 	    kattr->test.ctx_out || kattr->test.duration ||
1229 	    kattr->test.repeat || kattr->test.flags)
1230 		return -EINVAL;
1231 
1232 	if (ctx_size_in < prog->aux->max_ctx_offset ||
1233 	    ctx_size_in > U16_MAX)
1234 		return -EINVAL;
1235 
1236 	if (ctx_size_in) {
1237 		ctx = memdup_user(ctx_in, ctx_size_in);
1238 		if (IS_ERR(ctx))
1239 			return PTR_ERR(ctx);
1240 	}
1241 
1242 	rcu_read_lock_trace();
1243 	retval = bpf_prog_run_pin_on_cpu(prog, ctx);
1244 	rcu_read_unlock_trace();
1245 
1246 	if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) {
1247 		err = -EFAULT;
1248 		goto out;
1249 	}
1250 	if (ctx_size_in)
1251 		if (copy_to_user(ctx_in, ctx, ctx_size_in))
1252 			err = -EFAULT;
1253 out:
1254 	kfree(ctx);
1255 	return err;
1256 }
1257 
1258 static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = {
1259 	.owner        = THIS_MODULE,
1260 	.check_set    = &test_sk_check_kfunc_ids,
1261 	.acquire_set  = &test_sk_acquire_kfunc_ids,
1262 	.release_set  = &test_sk_release_kfunc_ids,
1263 	.ret_null_set = &test_sk_ret_null_kfunc_ids,
1264 };
1265 
1266 static int __init bpf_prog_test_run_init(void)
1267 {
1268 	return register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_prog_test_kfunc_set);
1269 }
1270 late_initcall(bpf_prog_test_run_init);
1271