xref: /linux/net/bpf/test_run.c (revision c1ff181ffabc292abcd1832a1c83aac2bc499e71)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2017 Facebook
3  */
4 #include <linux/bpf.h>
5 #include <linux/btf.h>
6 #include <linux/btf_ids.h>
7 #include <linux/slab.h>
8 #include <linux/init.h>
9 #include <linux/vmalloc.h>
10 #include <linux/etherdevice.h>
11 #include <linux/filter.h>
12 #include <linux/rcupdate_trace.h>
13 #include <linux/sched/signal.h>
14 #include <net/bpf_sk_storage.h>
15 #include <net/sock.h>
16 #include <net/tcp.h>
17 #include <net/net_namespace.h>
18 #include <linux/error-injection.h>
19 #include <linux/smp.h>
20 #include <linux/sock_diag.h>
21 #include <net/xdp.h>
22 
23 #define CREATE_TRACE_POINTS
24 #include <trace/events/bpf_test_run.h>
25 
26 struct bpf_test_timer {
27 	enum { NO_PREEMPT, NO_MIGRATE } mode;
28 	u32 i;
29 	u64 time_start, time_spent;
30 };
31 
32 static void bpf_test_timer_enter(struct bpf_test_timer *t)
33 	__acquires(rcu)
34 {
35 	rcu_read_lock();
36 	if (t->mode == NO_PREEMPT)
37 		preempt_disable();
38 	else
39 		migrate_disable();
40 
41 	t->time_start = ktime_get_ns();
42 }
43 
44 static void bpf_test_timer_leave(struct bpf_test_timer *t)
45 	__releases(rcu)
46 {
47 	t->time_start = 0;
48 
49 	if (t->mode == NO_PREEMPT)
50 		preempt_enable();
51 	else
52 		migrate_enable();
53 	rcu_read_unlock();
54 }
55 
56 static bool bpf_test_timer_continue(struct bpf_test_timer *t, u32 repeat, int *err, u32 *duration)
57 	__must_hold(rcu)
58 {
59 	t->i++;
60 	if (t->i >= repeat) {
61 		/* We're done. */
62 		t->time_spent += ktime_get_ns() - t->time_start;
63 		do_div(t->time_spent, t->i);
64 		*duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent;
65 		*err = 0;
66 		goto reset;
67 	}
68 
69 	if (signal_pending(current)) {
70 		/* During iteration: we've been cancelled, abort. */
71 		*err = -EINTR;
72 		goto reset;
73 	}
74 
75 	if (need_resched()) {
76 		/* During iteration: we need to reschedule between runs. */
77 		t->time_spent += ktime_get_ns() - t->time_start;
78 		bpf_test_timer_leave(t);
79 		cond_resched();
80 		bpf_test_timer_enter(t);
81 	}
82 
83 	/* Do another round. */
84 	return true;
85 
86 reset:
87 	t->i = 0;
88 	return false;
89 }
90 
91 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
92 			u32 *retval, u32 *time, bool xdp)
93 {
94 	struct bpf_prog_array_item item = {.prog = prog};
95 	struct bpf_run_ctx *old_ctx;
96 	struct bpf_cg_run_ctx run_ctx;
97 	struct bpf_test_timer t = { NO_MIGRATE };
98 	enum bpf_cgroup_storage_type stype;
99 	int ret;
100 
101 	for_each_cgroup_storage_type(stype) {
102 		item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
103 		if (IS_ERR(item.cgroup_storage[stype])) {
104 			item.cgroup_storage[stype] = NULL;
105 			for_each_cgroup_storage_type(stype)
106 				bpf_cgroup_storage_free(item.cgroup_storage[stype]);
107 			return -ENOMEM;
108 		}
109 	}
110 
111 	if (!repeat)
112 		repeat = 1;
113 
114 	bpf_test_timer_enter(&t);
115 	old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
116 	do {
117 		run_ctx.prog_item = &item;
118 		if (xdp)
119 			*retval = bpf_prog_run_xdp(prog, ctx);
120 		else
121 			*retval = bpf_prog_run(prog, ctx);
122 	} while (bpf_test_timer_continue(&t, repeat, &ret, time));
123 	bpf_reset_run_ctx(old_ctx);
124 	bpf_test_timer_leave(&t);
125 
126 	for_each_cgroup_storage_type(stype)
127 		bpf_cgroup_storage_free(item.cgroup_storage[stype]);
128 
129 	return ret;
130 }
131 
132 static int bpf_test_finish(const union bpf_attr *kattr,
133 			   union bpf_attr __user *uattr, const void *data,
134 			   u32 size, u32 retval, u32 duration)
135 {
136 	void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
137 	int err = -EFAULT;
138 	u32 copy_size = size;
139 
140 	/* Clamp copy if the user has provided a size hint, but copy the full
141 	 * buffer if not to retain old behaviour.
142 	 */
143 	if (kattr->test.data_size_out &&
144 	    copy_size > kattr->test.data_size_out) {
145 		copy_size = kattr->test.data_size_out;
146 		err = -ENOSPC;
147 	}
148 
149 	if (data_out && copy_to_user(data_out, data, copy_size))
150 		goto out;
151 	if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
152 		goto out;
153 	if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
154 		goto out;
155 	if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
156 		goto out;
157 	if (err != -ENOSPC)
158 		err = 0;
159 out:
160 	trace_bpf_test_finish(&err);
161 	return err;
162 }
163 
164 /* Integer types of various sizes and pointer combinations cover variety of
165  * architecture dependent calling conventions. 7+ can be supported in the
166  * future.
167  */
168 __diag_push();
169 __diag_ignore(GCC, 8, "-Wmissing-prototypes",
170 	      "Global functions as their definitions will be in vmlinux BTF");
171 int noinline bpf_fentry_test1(int a)
172 {
173 	return a + 1;
174 }
175 
176 int noinline bpf_fentry_test2(int a, u64 b)
177 {
178 	return a + b;
179 }
180 
181 int noinline bpf_fentry_test3(char a, int b, u64 c)
182 {
183 	return a + b + c;
184 }
185 
186 int noinline bpf_fentry_test4(void *a, char b, int c, u64 d)
187 {
188 	return (long)a + b + c + d;
189 }
190 
191 int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e)
192 {
193 	return a + (long)b + c + d + e;
194 }
195 
196 int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f)
197 {
198 	return a + (long)b + c + d + (long)e + f;
199 }
200 
201 struct bpf_fentry_test_t {
202 	struct bpf_fentry_test_t *a;
203 };
204 
205 int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg)
206 {
207 	return (long)arg;
208 }
209 
210 int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg)
211 {
212 	return (long)arg->a;
213 }
214 
215 int noinline bpf_modify_return_test(int a, int *b)
216 {
217 	*b += 1;
218 	return a + *b;
219 }
220 
221 u64 noinline bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d)
222 {
223 	return a + b + c + d;
224 }
225 
226 int noinline bpf_kfunc_call_test2(struct sock *sk, u32 a, u32 b)
227 {
228 	return a + b;
229 }
230 
231 struct sock * noinline bpf_kfunc_call_test3(struct sock *sk)
232 {
233 	return sk;
234 }
235 
236 struct prog_test_ref_kfunc {
237 	int a;
238 	int b;
239 	struct prog_test_ref_kfunc *next;
240 };
241 
242 static struct prog_test_ref_kfunc prog_test_struct = {
243 	.a = 42,
244 	.b = 108,
245 	.next = &prog_test_struct,
246 };
247 
248 noinline struct prog_test_ref_kfunc *
249 bpf_kfunc_call_test_acquire(unsigned long *scalar_ptr)
250 {
251 	/* randomly return NULL */
252 	if (get_jiffies_64() % 2)
253 		return NULL;
254 	return &prog_test_struct;
255 }
256 
257 noinline void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p)
258 {
259 }
260 
261 struct prog_test_pass1 {
262 	int x0;
263 	struct {
264 		int x1;
265 		struct {
266 			int x2;
267 			struct {
268 				int x3;
269 			};
270 		};
271 	};
272 };
273 
274 struct prog_test_pass2 {
275 	int len;
276 	short arr1[4];
277 	struct {
278 		char arr2[4];
279 		unsigned long arr3[8];
280 	} x;
281 };
282 
283 struct prog_test_fail1 {
284 	void *p;
285 	int x;
286 };
287 
288 struct prog_test_fail2 {
289 	int x8;
290 	struct prog_test_pass1 x;
291 };
292 
293 struct prog_test_fail3 {
294 	int len;
295 	char arr1[2];
296 	char arr2[0];
297 };
298 
299 noinline void bpf_kfunc_call_test_pass_ctx(struct __sk_buff *skb)
300 {
301 }
302 
303 noinline void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p)
304 {
305 }
306 
307 noinline void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p)
308 {
309 }
310 
311 noinline void bpf_kfunc_call_test_fail1(struct prog_test_fail1 *p)
312 {
313 }
314 
315 noinline void bpf_kfunc_call_test_fail2(struct prog_test_fail2 *p)
316 {
317 }
318 
319 noinline void bpf_kfunc_call_test_fail3(struct prog_test_fail3 *p)
320 {
321 }
322 
323 noinline void bpf_kfunc_call_test_mem_len_pass1(void *mem, int mem__sz)
324 {
325 }
326 
327 noinline void bpf_kfunc_call_test_mem_len_fail1(void *mem, int len)
328 {
329 }
330 
331 noinline void bpf_kfunc_call_test_mem_len_fail2(u64 *mem, int len)
332 {
333 }
334 
335 __diag_pop();
336 
337 ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO);
338 
339 BTF_SET_START(test_sk_check_kfunc_ids)
340 BTF_ID(func, bpf_kfunc_call_test1)
341 BTF_ID(func, bpf_kfunc_call_test2)
342 BTF_ID(func, bpf_kfunc_call_test3)
343 BTF_ID(func, bpf_kfunc_call_test_acquire)
344 BTF_ID(func, bpf_kfunc_call_test_release)
345 BTF_ID(func, bpf_kfunc_call_test_pass_ctx)
346 BTF_ID(func, bpf_kfunc_call_test_pass1)
347 BTF_ID(func, bpf_kfunc_call_test_pass2)
348 BTF_ID(func, bpf_kfunc_call_test_fail1)
349 BTF_ID(func, bpf_kfunc_call_test_fail2)
350 BTF_ID(func, bpf_kfunc_call_test_fail3)
351 BTF_ID(func, bpf_kfunc_call_test_mem_len_pass1)
352 BTF_ID(func, bpf_kfunc_call_test_mem_len_fail1)
353 BTF_ID(func, bpf_kfunc_call_test_mem_len_fail2)
354 BTF_SET_END(test_sk_check_kfunc_ids)
355 
356 BTF_SET_START(test_sk_acquire_kfunc_ids)
357 BTF_ID(func, bpf_kfunc_call_test_acquire)
358 BTF_SET_END(test_sk_acquire_kfunc_ids)
359 
360 BTF_SET_START(test_sk_release_kfunc_ids)
361 BTF_ID(func, bpf_kfunc_call_test_release)
362 BTF_SET_END(test_sk_release_kfunc_ids)
363 
364 BTF_SET_START(test_sk_ret_null_kfunc_ids)
365 BTF_ID(func, bpf_kfunc_call_test_acquire)
366 BTF_SET_END(test_sk_ret_null_kfunc_ids)
367 
368 static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
369 			   u32 headroom, u32 tailroom)
370 {
371 	void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
372 	u32 user_size = kattr->test.data_size_in;
373 	void *data;
374 
375 	if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
376 		return ERR_PTR(-EINVAL);
377 
378 	if (user_size > size)
379 		return ERR_PTR(-EMSGSIZE);
380 
381 	data = kzalloc(size + headroom + tailroom, GFP_USER);
382 	if (!data)
383 		return ERR_PTR(-ENOMEM);
384 
385 	if (copy_from_user(data + headroom, data_in, user_size)) {
386 		kfree(data);
387 		return ERR_PTR(-EFAULT);
388 	}
389 
390 	return data;
391 }
392 
393 int bpf_prog_test_run_tracing(struct bpf_prog *prog,
394 			      const union bpf_attr *kattr,
395 			      union bpf_attr __user *uattr)
396 {
397 	struct bpf_fentry_test_t arg = {};
398 	u16 side_effect = 0, ret = 0;
399 	int b = 2, err = -EFAULT;
400 	u32 retval = 0;
401 
402 	if (kattr->test.flags || kattr->test.cpu)
403 		return -EINVAL;
404 
405 	switch (prog->expected_attach_type) {
406 	case BPF_TRACE_FENTRY:
407 	case BPF_TRACE_FEXIT:
408 		if (bpf_fentry_test1(1) != 2 ||
409 		    bpf_fentry_test2(2, 3) != 5 ||
410 		    bpf_fentry_test3(4, 5, 6) != 15 ||
411 		    bpf_fentry_test4((void *)7, 8, 9, 10) != 34 ||
412 		    bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 ||
413 		    bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 ||
414 		    bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 ||
415 		    bpf_fentry_test8(&arg) != 0)
416 			goto out;
417 		break;
418 	case BPF_MODIFY_RETURN:
419 		ret = bpf_modify_return_test(1, &b);
420 		if (b != 2)
421 			side_effect = 1;
422 		break;
423 	default:
424 		goto out;
425 	}
426 
427 	retval = ((u32)side_effect << 16) | ret;
428 	if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
429 		goto out;
430 
431 	err = 0;
432 out:
433 	trace_bpf_test_finish(&err);
434 	return err;
435 }
436 
437 struct bpf_raw_tp_test_run_info {
438 	struct bpf_prog *prog;
439 	void *ctx;
440 	u32 retval;
441 };
442 
443 static void
444 __bpf_prog_test_run_raw_tp(void *data)
445 {
446 	struct bpf_raw_tp_test_run_info *info = data;
447 
448 	rcu_read_lock();
449 	info->retval = bpf_prog_run(info->prog, info->ctx);
450 	rcu_read_unlock();
451 }
452 
453 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
454 			     const union bpf_attr *kattr,
455 			     union bpf_attr __user *uattr)
456 {
457 	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
458 	__u32 ctx_size_in = kattr->test.ctx_size_in;
459 	struct bpf_raw_tp_test_run_info info;
460 	int cpu = kattr->test.cpu, err = 0;
461 	int current_cpu;
462 
463 	/* doesn't support data_in/out, ctx_out, duration, or repeat */
464 	if (kattr->test.data_in || kattr->test.data_out ||
465 	    kattr->test.ctx_out || kattr->test.duration ||
466 	    kattr->test.repeat)
467 		return -EINVAL;
468 
469 	if (ctx_size_in < prog->aux->max_ctx_offset ||
470 	    ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64))
471 		return -EINVAL;
472 
473 	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)
474 		return -EINVAL;
475 
476 	if (ctx_size_in) {
477 		info.ctx = memdup_user(ctx_in, ctx_size_in);
478 		if (IS_ERR(info.ctx))
479 			return PTR_ERR(info.ctx);
480 	} else {
481 		info.ctx = NULL;
482 	}
483 
484 	info.prog = prog;
485 
486 	current_cpu = get_cpu();
487 	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||
488 	    cpu == current_cpu) {
489 		__bpf_prog_test_run_raw_tp(&info);
490 	} else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
491 		/* smp_call_function_single() also checks cpu_online()
492 		 * after csd_lock(). However, since cpu is from user
493 		 * space, let's do an extra quick check to filter out
494 		 * invalid value before smp_call_function_single().
495 		 */
496 		err = -ENXIO;
497 	} else {
498 		err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp,
499 					       &info, 1);
500 	}
501 	put_cpu();
502 
503 	if (!err &&
504 	    copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32)))
505 		err = -EFAULT;
506 
507 	kfree(info.ctx);
508 	return err;
509 }
510 
511 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
512 {
513 	void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
514 	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
515 	u32 size = kattr->test.ctx_size_in;
516 	void *data;
517 	int err;
518 
519 	if (!data_in && !data_out)
520 		return NULL;
521 
522 	data = kzalloc(max_size, GFP_USER);
523 	if (!data)
524 		return ERR_PTR(-ENOMEM);
525 
526 	if (data_in) {
527 		err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size);
528 		if (err) {
529 			kfree(data);
530 			return ERR_PTR(err);
531 		}
532 
533 		size = min_t(u32, max_size, size);
534 		if (copy_from_user(data, data_in, size)) {
535 			kfree(data);
536 			return ERR_PTR(-EFAULT);
537 		}
538 	}
539 	return data;
540 }
541 
542 static int bpf_ctx_finish(const union bpf_attr *kattr,
543 			  union bpf_attr __user *uattr, const void *data,
544 			  u32 size)
545 {
546 	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
547 	int err = -EFAULT;
548 	u32 copy_size = size;
549 
550 	if (!data || !data_out)
551 		return 0;
552 
553 	if (copy_size > kattr->test.ctx_size_out) {
554 		copy_size = kattr->test.ctx_size_out;
555 		err = -ENOSPC;
556 	}
557 
558 	if (copy_to_user(data_out, data, copy_size))
559 		goto out;
560 	if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
561 		goto out;
562 	if (err != -ENOSPC)
563 		err = 0;
564 out:
565 	return err;
566 }
567 
568 /**
569  * range_is_zero - test whether buffer is initialized
570  * @buf: buffer to check
571  * @from: check from this position
572  * @to: check up until (excluding) this position
573  *
574  * This function returns true if the there is a non-zero byte
575  * in the buf in the range [from,to).
576  */
577 static inline bool range_is_zero(void *buf, size_t from, size_t to)
578 {
579 	return !memchr_inv((u8 *)buf + from, 0, to - from);
580 }
581 
582 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
583 {
584 	struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
585 
586 	if (!__skb)
587 		return 0;
588 
589 	/* make sure the fields we don't use are zeroed */
590 	if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark)))
591 		return -EINVAL;
592 
593 	/* mark is allowed */
594 
595 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark),
596 			   offsetof(struct __sk_buff, priority)))
597 		return -EINVAL;
598 
599 	/* priority is allowed */
600 	/* ingress_ifindex is allowed */
601 	/* ifindex is allowed */
602 
603 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex),
604 			   offsetof(struct __sk_buff, cb)))
605 		return -EINVAL;
606 
607 	/* cb is allowed */
608 
609 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb),
610 			   offsetof(struct __sk_buff, tstamp)))
611 		return -EINVAL;
612 
613 	/* tstamp is allowed */
614 	/* wire_len is allowed */
615 	/* gso_segs is allowed */
616 
617 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs),
618 			   offsetof(struct __sk_buff, gso_size)))
619 		return -EINVAL;
620 
621 	/* gso_size is allowed */
622 
623 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size),
624 			   offsetof(struct __sk_buff, hwtstamp)))
625 		return -EINVAL;
626 
627 	/* hwtstamp is allowed */
628 
629 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp),
630 			   sizeof(struct __sk_buff)))
631 		return -EINVAL;
632 
633 	skb->mark = __skb->mark;
634 	skb->priority = __skb->priority;
635 	skb->skb_iif = __skb->ingress_ifindex;
636 	skb->tstamp = __skb->tstamp;
637 	memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
638 
639 	if (__skb->wire_len == 0) {
640 		cb->pkt_len = skb->len;
641 	} else {
642 		if (__skb->wire_len < skb->len ||
643 		    __skb->wire_len > GSO_MAX_SIZE)
644 			return -EINVAL;
645 		cb->pkt_len = __skb->wire_len;
646 	}
647 
648 	if (__skb->gso_segs > GSO_MAX_SEGS)
649 		return -EINVAL;
650 	skb_shinfo(skb)->gso_segs = __skb->gso_segs;
651 	skb_shinfo(skb)->gso_size = __skb->gso_size;
652 	skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp;
653 
654 	return 0;
655 }
656 
657 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
658 {
659 	struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
660 
661 	if (!__skb)
662 		return;
663 
664 	__skb->mark = skb->mark;
665 	__skb->priority = skb->priority;
666 	__skb->ingress_ifindex = skb->skb_iif;
667 	__skb->ifindex = skb->dev->ifindex;
668 	__skb->tstamp = skb->tstamp;
669 	memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
670 	__skb->wire_len = cb->pkt_len;
671 	__skb->gso_segs = skb_shinfo(skb)->gso_segs;
672 	__skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp;
673 }
674 
675 static struct proto bpf_dummy_proto = {
676 	.name   = "bpf_dummy",
677 	.owner  = THIS_MODULE,
678 	.obj_size = sizeof(struct sock),
679 };
680 
681 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
682 			  union bpf_attr __user *uattr)
683 {
684 	bool is_l2 = false, is_direct_pkt_access = false;
685 	struct net *net = current->nsproxy->net_ns;
686 	struct net_device *dev = net->loopback_dev;
687 	u32 size = kattr->test.data_size_in;
688 	u32 repeat = kattr->test.repeat;
689 	struct __sk_buff *ctx = NULL;
690 	u32 retval, duration;
691 	int hh_len = ETH_HLEN;
692 	struct sk_buff *skb;
693 	struct sock *sk;
694 	void *data;
695 	int ret;
696 
697 	if (kattr->test.flags || kattr->test.cpu)
698 		return -EINVAL;
699 
700 	data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
701 			     SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
702 	if (IS_ERR(data))
703 		return PTR_ERR(data);
704 
705 	ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
706 	if (IS_ERR(ctx)) {
707 		kfree(data);
708 		return PTR_ERR(ctx);
709 	}
710 
711 	switch (prog->type) {
712 	case BPF_PROG_TYPE_SCHED_CLS:
713 	case BPF_PROG_TYPE_SCHED_ACT:
714 		is_l2 = true;
715 		fallthrough;
716 	case BPF_PROG_TYPE_LWT_IN:
717 	case BPF_PROG_TYPE_LWT_OUT:
718 	case BPF_PROG_TYPE_LWT_XMIT:
719 		is_direct_pkt_access = true;
720 		break;
721 	default:
722 		break;
723 	}
724 
725 	sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1);
726 	if (!sk) {
727 		kfree(data);
728 		kfree(ctx);
729 		return -ENOMEM;
730 	}
731 	sock_init_data(NULL, sk);
732 
733 	skb = build_skb(data, 0);
734 	if (!skb) {
735 		kfree(data);
736 		kfree(ctx);
737 		sk_free(sk);
738 		return -ENOMEM;
739 	}
740 	skb->sk = sk;
741 
742 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
743 	__skb_put(skb, size);
744 	if (ctx && ctx->ifindex > 1) {
745 		dev = dev_get_by_index(net, ctx->ifindex);
746 		if (!dev) {
747 			ret = -ENODEV;
748 			goto out;
749 		}
750 	}
751 	skb->protocol = eth_type_trans(skb, dev);
752 	skb_reset_network_header(skb);
753 
754 	switch (skb->protocol) {
755 	case htons(ETH_P_IP):
756 		sk->sk_family = AF_INET;
757 		if (sizeof(struct iphdr) <= skb_headlen(skb)) {
758 			sk->sk_rcv_saddr = ip_hdr(skb)->saddr;
759 			sk->sk_daddr = ip_hdr(skb)->daddr;
760 		}
761 		break;
762 #if IS_ENABLED(CONFIG_IPV6)
763 	case htons(ETH_P_IPV6):
764 		sk->sk_family = AF_INET6;
765 		if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) {
766 			sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr;
767 			sk->sk_v6_daddr = ipv6_hdr(skb)->daddr;
768 		}
769 		break;
770 #endif
771 	default:
772 		break;
773 	}
774 
775 	if (is_l2)
776 		__skb_push(skb, hh_len);
777 	if (is_direct_pkt_access)
778 		bpf_compute_data_pointers(skb);
779 	ret = convert___skb_to_skb(skb, ctx);
780 	if (ret)
781 		goto out;
782 	ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false);
783 	if (ret)
784 		goto out;
785 	if (!is_l2) {
786 		if (skb_headroom(skb) < hh_len) {
787 			int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
788 
789 			if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
790 				ret = -ENOMEM;
791 				goto out;
792 			}
793 		}
794 		memset(__skb_push(skb, hh_len), 0, hh_len);
795 	}
796 	convert_skb_to___skb(skb, ctx);
797 
798 	size = skb->len;
799 	/* bpf program can never convert linear skb to non-linear */
800 	if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
801 		size = skb_headlen(skb);
802 	ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
803 	if (!ret)
804 		ret = bpf_ctx_finish(kattr, uattr, ctx,
805 				     sizeof(struct __sk_buff));
806 out:
807 	if (dev && dev != net->loopback_dev)
808 		dev_put(dev);
809 	kfree_skb(skb);
810 	sk_free(sk);
811 	kfree(ctx);
812 	return ret;
813 }
814 
815 static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp)
816 {
817 	unsigned int ingress_ifindex, rx_queue_index;
818 	struct netdev_rx_queue *rxqueue;
819 	struct net_device *device;
820 
821 	if (!xdp_md)
822 		return 0;
823 
824 	if (xdp_md->egress_ifindex != 0)
825 		return -EINVAL;
826 
827 	ingress_ifindex = xdp_md->ingress_ifindex;
828 	rx_queue_index = xdp_md->rx_queue_index;
829 
830 	if (!ingress_ifindex && rx_queue_index)
831 		return -EINVAL;
832 
833 	if (ingress_ifindex) {
834 		device = dev_get_by_index(current->nsproxy->net_ns,
835 					  ingress_ifindex);
836 		if (!device)
837 			return -ENODEV;
838 
839 		if (rx_queue_index >= device->real_num_rx_queues)
840 			goto free_dev;
841 
842 		rxqueue = __netif_get_rx_queue(device, rx_queue_index);
843 
844 		if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq))
845 			goto free_dev;
846 
847 		xdp->rxq = &rxqueue->xdp_rxq;
848 		/* The device is now tracked in the xdp->rxq for later
849 		 * dev_put()
850 		 */
851 	}
852 
853 	xdp->data = xdp->data_meta + xdp_md->data;
854 	return 0;
855 
856 free_dev:
857 	dev_put(device);
858 	return -EINVAL;
859 }
860 
861 static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md)
862 {
863 	if (!xdp_md)
864 		return;
865 
866 	xdp_md->data = xdp->data - xdp->data_meta;
867 	xdp_md->data_end = xdp->data_end - xdp->data_meta;
868 
869 	if (xdp_md->ingress_ifindex)
870 		dev_put(xdp->rxq->dev);
871 }
872 
873 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
874 			  union bpf_attr __user *uattr)
875 {
876 	u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
877 	u32 headroom = XDP_PACKET_HEADROOM;
878 	u32 size = kattr->test.data_size_in;
879 	u32 repeat = kattr->test.repeat;
880 	struct netdev_rx_queue *rxqueue;
881 	struct xdp_buff xdp = {};
882 	u32 retval, duration;
883 	struct xdp_md *ctx;
884 	u32 max_data_sz;
885 	void *data;
886 	int ret = -EINVAL;
887 
888 	if (prog->expected_attach_type == BPF_XDP_DEVMAP ||
889 	    prog->expected_attach_type == BPF_XDP_CPUMAP)
890 		return -EINVAL;
891 
892 	ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md));
893 	if (IS_ERR(ctx))
894 		return PTR_ERR(ctx);
895 
896 	if (ctx) {
897 		/* There can't be user provided data before the meta data */
898 		if (ctx->data_meta || ctx->data_end != size ||
899 		    ctx->data > ctx->data_end ||
900 		    unlikely(xdp_metalen_invalid(ctx->data)))
901 			goto free_ctx;
902 		/* Meta data is allocated from the headroom */
903 		headroom -= ctx->data;
904 	}
905 
906 	/* XDP have extra tailroom as (most) drivers use full page */
907 	max_data_sz = 4096 - headroom - tailroom;
908 
909 	data = bpf_test_init(kattr, max_data_sz, headroom, tailroom);
910 	if (IS_ERR(data)) {
911 		ret = PTR_ERR(data);
912 		goto free_ctx;
913 	}
914 
915 	rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
916 	xdp_init_buff(&xdp, headroom + max_data_sz + tailroom,
917 		      &rxqueue->xdp_rxq);
918 	xdp_prepare_buff(&xdp, data, headroom, size, true);
919 
920 	ret = xdp_convert_md_to_buff(ctx, &xdp);
921 	if (ret)
922 		goto free_data;
923 
924 	if (repeat > 1)
925 		bpf_prog_change_xdp(NULL, prog);
926 	ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true);
927 	/* We convert the xdp_buff back to an xdp_md before checking the return
928 	 * code so the reference count of any held netdevice will be decremented
929 	 * even if the test run failed.
930 	 */
931 	xdp_convert_buff_to_md(&xdp, ctx);
932 	if (ret)
933 		goto out;
934 
935 	if (xdp.data_meta != data + headroom ||
936 	    xdp.data_end != xdp.data_meta + size)
937 		size = xdp.data_end - xdp.data_meta;
938 
939 	ret = bpf_test_finish(kattr, uattr, xdp.data_meta, size, retval,
940 			      duration);
941 	if (!ret)
942 		ret = bpf_ctx_finish(kattr, uattr, ctx,
943 				     sizeof(struct xdp_md));
944 
945 out:
946 	if (repeat > 1)
947 		bpf_prog_change_xdp(prog, NULL);
948 free_data:
949 	kfree(data);
950 free_ctx:
951 	kfree(ctx);
952 	return ret;
953 }
954 
955 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
956 {
957 	/* make sure the fields we don't use are zeroed */
958 	if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags)))
959 		return -EINVAL;
960 
961 	/* flags is allowed */
962 
963 	if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags),
964 			   sizeof(struct bpf_flow_keys)))
965 		return -EINVAL;
966 
967 	return 0;
968 }
969 
970 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
971 				     const union bpf_attr *kattr,
972 				     union bpf_attr __user *uattr)
973 {
974 	struct bpf_test_timer t = { NO_PREEMPT };
975 	u32 size = kattr->test.data_size_in;
976 	struct bpf_flow_dissector ctx = {};
977 	u32 repeat = kattr->test.repeat;
978 	struct bpf_flow_keys *user_ctx;
979 	struct bpf_flow_keys flow_keys;
980 	const struct ethhdr *eth;
981 	unsigned int flags = 0;
982 	u32 retval, duration;
983 	void *data;
984 	int ret;
985 
986 	if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
987 		return -EINVAL;
988 
989 	if (kattr->test.flags || kattr->test.cpu)
990 		return -EINVAL;
991 
992 	if (size < ETH_HLEN)
993 		return -EINVAL;
994 
995 	data = bpf_test_init(kattr, size, 0, 0);
996 	if (IS_ERR(data))
997 		return PTR_ERR(data);
998 
999 	eth = (struct ethhdr *)data;
1000 
1001 	if (!repeat)
1002 		repeat = 1;
1003 
1004 	user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys));
1005 	if (IS_ERR(user_ctx)) {
1006 		kfree(data);
1007 		return PTR_ERR(user_ctx);
1008 	}
1009 	if (user_ctx) {
1010 		ret = verify_user_bpf_flow_keys(user_ctx);
1011 		if (ret)
1012 			goto out;
1013 		flags = user_ctx->flags;
1014 	}
1015 
1016 	ctx.flow_keys = &flow_keys;
1017 	ctx.data = data;
1018 	ctx.data_end = (__u8 *)data + size;
1019 
1020 	bpf_test_timer_enter(&t);
1021 	do {
1022 		retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
1023 					  size, flags);
1024 	} while (bpf_test_timer_continue(&t, repeat, &ret, &duration));
1025 	bpf_test_timer_leave(&t);
1026 
1027 	if (ret < 0)
1028 		goto out;
1029 
1030 	ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),
1031 			      retval, duration);
1032 	if (!ret)
1033 		ret = bpf_ctx_finish(kattr, uattr, user_ctx,
1034 				     sizeof(struct bpf_flow_keys));
1035 
1036 out:
1037 	kfree(user_ctx);
1038 	kfree(data);
1039 	return ret;
1040 }
1041 
1042 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,
1043 				union bpf_attr __user *uattr)
1044 {
1045 	struct bpf_test_timer t = { NO_PREEMPT };
1046 	struct bpf_prog_array *progs = NULL;
1047 	struct bpf_sk_lookup_kern ctx = {};
1048 	u32 repeat = kattr->test.repeat;
1049 	struct bpf_sk_lookup *user_ctx;
1050 	u32 retval, duration;
1051 	int ret = -EINVAL;
1052 
1053 	if (prog->type != BPF_PROG_TYPE_SK_LOOKUP)
1054 		return -EINVAL;
1055 
1056 	if (kattr->test.flags || kattr->test.cpu)
1057 		return -EINVAL;
1058 
1059 	if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||
1060 	    kattr->test.data_size_out)
1061 		return -EINVAL;
1062 
1063 	if (!repeat)
1064 		repeat = 1;
1065 
1066 	user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx));
1067 	if (IS_ERR(user_ctx))
1068 		return PTR_ERR(user_ctx);
1069 
1070 	if (!user_ctx)
1071 		return -EINVAL;
1072 
1073 	if (user_ctx->sk)
1074 		goto out;
1075 
1076 	if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx)))
1077 		goto out;
1078 
1079 	if (user_ctx->local_port > U16_MAX || user_ctx->remote_port > U16_MAX) {
1080 		ret = -ERANGE;
1081 		goto out;
1082 	}
1083 
1084 	ctx.family = (u16)user_ctx->family;
1085 	ctx.protocol = (u16)user_ctx->protocol;
1086 	ctx.dport = (u16)user_ctx->local_port;
1087 	ctx.sport = (__force __be16)user_ctx->remote_port;
1088 
1089 	switch (ctx.family) {
1090 	case AF_INET:
1091 		ctx.v4.daddr = (__force __be32)user_ctx->local_ip4;
1092 		ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4;
1093 		break;
1094 
1095 #if IS_ENABLED(CONFIG_IPV6)
1096 	case AF_INET6:
1097 		ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6;
1098 		ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6;
1099 		break;
1100 #endif
1101 
1102 	default:
1103 		ret = -EAFNOSUPPORT;
1104 		goto out;
1105 	}
1106 
1107 	progs = bpf_prog_array_alloc(1, GFP_KERNEL);
1108 	if (!progs) {
1109 		ret = -ENOMEM;
1110 		goto out;
1111 	}
1112 
1113 	progs->items[0].prog = prog;
1114 
1115 	bpf_test_timer_enter(&t);
1116 	do {
1117 		ctx.selected_sk = NULL;
1118 		retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run);
1119 	} while (bpf_test_timer_continue(&t, repeat, &ret, &duration));
1120 	bpf_test_timer_leave(&t);
1121 
1122 	if (ret < 0)
1123 		goto out;
1124 
1125 	user_ctx->cookie = 0;
1126 	if (ctx.selected_sk) {
1127 		if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) {
1128 			ret = -EOPNOTSUPP;
1129 			goto out;
1130 		}
1131 
1132 		user_ctx->cookie = sock_gen_cookie(ctx.selected_sk);
1133 	}
1134 
1135 	ret = bpf_test_finish(kattr, uattr, NULL, 0, retval, duration);
1136 	if (!ret)
1137 		ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx));
1138 
1139 out:
1140 	bpf_prog_array_free(progs);
1141 	kfree(user_ctx);
1142 	return ret;
1143 }
1144 
1145 int bpf_prog_test_run_syscall(struct bpf_prog *prog,
1146 			      const union bpf_attr *kattr,
1147 			      union bpf_attr __user *uattr)
1148 {
1149 	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
1150 	__u32 ctx_size_in = kattr->test.ctx_size_in;
1151 	void *ctx = NULL;
1152 	u32 retval;
1153 	int err = 0;
1154 
1155 	/* doesn't support data_in/out, ctx_out, duration, or repeat or flags */
1156 	if (kattr->test.data_in || kattr->test.data_out ||
1157 	    kattr->test.ctx_out || kattr->test.duration ||
1158 	    kattr->test.repeat || kattr->test.flags)
1159 		return -EINVAL;
1160 
1161 	if (ctx_size_in < prog->aux->max_ctx_offset ||
1162 	    ctx_size_in > U16_MAX)
1163 		return -EINVAL;
1164 
1165 	if (ctx_size_in) {
1166 		ctx = memdup_user(ctx_in, ctx_size_in);
1167 		if (IS_ERR(ctx))
1168 			return PTR_ERR(ctx);
1169 	}
1170 
1171 	rcu_read_lock_trace();
1172 	retval = bpf_prog_run_pin_on_cpu(prog, ctx);
1173 	rcu_read_unlock_trace();
1174 
1175 	if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) {
1176 		err = -EFAULT;
1177 		goto out;
1178 	}
1179 	if (ctx_size_in)
1180 		if (copy_to_user(ctx_in, ctx, ctx_size_in))
1181 			err = -EFAULT;
1182 out:
1183 	kfree(ctx);
1184 	return err;
1185 }
1186 
1187 static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = {
1188 	.owner        = THIS_MODULE,
1189 	.check_set    = &test_sk_check_kfunc_ids,
1190 	.acquire_set  = &test_sk_acquire_kfunc_ids,
1191 	.release_set  = &test_sk_release_kfunc_ids,
1192 	.ret_null_set = &test_sk_ret_null_kfunc_ids,
1193 };
1194 
1195 static int __init bpf_prog_test_run_init(void)
1196 {
1197 	return register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_prog_test_kfunc_set);
1198 }
1199 late_initcall(bpf_prog_test_run_init);
1200