xref: /linux/net/bpf/test_run.c (revision 4656569643409568fa7c162614c17277abdf84de) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2017 Facebook
3  */
4 #include <linux/bpf.h>
5 #include <linux/btf.h>
6 #include <linux/btf_ids.h>
7 #include <linux/slab.h>
8 #include <linux/init.h>
9 #include <linux/vmalloc.h>
10 #include <linux/etherdevice.h>
11 #include <linux/filter.h>
12 #include <linux/rcupdate_trace.h>
13 #include <linux/sched/signal.h>
14 #include <net/bpf_sk_storage.h>
15 #include <net/sock.h>
16 #include <net/tcp.h>
17 #include <net/net_namespace.h>
18 #include <linux/error-injection.h>
19 #include <linux/smp.h>
20 #include <linux/sock_diag.h>
21 #include <net/xdp.h>
22 
23 #define CREATE_TRACE_POINTS
24 #include <trace/events/bpf_test_run.h>
25 
26 struct bpf_test_timer {
27 	enum { NO_PREEMPT, NO_MIGRATE } mode;
28 	u32 i;
29 	u64 time_start, time_spent;
30 };
31 
32 static void bpf_test_timer_enter(struct bpf_test_timer *t)
33 	__acquires(rcu)
34 {
35 	rcu_read_lock();
36 	if (t->mode == NO_PREEMPT)
37 		preempt_disable();
38 	else
39 		migrate_disable();
40 
41 	t->time_start = ktime_get_ns();
42 }
43 
44 static void bpf_test_timer_leave(struct bpf_test_timer *t)
45 	__releases(rcu)
46 {
47 	t->time_start = 0;
48 
49 	if (t->mode == NO_PREEMPT)
50 		preempt_enable();
51 	else
52 		migrate_enable();
53 	rcu_read_unlock();
54 }
55 
56 static bool bpf_test_timer_continue(struct bpf_test_timer *t, u32 repeat, int *err, u32 *duration)
57 	__must_hold(rcu)
58 {
59 	t->i++;
60 	if (t->i >= repeat) {
61 		/* We're done. */
62 		t->time_spent += ktime_get_ns() - t->time_start;
63 		do_div(t->time_spent, t->i);
64 		*duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent;
65 		*err = 0;
66 		goto reset;
67 	}
68 
69 	if (signal_pending(current)) {
70 		/* During iteration: we've been cancelled, abort. */
71 		*err = -EINTR;
72 		goto reset;
73 	}
74 
75 	if (need_resched()) {
76 		/* During iteration: we need to reschedule between runs. */
77 		t->time_spent += ktime_get_ns() - t->time_start;
78 		bpf_test_timer_leave(t);
79 		cond_resched();
80 		bpf_test_timer_enter(t);
81 	}
82 
83 	/* Do another round. */
84 	return true;
85 
86 reset:
87 	t->i = 0;
88 	return false;
89 }
90 
91 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
92 			u32 *retval, u32 *time, bool xdp)
93 {
94 	struct bpf_prog_array_item item = {.prog = prog};
95 	struct bpf_run_ctx *old_ctx;
96 	struct bpf_cg_run_ctx run_ctx;
97 	struct bpf_test_timer t = { NO_MIGRATE };
98 	enum bpf_cgroup_storage_type stype;
99 	int ret;
100 
101 	for_each_cgroup_storage_type(stype) {
102 		item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
103 		if (IS_ERR(item.cgroup_storage[stype])) {
104 			item.cgroup_storage[stype] = NULL;
105 			for_each_cgroup_storage_type(stype)
106 				bpf_cgroup_storage_free(item.cgroup_storage[stype]);
107 			return -ENOMEM;
108 		}
109 	}
110 
111 	if (!repeat)
112 		repeat = 1;
113 
114 	bpf_test_timer_enter(&t);
115 	old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
116 	do {
117 		run_ctx.prog_item = &item;
118 		if (xdp)
119 			*retval = bpf_prog_run_xdp(prog, ctx);
120 		else
121 			*retval = bpf_prog_run(prog, ctx);
122 	} while (bpf_test_timer_continue(&t, repeat, &ret, time));
123 	bpf_reset_run_ctx(old_ctx);
124 	bpf_test_timer_leave(&t);
125 
126 	for_each_cgroup_storage_type(stype)
127 		bpf_cgroup_storage_free(item.cgroup_storage[stype]);
128 
129 	return ret;
130 }
131 
132 static int bpf_test_finish(const union bpf_attr *kattr,
133 			   union bpf_attr __user *uattr, const void *data,
134 			   u32 size, u32 retval, u32 duration)
135 {
136 	void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
137 	int err = -EFAULT;
138 	u32 copy_size = size;
139 
140 	/* Clamp copy if the user has provided a size hint, but copy the full
141 	 * buffer if not to retain old behaviour.
142 	 */
143 	if (kattr->test.data_size_out &&
144 	    copy_size > kattr->test.data_size_out) {
145 		copy_size = kattr->test.data_size_out;
146 		err = -ENOSPC;
147 	}
148 
149 	if (data_out && copy_to_user(data_out, data, copy_size))
150 		goto out;
151 	if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
152 		goto out;
153 	if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
154 		goto out;
155 	if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
156 		goto out;
157 	if (err != -ENOSPC)
158 		err = 0;
159 out:
160 	trace_bpf_test_finish(&err);
161 	return err;
162 }
163 
164 /* Integer types of various sizes and pointer combinations cover variety of
165  * architecture dependent calling conventions. 7+ can be supported in the
166  * future.
167  */
168 __diag_push();
169 __diag_ignore(GCC, 8, "-Wmissing-prototypes",
170 	      "Global functions as their definitions will be in vmlinux BTF");
171 int noinline bpf_fentry_test1(int a)
172 {
173 	return a + 1;
174 }
175 EXPORT_SYMBOL_GPL(bpf_fentry_test1);
176 ALLOW_ERROR_INJECTION(bpf_fentry_test1, ERRNO);
177 
178 int noinline bpf_fentry_test2(int a, u64 b)
179 {
180 	return a + b;
181 }
182 
183 int noinline bpf_fentry_test3(char a, int b, u64 c)
184 {
185 	return a + b + c;
186 }
187 
188 int noinline bpf_fentry_test4(void *a, char b, int c, u64 d)
189 {
190 	return (long)a + b + c + d;
191 }
192 
193 int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e)
194 {
195 	return a + (long)b + c + d + e;
196 }
197 
198 int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f)
199 {
200 	return a + (long)b + c + d + (long)e + f;
201 }
202 
203 struct bpf_fentry_test_t {
204 	struct bpf_fentry_test_t *a;
205 };
206 
207 int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg)
208 {
209 	return (long)arg;
210 }
211 
212 int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg)
213 {
214 	return (long)arg->a;
215 }
216 
217 int noinline bpf_modify_return_test(int a, int *b)
218 {
219 	*b += 1;
220 	return a + *b;
221 }
222 
223 u64 noinline bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d)
224 {
225 	return a + b + c + d;
226 }
227 
228 int noinline bpf_kfunc_call_test2(struct sock *sk, u32 a, u32 b)
229 {
230 	return a + b;
231 }
232 
233 struct sock * noinline bpf_kfunc_call_test3(struct sock *sk)
234 {
235 	return sk;
236 }
237 
238 struct prog_test_ref_kfunc {
239 	int a;
240 	int b;
241 	struct prog_test_ref_kfunc *next;
242 };
243 
244 static struct prog_test_ref_kfunc prog_test_struct = {
245 	.a = 42,
246 	.b = 108,
247 	.next = &prog_test_struct,
248 };
249 
250 noinline struct prog_test_ref_kfunc *
251 bpf_kfunc_call_test_acquire(unsigned long *scalar_ptr)
252 {
253 	/* randomly return NULL */
254 	if (get_jiffies_64() % 2)
255 		return NULL;
256 	return &prog_test_struct;
257 }
258 
259 noinline void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p)
260 {
261 }
262 
263 struct prog_test_pass1 {
264 	int x0;
265 	struct {
266 		int x1;
267 		struct {
268 			int x2;
269 			struct {
270 				int x3;
271 			};
272 		};
273 	};
274 };
275 
276 struct prog_test_pass2 {
277 	int len;
278 	short arr1[4];
279 	struct {
280 		char arr2[4];
281 		unsigned long arr3[8];
282 	} x;
283 };
284 
285 struct prog_test_fail1 {
286 	void *p;
287 	int x;
288 };
289 
290 struct prog_test_fail2 {
291 	int x8;
292 	struct prog_test_pass1 x;
293 };
294 
295 struct prog_test_fail3 {
296 	int len;
297 	char arr1[2];
298 	char arr2[0];
299 };
300 
301 noinline void bpf_kfunc_call_test_pass_ctx(struct __sk_buff *skb)
302 {
303 }
304 
305 noinline void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p)
306 {
307 }
308 
309 noinline void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p)
310 {
311 }
312 
313 noinline void bpf_kfunc_call_test_fail1(struct prog_test_fail1 *p)
314 {
315 }
316 
317 noinline void bpf_kfunc_call_test_fail2(struct prog_test_fail2 *p)
318 {
319 }
320 
321 noinline void bpf_kfunc_call_test_fail3(struct prog_test_fail3 *p)
322 {
323 }
324 
325 noinline void bpf_kfunc_call_test_mem_len_pass1(void *mem, int mem__sz)
326 {
327 }
328 
329 noinline void bpf_kfunc_call_test_mem_len_fail1(void *mem, int len)
330 {
331 }
332 
333 noinline void bpf_kfunc_call_test_mem_len_fail2(u64 *mem, int len)
334 {
335 }
336 
337 __diag_pop();
338 
339 ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO);
340 
341 BTF_SET_START(test_sk_check_kfunc_ids)
342 BTF_ID(func, bpf_kfunc_call_test1)
343 BTF_ID(func, bpf_kfunc_call_test2)
344 BTF_ID(func, bpf_kfunc_call_test3)
345 BTF_ID(func, bpf_kfunc_call_test_acquire)
346 BTF_ID(func, bpf_kfunc_call_test_release)
347 BTF_ID(func, bpf_kfunc_call_test_pass_ctx)
348 BTF_ID(func, bpf_kfunc_call_test_pass1)
349 BTF_ID(func, bpf_kfunc_call_test_pass2)
350 BTF_ID(func, bpf_kfunc_call_test_fail1)
351 BTF_ID(func, bpf_kfunc_call_test_fail2)
352 BTF_ID(func, bpf_kfunc_call_test_fail3)
353 BTF_ID(func, bpf_kfunc_call_test_mem_len_pass1)
354 BTF_ID(func, bpf_kfunc_call_test_mem_len_fail1)
355 BTF_ID(func, bpf_kfunc_call_test_mem_len_fail2)
356 BTF_SET_END(test_sk_check_kfunc_ids)
357 
358 BTF_SET_START(test_sk_acquire_kfunc_ids)
359 BTF_ID(func, bpf_kfunc_call_test_acquire)
360 BTF_SET_END(test_sk_acquire_kfunc_ids)
361 
362 BTF_SET_START(test_sk_release_kfunc_ids)
363 BTF_ID(func, bpf_kfunc_call_test_release)
364 BTF_SET_END(test_sk_release_kfunc_ids)
365 
366 BTF_SET_START(test_sk_ret_null_kfunc_ids)
367 BTF_ID(func, bpf_kfunc_call_test_acquire)
368 BTF_SET_END(test_sk_ret_null_kfunc_ids)
369 
370 static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
371 			   u32 headroom, u32 tailroom)
372 {
373 	void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
374 	u32 user_size = kattr->test.data_size_in;
375 	void *data;
376 
377 	if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
378 		return ERR_PTR(-EINVAL);
379 
380 	if (user_size > size)
381 		return ERR_PTR(-EMSGSIZE);
382 
383 	data = kzalloc(size + headroom + tailroom, GFP_USER);
384 	if (!data)
385 		return ERR_PTR(-ENOMEM);
386 
387 	if (copy_from_user(data + headroom, data_in, user_size)) {
388 		kfree(data);
389 		return ERR_PTR(-EFAULT);
390 	}
391 
392 	return data;
393 }
394 
395 int bpf_prog_test_run_tracing(struct bpf_prog *prog,
396 			      const union bpf_attr *kattr,
397 			      union bpf_attr __user *uattr)
398 {
399 	struct bpf_fentry_test_t arg = {};
400 	u16 side_effect = 0, ret = 0;
401 	int b = 2, err = -EFAULT;
402 	u32 retval = 0;
403 
404 	if (kattr->test.flags || kattr->test.cpu)
405 		return -EINVAL;
406 
407 	switch (prog->expected_attach_type) {
408 	case BPF_TRACE_FENTRY:
409 	case BPF_TRACE_FEXIT:
410 		if (bpf_fentry_test1(1) != 2 ||
411 		    bpf_fentry_test2(2, 3) != 5 ||
412 		    bpf_fentry_test3(4, 5, 6) != 15 ||
413 		    bpf_fentry_test4((void *)7, 8, 9, 10) != 34 ||
414 		    bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 ||
415 		    bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 ||
416 		    bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 ||
417 		    bpf_fentry_test8(&arg) != 0)
418 			goto out;
419 		break;
420 	case BPF_MODIFY_RETURN:
421 		ret = bpf_modify_return_test(1, &b);
422 		if (b != 2)
423 			side_effect = 1;
424 		break;
425 	default:
426 		goto out;
427 	}
428 
429 	retval = ((u32)side_effect << 16) | ret;
430 	if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
431 		goto out;
432 
433 	err = 0;
434 out:
435 	trace_bpf_test_finish(&err);
436 	return err;
437 }
438 
439 struct bpf_raw_tp_test_run_info {
440 	struct bpf_prog *prog;
441 	void *ctx;
442 	u32 retval;
443 };
444 
445 static void
446 __bpf_prog_test_run_raw_tp(void *data)
447 {
448 	struct bpf_raw_tp_test_run_info *info = data;
449 
450 	rcu_read_lock();
451 	info->retval = bpf_prog_run(info->prog, info->ctx);
452 	rcu_read_unlock();
453 }
454 
455 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
456 			     const union bpf_attr *kattr,
457 			     union bpf_attr __user *uattr)
458 {
459 	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
460 	__u32 ctx_size_in = kattr->test.ctx_size_in;
461 	struct bpf_raw_tp_test_run_info info;
462 	int cpu = kattr->test.cpu, err = 0;
463 	int current_cpu;
464 
465 	/* doesn't support data_in/out, ctx_out, duration, or repeat */
466 	if (kattr->test.data_in || kattr->test.data_out ||
467 	    kattr->test.ctx_out || kattr->test.duration ||
468 	    kattr->test.repeat)
469 		return -EINVAL;
470 
471 	if (ctx_size_in < prog->aux->max_ctx_offset ||
472 	    ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64))
473 		return -EINVAL;
474 
475 	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)
476 		return -EINVAL;
477 
478 	if (ctx_size_in) {
479 		info.ctx = memdup_user(ctx_in, ctx_size_in);
480 		if (IS_ERR(info.ctx))
481 			return PTR_ERR(info.ctx);
482 	} else {
483 		info.ctx = NULL;
484 	}
485 
486 	info.prog = prog;
487 
488 	current_cpu = get_cpu();
489 	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||
490 	    cpu == current_cpu) {
491 		__bpf_prog_test_run_raw_tp(&info);
492 	} else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
493 		/* smp_call_function_single() also checks cpu_online()
494 		 * after csd_lock(). However, since cpu is from user
495 		 * space, let's do an extra quick check to filter out
496 		 * invalid value before smp_call_function_single().
497 		 */
498 		err = -ENXIO;
499 	} else {
500 		err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp,
501 					       &info, 1);
502 	}
503 	put_cpu();
504 
505 	if (!err &&
506 	    copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32)))
507 		err = -EFAULT;
508 
509 	kfree(info.ctx);
510 	return err;
511 }
512 
513 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
514 {
515 	void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
516 	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
517 	u32 size = kattr->test.ctx_size_in;
518 	void *data;
519 	int err;
520 
521 	if (!data_in && !data_out)
522 		return NULL;
523 
524 	data = kzalloc(max_size, GFP_USER);
525 	if (!data)
526 		return ERR_PTR(-ENOMEM);
527 
528 	if (data_in) {
529 		err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size);
530 		if (err) {
531 			kfree(data);
532 			return ERR_PTR(err);
533 		}
534 
535 		size = min_t(u32, max_size, size);
536 		if (copy_from_user(data, data_in, size)) {
537 			kfree(data);
538 			return ERR_PTR(-EFAULT);
539 		}
540 	}
541 	return data;
542 }
543 
544 static int bpf_ctx_finish(const union bpf_attr *kattr,
545 			  union bpf_attr __user *uattr, const void *data,
546 			  u32 size)
547 {
548 	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
549 	int err = -EFAULT;
550 	u32 copy_size = size;
551 
552 	if (!data || !data_out)
553 		return 0;
554 
555 	if (copy_size > kattr->test.ctx_size_out) {
556 		copy_size = kattr->test.ctx_size_out;
557 		err = -ENOSPC;
558 	}
559 
560 	if (copy_to_user(data_out, data, copy_size))
561 		goto out;
562 	if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
563 		goto out;
564 	if (err != -ENOSPC)
565 		err = 0;
566 out:
567 	return err;
568 }
569 
570 /**
571  * range_is_zero - test whether buffer is initialized
572  * @buf: buffer to check
573  * @from: check from this position
574  * @to: check up until (excluding) this position
575  *
576  * This function returns true if the there is a non-zero byte
577  * in the buf in the range [from,to).
578  */
579 static inline bool range_is_zero(void *buf, size_t from, size_t to)
580 {
581 	return !memchr_inv((u8 *)buf + from, 0, to - from);
582 }
583 
584 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
585 {
586 	struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
587 
588 	if (!__skb)
589 		return 0;
590 
591 	/* make sure the fields we don't use are zeroed */
592 	if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark)))
593 		return -EINVAL;
594 
595 	/* mark is allowed */
596 
597 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark),
598 			   offsetof(struct __sk_buff, priority)))
599 		return -EINVAL;
600 
601 	/* priority is allowed */
602 	/* ingress_ifindex is allowed */
603 	/* ifindex is allowed */
604 
605 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex),
606 			   offsetof(struct __sk_buff, cb)))
607 		return -EINVAL;
608 
609 	/* cb is allowed */
610 
611 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb),
612 			   offsetof(struct __sk_buff, tstamp)))
613 		return -EINVAL;
614 
615 	/* tstamp is allowed */
616 	/* wire_len is allowed */
617 	/* gso_segs is allowed */
618 
619 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs),
620 			   offsetof(struct __sk_buff, gso_size)))
621 		return -EINVAL;
622 
623 	/* gso_size is allowed */
624 
625 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size),
626 			   offsetof(struct __sk_buff, hwtstamp)))
627 		return -EINVAL;
628 
629 	/* hwtstamp is allowed */
630 
631 	if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp),
632 			   sizeof(struct __sk_buff)))
633 		return -EINVAL;
634 
635 	skb->mark = __skb->mark;
636 	skb->priority = __skb->priority;
637 	skb->skb_iif = __skb->ingress_ifindex;
638 	skb->tstamp = __skb->tstamp;
639 	memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
640 
641 	if (__skb->wire_len == 0) {
642 		cb->pkt_len = skb->len;
643 	} else {
644 		if (__skb->wire_len < skb->len ||
645 		    __skb->wire_len > GSO_MAX_SIZE)
646 			return -EINVAL;
647 		cb->pkt_len = __skb->wire_len;
648 	}
649 
650 	if (__skb->gso_segs > GSO_MAX_SEGS)
651 		return -EINVAL;
652 	skb_shinfo(skb)->gso_segs = __skb->gso_segs;
653 	skb_shinfo(skb)->gso_size = __skb->gso_size;
654 	skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp;
655 
656 	return 0;
657 }
658 
659 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
660 {
661 	struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
662 
663 	if (!__skb)
664 		return;
665 
666 	__skb->mark = skb->mark;
667 	__skb->priority = skb->priority;
668 	__skb->ingress_ifindex = skb->skb_iif;
669 	__skb->ifindex = skb->dev->ifindex;
670 	__skb->tstamp = skb->tstamp;
671 	memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
672 	__skb->wire_len = cb->pkt_len;
673 	__skb->gso_segs = skb_shinfo(skb)->gso_segs;
674 	__skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp;
675 }
676 
677 static struct proto bpf_dummy_proto = {
678 	.name   = "bpf_dummy",
679 	.owner  = THIS_MODULE,
680 	.obj_size = sizeof(struct sock),
681 };
682 
683 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
684 			  union bpf_attr __user *uattr)
685 {
686 	bool is_l2 = false, is_direct_pkt_access = false;
687 	struct net *net = current->nsproxy->net_ns;
688 	struct net_device *dev = net->loopback_dev;
689 	u32 size = kattr->test.data_size_in;
690 	u32 repeat = kattr->test.repeat;
691 	struct __sk_buff *ctx = NULL;
692 	u32 retval, duration;
693 	int hh_len = ETH_HLEN;
694 	struct sk_buff *skb;
695 	struct sock *sk;
696 	void *data;
697 	int ret;
698 
699 	if (kattr->test.flags || kattr->test.cpu)
700 		return -EINVAL;
701 
702 	data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
703 			     SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
704 	if (IS_ERR(data))
705 		return PTR_ERR(data);
706 
707 	ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
708 	if (IS_ERR(ctx)) {
709 		kfree(data);
710 		return PTR_ERR(ctx);
711 	}
712 
713 	switch (prog->type) {
714 	case BPF_PROG_TYPE_SCHED_CLS:
715 	case BPF_PROG_TYPE_SCHED_ACT:
716 		is_l2 = true;
717 		fallthrough;
718 	case BPF_PROG_TYPE_LWT_IN:
719 	case BPF_PROG_TYPE_LWT_OUT:
720 	case BPF_PROG_TYPE_LWT_XMIT:
721 		is_direct_pkt_access = true;
722 		break;
723 	default:
724 		break;
725 	}
726 
727 	sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1);
728 	if (!sk) {
729 		kfree(data);
730 		kfree(ctx);
731 		return -ENOMEM;
732 	}
733 	sock_init_data(NULL, sk);
734 
735 	skb = build_skb(data, 0);
736 	if (!skb) {
737 		kfree(data);
738 		kfree(ctx);
739 		sk_free(sk);
740 		return -ENOMEM;
741 	}
742 	skb->sk = sk;
743 
744 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
745 	__skb_put(skb, size);
746 	if (ctx && ctx->ifindex > 1) {
747 		dev = dev_get_by_index(net, ctx->ifindex);
748 		if (!dev) {
749 			ret = -ENODEV;
750 			goto out;
751 		}
752 	}
753 	skb->protocol = eth_type_trans(skb, dev);
754 	skb_reset_network_header(skb);
755 
756 	switch (skb->protocol) {
757 	case htons(ETH_P_IP):
758 		sk->sk_family = AF_INET;
759 		if (sizeof(struct iphdr) <= skb_headlen(skb)) {
760 			sk->sk_rcv_saddr = ip_hdr(skb)->saddr;
761 			sk->sk_daddr = ip_hdr(skb)->daddr;
762 		}
763 		break;
764 #if IS_ENABLED(CONFIG_IPV6)
765 	case htons(ETH_P_IPV6):
766 		sk->sk_family = AF_INET6;
767 		if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) {
768 			sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr;
769 			sk->sk_v6_daddr = ipv6_hdr(skb)->daddr;
770 		}
771 		break;
772 #endif
773 	default:
774 		break;
775 	}
776 
777 	if (is_l2)
778 		__skb_push(skb, hh_len);
779 	if (is_direct_pkt_access)
780 		bpf_compute_data_pointers(skb);
781 	ret = convert___skb_to_skb(skb, ctx);
782 	if (ret)
783 		goto out;
784 	ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false);
785 	if (ret)
786 		goto out;
787 	if (!is_l2) {
788 		if (skb_headroom(skb) < hh_len) {
789 			int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
790 
791 			if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
792 				ret = -ENOMEM;
793 				goto out;
794 			}
795 		}
796 		memset(__skb_push(skb, hh_len), 0, hh_len);
797 	}
798 	convert_skb_to___skb(skb, ctx);
799 
800 	size = skb->len;
801 	/* bpf program can never convert linear skb to non-linear */
802 	if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
803 		size = skb_headlen(skb);
804 	ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
805 	if (!ret)
806 		ret = bpf_ctx_finish(kattr, uattr, ctx,
807 				     sizeof(struct __sk_buff));
808 out:
809 	if (dev && dev != net->loopback_dev)
810 		dev_put(dev);
811 	kfree_skb(skb);
812 	sk_free(sk);
813 	kfree(ctx);
814 	return ret;
815 }
816 
817 static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp)
818 {
819 	unsigned int ingress_ifindex, rx_queue_index;
820 	struct netdev_rx_queue *rxqueue;
821 	struct net_device *device;
822 
823 	if (!xdp_md)
824 		return 0;
825 
826 	if (xdp_md->egress_ifindex != 0)
827 		return -EINVAL;
828 
829 	ingress_ifindex = xdp_md->ingress_ifindex;
830 	rx_queue_index = xdp_md->rx_queue_index;
831 
832 	if (!ingress_ifindex && rx_queue_index)
833 		return -EINVAL;
834 
835 	if (ingress_ifindex) {
836 		device = dev_get_by_index(current->nsproxy->net_ns,
837 					  ingress_ifindex);
838 		if (!device)
839 			return -ENODEV;
840 
841 		if (rx_queue_index >= device->real_num_rx_queues)
842 			goto free_dev;
843 
844 		rxqueue = __netif_get_rx_queue(device, rx_queue_index);
845 
846 		if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq))
847 			goto free_dev;
848 
849 		xdp->rxq = &rxqueue->xdp_rxq;
850 		/* The device is now tracked in the xdp->rxq for later
851 		 * dev_put()
852 		 */
853 	}
854 
855 	xdp->data = xdp->data_meta + xdp_md->data;
856 	return 0;
857 
858 free_dev:
859 	dev_put(device);
860 	return -EINVAL;
861 }
862 
863 static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md)
864 {
865 	if (!xdp_md)
866 		return;
867 
868 	xdp_md->data = xdp->data - xdp->data_meta;
869 	xdp_md->data_end = xdp->data_end - xdp->data_meta;
870 
871 	if (xdp_md->ingress_ifindex)
872 		dev_put(xdp->rxq->dev);
873 }
874 
875 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
876 			  union bpf_attr __user *uattr)
877 {
878 	u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
879 	u32 headroom = XDP_PACKET_HEADROOM;
880 	u32 size = kattr->test.data_size_in;
881 	u32 repeat = kattr->test.repeat;
882 	struct netdev_rx_queue *rxqueue;
883 	struct xdp_buff xdp = {};
884 	u32 retval, duration;
885 	struct xdp_md *ctx;
886 	u32 max_data_sz;
887 	void *data;
888 	int ret = -EINVAL;
889 
890 	if (prog->expected_attach_type == BPF_XDP_DEVMAP ||
891 	    prog->expected_attach_type == BPF_XDP_CPUMAP)
892 		return -EINVAL;
893 
894 	ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md));
895 	if (IS_ERR(ctx))
896 		return PTR_ERR(ctx);
897 
898 	if (ctx) {
899 		/* There can't be user provided data before the meta data */
900 		if (ctx->data_meta || ctx->data_end != size ||
901 		    ctx->data > ctx->data_end ||
902 		    unlikely(xdp_metalen_invalid(ctx->data)))
903 			goto free_ctx;
904 		/* Meta data is allocated from the headroom */
905 		headroom -= ctx->data;
906 	}
907 
908 	/* XDP have extra tailroom as (most) drivers use full page */
909 	max_data_sz = 4096 - headroom - tailroom;
910 
911 	data = bpf_test_init(kattr, max_data_sz, headroom, tailroom);
912 	if (IS_ERR(data)) {
913 		ret = PTR_ERR(data);
914 		goto free_ctx;
915 	}
916 
917 	rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
918 	xdp_init_buff(&xdp, headroom + max_data_sz + tailroom,
919 		      &rxqueue->xdp_rxq);
920 	xdp_prepare_buff(&xdp, data, headroom, size, true);
921 
922 	ret = xdp_convert_md_to_buff(ctx, &xdp);
923 	if (ret)
924 		goto free_data;
925 
926 	if (repeat > 1)
927 		bpf_prog_change_xdp(NULL, prog);
928 	ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true);
929 	/* We convert the xdp_buff back to an xdp_md before checking the return
930 	 * code so the reference count of any held netdevice will be decremented
931 	 * even if the test run failed.
932 	 */
933 	xdp_convert_buff_to_md(&xdp, ctx);
934 	if (ret)
935 		goto out;
936 
937 	if (xdp.data_meta != data + headroom ||
938 	    xdp.data_end != xdp.data_meta + size)
939 		size = xdp.data_end - xdp.data_meta;
940 
941 	ret = bpf_test_finish(kattr, uattr, xdp.data_meta, size, retval,
942 			      duration);
943 	if (!ret)
944 		ret = bpf_ctx_finish(kattr, uattr, ctx,
945 				     sizeof(struct xdp_md));
946 
947 out:
948 	if (repeat > 1)
949 		bpf_prog_change_xdp(prog, NULL);
950 free_data:
951 	kfree(data);
952 free_ctx:
953 	kfree(ctx);
954 	return ret;
955 }
956 
957 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
958 {
959 	/* make sure the fields we don't use are zeroed */
960 	if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags)))
961 		return -EINVAL;
962 
963 	/* flags is allowed */
964 
965 	if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags),
966 			   sizeof(struct bpf_flow_keys)))
967 		return -EINVAL;
968 
969 	return 0;
970 }
971 
972 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
973 				     const union bpf_attr *kattr,
974 				     union bpf_attr __user *uattr)
975 {
976 	struct bpf_test_timer t = { NO_PREEMPT };
977 	u32 size = kattr->test.data_size_in;
978 	struct bpf_flow_dissector ctx = {};
979 	u32 repeat = kattr->test.repeat;
980 	struct bpf_flow_keys *user_ctx;
981 	struct bpf_flow_keys flow_keys;
982 	const struct ethhdr *eth;
983 	unsigned int flags = 0;
984 	u32 retval, duration;
985 	void *data;
986 	int ret;
987 
988 	if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
989 		return -EINVAL;
990 
991 	if (kattr->test.flags || kattr->test.cpu)
992 		return -EINVAL;
993 
994 	if (size < ETH_HLEN)
995 		return -EINVAL;
996 
997 	data = bpf_test_init(kattr, size, 0, 0);
998 	if (IS_ERR(data))
999 		return PTR_ERR(data);
1000 
1001 	eth = (struct ethhdr *)data;
1002 
1003 	if (!repeat)
1004 		repeat = 1;
1005 
1006 	user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys));
1007 	if (IS_ERR(user_ctx)) {
1008 		kfree(data);
1009 		return PTR_ERR(user_ctx);
1010 	}
1011 	if (user_ctx) {
1012 		ret = verify_user_bpf_flow_keys(user_ctx);
1013 		if (ret)
1014 			goto out;
1015 		flags = user_ctx->flags;
1016 	}
1017 
1018 	ctx.flow_keys = &flow_keys;
1019 	ctx.data = data;
1020 	ctx.data_end = (__u8 *)data + size;
1021 
1022 	bpf_test_timer_enter(&t);
1023 	do {
1024 		retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
1025 					  size, flags);
1026 	} while (bpf_test_timer_continue(&t, repeat, &ret, &duration));
1027 	bpf_test_timer_leave(&t);
1028 
1029 	if (ret < 0)
1030 		goto out;
1031 
1032 	ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),
1033 			      retval, duration);
1034 	if (!ret)
1035 		ret = bpf_ctx_finish(kattr, uattr, user_ctx,
1036 				     sizeof(struct bpf_flow_keys));
1037 
1038 out:
1039 	kfree(user_ctx);
1040 	kfree(data);
1041 	return ret;
1042 }
1043 
1044 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,
1045 				union bpf_attr __user *uattr)
1046 {
1047 	struct bpf_test_timer t = { NO_PREEMPT };
1048 	struct bpf_prog_array *progs = NULL;
1049 	struct bpf_sk_lookup_kern ctx = {};
1050 	u32 repeat = kattr->test.repeat;
1051 	struct bpf_sk_lookup *user_ctx;
1052 	u32 retval, duration;
1053 	int ret = -EINVAL;
1054 
1055 	if (prog->type != BPF_PROG_TYPE_SK_LOOKUP)
1056 		return -EINVAL;
1057 
1058 	if (kattr->test.flags || kattr->test.cpu)
1059 		return -EINVAL;
1060 
1061 	if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||
1062 	    kattr->test.data_size_out)
1063 		return -EINVAL;
1064 
1065 	if (!repeat)
1066 		repeat = 1;
1067 
1068 	user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx));
1069 	if (IS_ERR(user_ctx))
1070 		return PTR_ERR(user_ctx);
1071 
1072 	if (!user_ctx)
1073 		return -EINVAL;
1074 
1075 	if (user_ctx->sk)
1076 		goto out;
1077 
1078 	if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx)))
1079 		goto out;
1080 
1081 	if (user_ctx->local_port > U16_MAX || user_ctx->remote_port > U16_MAX) {
1082 		ret = -ERANGE;
1083 		goto out;
1084 	}
1085 
1086 	ctx.family = (u16)user_ctx->family;
1087 	ctx.protocol = (u16)user_ctx->protocol;
1088 	ctx.dport = (u16)user_ctx->local_port;
1089 	ctx.sport = (__force __be16)user_ctx->remote_port;
1090 
1091 	switch (ctx.family) {
1092 	case AF_INET:
1093 		ctx.v4.daddr = (__force __be32)user_ctx->local_ip4;
1094 		ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4;
1095 		break;
1096 
1097 #if IS_ENABLED(CONFIG_IPV6)
1098 	case AF_INET6:
1099 		ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6;
1100 		ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6;
1101 		break;
1102 #endif
1103 
1104 	default:
1105 		ret = -EAFNOSUPPORT;
1106 		goto out;
1107 	}
1108 
1109 	progs = bpf_prog_array_alloc(1, GFP_KERNEL);
1110 	if (!progs) {
1111 		ret = -ENOMEM;
1112 		goto out;
1113 	}
1114 
1115 	progs->items[0].prog = prog;
1116 
1117 	bpf_test_timer_enter(&t);
1118 	do {
1119 		ctx.selected_sk = NULL;
1120 		retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run);
1121 	} while (bpf_test_timer_continue(&t, repeat, &ret, &duration));
1122 	bpf_test_timer_leave(&t);
1123 
1124 	if (ret < 0)
1125 		goto out;
1126 
1127 	user_ctx->cookie = 0;
1128 	if (ctx.selected_sk) {
1129 		if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) {
1130 			ret = -EOPNOTSUPP;
1131 			goto out;
1132 		}
1133 
1134 		user_ctx->cookie = sock_gen_cookie(ctx.selected_sk);
1135 	}
1136 
1137 	ret = bpf_test_finish(kattr, uattr, NULL, 0, retval, duration);
1138 	if (!ret)
1139 		ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx));
1140 
1141 out:
1142 	bpf_prog_array_free(progs);
1143 	kfree(user_ctx);
1144 	return ret;
1145 }
1146 
1147 int bpf_prog_test_run_syscall(struct bpf_prog *prog,
1148 			      const union bpf_attr *kattr,
1149 			      union bpf_attr __user *uattr)
1150 {
1151 	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
1152 	__u32 ctx_size_in = kattr->test.ctx_size_in;
1153 	void *ctx = NULL;
1154 	u32 retval;
1155 	int err = 0;
1156 
1157 	/* doesn't support data_in/out, ctx_out, duration, or repeat or flags */
1158 	if (kattr->test.data_in || kattr->test.data_out ||
1159 	    kattr->test.ctx_out || kattr->test.duration ||
1160 	    kattr->test.repeat || kattr->test.flags)
1161 		return -EINVAL;
1162 
1163 	if (ctx_size_in < prog->aux->max_ctx_offset ||
1164 	    ctx_size_in > U16_MAX)
1165 		return -EINVAL;
1166 
1167 	if (ctx_size_in) {
1168 		ctx = memdup_user(ctx_in, ctx_size_in);
1169 		if (IS_ERR(ctx))
1170 			return PTR_ERR(ctx);
1171 	}
1172 
1173 	rcu_read_lock_trace();
1174 	retval = bpf_prog_run_pin_on_cpu(prog, ctx);
1175 	rcu_read_unlock_trace();
1176 
1177 	if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) {
1178 		err = -EFAULT;
1179 		goto out;
1180 	}
1181 	if (ctx_size_in)
1182 		if (copy_to_user(ctx_in, ctx, ctx_size_in))
1183 			err = -EFAULT;
1184 out:
1185 	kfree(ctx);
1186 	return err;
1187 }
1188 
1189 static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = {
1190 	.owner        = THIS_MODULE,
1191 	.check_set    = &test_sk_check_kfunc_ids,
1192 	.acquire_set  = &test_sk_acquire_kfunc_ids,
1193 	.release_set  = &test_sk_release_kfunc_ids,
1194 	.ret_null_set = &test_sk_ret_null_kfunc_ids,
1195 };
1196 
1197 static int __init bpf_prog_test_run_init(void)
1198 {
1199 	return register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_prog_test_kfunc_set);
1200 }
1201 late_initcall(bpf_prog_test_run_init);
1202