xref: /linux/kernel/trace/bpf_trace.c (revision 84afebd853437e44cb71f718be11d90e4c1a59dd)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com
3  * Copyright (c) 2016 Facebook
4  */
5 #include <linux/kernel.h>
6 #include <linux/types.h>
7 #include <linux/slab.h>
8 #include <linux/bpf.h>
9 #include <linux/bpf_verifier.h>
10 #include <linux/bpf_perf_event.h>
11 #include <linux/btf.h>
12 #include <linux/filter.h>
13 #include <linux/uaccess.h>
14 #include <linux/ctype.h>
15 #include <linux/kprobes.h>
16 #include <linux/spinlock.h>
17 #include <linux/syscalls.h>
18 #include <linux/error-injection.h>
19 #include <linux/btf_ids.h>
20 #include <linux/bpf_lsm.h>
21 #include <linux/fprobe.h>
22 #include <linux/bsearch.h>
23 #include <linux/sort.h>
24 #include <linux/key.h>
25 #include <linux/namei.h>
26 #include <linux/file.h>
27 
28 #include <net/bpf_sk_storage.h>
29 
30 #include <uapi/linux/bpf.h>
31 #include <uapi/linux/btf.h>
32 
33 #include <asm/tlb.h>
34 
35 #include "trace_probe.h"
36 #include "trace.h"
37 
38 #define CREATE_TRACE_POINTS
39 #include "bpf_trace.h"
40 
41 #define bpf_event_rcu_dereference(p)					\
42 	rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex))
43 
44 #define MAX_UPROBE_MULTI_CNT (1U << 20)
45 #define MAX_KPROBE_MULTI_CNT (1U << 20)
46 #define MAX_TRACING_MULTI_CNT (1U << 20)
47 
48 #ifdef CONFIG_MODULES
49 struct bpf_trace_module {
50 	struct module *module;
51 	struct list_head list;
52 };
53 
54 static LIST_HEAD(bpf_trace_modules);
55 static DEFINE_MUTEX(bpf_module_mutex);
56 
57 static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name)
58 {
59 	struct bpf_raw_event_map *btp, *ret = NULL;
60 	struct bpf_trace_module *btm;
61 	unsigned int i;
62 
63 	mutex_lock(&bpf_module_mutex);
64 	list_for_each_entry(btm, &bpf_trace_modules, list) {
65 		for (i = 0; i < btm->module->num_bpf_raw_events; ++i) {
66 			btp = &btm->module->bpf_raw_events[i];
67 			if (!strcmp(btp->tp->name, name)) {
68 				if (try_module_get(btm->module))
69 					ret = btp;
70 				goto out;
71 			}
72 		}
73 	}
74 out:
75 	mutex_unlock(&bpf_module_mutex);
76 	return ret;
77 }
78 #else
79 static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name)
80 {
81 	return NULL;
82 }
83 #endif /* CONFIG_MODULES */
84 
85 u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
86 u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
87 
88 static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,
89 				  u64 flags, const struct btf **btf,
90 				  s32 *btf_id);
91 static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx);
92 static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx);
93 
94 static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx);
95 static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx);
96 
97 /**
98  * trace_call_bpf - invoke BPF program
99  * @call: tracepoint event
100  * @ctx: opaque context pointer
101  *
102  * kprobe handlers execute BPF programs via this helper.
103  * Can be used from static tracepoints in the future.
104  *
105  * Return: BPF programs always return an integer which is interpreted by
106  * kprobe handler as:
107  * 0 - return from kprobe (event is filtered out)
108  * 1 - store kprobe event into ring buffer
109  * Other values are reserved and currently alias to 1
110  */
111 unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
112 {
113 	unsigned int ret;
114 
115 	cant_sleep();
116 
117 	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
118 		/*
119 		 * since some bpf program is already running on this cpu,
120 		 * don't call into another bpf program (same or different)
121 		 * and don't send kprobe event into ring-buffer,
122 		 * so return zero here
123 		 */
124 		rcu_read_lock();
125 		bpf_prog_inc_misses_counters(rcu_dereference(call->prog_array));
126 		rcu_read_unlock();
127 		ret = 0;
128 		goto out;
129 	}
130 
131 	/*
132 	 * Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock
133 	 * to all call sites, we did a bpf_prog_array_valid() there to check
134 	 * whether call->prog_array is empty or not, which is
135 	 * a heuristic to speed up execution.
136 	 *
137 	 * If bpf_prog_array_valid() fetched prog_array was
138 	 * non-NULL, we go into trace_call_bpf() and do the actual
139 	 * proper rcu_dereference() under RCU lock.
140 	 * If it turns out that prog_array is NULL then, we bail out.
141 	 * For the opposite, if the bpf_prog_array_valid() fetched pointer
142 	 * was NULL, you'll skip the prog_array with the risk of missing
143 	 * out of events when it was updated in between this and the
144 	 * rcu_dereference() which is accepted risk.
145 	 */
146 	rcu_read_lock();
147 	ret = bpf_prog_run_array(rcu_dereference(call->prog_array),
148 				 ctx, bpf_prog_run);
149 	rcu_read_unlock();
150 
151  out:
152 	__this_cpu_dec(bpf_prog_active);
153 
154 	return ret;
155 }
156 
157 /**
158  * trace_call_bpf_faultable - invoke BPF program in faultable context
159  * @call: tracepoint event
160  * @ctx: opaque context pointer
161  *
162  * Variant of trace_call_bpf() for faultable tracepoints (syscall
163  * tracepoints). Supports sleepable BPF programs by using rcu_tasks_trace
164  * for lifetime protection and bpf_prog_run_array_sleepable() for per-program
165  * RCU flavor selection, following the uprobe pattern.
166  *
167  * Per-program recursion protection is provided by
168  * bpf_prog_run_array_sleepable(). Global bpf_prog_active is not
169  * needed because syscall tracepoints cannot self-recurse.
170  *
171  * Must be called from a faultable/preemptible context.
172  */
173 unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx)
174 {
175 	struct bpf_prog_array *prog_array;
176 
177 	might_fault();
178 	guard(rcu_tasks_trace)();
179 
180 	prog_array = rcu_dereference_check(call->prog_array,
181 					   rcu_read_lock_trace_held());
182 	return bpf_prog_run_array_sleepable(prog_array, ctx, bpf_prog_run);
183 }
184 
185 #ifdef CONFIG_BPF_KPROBE_OVERRIDE
186 BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc)
187 {
188 	regs_set_return_value(regs, rc);
189 	override_function_with_return(regs);
190 	return 0;
191 }
192 
193 static const struct bpf_func_proto bpf_override_return_proto = {
194 	.func		= bpf_override_return,
195 	.gpl_only	= true,
196 	.ret_type	= RET_INTEGER,
197 	.arg1_type	= ARG_PTR_TO_CTX,
198 	.arg2_type	= ARG_ANYTHING,
199 };
200 #endif
201 
202 static __always_inline int
203 bpf_probe_read_user_common(void *dst, u32 size, const void __user *unsafe_ptr)
204 {
205 	int ret;
206 
207 	ret = copy_from_user_nofault(dst, unsafe_ptr, size);
208 	if (unlikely(ret < 0))
209 		memset(dst, 0, size);
210 	return ret;
211 }
212 
213 BPF_CALL_3(bpf_probe_read_user, void *, dst, u32, size,
214 	   const void __user *, unsafe_ptr)
215 {
216 	return bpf_probe_read_user_common(dst, size, unsafe_ptr);
217 }
218 
219 const struct bpf_func_proto bpf_probe_read_user_proto = {
220 	.func		= bpf_probe_read_user,
221 	.gpl_only	= true,
222 	.ret_type	= RET_INTEGER,
223 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
224 	.arg2_type	= ARG_MEM_SIZE_OR_ZERO,
225 	.arg3_type	= ARG_ANYTHING,
226 };
227 
228 static __always_inline int
229 bpf_probe_read_user_str_common(void *dst, u32 size,
230 			       const void __user *unsafe_ptr)
231 {
232 	int ret;
233 
234 	/*
235 	 * NB: We rely on strncpy_from_user() not copying junk past the NUL
236 	 * terminator into `dst`.
237 	 *
238 	 * strncpy_from_user() does long-sized strides in the fast path. If the
239 	 * strncpy does not mask out the bytes after the NUL in `unsafe_ptr`,
240 	 * then there could be junk after the NUL in `dst`. If user takes `dst`
241 	 * and keys a hash map with it, then semantically identical strings can
242 	 * occupy multiple entries in the map.
243 	 */
244 	ret = strncpy_from_user_nofault(dst, unsafe_ptr, size);
245 	if (unlikely(ret < 0))
246 		memset(dst, 0, size);
247 	return ret;
248 }
249 
250 BPF_CALL_3(bpf_probe_read_user_str, void *, dst, u32, size,
251 	   const void __user *, unsafe_ptr)
252 {
253 	return bpf_probe_read_user_str_common(dst, size, unsafe_ptr);
254 }
255 
256 const struct bpf_func_proto bpf_probe_read_user_str_proto = {
257 	.func		= bpf_probe_read_user_str,
258 	.gpl_only	= true,
259 	.ret_type	= RET_INTEGER,
260 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
261 	.arg2_type	= ARG_MEM_SIZE_OR_ZERO,
262 	.arg3_type	= ARG_ANYTHING,
263 };
264 
265 BPF_CALL_3(bpf_probe_read_kernel, void *, dst, u32, size,
266 	   const void *, unsafe_ptr)
267 {
268 	return bpf_probe_read_kernel_common(dst, size, unsafe_ptr);
269 }
270 
271 const struct bpf_func_proto bpf_probe_read_kernel_proto = {
272 	.func		= bpf_probe_read_kernel,
273 	.gpl_only	= true,
274 	.ret_type	= RET_INTEGER,
275 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
276 	.arg2_type	= ARG_MEM_SIZE_OR_ZERO,
277 	.arg3_type	= ARG_ANYTHING,
278 };
279 
280 static __always_inline int
281 bpf_probe_read_kernel_str_common(void *dst, u32 size, const void *unsafe_ptr)
282 {
283 	int ret;
284 
285 	/*
286 	 * The strncpy_from_kernel_nofault() call will likely not fill the
287 	 * entire buffer, but that's okay in this circumstance as we're probing
288 	 * arbitrary memory anyway similar to bpf_probe_read_*() and might
289 	 * as well probe the stack. Thus, memory is explicitly cleared
290 	 * only in error case, so that improper users ignoring return
291 	 * code altogether don't copy garbage; otherwise length of string
292 	 * is returned that can be used for bpf_perf_event_output() et al.
293 	 */
294 	ret = strncpy_from_kernel_nofault(dst, unsafe_ptr, size);
295 	if (unlikely(ret < 0))
296 		memset(dst, 0, size);
297 	return ret;
298 }
299 
300 BPF_CALL_3(bpf_probe_read_kernel_str, void *, dst, u32, size,
301 	   const void *, unsafe_ptr)
302 {
303 	return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr);
304 }
305 
306 const struct bpf_func_proto bpf_probe_read_kernel_str_proto = {
307 	.func		= bpf_probe_read_kernel_str,
308 	.gpl_only	= true,
309 	.ret_type	= RET_INTEGER,
310 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
311 	.arg2_type	= ARG_MEM_SIZE_OR_ZERO,
312 	.arg3_type	= ARG_ANYTHING,
313 };
314 
315 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
316 BPF_CALL_3(bpf_probe_read_compat, void *, dst, u32, size,
317 	   const void *, unsafe_ptr)
318 {
319 	if ((unsigned long)unsafe_ptr < TASK_SIZE) {
320 		return bpf_probe_read_user_common(dst, size,
321 				(__force void __user *)unsafe_ptr);
322 	}
323 	return bpf_probe_read_kernel_common(dst, size, unsafe_ptr);
324 }
325 
326 static const struct bpf_func_proto bpf_probe_read_compat_proto = {
327 	.func		= bpf_probe_read_compat,
328 	.gpl_only	= true,
329 	.ret_type	= RET_INTEGER,
330 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
331 	.arg2_type	= ARG_MEM_SIZE_OR_ZERO,
332 	.arg3_type	= ARG_ANYTHING,
333 };
334 
335 BPF_CALL_3(bpf_probe_read_compat_str, void *, dst, u32, size,
336 	   const void *, unsafe_ptr)
337 {
338 	if ((unsigned long)unsafe_ptr < TASK_SIZE) {
339 		return bpf_probe_read_user_str_common(dst, size,
340 				(__force void __user *)unsafe_ptr);
341 	}
342 	return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr);
343 }
344 
345 static const struct bpf_func_proto bpf_probe_read_compat_str_proto = {
346 	.func		= bpf_probe_read_compat_str,
347 	.gpl_only	= true,
348 	.ret_type	= RET_INTEGER,
349 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
350 	.arg2_type	= ARG_MEM_SIZE_OR_ZERO,
351 	.arg3_type	= ARG_ANYTHING,
352 };
353 #endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */
354 
355 BPF_CALL_3(bpf_probe_write_user, void __user *, unsafe_ptr, const void *, src,
356 	   u32, size)
357 {
358 	/*
359 	 * Ensure we're in user context which is safe for the helper to
360 	 * run. This helper has no business in a kthread.
361 	 *
362 	 * access_ok() should prevent writing to non-user memory, but in
363 	 * some situations (nommu, temporary switch, etc) access_ok() does
364 	 * not provide enough validation, hence the check on KERNEL_DS.
365 	 *
366 	 * nmi_uaccess_okay() ensures the probe is not run in an interim
367 	 * state, when the task or mm are switched. This is specifically
368 	 * required to prevent the use of temporary mm.
369 	 */
370 
371 	if (unlikely(in_interrupt() ||
372 		     current->flags & (PF_KTHREAD | PF_EXITING)))
373 		return -EPERM;
374 	if (unlikely(!nmi_uaccess_okay()))
375 		return -EPERM;
376 
377 	return copy_to_user_nofault(unsafe_ptr, src, size);
378 }
379 
380 static const struct bpf_func_proto bpf_probe_write_user_proto = {
381 	.func		= bpf_probe_write_user,
382 	.gpl_only	= true,
383 	.ret_type	= RET_INTEGER,
384 	.arg1_type	= ARG_ANYTHING,
385 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
386 	.arg3_type	= ARG_MEM_SIZE,
387 };
388 
389 #define MAX_TRACE_PRINTK_VARARGS	3
390 #define BPF_TRACE_PRINTK_SIZE		1024
391 
392 BPF_CALL_5(bpf_trace_printk, char *, fmt, u32, fmt_size, u64, arg1,
393 	   u64, arg2, u64, arg3)
394 {
395 	u64 args[MAX_TRACE_PRINTK_VARARGS] = { arg1, arg2, arg3 };
396 	struct bpf_bprintf_data data = {
397 		.get_bin_args	= true,
398 		.get_buf	= true,
399 	};
400 	int ret;
401 
402 	ret = bpf_bprintf_prepare(fmt, fmt_size, args,
403 				  MAX_TRACE_PRINTK_VARARGS, &data);
404 	if (ret < 0)
405 		return ret;
406 
407 	ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt, data.bin_args);
408 
409 	trace_bpf_trace_printk(data.buf);
410 
411 	bpf_bprintf_cleanup(&data);
412 
413 	return ret;
414 }
415 
416 static const struct bpf_func_proto bpf_trace_printk_proto = {
417 	.func		= bpf_trace_printk,
418 	.gpl_only	= true,
419 	.ret_type	= RET_INTEGER,
420 	.arg1_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
421 	.arg2_type	= ARG_MEM_SIZE,
422 };
423 
424 static void __set_printk_clr_event(struct work_struct *work)
425 {
426 	/*
427 	 * This program might be calling bpf_trace_printk,
428 	 * so enable the associated bpf_trace/bpf_trace_printk event.
429 	 * Repeat this each time as it is possible a user has
430 	 * disabled bpf_trace_printk events.  By loading a program
431 	 * calling bpf_trace_printk() however the user has expressed
432 	 * the intent to see such events.
433 	 */
434 	if (trace_set_clr_event("bpf_trace", "bpf_trace_printk", 1))
435 		pr_warn_ratelimited("could not enable bpf_trace_printk events");
436 }
437 static DECLARE_WORK(set_printk_work, __set_printk_clr_event);
438 
439 const struct bpf_func_proto *bpf_get_trace_printk_proto(void)
440 {
441 	schedule_work(&set_printk_work);
442 	return &bpf_trace_printk_proto;
443 }
444 
445 BPF_CALL_4(bpf_trace_vprintk, char *, fmt, u32, fmt_size, const void *, args,
446 	   u32, data_len)
447 {
448 	struct bpf_bprintf_data data = {
449 		.get_bin_args	= true,
450 		.get_buf	= true,
451 	};
452 	int ret, num_args;
453 
454 	if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 ||
455 	    (data_len && !args))
456 		return -EINVAL;
457 	num_args = data_len / 8;
458 
459 	ret = bpf_bprintf_prepare(fmt, fmt_size, args, num_args, &data);
460 	if (ret < 0)
461 		return ret;
462 
463 	ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt, data.bin_args);
464 
465 	trace_bpf_trace_printk(data.buf);
466 
467 	bpf_bprintf_cleanup(&data);
468 
469 	return ret;
470 }
471 
472 static const struct bpf_func_proto bpf_trace_vprintk_proto = {
473 	.func		= bpf_trace_vprintk,
474 	.gpl_only	= true,
475 	.ret_type	= RET_INTEGER,
476 	.arg1_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
477 	.arg2_type	= ARG_MEM_SIZE,
478 	.arg3_type	= ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
479 	.arg4_type	= ARG_MEM_SIZE_OR_ZERO,
480 };
481 
482 const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void)
483 {
484 	schedule_work(&set_printk_work);
485 	return &bpf_trace_vprintk_proto;
486 }
487 
488 BPF_CALL_5(bpf_seq_printf, struct seq_file *, m, char *, fmt, u32, fmt_size,
489 	   const void *, args, u32, data_len)
490 {
491 	struct bpf_bprintf_data data = {
492 		.get_bin_args	= true,
493 	};
494 	int err, num_args;
495 
496 	if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 ||
497 	    (data_len && !args))
498 		return -EINVAL;
499 	num_args = data_len / 8;
500 
501 	err = bpf_bprintf_prepare(fmt, fmt_size, args, num_args, &data);
502 	if (err < 0)
503 		return err;
504 
505 	seq_bprintf(m, fmt, data.bin_args);
506 
507 	bpf_bprintf_cleanup(&data);
508 
509 	return seq_has_overflowed(m) ? -EOVERFLOW : 0;
510 }
511 
512 BTF_ID_LIST_SINGLE(btf_seq_file_ids, struct, seq_file)
513 
514 static const struct bpf_func_proto bpf_seq_printf_proto = {
515 	.func		= bpf_seq_printf,
516 	.gpl_only	= true,
517 	.ret_type	= RET_INTEGER,
518 	.arg1_type	= ARG_PTR_TO_BTF_ID,
519 	.arg1_btf_id	= &btf_seq_file_ids[0],
520 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
521 	.arg3_type	= ARG_MEM_SIZE,
522 	.arg4_type      = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
523 	.arg5_type      = ARG_MEM_SIZE_OR_ZERO,
524 };
525 
526 BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len)
527 {
528 	return seq_write(m, data, len) ? -EOVERFLOW : 0;
529 }
530 
531 static const struct bpf_func_proto bpf_seq_write_proto = {
532 	.func		= bpf_seq_write,
533 	.gpl_only	= true,
534 	.ret_type	= RET_INTEGER,
535 	.arg1_type	= ARG_PTR_TO_BTF_ID,
536 	.arg1_btf_id	= &btf_seq_file_ids[0],
537 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
538 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
539 };
540 
541 BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr,
542 	   u32, btf_ptr_size, u64, flags)
543 {
544 	const struct btf *btf;
545 	s32 btf_id;
546 	int ret;
547 
548 	ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id);
549 	if (ret)
550 		return ret;
551 
552 	return btf_type_seq_show_flags(btf, btf_id, ptr->ptr, m, flags);
553 }
554 
555 static const struct bpf_func_proto bpf_seq_printf_btf_proto = {
556 	.func		= bpf_seq_printf_btf,
557 	.gpl_only	= true,
558 	.ret_type	= RET_INTEGER,
559 	.arg1_type	= ARG_PTR_TO_BTF_ID,
560 	.arg1_btf_id	= &btf_seq_file_ids[0],
561 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
562 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
563 	.arg4_type	= ARG_ANYTHING,
564 };
565 
566 static __always_inline int
567 get_map_perf_counter(struct bpf_map *map, u64 flags,
568 		     u64 *value, u64 *enabled, u64 *running)
569 {
570 	struct bpf_array *array = container_of(map, struct bpf_array, map);
571 	unsigned int cpu = smp_processor_id();
572 	u64 index = flags & BPF_F_INDEX_MASK;
573 	struct bpf_event_entry *ee;
574 
575 	if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
576 		return -EINVAL;
577 	if (index == BPF_F_CURRENT_CPU)
578 		index = cpu;
579 	if (unlikely(index >= array->map.max_entries))
580 		return -E2BIG;
581 
582 	ee = READ_ONCE(array->ptrs[index]);
583 	if (!ee)
584 		return -ENOENT;
585 
586 	return perf_event_read_local(ee->event, value, enabled, running);
587 }
588 
589 BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags)
590 {
591 	u64 value = 0;
592 	int err;
593 
594 	err = get_map_perf_counter(map, flags, &value, NULL, NULL);
595 	/*
596 	 * this api is ugly since we miss [-22..-2] range of valid
597 	 * counter values, but that's uapi
598 	 */
599 	if (err)
600 		return err;
601 	return value;
602 }
603 
604 const struct bpf_func_proto bpf_perf_event_read_proto = {
605 	.func		= bpf_perf_event_read,
606 	.gpl_only	= true,
607 	.ret_type	= RET_INTEGER,
608 	.arg1_type	= ARG_CONST_MAP_PTR,
609 	.arg2_type	= ARG_ANYTHING,
610 };
611 
612 BPF_CALL_4(bpf_perf_event_read_value, struct bpf_map *, map, u64, flags,
613 	   struct bpf_perf_event_value *, buf, u32, size)
614 {
615 	int err = -EINVAL;
616 
617 	if (unlikely(size != sizeof(struct bpf_perf_event_value)))
618 		goto clear;
619 	err = get_map_perf_counter(map, flags, &buf->counter, &buf->enabled,
620 				   &buf->running);
621 	if (unlikely(err))
622 		goto clear;
623 	return 0;
624 clear:
625 	memset(buf, 0, size);
626 	return err;
627 }
628 
629 static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
630 	.func		= bpf_perf_event_read_value,
631 	.gpl_only	= true,
632 	.ret_type	= RET_INTEGER,
633 	.arg1_type	= ARG_CONST_MAP_PTR,
634 	.arg2_type	= ARG_ANYTHING,
635 	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,
636 	.arg4_type	= ARG_MEM_SIZE,
637 };
638 
639 const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void)
640 {
641 	return &bpf_perf_event_read_value_proto;
642 }
643 
644 static __always_inline u64
645 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
646 			u64 flags, struct perf_raw_record *raw,
647 			struct perf_sample_data *sd)
648 {
649 	struct bpf_array *array = container_of(map, struct bpf_array, map);
650 	unsigned int cpu = smp_processor_id();
651 	u64 index = flags & BPF_F_INDEX_MASK;
652 	struct bpf_event_entry *ee;
653 	struct perf_event *event;
654 
655 	if (index == BPF_F_CURRENT_CPU)
656 		index = cpu;
657 	if (unlikely(index >= array->map.max_entries))
658 		return -E2BIG;
659 
660 	ee = READ_ONCE(array->ptrs[index]);
661 	if (!ee)
662 		return -ENOENT;
663 
664 	event = ee->event;
665 	if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE ||
666 		     event->attr.config != PERF_COUNT_SW_BPF_OUTPUT))
667 		return -EINVAL;
668 
669 	if (unlikely(event->oncpu != cpu))
670 		return -EOPNOTSUPP;
671 
672 	perf_sample_save_raw_data(sd, event, raw);
673 
674 	return perf_event_output(event, sd, regs);
675 }
676 
677 /*
678  * Support executing tracepoints in normal, irq, and nmi context that each call
679  * bpf_perf_event_output
680  */
681 struct bpf_trace_sample_data {
682 	struct perf_sample_data sds[3];
683 };
684 
685 static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds);
686 static DEFINE_PER_CPU(int, bpf_trace_nest_level);
687 BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
688 	   u64, flags, void *, data, u64, size)
689 {
690 	struct bpf_trace_sample_data *sds;
691 	struct perf_raw_record raw = {
692 		.frag = {
693 			.size = size,
694 			.data = data,
695 		},
696 	};
697 	struct perf_sample_data *sd;
698 	int nest_level, err;
699 
700 	preempt_disable();
701 	sds = this_cpu_ptr(&bpf_trace_sds);
702 	nest_level = this_cpu_inc_return(bpf_trace_nest_level);
703 
704 	if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds))) {
705 		err = -EBUSY;
706 		goto out;
707 	}
708 
709 	sd = &sds->sds[nest_level - 1];
710 
711 	if (unlikely(flags & ~(BPF_F_INDEX_MASK))) {
712 		err = -EINVAL;
713 		goto out;
714 	}
715 
716 	perf_sample_data_init(sd, 0, 0);
717 
718 	err = __bpf_perf_event_output(regs, map, flags, &raw, sd);
719 out:
720 	this_cpu_dec(bpf_trace_nest_level);
721 	preempt_enable();
722 	return err;
723 }
724 
725 static const struct bpf_func_proto bpf_perf_event_output_proto = {
726 	.func		= bpf_perf_event_output,
727 	.gpl_only	= true,
728 	.ret_type	= RET_INTEGER,
729 	.arg1_type	= ARG_PTR_TO_CTX,
730 	.arg2_type	= ARG_CONST_MAP_PTR,
731 	.arg3_type	= ARG_ANYTHING,
732 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
733 	.arg5_type	= ARG_MEM_SIZE_OR_ZERO,
734 };
735 
736 static DEFINE_PER_CPU(int, bpf_event_output_nest_level);
737 struct bpf_nested_pt_regs {
738 	struct pt_regs regs[3];
739 };
740 static DEFINE_PER_CPU(struct bpf_nested_pt_regs, bpf_pt_regs);
741 static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_misc_sds);
742 
743 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
744 		     void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)
745 {
746 	struct perf_raw_frag frag = {
747 		.copy		= ctx_copy,
748 		.size		= ctx_size,
749 		.data		= ctx,
750 	};
751 	struct perf_raw_record raw = {
752 		.frag = {
753 			{
754 				.next	= ctx_size ? &frag : NULL,
755 			},
756 			.size	= meta_size,
757 			.data	= meta,
758 		},
759 	};
760 	struct perf_sample_data *sd;
761 	struct pt_regs *regs;
762 	int nest_level;
763 	u64 ret;
764 
765 	preempt_disable();
766 	nest_level = this_cpu_inc_return(bpf_event_output_nest_level);
767 
768 	if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bpf_misc_sds.sds))) {
769 		ret = -EBUSY;
770 		goto out;
771 	}
772 	sd = this_cpu_ptr(&bpf_misc_sds.sds[nest_level - 1]);
773 	regs = this_cpu_ptr(&bpf_pt_regs.regs[nest_level - 1]);
774 
775 	perf_fetch_caller_regs(regs);
776 	perf_sample_data_init(sd, 0, 0);
777 
778 	ret = __bpf_perf_event_output(regs, map, flags, &raw, sd);
779 out:
780 	this_cpu_dec(bpf_event_output_nest_level);
781 	preempt_enable();
782 	return ret;
783 }
784 
785 BPF_CALL_0(bpf_get_current_task)
786 {
787 	return (long) current;
788 }
789 
790 const struct bpf_func_proto bpf_get_current_task_proto = {
791 	.func		= bpf_get_current_task,
792 	.gpl_only	= true,
793 	.ret_type	= RET_INTEGER,
794 };
795 
796 BPF_CALL_0(bpf_get_current_task_btf)
797 {
798 	return (unsigned long) current;
799 }
800 
801 const struct bpf_func_proto bpf_get_current_task_btf_proto = {
802 	.func		= bpf_get_current_task_btf,
803 	.gpl_only	= true,
804 	.ret_type	= RET_PTR_TO_BTF_ID_TRUSTED,
805 	.ret_btf_id	= &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
806 };
807 
808 BPF_CALL_1(bpf_task_pt_regs, struct task_struct *, task)
809 {
810 	return (unsigned long) task_pt_regs(task);
811 }
812 
813 BTF_ID_LIST_SINGLE(bpf_task_pt_regs_ids, struct, pt_regs)
814 
815 const struct bpf_func_proto bpf_task_pt_regs_proto = {
816 	.func		= bpf_task_pt_regs,
817 	.gpl_only	= true,
818 	.arg1_type	= ARG_PTR_TO_BTF_ID,
819 	.arg1_btf_id	= &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
820 	.ret_type	= RET_PTR_TO_BTF_ID,
821 	.ret_btf_id	= &bpf_task_pt_regs_ids[0],
822 };
823 
824 struct send_signal_irq_work {
825 	struct irq_work irq_work;
826 	/* The claim: NULL marks the slot free for the next caller. */
827 	struct task_struct *task;
828 	u32 sig;
829 	enum pid_type type;
830 	bool has_siginfo;
831 	struct kernel_siginfo info;
832 };
833 
834 static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work);
835 
836 static void do_bpf_send_signal(struct irq_work *entry)
837 {
838 	struct send_signal_irq_work *work;
839 	struct kernel_siginfo *siginfo;
840 
841 	work = container_of(entry, struct send_signal_irq_work, irq_work);
842 	siginfo = work->has_siginfo ? &work->info : SEND_SIG_PRIV;
843 
844 	group_send_sig_info(work->sig, siginfo, work->task, work->type);
845 	put_task_struct(work->task);
846 	/* Release once the fields are consumed. */
847 	smp_store_release(&work->task, NULL);
848 }
849 
850 static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value)
851 {
852 	struct send_signal_irq_work *work = NULL;
853 	struct kernel_siginfo info;
854 	struct kernel_siginfo *siginfo;
855 
856 	if (!task) {
857 		task = current;
858 		siginfo = SEND_SIG_PRIV;
859 	} else {
860 		clear_siginfo(&info);
861 		info.si_signo = sig;
862 		info.si_errno = 0;
863 		info.si_code = SI_KERNEL;
864 		info.si_pid = 0;
865 		info.si_uid = 0;
866 		info.si_value.sival_ptr = (void __user __force *)(unsigned long)value;
867 		siginfo = &info;
868 	}
869 
870 	/*
871 	 * Similar to bpf_probe_write_user, task needs to be
872 	 * in a sound condition.
873 	 */
874 	if (unlikely(task->flags & (PF_KTHREAD | PF_EXITING)))
875 		return -EPERM;
876 	/* Task should not be pid=1 to avoid kernel panic. */
877 	if (unlikely(is_global_init(task)))
878 		return -EPERM;
879 
880 	if (preempt_count() != 0 || irqs_disabled()) {
881 		/* Do an early check on signal validity. Otherwise,
882 		 * the error is lost in deferred irq_work.
883 		 */
884 		if (unlikely(!valid_signal(sig)))
885 			return -EINVAL;
886 
887 		work = this_cpu_ptr(&send_signal_work);
888 		if (cmpxchg(&work->task, NULL, task))
889 			return -EBUSY;
890 
891 		/* Add the current task, which is the target of sending signal,
892 		 * to the irq_work. The current task may change when queued
893 		 * irq works get executed.
894 		 */
895 		get_task_struct(task);
896 		work->has_siginfo = siginfo == &info;
897 		if (work->has_siginfo)
898 			copy_siginfo(&work->info, &info);
899 		work->sig = sig;
900 		work->type = type;
901 		irq_work_queue(&work->irq_work);
902 		return 0;
903 	}
904 
905 	return group_send_sig_info(sig, siginfo, task, type);
906 }
907 
908 BPF_CALL_1(bpf_send_signal, u32, sig)
909 {
910 	return bpf_send_signal_common(sig, PIDTYPE_TGID, NULL, 0);
911 }
912 
913 const struct bpf_func_proto bpf_send_signal_proto = {
914 	.func		= bpf_send_signal,
915 	.gpl_only	= false,
916 	.ret_type	= RET_INTEGER,
917 	.arg1_type	= ARG_ANYTHING,
918 };
919 
920 BPF_CALL_1(bpf_send_signal_thread, u32, sig)
921 {
922 	return bpf_send_signal_common(sig, PIDTYPE_PID, NULL, 0);
923 }
924 
925 const struct bpf_func_proto bpf_send_signal_thread_proto = {
926 	.func		= bpf_send_signal_thread,
927 	.gpl_only	= false,
928 	.ret_type	= RET_INTEGER,
929 	.arg1_type	= ARG_ANYTHING,
930 };
931 
932 BPF_CALL_3(bpf_d_path, const struct path *, path, char *, buf, u32, sz)
933 {
934 	struct path copy;
935 	long len;
936 	char *p;
937 
938 	if (!sz)
939 		return 0;
940 
941 	/*
942 	 * The path pointer is verified as trusted and safe to use,
943 	 * but let's double check it's valid anyway to workaround
944 	 * potentially broken verifier.
945 	 */
946 	len = copy_from_kernel_nofault(&copy, path, sizeof(*path));
947 	if (len < 0)
948 		return len;
949 
950 	p = d_path(&copy, buf, sz);
951 	if (IS_ERR(p)) {
952 		len = PTR_ERR(p);
953 	} else {
954 		len = buf + sz - p;
955 		memmove(buf, p, len);
956 	}
957 
958 	return len;
959 }
960 
961 BTF_SET_START(btf_allowlist_d_path)
962 #ifdef CONFIG_SECURITY
963 BTF_ID(func, security_file_permission)
964 BTF_ID(func, security_inode_getattr)
965 BTF_ID(func, security_file_open)
966 #endif
967 #ifdef CONFIG_SECURITY_PATH
968 BTF_ID(func, security_path_truncate)
969 #endif
970 BTF_ID(func, vfs_truncate)
971 BTF_ID(func, vfs_fallocate)
972 BTF_ID(func, dentry_open)
973 BTF_ID(func, vfs_getattr)
974 BTF_ID(func, filp_close)
975 BTF_SET_END(btf_allowlist_d_path)
976 
977 BTF_SET_START(btf_allowlist_d_path_fentry)
978 BTF_ID(func, filp_close_sync)
979 BTF_SET_END(btf_allowlist_d_path_fentry)
980 
981 static bool bpf_d_path_allowed(const struct bpf_prog *prog)
982 {
983 	if (prog->type == BPF_PROG_TYPE_TRACING &&
984 	    prog->expected_attach_type == BPF_TRACE_ITER)
985 		return true;
986 
987 	if (prog->type == BPF_PROG_TYPE_LSM)
988 		return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id);
989 
990 	if (btf_id_set_contains(&btf_allowlist_d_path,
991 				prog->aux->attach_btf_id))
992 		return true;
993 
994 	if (btf_id_set_contains(&btf_allowlist_d_path_fentry,
995 				prog->aux->attach_btf_id))
996 		return prog->expected_attach_type == BPF_TRACE_FENTRY;
997 
998 	return false;
999 }
1000 
1001 BTF_ID_LIST_SINGLE(bpf_d_path_btf_ids, struct, path)
1002 
1003 static const struct bpf_func_proto bpf_d_path_proto = {
1004 	.func		= bpf_d_path,
1005 	.gpl_only	= false,
1006 	.ret_type	= RET_INTEGER,
1007 	.arg1_type	= ARG_PTR_TO_BTF_ID,
1008 	.arg1_btf_id	= &bpf_d_path_btf_ids[0],
1009 	.arg2_type	= ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE,
1010 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
1011 	.allowed	= bpf_d_path_allowed,
1012 };
1013 
1014 #define BTF_F_ALL	(BTF_F_COMPACT  | BTF_F_NONAME | \
1015 			 BTF_F_PTR_RAW | BTF_F_ZERO)
1016 
1017 static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,
1018 				  u64 flags, const struct btf **btf,
1019 				  s32 *btf_id)
1020 {
1021 	const struct btf_type *t;
1022 
1023 	if (unlikely(flags & ~(BTF_F_ALL)))
1024 		return -EINVAL;
1025 
1026 	if (btf_ptr_size != sizeof(struct btf_ptr))
1027 		return -EINVAL;
1028 
1029 	*btf = bpf_get_btf_vmlinux();
1030 
1031 	if (IS_ERR_OR_NULL(*btf))
1032 		return IS_ERR(*btf) ? PTR_ERR(*btf) : -EINVAL;
1033 
1034 	if (ptr->type_id > 0)
1035 		*btf_id = ptr->type_id;
1036 	else
1037 		return -EINVAL;
1038 
1039 	if (*btf_id > 0)
1040 		t = btf_type_by_id(*btf, *btf_id);
1041 	if (*btf_id <= 0 || !t)
1042 		return -ENOENT;
1043 
1044 	return 0;
1045 }
1046 
1047 BPF_CALL_5(bpf_snprintf_btf, char *, str, u32, str_size, struct btf_ptr *, ptr,
1048 	   u32, btf_ptr_size, u64, flags)
1049 {
1050 	const struct btf *btf;
1051 	s32 btf_id;
1052 	int ret;
1053 
1054 	ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id);
1055 	if (ret)
1056 		return ret;
1057 
1058 	return btf_type_snprintf_show(btf, btf_id, ptr->ptr, str, str_size,
1059 				      flags);
1060 }
1061 
1062 const struct bpf_func_proto bpf_snprintf_btf_proto = {
1063 	.func		= bpf_snprintf_btf,
1064 	.gpl_only	= false,
1065 	.ret_type	= RET_INTEGER,
1066 	.arg1_type	= ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE,
1067 	.arg2_type	= ARG_MEM_SIZE,
1068 	.arg3_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
1069 	.arg4_type	= ARG_MEM_SIZE,
1070 	.arg5_type	= ARG_ANYTHING,
1071 };
1072 
1073 BPF_CALL_1(bpf_get_func_ip_tracing, void *, ctx)
1074 {
1075 	/* This helper call is inlined by verifier. */
1076 	return ((u64 *)ctx)[-2];
1077 }
1078 
1079 static const struct bpf_func_proto bpf_get_func_ip_proto_tracing = {
1080 	.func		= bpf_get_func_ip_tracing,
1081 	.gpl_only	= true,
1082 	.ret_type	= RET_INTEGER,
1083 	.arg1_type	= ARG_PTR_TO_CTX,
1084 };
1085 
1086 static inline unsigned long get_entry_ip(unsigned long fentry_ip)
1087 {
1088 #ifdef CONFIG_X86_KERNEL_IBT
1089 	if (is_endbr((void *)(fentry_ip - ENDBR_INSN_SIZE)))
1090 		fentry_ip -= ENDBR_INSN_SIZE;
1091 #endif
1092 	return fentry_ip;
1093 }
1094 
1095 BPF_CALL_1(bpf_get_func_ip_kprobe, struct pt_regs *, regs)
1096 {
1097 	struct bpf_trace_run_ctx *run_ctx __maybe_unused;
1098 	struct kprobe *kp;
1099 
1100 #ifdef CONFIG_UPROBES
1101 	run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx);
1102 	if (run_ctx->is_uprobe)
1103 		return ((struct uprobe_dispatch_data *)current->utask->vaddr)->bp_addr;
1104 #endif
1105 
1106 	kp = kprobe_running();
1107 
1108 	if (!kp || !(kp->flags & KPROBE_FLAG_ON_FUNC_ENTRY))
1109 		return 0;
1110 
1111 	return get_entry_ip((uintptr_t)kp->addr);
1112 }
1113 
1114 static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe = {
1115 	.func		= bpf_get_func_ip_kprobe,
1116 	.gpl_only	= true,
1117 	.ret_type	= RET_INTEGER,
1118 	.arg1_type	= ARG_PTR_TO_CTX,
1119 };
1120 
1121 BPF_CALL_1(bpf_get_func_ip_kprobe_multi, struct pt_regs *, regs)
1122 {
1123 	return bpf_kprobe_multi_entry_ip(current->bpf_ctx);
1124 }
1125 
1126 static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe_multi = {
1127 	.func		= bpf_get_func_ip_kprobe_multi,
1128 	.gpl_only	= false,
1129 	.ret_type	= RET_INTEGER,
1130 	.arg1_type	= ARG_PTR_TO_CTX,
1131 };
1132 
1133 BPF_CALL_1(bpf_get_attach_cookie_kprobe_multi, struct pt_regs *, regs)
1134 {
1135 	return bpf_kprobe_multi_cookie(current->bpf_ctx);
1136 }
1137 
1138 static const struct bpf_func_proto bpf_get_attach_cookie_proto_kmulti = {
1139 	.func		= bpf_get_attach_cookie_kprobe_multi,
1140 	.gpl_only	= false,
1141 	.ret_type	= RET_INTEGER,
1142 	.arg1_type	= ARG_PTR_TO_CTX,
1143 };
1144 
1145 BPF_CALL_1(bpf_get_func_ip_uprobe_multi, struct pt_regs *, regs)
1146 {
1147 	return bpf_uprobe_multi_entry_ip(current->bpf_ctx);
1148 }
1149 
1150 static const struct bpf_func_proto bpf_get_func_ip_proto_uprobe_multi = {
1151 	.func		= bpf_get_func_ip_uprobe_multi,
1152 	.gpl_only	= false,
1153 	.ret_type	= RET_INTEGER,
1154 	.arg1_type	= ARG_PTR_TO_CTX,
1155 };
1156 
1157 BPF_CALL_1(bpf_get_attach_cookie_uprobe_multi, struct pt_regs *, regs)
1158 {
1159 	return bpf_uprobe_multi_cookie(current->bpf_ctx);
1160 }
1161 
1162 static const struct bpf_func_proto bpf_get_attach_cookie_proto_umulti = {
1163 	.func		= bpf_get_attach_cookie_uprobe_multi,
1164 	.gpl_only	= false,
1165 	.ret_type	= RET_INTEGER,
1166 	.arg1_type	= ARG_PTR_TO_CTX,
1167 };
1168 
1169 BPF_CALL_1(bpf_get_attach_cookie_trace, void *, ctx)
1170 {
1171 	struct bpf_trace_run_ctx *run_ctx;
1172 
1173 	run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx);
1174 	return run_ctx->bpf_cookie;
1175 }
1176 
1177 static const struct bpf_func_proto bpf_get_attach_cookie_proto_trace = {
1178 	.func		= bpf_get_attach_cookie_trace,
1179 	.gpl_only	= false,
1180 	.ret_type	= RET_INTEGER,
1181 	.arg1_type	= ARG_PTR_TO_CTX,
1182 };
1183 
1184 BPF_CALL_1(bpf_get_attach_cookie_pe, struct bpf_perf_event_data_kern *, ctx)
1185 {
1186 	return ctx->event->bpf_cookie;
1187 }
1188 
1189 static const struct bpf_func_proto bpf_get_attach_cookie_proto_pe = {
1190 	.func		= bpf_get_attach_cookie_pe,
1191 	.gpl_only	= false,
1192 	.ret_type	= RET_INTEGER,
1193 	.arg1_type	= ARG_PTR_TO_CTX,
1194 };
1195 
1196 BPF_CALL_1(bpf_get_attach_cookie_tracing, void *, ctx)
1197 {
1198 	struct bpf_trace_run_ctx *run_ctx;
1199 
1200 	run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx);
1201 	return run_ctx->bpf_cookie;
1202 }
1203 
1204 static const struct bpf_func_proto bpf_get_attach_cookie_proto_tracing = {
1205 	.func		= bpf_get_attach_cookie_tracing,
1206 	.gpl_only	= false,
1207 	.ret_type	= RET_INTEGER,
1208 	.arg1_type	= ARG_PTR_TO_CTX,
1209 };
1210 
1211 BPF_CALL_3(bpf_get_branch_snapshot, void *, buf, u32, size, u64, flags)
1212 {
1213 	static const u32 br_entry_size = sizeof(struct perf_branch_entry);
1214 	u32 entry_cnt = size / br_entry_size;
1215 
1216 	entry_cnt = static_call(perf_snapshot_branch_stack)(buf, entry_cnt);
1217 
1218 	if (unlikely(flags))
1219 		return -EINVAL;
1220 
1221 	if (!entry_cnt)
1222 		return -ENOENT;
1223 
1224 	return entry_cnt * br_entry_size;
1225 }
1226 
1227 const struct bpf_func_proto bpf_get_branch_snapshot_proto = {
1228 	.func		= bpf_get_branch_snapshot,
1229 	.gpl_only	= true,
1230 	.ret_type	= RET_INTEGER,
1231 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
1232 	.arg2_type	= ARG_MEM_SIZE_OR_ZERO,
1233 };
1234 
1235 BPF_CALL_3(get_func_arg, void *, ctx, u32, n, u64 *, value)
1236 {
1237 	/* This helper call is inlined by verifier. */
1238 	u64 nr_args = ((u64 *)ctx)[-1] & 0xFF;
1239 
1240 	if ((u64) n >= nr_args)
1241 		return -EINVAL;
1242 	*value = ((u64 *)ctx)[n];
1243 	return 0;
1244 }
1245 
1246 static const struct bpf_func_proto bpf_get_func_arg_proto = {
1247 	.func		= get_func_arg,
1248 	.ret_type	= RET_INTEGER,
1249 	.arg1_type	= ARG_PTR_TO_CTX,
1250 	.arg2_type	= ARG_ANYTHING,
1251 	.arg3_type	= ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED,
1252 	.arg3_size	= sizeof(u64),
1253 };
1254 
1255 BPF_CALL_2(get_func_ret, void *, ctx, u64 *, value)
1256 {
1257 	/* This helper call is inlined by verifier. */
1258 	u64 nr_args = ((u64 *)ctx)[-1] & 0xFF;
1259 
1260 	*value = ((u64 *)ctx)[nr_args];
1261 	return 0;
1262 }
1263 
1264 static const struct bpf_func_proto bpf_get_func_ret_proto = {
1265 	.func		= get_func_ret,
1266 	.ret_type	= RET_INTEGER,
1267 	.arg1_type	= ARG_PTR_TO_CTX,
1268 	.arg2_type	= ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED,
1269 	.arg2_size	= sizeof(u64),
1270 };
1271 
1272 BPF_CALL_1(get_func_arg_cnt, void *, ctx)
1273 {
1274 	/* This helper call is inlined by verifier. */
1275 	return ((u64 *)ctx)[-1] & 0xFF;
1276 }
1277 
1278 static const struct bpf_func_proto bpf_get_func_arg_cnt_proto = {
1279 	.func		= get_func_arg_cnt,
1280 	.ret_type	= RET_INTEGER,
1281 	.arg1_type	= ARG_PTR_TO_CTX,
1282 };
1283 
1284 static const struct bpf_func_proto *
1285 bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1286 {
1287 	const struct bpf_func_proto *func_proto;
1288 
1289 	switch (func_id) {
1290 	case BPF_FUNC_get_smp_processor_id:
1291 		return &bpf_get_smp_processor_id_proto;
1292 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
1293 	case BPF_FUNC_probe_read:
1294 		return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
1295 		       NULL : &bpf_probe_read_compat_proto;
1296 	case BPF_FUNC_probe_read_str:
1297 		return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
1298 		       NULL : &bpf_probe_read_compat_str_proto;
1299 #endif
1300 	case BPF_FUNC_get_func_ip:
1301 		return &bpf_get_func_ip_proto_tracing;
1302 	default:
1303 		break;
1304 	}
1305 
1306 	func_proto = bpf_base_func_proto(func_id, prog);
1307 	if (func_proto)
1308 		return func_proto;
1309 
1310 	if (!bpf_token_capable(prog->aux->token, CAP_SYS_ADMIN))
1311 		return NULL;
1312 
1313 	switch (func_id) {
1314 	case BPF_FUNC_probe_write_user:
1315 		return security_locked_down(LOCKDOWN_BPF_WRITE_USER) < 0 ?
1316 		       NULL : &bpf_probe_write_user_proto;
1317 	default:
1318 		return NULL;
1319 	}
1320 }
1321 
1322 static bool is_kprobe_multi(const struct bpf_prog *prog)
1323 {
1324 	return prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI ||
1325 	       prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;
1326 }
1327 
1328 static inline bool is_kprobe_session(const struct bpf_prog *prog)
1329 {
1330 	return prog->type == BPF_PROG_TYPE_KPROBE &&
1331 	       prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;
1332 }
1333 
1334 static inline bool is_uprobe_multi(const struct bpf_prog *prog)
1335 {
1336 	return prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI ||
1337 	       prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1338 }
1339 
1340 static inline bool is_uprobe_session(const struct bpf_prog *prog)
1341 {
1342 	return prog->type == BPF_PROG_TYPE_KPROBE &&
1343 	       prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1344 }
1345 
1346 static inline bool is_trace_fsession(const struct bpf_prog *prog)
1347 {
1348 	return prog->type == BPF_PROG_TYPE_TRACING &&
1349 	       (prog->expected_attach_type == BPF_TRACE_FSESSION ||
1350 		prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI);
1351 }
1352 
1353 static const struct bpf_func_proto *
1354 kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1355 {
1356 	switch (func_id) {
1357 	case BPF_FUNC_perf_event_output:
1358 		return &bpf_perf_event_output_proto;
1359 	case BPF_FUNC_get_stackid:
1360 		return &bpf_get_stackid_proto;
1361 	case BPF_FUNC_get_stack:
1362 		return prog->sleepable ? &bpf_get_stack_sleepable_proto : &bpf_get_stack_proto;
1363 #ifdef CONFIG_BPF_KPROBE_OVERRIDE
1364 	case BPF_FUNC_override_return:
1365 		return &bpf_override_return_proto;
1366 #endif
1367 	case BPF_FUNC_get_func_ip:
1368 		if (is_kprobe_multi(prog))
1369 			return &bpf_get_func_ip_proto_kprobe_multi;
1370 		if (is_uprobe_multi(prog))
1371 			return &bpf_get_func_ip_proto_uprobe_multi;
1372 		return &bpf_get_func_ip_proto_kprobe;
1373 	case BPF_FUNC_get_attach_cookie:
1374 		if (is_kprobe_multi(prog))
1375 			return &bpf_get_attach_cookie_proto_kmulti;
1376 		if (is_uprobe_multi(prog))
1377 			return &bpf_get_attach_cookie_proto_umulti;
1378 		return &bpf_get_attach_cookie_proto_trace;
1379 	default:
1380 		return bpf_tracing_func_proto(func_id, prog);
1381 	}
1382 }
1383 
1384 /* bpf+kprobe programs can access fields of 'struct pt_regs' */
1385 static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type type,
1386 					const struct bpf_prog *prog,
1387 					struct bpf_insn_access_aux *info)
1388 {
1389 	if (off < 0 || off >= sizeof(struct pt_regs))
1390 		return false;
1391 	if (off % size != 0)
1392 		return false;
1393 	/*
1394 	 * Assertion for 32 bit to make sure last 8 byte access
1395 	 * (BPF_DW) to the last 4 byte member is disallowed.
1396 	 */
1397 	if (off + size > sizeof(struct pt_regs))
1398 		return false;
1399 
1400 	if (type == BPF_WRITE) {
1401 		/*
1402 		 * pt_regs also holds what the kernel restores on the way to
1403 		 * user space: cs and flags on x86, pstate on arm64, status on
1404 		 * riscv. A prog can write the registers of the task only.
1405 		 */
1406 		if (!arch_uprobe_regs_writable(off, size))
1407 			return false;
1408 		prog->aux->kprobe_write_ctx = true;
1409 	}
1410 
1411 	return true;
1412 }
1413 
1414 const struct bpf_verifier_ops kprobe_verifier_ops = {
1415 	.get_func_proto  = kprobe_prog_func_proto,
1416 	.is_valid_access = kprobe_prog_is_valid_access,
1417 };
1418 
1419 const struct bpf_prog_ops kprobe_prog_ops = {
1420 };
1421 
1422 BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map,
1423 	   u64, flags, void *, data, u64, size)
1424 {
1425 	struct pt_regs *regs = *(struct pt_regs **)tp_buff;
1426 
1427 	/*
1428 	 * r1 points to perf tracepoint buffer where first 8 bytes are hidden
1429 	 * from bpf program and contain a pointer to 'struct pt_regs'. Fetch it
1430 	 * from there and call the same bpf_perf_event_output() helper inline.
1431 	 */
1432 	return ____bpf_perf_event_output(regs, map, flags, data, size);
1433 }
1434 
1435 static const struct bpf_func_proto bpf_perf_event_output_proto_tp = {
1436 	.func		= bpf_perf_event_output_tp,
1437 	.gpl_only	= true,
1438 	.ret_type	= RET_INTEGER,
1439 	.arg1_type	= ARG_PTR_TO_CTX,
1440 	.arg2_type	= ARG_CONST_MAP_PTR,
1441 	.arg3_type	= ARG_ANYTHING,
1442 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
1443 	.arg5_type	= ARG_MEM_SIZE_OR_ZERO,
1444 };
1445 
1446 BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map,
1447 	   u64, flags)
1448 {
1449 	struct pt_regs *regs = *(struct pt_regs **)tp_buff;
1450 
1451 	/*
1452 	 * Same comment as in bpf_perf_event_output_tp(), only that this time
1453 	 * the other helper's function body cannot be inlined due to being
1454 	 * external, thus we need to call raw helper function.
1455 	 */
1456 	return bpf_get_stackid((unsigned long) regs, (unsigned long) map,
1457 			       flags, 0, 0);
1458 }
1459 
1460 static const struct bpf_func_proto bpf_get_stackid_proto_tp = {
1461 	.func		= bpf_get_stackid_tp,
1462 	.gpl_only	= true,
1463 	.ret_type	= RET_INTEGER,
1464 	.arg1_type	= ARG_PTR_TO_CTX,
1465 	.arg2_type	= ARG_CONST_MAP_PTR,
1466 	.arg3_type	= ARG_ANYTHING,
1467 };
1468 
1469 BPF_CALL_4(bpf_get_stack_tp, void *, tp_buff, void *, buf, u32, size,
1470 	   u64, flags)
1471 {
1472 	struct pt_regs *regs = *(struct pt_regs **)tp_buff;
1473 
1474 	return bpf_get_stack((unsigned long) regs, (unsigned long) buf,
1475 			     (unsigned long) size, flags, 0);
1476 }
1477 
1478 static const struct bpf_func_proto bpf_get_stack_proto_tp = {
1479 	.func		= bpf_get_stack_tp,
1480 	.gpl_only	= true,
1481 	.ret_type	= RET_INTEGER,
1482 	.arg1_type	= ARG_PTR_TO_CTX,
1483 	.arg2_type	= ARG_PTR_TO_UNINIT_MEM,
1484 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
1485 	.arg4_type	= ARG_ANYTHING,
1486 };
1487 
1488 static const struct bpf_func_proto *
1489 tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1490 {
1491 	switch (func_id) {
1492 	case BPF_FUNC_perf_event_output:
1493 		return &bpf_perf_event_output_proto_tp;
1494 	case BPF_FUNC_get_stackid:
1495 		return &bpf_get_stackid_proto_tp;
1496 	case BPF_FUNC_get_stack:
1497 		return &bpf_get_stack_proto_tp;
1498 	case BPF_FUNC_get_attach_cookie:
1499 		return &bpf_get_attach_cookie_proto_trace;
1500 	default:
1501 		return bpf_tracing_func_proto(func_id, prog);
1502 	}
1503 }
1504 
1505 static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type,
1506 				    const struct bpf_prog *prog,
1507 				    struct bpf_insn_access_aux *info)
1508 {
1509 	if (off < sizeof(void *) || off >= PERF_MAX_TRACE_SIZE)
1510 		return false;
1511 	if (type != BPF_READ)
1512 		return false;
1513 	if (off % size != 0)
1514 		return false;
1515 
1516 	BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(__u64));
1517 	return true;
1518 }
1519 
1520 const struct bpf_verifier_ops tracepoint_verifier_ops = {
1521 	.get_func_proto  = tp_prog_func_proto,
1522 	.is_valid_access = tp_prog_is_valid_access,
1523 };
1524 
1525 const struct bpf_prog_ops tracepoint_prog_ops = {
1526 };
1527 
1528 BPF_CALL_3(bpf_perf_prog_read_value, struct bpf_perf_event_data_kern *, ctx,
1529 	   struct bpf_perf_event_value *, buf, u32, size)
1530 {
1531 	int err = -EINVAL;
1532 
1533 	if (unlikely(size != sizeof(struct bpf_perf_event_value)))
1534 		goto clear;
1535 	err = perf_event_read_local(ctx->event, &buf->counter, &buf->enabled,
1536 				    &buf->running);
1537 	if (unlikely(err))
1538 		goto clear;
1539 	return 0;
1540 clear:
1541 	memset(buf, 0, size);
1542 	return err;
1543 }
1544 
1545 static const struct bpf_func_proto bpf_perf_prog_read_value_proto = {
1546 	.func           = bpf_perf_prog_read_value,
1547 	.gpl_only       = true,
1548 	.ret_type       = RET_INTEGER,
1549 	.arg1_type      = ARG_PTR_TO_CTX,
1550 	.arg2_type      = ARG_PTR_TO_UNINIT_MEM,
1551 	.arg3_type      = ARG_MEM_SIZE,
1552 };
1553 
1554 BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx,
1555 	   void *, buf, u32, size, u64, flags)
1556 {
1557 	static const u32 br_entry_size = sizeof(struct perf_branch_entry);
1558 	struct perf_branch_stack *br_stack = ctx->data->br_stack;
1559 	u32 to_copy;
1560 
1561 	if (unlikely(flags & ~BPF_F_GET_BRANCH_RECORDS_SIZE))
1562 		return -EINVAL;
1563 
1564 	if (unlikely(!(ctx->data->sample_flags & PERF_SAMPLE_BRANCH_STACK)))
1565 		return -ENOENT;
1566 
1567 	if (unlikely(!br_stack))
1568 		return -ENOENT;
1569 
1570 	if (flags & BPF_F_GET_BRANCH_RECORDS_SIZE)
1571 		return br_stack->nr * br_entry_size;
1572 
1573 	if (!buf || (size % br_entry_size != 0))
1574 		return -EINVAL;
1575 
1576 	to_copy = min_t(u32, br_stack->nr * br_entry_size, size);
1577 	memcpy(buf, br_stack->entries, to_copy);
1578 
1579 	return to_copy;
1580 }
1581 
1582 static const struct bpf_func_proto bpf_read_branch_records_proto = {
1583 	.func           = bpf_read_branch_records,
1584 	.gpl_only       = true,
1585 	.ret_type       = RET_INTEGER,
1586 	.arg1_type      = ARG_PTR_TO_CTX,
1587 	.arg2_type      = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE,
1588 	.arg3_type      = ARG_MEM_SIZE_OR_ZERO,
1589 	.arg4_type      = ARG_ANYTHING,
1590 };
1591 
1592 static const struct bpf_func_proto *
1593 pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1594 {
1595 	switch (func_id) {
1596 	case BPF_FUNC_perf_event_output:
1597 		return &bpf_perf_event_output_proto_tp;
1598 	case BPF_FUNC_get_stackid:
1599 		return &bpf_get_stackid_proto_pe;
1600 	case BPF_FUNC_get_stack:
1601 		return &bpf_get_stack_proto_pe;
1602 	case BPF_FUNC_perf_prog_read_value:
1603 		return &bpf_perf_prog_read_value_proto;
1604 	case BPF_FUNC_read_branch_records:
1605 		return &bpf_read_branch_records_proto;
1606 	case BPF_FUNC_get_attach_cookie:
1607 		return &bpf_get_attach_cookie_proto_pe;
1608 	default:
1609 		return bpf_tracing_func_proto(func_id, prog);
1610 	}
1611 }
1612 
1613 /*
1614  * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp
1615  * to avoid potential recursive reuse issue when/if tracepoints are added
1616  * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack.
1617  *
1618  * Since raw tracepoints run despite bpf_prog_active, support concurrent usage
1619  * in normal, irq, and nmi context.
1620  */
1621 struct bpf_raw_tp_regs {
1622 	struct pt_regs regs[3];
1623 };
1624 static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs);
1625 static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level);
1626 static struct pt_regs *get_bpf_raw_tp_regs(void)
1627 {
1628 	struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs);
1629 	int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level);
1630 
1631 	if (nest_level > ARRAY_SIZE(tp_regs->regs)) {
1632 		this_cpu_dec(bpf_raw_tp_nest_level);
1633 		return ERR_PTR(-EBUSY);
1634 	}
1635 
1636 	return &tp_regs->regs[nest_level - 1];
1637 }
1638 
1639 static void put_bpf_raw_tp_regs(void)
1640 {
1641 	this_cpu_dec(bpf_raw_tp_nest_level);
1642 }
1643 
1644 BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args,
1645 	   struct bpf_map *, map, u64, flags, void *, data, u64, size)
1646 {
1647 	struct pt_regs *regs = get_bpf_raw_tp_regs();
1648 	int ret;
1649 
1650 	if (IS_ERR(regs))
1651 		return PTR_ERR(regs);
1652 
1653 	perf_fetch_caller_regs(regs);
1654 	ret = ____bpf_perf_event_output(regs, map, flags, data, size);
1655 
1656 	put_bpf_raw_tp_regs();
1657 	return ret;
1658 }
1659 
1660 static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
1661 	.func		= bpf_perf_event_output_raw_tp,
1662 	.gpl_only	= true,
1663 	.ret_type	= RET_INTEGER,
1664 	.arg1_type	= ARG_PTR_TO_CTX,
1665 	.arg2_type	= ARG_CONST_MAP_PTR,
1666 	.arg3_type	= ARG_ANYTHING,
1667 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
1668 	.arg5_type	= ARG_MEM_SIZE_OR_ZERO,
1669 };
1670 
1671 extern const struct bpf_func_proto bpf_skb_output_proto;
1672 extern const struct bpf_func_proto bpf_xdp_output_proto;
1673 extern const struct bpf_func_proto bpf_xdp_get_buff_len_trace_proto;
1674 
1675 BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args,
1676 	   struct bpf_map *, map, u64, flags)
1677 {
1678 	struct pt_regs *regs = get_bpf_raw_tp_regs();
1679 	int ret;
1680 
1681 	if (IS_ERR(regs))
1682 		return PTR_ERR(regs);
1683 
1684 	perf_fetch_caller_regs(regs);
1685 	/* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */
1686 	ret = bpf_get_stackid((unsigned long) regs, (unsigned long) map,
1687 			      flags, 0, 0);
1688 	put_bpf_raw_tp_regs();
1689 	return ret;
1690 }
1691 
1692 static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = {
1693 	.func		= bpf_get_stackid_raw_tp,
1694 	.gpl_only	= true,
1695 	.ret_type	= RET_INTEGER,
1696 	.arg1_type	= ARG_PTR_TO_CTX,
1697 	.arg2_type	= ARG_CONST_MAP_PTR,
1698 	.arg3_type	= ARG_ANYTHING,
1699 };
1700 
1701 BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args,
1702 	   void *, buf, u32, size, u64, flags)
1703 {
1704 	struct pt_regs *regs = get_bpf_raw_tp_regs();
1705 	int ret;
1706 
1707 	if (IS_ERR(regs))
1708 		return PTR_ERR(regs);
1709 
1710 	perf_fetch_caller_regs(regs);
1711 	ret = bpf_get_stack((unsigned long) regs, (unsigned long) buf,
1712 			    (unsigned long) size, flags, 0);
1713 	put_bpf_raw_tp_regs();
1714 	return ret;
1715 }
1716 
1717 static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = {
1718 	.func		= bpf_get_stack_raw_tp,
1719 	.gpl_only	= true,
1720 	.ret_type	= RET_INTEGER,
1721 	.arg1_type	= ARG_PTR_TO_CTX,
1722 	.arg2_type	= ARG_PTR_TO_UNINIT_MEM,
1723 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
1724 	.arg4_type	= ARG_ANYTHING,
1725 };
1726 
1727 static const struct bpf_func_proto *
1728 raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1729 {
1730 	switch (func_id) {
1731 	case BPF_FUNC_perf_event_output:
1732 		return &bpf_perf_event_output_proto_raw_tp;
1733 	case BPF_FUNC_get_stackid:
1734 		return &bpf_get_stackid_proto_raw_tp;
1735 	case BPF_FUNC_get_stack:
1736 		return &bpf_get_stack_proto_raw_tp;
1737 	case BPF_FUNC_get_attach_cookie:
1738 		return &bpf_get_attach_cookie_proto_tracing;
1739 	default:
1740 		return bpf_tracing_func_proto(func_id, prog);
1741 	}
1742 }
1743 
1744 const struct bpf_func_proto *
1745 tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1746 {
1747 	const struct bpf_func_proto *fn;
1748 
1749 	switch (func_id) {
1750 #ifdef CONFIG_NET
1751 	case BPF_FUNC_skb_output:
1752 		return &bpf_skb_output_proto;
1753 	case BPF_FUNC_xdp_output:
1754 		return &bpf_xdp_output_proto;
1755 	case BPF_FUNC_skc_to_tcp6_sock:
1756 		return &bpf_skc_to_tcp6_sock_proto;
1757 	case BPF_FUNC_skc_to_tcp_sock:
1758 		return &bpf_skc_to_tcp_sock_proto;
1759 	case BPF_FUNC_skc_to_tcp_timewait_sock:
1760 		return &bpf_skc_to_tcp_timewait_sock_proto;
1761 	case BPF_FUNC_skc_to_tcp_request_sock:
1762 		return &bpf_skc_to_tcp_request_sock_proto;
1763 	case BPF_FUNC_skc_to_udp6_sock:
1764 		return &bpf_skc_to_udp6_sock_proto;
1765 	case BPF_FUNC_skc_to_unix_sock:
1766 		return &bpf_skc_to_unix_sock_proto;
1767 	case BPF_FUNC_skc_to_mptcp_sock:
1768 		return &bpf_skc_to_mptcp_sock_proto;
1769 	case BPF_FUNC_sk_storage_get:
1770 		return &bpf_sk_storage_get_tracing_proto;
1771 	case BPF_FUNC_sk_storage_delete:
1772 		return &bpf_sk_storage_delete_tracing_proto;
1773 	case BPF_FUNC_sock_from_file:
1774 		return &bpf_sock_from_file_proto;
1775 	case BPF_FUNC_get_socket_cookie:
1776 		return &bpf_get_socket_ptr_cookie_proto;
1777 	case BPF_FUNC_xdp_get_buff_len:
1778 		return &bpf_xdp_get_buff_len_trace_proto;
1779 #endif
1780 	case BPF_FUNC_seq_printf:
1781 		return prog->expected_attach_type == BPF_TRACE_ITER ?
1782 		       &bpf_seq_printf_proto :
1783 		       NULL;
1784 	case BPF_FUNC_seq_write:
1785 		return prog->expected_attach_type == BPF_TRACE_ITER ?
1786 		       &bpf_seq_write_proto :
1787 		       NULL;
1788 	case BPF_FUNC_seq_printf_btf:
1789 		return prog->expected_attach_type == BPF_TRACE_ITER ?
1790 		       &bpf_seq_printf_btf_proto :
1791 		       NULL;
1792 	case BPF_FUNC_d_path:
1793 		return &bpf_d_path_proto;
1794 	case BPF_FUNC_get_func_arg:
1795 		if (bpf_prog_has_trampoline(prog) ||
1796 		    prog->expected_attach_type == BPF_TRACE_RAW_TP)
1797 			return &bpf_get_func_arg_proto;
1798 		return NULL;
1799 	case BPF_FUNC_get_func_ret:
1800 		return bpf_prog_has_trampoline(prog) ? &bpf_get_func_ret_proto : NULL;
1801 	case BPF_FUNC_get_func_arg_cnt:
1802 		if (bpf_prog_has_trampoline(prog) ||
1803 		    prog->expected_attach_type == BPF_TRACE_RAW_TP)
1804 			return &bpf_get_func_arg_cnt_proto;
1805 		return NULL;
1806 	case BPF_FUNC_get_attach_cookie:
1807 		if (prog->type == BPF_PROG_TYPE_TRACING &&
1808 		    prog->expected_attach_type == BPF_TRACE_RAW_TP)
1809 			return &bpf_get_attach_cookie_proto_tracing;
1810 		return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto_tracing : NULL;
1811 	default:
1812 		fn = raw_tp_prog_func_proto(func_id, prog);
1813 		if (!fn && prog->expected_attach_type == BPF_TRACE_ITER)
1814 			fn = bpf_iter_get_func_proto(func_id, prog);
1815 		return fn;
1816 	}
1817 }
1818 
1819 static bool raw_tp_prog_is_valid_access(int off, int size,
1820 					enum bpf_access_type type,
1821 					const struct bpf_prog *prog,
1822 					struct bpf_insn_access_aux *info)
1823 {
1824 	return bpf_tracing_ctx_access(off, size, type);
1825 }
1826 
1827 static bool tracing_prog_is_valid_access(int off, int size,
1828 					 enum bpf_access_type type,
1829 					 const struct bpf_prog *prog,
1830 					 struct bpf_insn_access_aux *info)
1831 {
1832 	return bpf_tracing_btf_ctx_access(off, size, type, prog, info);
1833 }
1834 
1835 int __weak bpf_prog_test_run_tracing(struct bpf_prog *prog,
1836 				     const union bpf_attr *kattr,
1837 				     union bpf_attr __user *uattr)
1838 {
1839 	return -ENOTSUPP;
1840 }
1841 
1842 const struct bpf_verifier_ops raw_tracepoint_verifier_ops = {
1843 	.get_func_proto  = raw_tp_prog_func_proto,
1844 	.is_valid_access = raw_tp_prog_is_valid_access,
1845 };
1846 
1847 const struct bpf_prog_ops raw_tracepoint_prog_ops = {
1848 #ifdef CONFIG_NET
1849 	.test_run = bpf_prog_test_run_raw_tp,
1850 #endif
1851 };
1852 
1853 const struct bpf_verifier_ops tracing_verifier_ops = {
1854 	.get_func_proto  = tracing_prog_func_proto,
1855 	.is_valid_access = tracing_prog_is_valid_access,
1856 };
1857 
1858 const struct bpf_prog_ops tracing_prog_ops = {
1859 	.test_run = bpf_prog_test_run_tracing,
1860 };
1861 
1862 static bool raw_tp_writable_prog_is_valid_access(int off, int size,
1863 						 enum bpf_access_type type,
1864 						 const struct bpf_prog *prog,
1865 						 struct bpf_insn_access_aux *info)
1866 {
1867 	if (off == 0) {
1868 		if (size != sizeof(u64) || type != BPF_READ)
1869 			return false;
1870 		info->reg_type = PTR_TO_TP_BUFFER;
1871 	}
1872 	return raw_tp_prog_is_valid_access(off, size, type, prog, info);
1873 }
1874 
1875 const struct bpf_verifier_ops raw_tracepoint_writable_verifier_ops = {
1876 	.get_func_proto  = raw_tp_prog_func_proto,
1877 	.is_valid_access = raw_tp_writable_prog_is_valid_access,
1878 };
1879 
1880 const struct bpf_prog_ops raw_tracepoint_writable_prog_ops = {
1881 };
1882 
1883 static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type,
1884 				    const struct bpf_prog *prog,
1885 				    struct bpf_insn_access_aux *info)
1886 {
1887 	const int size_u64 = sizeof(u64);
1888 
1889 	if (off < 0 || off >= sizeof(struct bpf_perf_event_data))
1890 		return false;
1891 	if (type != BPF_READ)
1892 		return false;
1893 	if (off % size != 0) {
1894 		if (sizeof(unsigned long) != 4)
1895 			return false;
1896 		if (size != 8)
1897 			return false;
1898 		if (off % size != 4)
1899 			return false;
1900 	}
1901 
1902 	switch (off) {
1903 	case bpf_ctx_range(struct bpf_perf_event_data, sample_period):
1904 		bpf_ctx_record_field_size(info, size_u64);
1905 		if (!bpf_ctx_narrow_access_ok(off, size, size_u64))
1906 			return false;
1907 		break;
1908 	case bpf_ctx_range(struct bpf_perf_event_data, addr):
1909 		bpf_ctx_record_field_size(info, size_u64);
1910 		if (!bpf_ctx_narrow_access_ok(off, size, size_u64))
1911 			return false;
1912 		break;
1913 	default:
1914 		if (size != sizeof(long))
1915 			return false;
1916 	}
1917 
1918 	return true;
1919 }
1920 
1921 static u32 pe_prog_convert_ctx_access(enum bpf_access_type type,
1922 				      const struct bpf_insn *si,
1923 				      struct bpf_insn *insn_buf,
1924 				      struct bpf_prog *prog, u32 *target_size)
1925 {
1926 	struct bpf_insn *insn = insn_buf;
1927 
1928 	switch (si->off) {
1929 	case offsetof(struct bpf_perf_event_data, sample_period):
1930 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,
1931 						       data), si->dst_reg, si->src_reg,
1932 				      offsetof(struct bpf_perf_event_data_kern, data));
1933 		*insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg,
1934 				      bpf_target_off(struct perf_sample_data, period, 8,
1935 						     target_size));
1936 		break;
1937 	case offsetof(struct bpf_perf_event_data, addr):
1938 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,
1939 						       data), si->dst_reg, si->src_reg,
1940 				      offsetof(struct bpf_perf_event_data_kern, data));
1941 		*insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg,
1942 				      bpf_target_off(struct perf_sample_data, addr, 8,
1943 						     target_size));
1944 		break;
1945 	default:
1946 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,
1947 						       regs), si->dst_reg, si->src_reg,
1948 				      offsetof(struct bpf_perf_event_data_kern, regs));
1949 		*insn++ = BPF_LDX_MEM(BPF_SIZEOF(long), si->dst_reg, si->dst_reg,
1950 				      si->off);
1951 		break;
1952 	}
1953 
1954 	return insn - insn_buf;
1955 }
1956 
1957 const struct bpf_verifier_ops perf_event_verifier_ops = {
1958 	.get_func_proto		= pe_prog_func_proto,
1959 	.is_valid_access	= pe_prog_is_valid_access,
1960 	.convert_ctx_access	= pe_prog_convert_ctx_access,
1961 };
1962 
1963 const struct bpf_prog_ops perf_event_prog_ops = {
1964 };
1965 
1966 static DEFINE_MUTEX(bpf_event_mutex);
1967 
1968 #define BPF_TRACE_MAX_PROGS 64
1969 
1970 int perf_event_attach_bpf_prog(struct perf_event *event,
1971 			       struct bpf_prog *prog,
1972 			       u64 bpf_cookie)
1973 {
1974 	struct bpf_prog_array *old_array;
1975 	struct bpf_prog_array *new_array;
1976 	int ret = -EEXIST;
1977 
1978 	/*
1979 	 * Kprobe override only works if they are on the function entry,
1980 	 * and only if they are on the opt-in list.
1981 	 */
1982 	if (prog->kprobe_override &&
1983 	    (!trace_kprobe_on_func_entry(event->tp_event) ||
1984 	     !trace_kprobe_error_injectable(event->tp_event)))
1985 		return -EINVAL;
1986 
1987 	mutex_lock(&bpf_event_mutex);
1988 
1989 	if (event->prog)
1990 		goto unlock;
1991 
1992 	old_array = bpf_event_rcu_dereference(event->tp_event->prog_array);
1993 	if (old_array &&
1994 	    bpf_prog_array_length(old_array) >= BPF_TRACE_MAX_PROGS) {
1995 		ret = -E2BIG;
1996 		goto unlock;
1997 	}
1998 
1999 	ret = bpf_prog_array_copy(old_array, NULL, prog, bpf_cookie, &new_array);
2000 	if (ret < 0)
2001 		goto unlock;
2002 
2003 	/* set the new array to event->tp_event and set event->prog */
2004 	event->prog = prog;
2005 	event->bpf_cookie = bpf_cookie;
2006 	rcu_assign_pointer(event->tp_event->prog_array, new_array);
2007 	bpf_prog_array_free_sleepable(old_array);
2008 
2009 unlock:
2010 	mutex_unlock(&bpf_event_mutex);
2011 	return ret;
2012 }
2013 
2014 void perf_event_detach_bpf_prog(struct perf_event *event)
2015 {
2016 	struct bpf_prog_array *old_array;
2017 	struct bpf_prog_array *new_array;
2018 	struct bpf_prog *prog = NULL;
2019 	int ret;
2020 
2021 	mutex_lock(&bpf_event_mutex);
2022 
2023 	if (!event->prog)
2024 		goto unlock;
2025 
2026 	old_array = bpf_event_rcu_dereference(event->tp_event->prog_array);
2027 	if (!old_array)
2028 		goto put;
2029 
2030 	ret = bpf_prog_array_copy(old_array, event->prog, NULL, 0, &new_array);
2031 	if (ret < 0) {
2032 		bpf_prog_array_delete_safe(old_array, event->prog);
2033 	} else {
2034 		rcu_assign_pointer(event->tp_event->prog_array, new_array);
2035 		bpf_prog_array_free_sleepable(old_array);
2036 	}
2037 
2038 put:
2039 	prog = event->prog;
2040 	event->prog = NULL;
2041 
2042 unlock:
2043 	mutex_unlock(&bpf_event_mutex);
2044 
2045 	if (prog) {
2046 		/*
2047 		 * It could be that the bpf_prog is not sleepable (and will be freed
2048 		 * via normal RCU), but is called from a point that supports sleepable
2049 		 * programs and uses tasks-trace-RCU.
2050 		 */
2051 		synchronize_rcu_tasks_trace();
2052 
2053 		bpf_prog_put(prog);
2054 	}
2055 }
2056 
2057 int perf_event_query_prog_array(struct perf_event *event, void __user *info)
2058 {
2059 	struct perf_event_query_bpf __user *uquery = info;
2060 	struct perf_event_query_bpf query = {};
2061 	struct bpf_prog_array *progs;
2062 	u32 *ids, prog_cnt, ids_len;
2063 	int ret;
2064 
2065 	if (!perfmon_capable())
2066 		return -EPERM;
2067 	if (event->attr.type != PERF_TYPE_TRACEPOINT)
2068 		return -EINVAL;
2069 	if (copy_from_user(&query, uquery, sizeof(query)))
2070 		return -EFAULT;
2071 
2072 	ids_len = query.ids_len;
2073 	if (ids_len > BPF_TRACE_MAX_PROGS)
2074 		return -E2BIG;
2075 	ids = kzalloc_objs(u32, ids_len, GFP_USER | __GFP_NOWARN);
2076 	if (!ids)
2077 		return -ENOMEM;
2078 	/*
2079 	 * The above kcalloc returns ZERO_SIZE_PTR when ids_len = 0, which
2080 	 * is required when user only wants to check for uquery->prog_cnt.
2081 	 * There is no need to check for it since the case is handled
2082 	 * gracefully in bpf_prog_array_copy_info.
2083 	 */
2084 
2085 	mutex_lock(&bpf_event_mutex);
2086 	progs = bpf_event_rcu_dereference(event->tp_event->prog_array);
2087 	ret = bpf_prog_array_copy_info(progs, ids, ids_len, &prog_cnt);
2088 	mutex_unlock(&bpf_event_mutex);
2089 
2090 	if (copy_to_user(&uquery->prog_cnt, &prog_cnt, sizeof(prog_cnt)) ||
2091 	    copy_to_user(uquery->ids, ids, ids_len * sizeof(u32)))
2092 		ret = -EFAULT;
2093 
2094 	kfree(ids);
2095 	return ret;
2096 }
2097 
2098 extern struct bpf_raw_event_map __start__bpf_raw_tp[];
2099 extern struct bpf_raw_event_map __stop__bpf_raw_tp[];
2100 
2101 struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name)
2102 {
2103 	struct bpf_raw_event_map *btp = __start__bpf_raw_tp;
2104 
2105 	for (; btp < __stop__bpf_raw_tp; btp++) {
2106 		if (!strcmp(btp->tp->name, name))
2107 			return btp;
2108 	}
2109 
2110 	return bpf_get_raw_tracepoint_module(name);
2111 }
2112 
2113 void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp)
2114 {
2115 	struct module *mod;
2116 
2117 	guard(rcu)();
2118 	mod = __module_address((unsigned long)btp);
2119 	module_put(mod);
2120 }
2121 
2122 static __always_inline
2123 void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args)
2124 {
2125 	struct srcu_ctr __percpu *scp = NULL;
2126 	struct bpf_prog *prog = link->link.prog;
2127 	bool sleepable = prog->sleepable;
2128 	struct bpf_run_ctx *old_run_ctx;
2129 	struct bpf_trace_run_ctx run_ctx;
2130 
2131 	if (sleepable) {
2132 		scp = rcu_read_lock_tasks_trace();
2133 		migrate_disable();
2134 	} else {
2135 		rcu_read_lock_dont_migrate();
2136 	}
2137 
2138 	if (unlikely(!bpf_prog_get_recursion_context(prog))) {
2139 		bpf_prog_inc_misses_counter(prog);
2140 		goto out;
2141 	}
2142 
2143 	run_ctx.bpf_cookie = link->cookie;
2144 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
2145 
2146 	(void)bpf_prog_run(prog, args);
2147 
2148 	bpf_reset_run_ctx(old_run_ctx);
2149 out:
2150 	bpf_prog_put_recursion_context(prog);
2151 
2152 	if (sleepable) {
2153 		migrate_enable();
2154 		rcu_read_unlock_tasks_trace(scp);
2155 	} else {
2156 		rcu_read_unlock_migrate();
2157 	}
2158 }
2159 
2160 #define UNPACK(...)			__VA_ARGS__
2161 #define REPEAT_1(FN, DL, X, ...)	FN(X)
2162 #define REPEAT_2(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_1(FN, DL, __VA_ARGS__)
2163 #define REPEAT_3(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_2(FN, DL, __VA_ARGS__)
2164 #define REPEAT_4(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_3(FN, DL, __VA_ARGS__)
2165 #define REPEAT_5(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_4(FN, DL, __VA_ARGS__)
2166 #define REPEAT_6(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_5(FN, DL, __VA_ARGS__)
2167 #define REPEAT_7(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_6(FN, DL, __VA_ARGS__)
2168 #define REPEAT_8(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_7(FN, DL, __VA_ARGS__)
2169 #define REPEAT_9(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_8(FN, DL, __VA_ARGS__)
2170 #define REPEAT_10(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_9(FN, DL, __VA_ARGS__)
2171 #define REPEAT_11(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_10(FN, DL, __VA_ARGS__)
2172 #define REPEAT_12(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_11(FN, DL, __VA_ARGS__)
2173 #define REPEAT(X, FN, DL, ...)		REPEAT_##X(FN, DL, __VA_ARGS__)
2174 
2175 #define SARG(X)		u64 arg##X
2176 #define COPY(X)		args[X] = arg##X
2177 
2178 #define __DL_COM	(,)
2179 #define __DL_SEM	(;)
2180 
2181 #define __SEQ_0_11	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
2182 
2183 #define BPF_TRACE_DEFN_x(x)						\
2184 	void bpf_trace_run##x(struct bpf_raw_tp_link *link,		\
2185 			      REPEAT(x, SARG, __DL_COM, __SEQ_0_11))	\
2186 	{								\
2187 		u64 args[x];						\
2188 		REPEAT(x, COPY, __DL_SEM, __SEQ_0_11);			\
2189 		__bpf_trace_run(link, args);				\
2190 	}								\
2191 	EXPORT_SYMBOL_GPL(bpf_trace_run##x)
2192 BPF_TRACE_DEFN_x(1);
2193 BPF_TRACE_DEFN_x(2);
2194 BPF_TRACE_DEFN_x(3);
2195 BPF_TRACE_DEFN_x(4);
2196 BPF_TRACE_DEFN_x(5);
2197 BPF_TRACE_DEFN_x(6);
2198 BPF_TRACE_DEFN_x(7);
2199 BPF_TRACE_DEFN_x(8);
2200 BPF_TRACE_DEFN_x(9);
2201 BPF_TRACE_DEFN_x(10);
2202 BPF_TRACE_DEFN_x(11);
2203 BPF_TRACE_DEFN_x(12);
2204 
2205 int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
2206 {
2207 	struct tracepoint *tp = btp->tp;
2208 	struct bpf_prog *prog = link->link.prog;
2209 
2210 	/*
2211 	 * check that program doesn't access arguments beyond what's
2212 	 * available in this tracepoint
2213 	 */
2214 	if (prog->aux->max_ctx_offset > btp->num_args * sizeof(u64))
2215 		return -EINVAL;
2216 
2217 	if (prog->aux->max_tp_access > btp->writable_size)
2218 		return -EINVAL;
2219 
2220 	return tracepoint_probe_register_may_exist(tp, (void *)btp->bpf_func, link);
2221 }
2222 
2223 int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
2224 {
2225 	return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, link);
2226 }
2227 
2228 /* On success the caller releases *buf with kfree_const() */
2229 int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
2230 			    u32 *fd_type, const char **buf,
2231 			    u64 *probe_offset, u64 *probe_addr,
2232 			    unsigned long *missed)
2233 {
2234 	bool is_tracepoint, is_syscall_tp;
2235 	struct bpf_prog *prog;
2236 	int flags, err = 0;
2237 
2238 	/*
2239 	 * The list of dynamic events is walked, and a probe which has siblings
2240 	 * can be deleted, with its strings, while its event is in use.
2241 	 */
2242 	guard(mutex)(&event_mutex);
2243 
2244 	prog = event->prog;
2245 	if (!prog)
2246 		return -ENOENT;
2247 
2248 	/* not supporting BPF_PROG_TYPE_PERF_EVENT yet */
2249 	if (prog->type == BPF_PROG_TYPE_PERF_EVENT)
2250 		return -EOPNOTSUPP;
2251 
2252 	*prog_id = prog->aux->id;
2253 	flags = event->tp_event->flags;
2254 	is_tracepoint = flags & TRACE_EVENT_FL_TRACEPOINT;
2255 	is_syscall_tp = is_syscall_trace_event(event->tp_event);
2256 
2257 	if (is_tracepoint || is_syscall_tp) {
2258 		*buf = is_tracepoint ? event->tp_event->tp->name
2259 				     : event->tp_event->name;
2260 		/* We allow NULL pointer for tracepoint */
2261 		if (fd_type)
2262 			*fd_type = BPF_FD_TYPE_TRACEPOINT;
2263 		if (probe_offset)
2264 			*probe_offset = 0x0;
2265 		if (probe_addr)
2266 			*probe_addr = 0x0;
2267 	} else {
2268 		/* kprobe/uprobe */
2269 		err = -EOPNOTSUPP;
2270 #ifdef CONFIG_KPROBE_EVENTS
2271 		if (flags & TRACE_EVENT_FL_KPROBE)
2272 			err = bpf_get_kprobe_info(event, fd_type, buf,
2273 						  probe_offset, probe_addr, missed,
2274 						  event->attr.type == PERF_TYPE_TRACEPOINT);
2275 #endif
2276 #ifdef CONFIG_UPROBE_EVENTS
2277 		if (flags & TRACE_EVENT_FL_UPROBE)
2278 			err = bpf_get_uprobe_info(event, fd_type, buf,
2279 						  probe_offset, probe_addr,
2280 						  event->attr.type == PERF_TYPE_TRACEPOINT);
2281 #endif
2282 	}
2283 
2284 	if (!err && *buf) {
2285 		*buf = kstrdup_const(*buf, GFP_KERNEL);
2286 		if (!*buf)
2287 			err = -ENOMEM;
2288 	}
2289 
2290 	return err;
2291 }
2292 
2293 static int __init send_signal_irq_work_init(void)
2294 {
2295 	int cpu;
2296 	struct send_signal_irq_work *work;
2297 
2298 	for_each_possible_cpu(cpu) {
2299 		work = per_cpu_ptr(&send_signal_work, cpu);
2300 		init_irq_work(&work->irq_work, do_bpf_send_signal);
2301 	}
2302 	return 0;
2303 }
2304 
2305 subsys_initcall(send_signal_irq_work_init);
2306 
2307 #ifdef CONFIG_MODULES
2308 static int bpf_event_notify(struct notifier_block *nb, unsigned long op,
2309 			    void *module)
2310 {
2311 	struct bpf_trace_module *btm, *tmp;
2312 	struct module *mod = module;
2313 	int ret = 0;
2314 
2315 	if (mod->num_bpf_raw_events == 0 ||
2316 	    (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING))
2317 		goto out;
2318 
2319 	mutex_lock(&bpf_module_mutex);
2320 
2321 	switch (op) {
2322 	case MODULE_STATE_COMING:
2323 		btm = kzalloc_obj(*btm);
2324 		if (btm) {
2325 			btm->module = module;
2326 			list_add(&btm->list, &bpf_trace_modules);
2327 		} else {
2328 			ret = -ENOMEM;
2329 		}
2330 		break;
2331 	case MODULE_STATE_GOING:
2332 		list_for_each_entry_safe(btm, tmp, &bpf_trace_modules, list) {
2333 			if (btm->module == module) {
2334 				list_del(&btm->list);
2335 				kfree(btm);
2336 				break;
2337 			}
2338 		}
2339 		break;
2340 	}
2341 
2342 	mutex_unlock(&bpf_module_mutex);
2343 
2344 out:
2345 	return notifier_from_errno(ret);
2346 }
2347 
2348 static struct notifier_block bpf_module_nb = {
2349 	.notifier_call = bpf_event_notify,
2350 };
2351 
2352 static int __init bpf_event_init(void)
2353 {
2354 	register_module_notifier(&bpf_module_nb);
2355 	return 0;
2356 }
2357 
2358 fs_initcall(bpf_event_init);
2359 #endif /* CONFIG_MODULES */
2360 
2361 struct bpf_session_run_ctx {
2362 	struct bpf_run_ctx run_ctx;
2363 	bool is_return;
2364 	void *data;
2365 };
2366 
2367 #ifdef CONFIG_FPROBE
2368 struct bpf_kprobe_multi_link {
2369 	struct bpf_link link;
2370 	struct fprobe fp;
2371 	unsigned long *addrs;
2372 	u64 *cookies;
2373 	u32 cnt;
2374 	u32 mods_cnt;
2375 	struct module **mods;
2376 };
2377 
2378 struct bpf_kprobe_multi_run_ctx {
2379 	struct bpf_session_run_ctx session_ctx;
2380 	struct bpf_kprobe_multi_link *link;
2381 	unsigned long entry_ip;
2382 };
2383 
2384 struct user_syms {
2385 	const char **syms;
2386 	char *buf;
2387 };
2388 
2389 #ifndef CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS
2390 static DEFINE_PER_CPU(struct pt_regs, bpf_kprobe_multi_pt_regs);
2391 #define bpf_kprobe_multi_pt_regs_ptr()	this_cpu_ptr(&bpf_kprobe_multi_pt_regs)
2392 #else
2393 #define bpf_kprobe_multi_pt_regs_ptr()	(NULL)
2394 #endif
2395 
2396 static unsigned long ftrace_get_entry_ip(unsigned long fentry_ip)
2397 {
2398 	unsigned long ip = ftrace_get_symaddr(fentry_ip);
2399 
2400 	return ip ? : fentry_ip;
2401 }
2402 
2403 static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 cnt)
2404 {
2405 	unsigned long __user usymbol;
2406 	const char **syms = NULL;
2407 	char *buf = NULL, *p;
2408 	int err = -ENOMEM;
2409 	unsigned int i;
2410 
2411 	if (!access_ok(usyms, cnt * sizeof(*usyms)))
2412 		return -EFAULT;
2413 
2414 	syms = kvmalloc_objs(*syms, cnt);
2415 	if (!syms)
2416 		return -ENOMEM;
2417 
2418 	buf = kvmalloc_array(cnt, KSYM_NAME_LEN, GFP_KERNEL);
2419 	if (!buf)
2420 		goto error;
2421 
2422 	for (p = buf, i = 0; i < cnt; i++) {
2423 		if (get_user(usymbol, usyms + i)) {
2424 			err = -EFAULT;
2425 			goto error;
2426 		}
2427 		err = strncpy_from_user(p, (const char __user *) usymbol, KSYM_NAME_LEN);
2428 		if (err == KSYM_NAME_LEN)
2429 			err = -E2BIG;
2430 		if (err < 0)
2431 			goto error;
2432 		syms[i] = p;
2433 		p += err + 1;
2434 	}
2435 
2436 	us->syms = syms;
2437 	us->buf = buf;
2438 	return 0;
2439 
2440 error:
2441 	kvfree(syms);
2442 	kvfree(buf);
2443 	return err;
2444 }
2445 
2446 static void kprobe_multi_put_modules(struct module **mods, u32 cnt)
2447 {
2448 	u32 i;
2449 
2450 	for (i = 0; i < cnt; i++)
2451 		module_put(mods[i]);
2452 }
2453 
2454 static void free_user_syms(struct user_syms *us)
2455 {
2456 	kvfree(us->syms);
2457 	kvfree(us->buf);
2458 }
2459 
2460 static void bpf_kprobe_multi_link_release(struct bpf_link *link)
2461 {
2462 	struct bpf_kprobe_multi_link *kmulti_link;
2463 
2464 	kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
2465 	/* Don't wait for RCU GP here. */
2466 	unregister_fprobe_async(&kmulti_link->fp);
2467 	kprobe_multi_put_modules(kmulti_link->mods, kmulti_link->mods_cnt);
2468 }
2469 
2470 static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link)
2471 {
2472 	struct bpf_kprobe_multi_link *kmulti_link;
2473 
2474 	kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
2475 	kvfree(kmulti_link->addrs);
2476 	kvfree(kmulti_link->cookies);
2477 	kfree(kmulti_link->mods);
2478 	kfree(kmulti_link);
2479 }
2480 
2481 static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link,
2482 						struct bpf_link_info *info)
2483 {
2484 	u64 __user *ucookies = u64_to_user_ptr(info->kprobe_multi.cookies);
2485 	u64 __user *uaddrs = u64_to_user_ptr(info->kprobe_multi.addrs);
2486 	struct bpf_kprobe_multi_link *kmulti_link;
2487 	u32 ucount = info->kprobe_multi.count;
2488 	int err = 0, i;
2489 
2490 	if (!uaddrs ^ !ucount)
2491 		return -EINVAL;
2492 	if (ucookies && !ucount)
2493 		return -EINVAL;
2494 
2495 	kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
2496 	info->kprobe_multi.count = kmulti_link->cnt;
2497 	info->kprobe_multi.flags = kmulti_link->link.flags;
2498 	info->kprobe_multi.missed = kmulti_link->fp.nmissed;
2499 
2500 	if (!uaddrs)
2501 		return 0;
2502 	if (ucount < kmulti_link->cnt)
2503 		err = -ENOSPC;
2504 	else
2505 		ucount = kmulti_link->cnt;
2506 
2507 	if (ucookies) {
2508 		if (kmulti_link->cookies) {
2509 			if (copy_to_user(ucookies, kmulti_link->cookies, ucount * sizeof(u64)))
2510 				return -EFAULT;
2511 		} else {
2512 			for (i = 0; i < ucount; i++) {
2513 				if (put_user(0, ucookies + i))
2514 					return -EFAULT;
2515 			}
2516 		}
2517 	}
2518 
2519 	if (kallsyms_show_value(current_cred())) {
2520 		if (copy_to_user(uaddrs, kmulti_link->addrs, ucount * sizeof(u64)))
2521 			return -EFAULT;
2522 	} else {
2523 		for (i = 0; i < ucount; i++) {
2524 			if (put_user(0, uaddrs + i))
2525 				return -EFAULT;
2526 		}
2527 	}
2528 	return err;
2529 }
2530 
2531 #ifdef CONFIG_PROC_FS
2532 static void bpf_kprobe_multi_show_fdinfo(const struct bpf_link *link,
2533 					 struct seq_file *seq)
2534 {
2535 	struct bpf_kprobe_multi_link *kmulti_link;
2536 	bool has_cookies;
2537 
2538 	kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
2539 	has_cookies = !!kmulti_link->cookies;
2540 
2541 	seq_printf(seq,
2542 		   "kprobe_cnt:\t%u\n"
2543 		   "missed:\t%lu\n",
2544 		   kmulti_link->cnt,
2545 		   kmulti_link->fp.nmissed);
2546 
2547 	seq_printf(seq, "%s\t %s\n", "cookie", "func");
2548 	for (int i = 0; i < kmulti_link->cnt; i++) {
2549 		seq_printf(seq,
2550 			   "%llu\t %pS\n",
2551 			   has_cookies ? kmulti_link->cookies[i] : 0,
2552 			   (void *)kmulti_link->addrs[i]);
2553 	}
2554 }
2555 #endif
2556 
2557 static const struct bpf_link_ops bpf_kprobe_multi_link_lops = {
2558 	.release = bpf_kprobe_multi_link_release,
2559 	.dealloc_deferred = bpf_kprobe_multi_link_dealloc,
2560 	.fill_link_info = bpf_kprobe_multi_link_fill_link_info,
2561 #ifdef CONFIG_PROC_FS
2562 	.show_fdinfo = bpf_kprobe_multi_show_fdinfo,
2563 #endif
2564 };
2565 
2566 static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv)
2567 {
2568 	const struct bpf_kprobe_multi_link *link = priv;
2569 	unsigned long *addr_a = a, *addr_b = b;
2570 	u64 *cookie_a, *cookie_b;
2571 
2572 	cookie_a = link->cookies + (addr_a - link->addrs);
2573 	cookie_b = link->cookies + (addr_b - link->addrs);
2574 
2575 	/* swap addr_a/addr_b and cookie_a/cookie_b values */
2576 	swap(*addr_a, *addr_b);
2577 	swap(*cookie_a, *cookie_b);
2578 }
2579 
2580 static int bpf_kprobe_multi_addrs_cmp(const void *a, const void *b)
2581 {
2582 	const unsigned long *addr_a = a, *addr_b = b;
2583 
2584 	if (*addr_a == *addr_b)
2585 		return 0;
2586 	return *addr_a < *addr_b ? -1 : 1;
2587 }
2588 
2589 static int bpf_kprobe_multi_cookie_cmp(const void *a, const void *b, const void *priv)
2590 {
2591 	return bpf_kprobe_multi_addrs_cmp(a, b);
2592 }
2593 
2594 static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx)
2595 {
2596 	struct bpf_kprobe_multi_run_ctx *run_ctx;
2597 	struct bpf_kprobe_multi_link *link;
2598 	u64 *cookie, entry_ip;
2599 	unsigned long *addr;
2600 
2601 	if (WARN_ON_ONCE(!ctx))
2602 		return 0;
2603 	run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx,
2604 			       session_ctx.run_ctx);
2605 	link = run_ctx->link;
2606 	if (!link->cookies)
2607 		return 0;
2608 	entry_ip = run_ctx->entry_ip;
2609 	addr = bsearch(&entry_ip, link->addrs, link->cnt, sizeof(entry_ip),
2610 		       bpf_kprobe_multi_addrs_cmp);
2611 	if (!addr)
2612 		return 0;
2613 	cookie = link->cookies + (addr - link->addrs);
2614 	return *cookie;
2615 }
2616 
2617 static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
2618 {
2619 	struct bpf_kprobe_multi_run_ctx *run_ctx;
2620 
2621 	run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx,
2622 			       session_ctx.run_ctx);
2623 	return run_ctx->entry_ip;
2624 }
2625 
2626 static __always_inline int
2627 kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,
2628 			   unsigned long entry_ip, struct ftrace_regs *fregs,
2629 			   bool is_return, void *data)
2630 {
2631 	struct bpf_kprobe_multi_run_ctx run_ctx = {
2632 		.session_ctx = {
2633 			.is_return = is_return,
2634 			.data = data,
2635 		},
2636 		.link = link,
2637 		.entry_ip = entry_ip,
2638 	};
2639 	struct bpf_run_ctx *old_run_ctx;
2640 	struct pt_regs *regs;
2641 	int err;
2642 
2643 	/*
2644 	 * graph tracer framework ensures we won't migrate, so there is no need
2645 	 * to use migrate_disable for bpf_prog_run again. The check here just for
2646 	 * __this_cpu_inc_return.
2647 	 */
2648 	cant_sleep();
2649 
2650 	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
2651 		bpf_prog_inc_misses_counter(link->link.prog);
2652 		err = 1;
2653 		goto out;
2654 	}
2655 
2656 	rcu_read_lock();
2657 	regs = ftrace_partial_regs(fregs, bpf_kprobe_multi_pt_regs_ptr());
2658 	old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx);
2659 	err = bpf_prog_run(link->link.prog, regs);
2660 	bpf_reset_run_ctx(old_run_ctx);
2661 	ftrace_partial_regs_update(fregs, bpf_kprobe_multi_pt_regs_ptr());
2662 	rcu_read_unlock();
2663 
2664  out:
2665 	__this_cpu_dec(bpf_prog_active);
2666 	return err;
2667 }
2668 
2669 static int
2670 kprobe_multi_link_handler(struct fprobe *fp, unsigned long fentry_ip,
2671 			  unsigned long ret_ip, struct ftrace_regs *fregs,
2672 			  void *data)
2673 {
2674 	struct bpf_kprobe_multi_link *link;
2675 	int err;
2676 
2677 	link = container_of(fp, struct bpf_kprobe_multi_link, fp);
2678 	err = kprobe_multi_link_prog_run(link, ftrace_get_entry_ip(fentry_ip),
2679 					 fregs, false, data);
2680 	return is_kprobe_session(link->link.prog) ? err : 0;
2681 }
2682 
2683 static void
2684 kprobe_multi_link_exit_handler(struct fprobe *fp, unsigned long fentry_ip,
2685 			       unsigned long ret_ip, struct ftrace_regs *fregs,
2686 			       void *data)
2687 {
2688 	struct bpf_kprobe_multi_link *link;
2689 
2690 	link = container_of(fp, struct bpf_kprobe_multi_link, fp);
2691 	kprobe_multi_link_prog_run(link, ftrace_get_entry_ip(fentry_ip),
2692 				   fregs, true, data);
2693 }
2694 
2695 static int symbols_cmp_r(const void *a, const void *b, const void *priv)
2696 {
2697 	const char **str_a = (const char **) a;
2698 	const char **str_b = (const char **) b;
2699 
2700 	return strcmp(*str_a, *str_b);
2701 }
2702 
2703 struct multi_symbols_sort {
2704 	const char **funcs;
2705 	u64 *cookies;
2706 };
2707 
2708 static void symbols_swap_r(void *a, void *b, int size, const void *priv)
2709 {
2710 	const struct multi_symbols_sort *data = priv;
2711 	const char **name_a = a, **name_b = b;
2712 
2713 	swap(*name_a, *name_b);
2714 
2715 	/* If defined, swap also related cookies. */
2716 	if (data->cookies) {
2717 		u64 *cookie_a, *cookie_b;
2718 
2719 		cookie_a = data->cookies + (name_a - data->funcs);
2720 		cookie_b = data->cookies + (name_b - data->funcs);
2721 		swap(*cookie_a, *cookie_b);
2722 	}
2723 }
2724 
2725 struct modules_array {
2726 	struct module **mods;
2727 	int mods_cnt;
2728 	int mods_cap;
2729 };
2730 
2731 static int add_module(struct modules_array *arr, struct module *mod)
2732 {
2733 	struct module **mods;
2734 
2735 	if (arr->mods_cnt == arr->mods_cap) {
2736 		arr->mods_cap = max(16, arr->mods_cap * 3 / 2);
2737 		mods = krealloc_array(arr->mods, arr->mods_cap, sizeof(*mods), GFP_KERNEL);
2738 		if (!mods)
2739 			return -ENOMEM;
2740 		arr->mods = mods;
2741 	}
2742 
2743 	arr->mods[arr->mods_cnt] = mod;
2744 	arr->mods_cnt++;
2745 	return 0;
2746 }
2747 
2748 static bool has_module(struct modules_array *arr, struct module *mod)
2749 {
2750 	int i;
2751 
2752 	for (i = arr->mods_cnt - 1; i >= 0; i--) {
2753 		if (arr->mods[i] == mod)
2754 			return true;
2755 	}
2756 	return false;
2757 }
2758 
2759 static int get_modules_for_addrs(struct module ***mods, unsigned long *addrs, u32 addrs_cnt)
2760 {
2761 	struct modules_array arr = {};
2762 	u32 i, err = 0;
2763 
2764 	for (i = 0; i < addrs_cnt; i++) {
2765 		bool skip_add = false;
2766 		struct module *mod;
2767 
2768 		scoped_guard(rcu) {
2769 			mod = __module_address(addrs[i]);
2770 			/* Either no module or it's already stored  */
2771 			if (!mod || has_module(&arr, mod)) {
2772 				skip_add = true;
2773 				break; /* scoped_guard */
2774 			}
2775 			if (!try_module_get(mod))
2776 				err = -EINVAL;
2777 		}
2778 		if (skip_add)
2779 			continue;
2780 		if (err)
2781 			break;
2782 		err = add_module(&arr, mod);
2783 		if (err) {
2784 			module_put(mod);
2785 			break;
2786 		}
2787 	}
2788 
2789 	/* We return either err < 0 in case of error, ... */
2790 	if (err) {
2791 		kprobe_multi_put_modules(arr.mods, arr.mods_cnt);
2792 		kfree(arr.mods);
2793 		return err;
2794 	}
2795 
2796 	/* or number of modules found if everything is ok. */
2797 	*mods = arr.mods;
2798 	return arr.mods_cnt;
2799 }
2800 
2801 static int addrs_check_error_injection_list(unsigned long *addrs, u32 cnt)
2802 {
2803 	u32 i;
2804 
2805 	for (i = 0; i < cnt; i++) {
2806 		if (!within_error_injection_list(addrs[i]))
2807 			return -EINVAL;
2808 	}
2809 	return 0;
2810 }
2811 
2812 int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
2813 {
2814 	struct bpf_kprobe_multi_link *link = NULL;
2815 	struct bpf_link_primer link_primer;
2816 	void __user *ucookies;
2817 	unsigned long *addrs;
2818 	u32 flags, cnt, size;
2819 	void __user *uaddrs;
2820 	u64 *cookies = NULL;
2821 	void __user *usyms;
2822 	int err;
2823 
2824 	/* no support for 32bit archs yet */
2825 	if (sizeof(u64) != sizeof(void *))
2826 		return -EOPNOTSUPP;
2827 
2828 	if (attr->link_create.flags)
2829 		return -EINVAL;
2830 
2831 	if (!is_kprobe_multi(prog))
2832 		return -EINVAL;
2833 
2834 	/* kprobe_multi is not allowed to be sleepable. */
2835 	if (prog->sleepable)
2836 		return -EINVAL;
2837 
2838 	/* Writing to context is not allowed for kprobes. */
2839 	if (prog->aux->kprobe_write_ctx)
2840 		return -EINVAL;
2841 
2842 	flags = attr->link_create.kprobe_multi.flags;
2843 	if (flags & ~BPF_F_KPROBE_MULTI_RETURN)
2844 		return -EINVAL;
2845 
2846 	uaddrs = u64_to_user_ptr(attr->link_create.kprobe_multi.addrs);
2847 	usyms = u64_to_user_ptr(attr->link_create.kprobe_multi.syms);
2848 	if (!!uaddrs == !!usyms)
2849 		return -EINVAL;
2850 
2851 	cnt = attr->link_create.kprobe_multi.cnt;
2852 	if (!cnt)
2853 		return -EINVAL;
2854 	if (cnt > MAX_KPROBE_MULTI_CNT)
2855 		return -E2BIG;
2856 
2857 	size = cnt * sizeof(*addrs);
2858 	addrs = kvmalloc_objs(*addrs, cnt);
2859 	if (!addrs)
2860 		return -ENOMEM;
2861 
2862 	ucookies = u64_to_user_ptr(attr->link_create.kprobe_multi.cookies);
2863 	if (ucookies) {
2864 		cookies = kvmalloc_array(cnt, sizeof(*addrs), GFP_KERNEL);
2865 		if (!cookies) {
2866 			err = -ENOMEM;
2867 			goto error;
2868 		}
2869 		if (copy_from_user(cookies, ucookies, size)) {
2870 			err = -EFAULT;
2871 			goto error;
2872 		}
2873 	}
2874 
2875 	if (uaddrs) {
2876 		if (copy_from_user(addrs, uaddrs, size)) {
2877 			err = -EFAULT;
2878 			goto error;
2879 		}
2880 	} else {
2881 		struct multi_symbols_sort data = {
2882 			.cookies = cookies,
2883 		};
2884 		struct user_syms us;
2885 
2886 		err = copy_user_syms(&us, usyms, cnt);
2887 		if (err)
2888 			goto error;
2889 
2890 		if (cookies)
2891 			data.funcs = us.syms;
2892 
2893 		sort_r(us.syms, cnt, sizeof(*us.syms), symbols_cmp_r,
2894 		       symbols_swap_r, &data);
2895 
2896 		err = ftrace_lookup_symbols(us.syms, cnt, addrs);
2897 		free_user_syms(&us);
2898 		if (err)
2899 			goto error;
2900 	}
2901 
2902 	if (prog->kprobe_override && addrs_check_error_injection_list(addrs, cnt)) {
2903 		err = -EINVAL;
2904 		goto error;
2905 	}
2906 
2907 	link = kzalloc_obj(*link);
2908 	if (!link) {
2909 		err = -ENOMEM;
2910 		goto error;
2911 	}
2912 
2913 	bpf_link_init(&link->link, BPF_LINK_TYPE_KPROBE_MULTI,
2914 		      &bpf_kprobe_multi_link_lops, prog, attr->link_create.attach_type);
2915 
2916 	err = bpf_link_prime(&link->link, &link_primer);
2917 	if (err)
2918 		goto error;
2919 
2920 	if (!(flags & BPF_F_KPROBE_MULTI_RETURN))
2921 		link->fp.entry_handler = kprobe_multi_link_handler;
2922 	if ((flags & BPF_F_KPROBE_MULTI_RETURN) || is_kprobe_session(prog))
2923 		link->fp.exit_handler = kprobe_multi_link_exit_handler;
2924 	if (is_kprobe_session(prog))
2925 		link->fp.entry_data_size = sizeof(u64);
2926 
2927 	link->addrs = addrs;
2928 	link->cookies = cookies;
2929 	link->cnt = cnt;
2930 	link->link.flags = flags;
2931 
2932 	if (cookies) {
2933 		/*
2934 		 * Sorting addresses will trigger sorting cookies as well
2935 		 * (check bpf_kprobe_multi_cookie_swap). This way we can
2936 		 * find cookie based on the address in bpf_get_attach_cookie
2937 		 * helper.
2938 		 */
2939 		sort_r(addrs, cnt, sizeof(*addrs),
2940 		       bpf_kprobe_multi_cookie_cmp,
2941 		       bpf_kprobe_multi_cookie_swap,
2942 		       link);
2943 	}
2944 
2945 	err = get_modules_for_addrs(&link->mods, addrs, cnt);
2946 	if (err < 0) {
2947 		bpf_link_cleanup(&link_primer);
2948 		return err;
2949 	}
2950 	link->mods_cnt = err;
2951 
2952 	err = register_fprobe_ips(&link->fp, addrs, cnt);
2953 	if (err) {
2954 		kprobe_multi_put_modules(link->mods, link->mods_cnt);
2955 		bpf_link_cleanup(&link_primer);
2956 		return err;
2957 	}
2958 
2959 	return bpf_link_settle(&link_primer);
2960 
2961 error:
2962 	kfree(link);
2963 	kvfree(addrs);
2964 	kvfree(cookies);
2965 	return err;
2966 }
2967 #else /* !CONFIG_FPROBE */
2968 int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
2969 {
2970 	return -EOPNOTSUPP;
2971 }
2972 static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx)
2973 {
2974 	return 0;
2975 }
2976 static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
2977 {
2978 	return 0;
2979 }
2980 #endif
2981 
2982 #ifdef CONFIG_UPROBES
2983 struct bpf_uprobe_multi_link;
2984 
2985 struct bpf_uprobe {
2986 	struct bpf_uprobe_multi_link *link;
2987 	loff_t offset;
2988 	unsigned long ref_ctr_offset;
2989 	u64 cookie;
2990 	struct uprobe *uprobe;
2991 	struct uprobe_consumer consumer;
2992 	bool session;
2993 };
2994 
2995 struct bpf_uprobe_multi_link {
2996 	struct path path;
2997 	struct bpf_link link;
2998 	u32 cnt;
2999 	struct bpf_uprobe *uprobes;
3000 	struct task_struct *task;
3001 };
3002 
3003 struct bpf_uprobe_multi_run_ctx {
3004 	struct bpf_session_run_ctx session_ctx;
3005 	unsigned long entry_ip;
3006 	struct bpf_uprobe *uprobe;
3007 };
3008 
3009 static void bpf_uprobe_unregister(struct bpf_uprobe *uprobes, u32 cnt)
3010 {
3011 	u32 i;
3012 
3013 	for (i = 0; i < cnt; i++)
3014 		uprobe_unregister_nosync(uprobes[i].uprobe, &uprobes[i].consumer);
3015 
3016 	if (cnt)
3017 		uprobe_unregister_sync();
3018 }
3019 
3020 static void bpf_uprobe_multi_link_release(struct bpf_link *link)
3021 {
3022 	struct bpf_uprobe_multi_link *umulti_link;
3023 
3024 	umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
3025 	bpf_uprobe_unregister(umulti_link->uprobes, umulti_link->cnt);
3026 	if (umulti_link->task)
3027 		put_task_struct(umulti_link->task);
3028 	path_put(&umulti_link->path);
3029 }
3030 
3031 static void bpf_uprobe_multi_link_dealloc(struct bpf_link *link)
3032 {
3033 	struct bpf_uprobe_multi_link *umulti_link;
3034 
3035 	umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
3036 	kvfree(umulti_link->uprobes);
3037 	kfree(umulti_link);
3038 }
3039 
3040 static int bpf_uprobe_multi_link_fill_link_info(const struct bpf_link *link,
3041 						struct bpf_link_info *info)
3042 {
3043 	u64 __user *uref_ctr_offsets = u64_to_user_ptr(info->uprobe_multi.ref_ctr_offsets);
3044 	u64 __user *ucookies = u64_to_user_ptr(info->uprobe_multi.cookies);
3045 	u64 __user *uoffsets = u64_to_user_ptr(info->uprobe_multi.offsets);
3046 	u64 __user *upath = u64_to_user_ptr(info->uprobe_multi.path);
3047 	u32 upath_size = info->uprobe_multi.path_size;
3048 	struct bpf_uprobe_multi_link *umulti_link;
3049 	u32 ucount = info->uprobe_multi.count;
3050 	int err = 0, i;
3051 	char *p, *buf;
3052 	long left = 0;
3053 
3054 	if (!upath ^ !upath_size)
3055 		return -EINVAL;
3056 
3057 	if ((uoffsets || uref_ctr_offsets || ucookies) && !ucount)
3058 		return -EINVAL;
3059 
3060 	umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
3061 	info->uprobe_multi.count = umulti_link->cnt;
3062 	info->uprobe_multi.flags = umulti_link->link.flags;
3063 	info->uprobe_multi.pid = umulti_link->task ?
3064 				 task_pid_nr_ns(umulti_link->task, task_active_pid_ns(current)) : 0;
3065 
3066 	upath_size = upath_size ? min_t(u32, upath_size, PATH_MAX) : PATH_MAX;
3067 	buf = kmalloc(upath_size, GFP_KERNEL);
3068 	if (!buf)
3069 		return -ENOMEM;
3070 	p = d_path(&umulti_link->path, buf, upath_size);
3071 	if (IS_ERR(p)) {
3072 		kfree(buf);
3073 		return PTR_ERR(p);
3074 	}
3075 	upath_size = buf + upath_size - p;
3076 
3077 	if (upath)
3078 		left = copy_to_user(upath, p, upath_size);
3079 	kfree(buf);
3080 	if (left)
3081 		return -EFAULT;
3082 	info->uprobe_multi.path_size = upath_size;
3083 
3084 	if (!uoffsets && !ucookies && !uref_ctr_offsets)
3085 		return 0;
3086 
3087 	if (ucount < umulti_link->cnt)
3088 		err = -ENOSPC;
3089 	else
3090 		ucount = umulti_link->cnt;
3091 
3092 	for (i = 0; i < ucount; i++) {
3093 		if (uoffsets &&
3094 		    put_user(umulti_link->uprobes[i].offset, uoffsets + i))
3095 			return -EFAULT;
3096 		if (uref_ctr_offsets &&
3097 		    put_user(umulti_link->uprobes[i].ref_ctr_offset, uref_ctr_offsets + i))
3098 			return -EFAULT;
3099 		if (ucookies &&
3100 		    put_user(umulti_link->uprobes[i].cookie, ucookies + i))
3101 			return -EFAULT;
3102 	}
3103 
3104 	return err;
3105 }
3106 
3107 #ifdef CONFIG_PROC_FS
3108 static void bpf_uprobe_multi_show_fdinfo(const struct bpf_link *link,
3109 					 struct seq_file *seq)
3110 {
3111 	struct bpf_uprobe_multi_link *umulti_link;
3112 	char *p, *buf;
3113 	pid_t pid;
3114 
3115 	umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
3116 
3117 	buf = kmalloc(PATH_MAX, GFP_KERNEL);
3118 	if (!buf)
3119 		return;
3120 
3121 	p = d_path(&umulti_link->path, buf, PATH_MAX);
3122 	if (IS_ERR(p)) {
3123 		kfree(buf);
3124 		return;
3125 	}
3126 
3127 	pid = umulti_link->task ?
3128 	      task_pid_nr_ns(umulti_link->task, task_active_pid_ns(current)) : 0;
3129 	seq_printf(seq,
3130 		   "uprobe_cnt:\t%u\n"
3131 		   "pid:\t%u\n"
3132 		   "path:\t%s\n",
3133 		   umulti_link->cnt, pid, p);
3134 
3135 	seq_printf(seq, "%s\t %s\t %s\n", "cookie", "offset", "ref_ctr_offset");
3136 	for (int i = 0; i < umulti_link->cnt; i++) {
3137 		seq_printf(seq,
3138 			   "%llu\t %#llx\t %#lx\n",
3139 			   umulti_link->uprobes[i].cookie,
3140 			   umulti_link->uprobes[i].offset,
3141 			   umulti_link->uprobes[i].ref_ctr_offset);
3142 	}
3143 
3144 	kfree(buf);
3145 }
3146 #endif
3147 
3148 static const struct bpf_link_ops bpf_uprobe_multi_link_lops = {
3149 	.release = bpf_uprobe_multi_link_release,
3150 	.dealloc_deferred = bpf_uprobe_multi_link_dealloc,
3151 	.fill_link_info = bpf_uprobe_multi_link_fill_link_info,
3152 #ifdef CONFIG_PROC_FS
3153 	.show_fdinfo = bpf_uprobe_multi_show_fdinfo,
3154 #endif
3155 };
3156 
3157 static int uprobe_prog_run(struct bpf_uprobe *uprobe,
3158 			   unsigned long entry_ip,
3159 			   struct pt_regs *regs,
3160 			   bool is_return, void *data)
3161 {
3162 	struct bpf_uprobe_multi_link *link = uprobe->link;
3163 	struct bpf_uprobe_multi_run_ctx run_ctx = {
3164 		.session_ctx = {
3165 			.is_return = is_return,
3166 			.data = data,
3167 		},
3168 		.entry_ip = entry_ip,
3169 		.uprobe = uprobe,
3170 	};
3171 	struct bpf_prog *prog = link->link.prog;
3172 	bool sleepable = prog->sleepable;
3173 	struct bpf_run_ctx *old_run_ctx;
3174 	int err;
3175 
3176 	if (link->task && !same_thread_group(current, link->task))
3177 		return 0;
3178 
3179 	if (sleepable)
3180 		rcu_read_lock_trace();
3181 	else
3182 		rcu_read_lock();
3183 
3184 	migrate_disable();
3185 
3186 	old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx);
3187 	err = bpf_prog_run(link->link.prog, regs);
3188 	bpf_reset_run_ctx(old_run_ctx);
3189 
3190 	migrate_enable();
3191 
3192 	if (sleepable)
3193 		rcu_read_unlock_trace();
3194 	else
3195 		rcu_read_unlock();
3196 	return err;
3197 }
3198 
3199 static bool
3200 uprobe_multi_link_filter(struct uprobe_consumer *con, struct mm_struct *mm)
3201 {
3202 	struct bpf_uprobe *uprobe;
3203 
3204 	uprobe = container_of(con, struct bpf_uprobe, consumer);
3205 	return uprobe->link->task->mm == mm;
3206 }
3207 
3208 static int
3209 uprobe_multi_link_handler(struct uprobe_consumer *con, struct pt_regs *regs,
3210 			  __u64 *data)
3211 {
3212 	struct bpf_uprobe *uprobe;
3213 	int ret;
3214 
3215 	uprobe = container_of(con, struct bpf_uprobe, consumer);
3216 	ret = uprobe_prog_run(uprobe, instruction_pointer(regs), regs, false, data);
3217 	if (uprobe->session)
3218 		return ret ? UPROBE_HANDLER_IGNORE : 0;
3219 	return 0;
3220 }
3221 
3222 static int
3223 uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, struct pt_regs *regs,
3224 			      __u64 *data)
3225 {
3226 	struct bpf_uprobe *uprobe;
3227 
3228 	uprobe = container_of(con, struct bpf_uprobe, consumer);
3229 	uprobe_prog_run(uprobe, func, regs, true, data);
3230 	return 0;
3231 }
3232 
3233 static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
3234 {
3235 	struct bpf_uprobe_multi_run_ctx *run_ctx;
3236 
3237 	run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3238 			       session_ctx.run_ctx);
3239 	return run_ctx->entry_ip;
3240 }
3241 
3242 static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx)
3243 {
3244 	struct bpf_uprobe_multi_run_ctx *run_ctx;
3245 
3246 	run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3247 			       session_ctx.run_ctx);
3248 	return run_ctx->uprobe->cookie;
3249 }
3250 
3251 static int bpf_uprobe_multi_get_path(const union bpf_attr *attr, struct path *path)
3252 {
3253 	void __user *upath = u64_to_user_ptr(attr->link_create.uprobe_multi.path);
3254 	u32 path_fd = attr->link_create.uprobe_multi.path_fd;
3255 	u32 flags = attr->link_create.uprobe_multi.flags;
3256 
3257 	if (flags & BPF_F_UPROBE_MULTI_PATH_FD) {
3258 		/*
3259 		 * When BPF_F_UPROBE_MULTI_PATH_FD is set, the executable is
3260 		 * identified by path_fd, upath must be NULL.
3261 		 */
3262 		if (upath)
3263 			return -EINVAL;
3264 
3265 		CLASS(fd, f)(path_fd);
3266 		if (fd_empty(f))
3267 			return -EBADF;
3268 		*path = fd_file(f)->f_path;
3269 		path_get(path);
3270 		return 0;
3271 	}
3272 
3273 	/*
3274 	 * When BPF_F_UPROBE_MULTI_PATH_FD is not set, the path is resolved
3275 	 * relative to the cwd (AT_FDCWD) or absolute using the upath string.
3276 	 */
3277 	if (!upath || path_fd)
3278 		return -EINVAL;
3279 
3280 	return user_path_at(AT_FDCWD, upath, LOOKUP_FOLLOW, path);
3281 }
3282 
3283 int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
3284 {
3285 	struct bpf_uprobe_multi_link *link = NULL;
3286 	unsigned long __user *uref_ctr_offsets;
3287 	struct bpf_link_primer link_primer;
3288 	struct bpf_uprobe *uprobes = NULL;
3289 	struct task_struct *task = NULL;
3290 	unsigned long __user *uoffsets;
3291 	u64 __user *ucookies;
3292 	unsigned long size;
3293 	u32 flags, cnt, i;
3294 	struct path path;
3295 	pid_t pid;
3296 	int err;
3297 
3298 	/* no support for 32bit archs yet */
3299 	if (sizeof(u64) != sizeof(void *))
3300 		return -EOPNOTSUPP;
3301 
3302 	if (attr->link_create.flags)
3303 		return -EINVAL;
3304 
3305 	if (!is_uprobe_multi(prog))
3306 		return -EINVAL;
3307 
3308 	flags = attr->link_create.uprobe_multi.flags;
3309 	if (flags & ~(BPF_F_UPROBE_MULTI_RETURN | BPF_F_UPROBE_MULTI_PATH_FD))
3310 		return -EINVAL;
3311 
3312 	/*
3313 	 * offsets and cnt are mandatory,
3314 	 * ref_ctr_offsets and cookies are optional
3315 	 */
3316 	uoffsets = u64_to_user_ptr(attr->link_create.uprobe_multi.offsets);
3317 	cnt = attr->link_create.uprobe_multi.cnt;
3318 	pid = attr->link_create.uprobe_multi.pid;
3319 
3320 	if (!uoffsets || !cnt || pid < 0)
3321 		return -EINVAL;
3322 	if (cnt > MAX_UPROBE_MULTI_CNT)
3323 		return -E2BIG;
3324 
3325 	uref_ctr_offsets = u64_to_user_ptr(attr->link_create.uprobe_multi.ref_ctr_offsets);
3326 	ucookies = u64_to_user_ptr(attr->link_create.uprobe_multi.cookies);
3327 
3328 	/*
3329 	 * All uoffsets/uref_ctr_offsets/ucookies arrays have the same value
3330 	 * size.  Check their address ranges up front, so that a bad pointer
3331 	 * fails before anything is looked up or allocated.
3332 	 */
3333 	size = sizeof(*uoffsets) * cnt;
3334 	if (!access_ok(uoffsets, size) || !access_ok(uref_ctr_offsets, size) ||
3335 	    !access_ok(ucookies, size))
3336 		return -EFAULT;
3337 
3338 	err = bpf_uprobe_multi_get_path(attr, &path);
3339 	if (err)
3340 		return err;
3341 
3342 	if (!d_is_reg(path.dentry)) {
3343 		err = -EBADF;
3344 		goto error_path_put;
3345 	}
3346 
3347 	if (pid) {
3348 		rcu_read_lock();
3349 		task = get_pid_task(find_vpid(pid), PIDTYPE_TGID);
3350 		rcu_read_unlock();
3351 		if (!task) {
3352 			err = -ESRCH;
3353 			goto error_path_put;
3354 		}
3355 	}
3356 
3357 	err = -ENOMEM;
3358 
3359 	link = kzalloc_obj(*link);
3360 	uprobes = kvzalloc_objs(*uprobes, cnt);
3361 
3362 	if (!uprobes || !link)
3363 		goto error_free;
3364 
3365 	for (i = 0; i < cnt; i++) {
3366 		if (get_user(uprobes[i].offset, uoffsets + i)) {
3367 			err = -EFAULT;
3368 			goto error_free;
3369 		}
3370 		if (uprobes[i].offset < 0) {
3371 			err = -EINVAL;
3372 			goto error_free;
3373 		}
3374 		if (uref_ctr_offsets && get_user(uprobes[i].ref_ctr_offset, uref_ctr_offsets + i)) {
3375 			err = -EFAULT;
3376 			goto error_free;
3377 		}
3378 		if (ucookies && get_user(uprobes[i].cookie, ucookies + i)) {
3379 			err = -EFAULT;
3380 			goto error_free;
3381 		}
3382 
3383 		uprobes[i].link = link;
3384 
3385 		if (!(flags & BPF_F_UPROBE_MULTI_RETURN))
3386 			uprobes[i].consumer.handler = uprobe_multi_link_handler;
3387 		if (flags & BPF_F_UPROBE_MULTI_RETURN || is_uprobe_session(prog))
3388 			uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3389 		if (is_uprobe_session(prog))
3390 			uprobes[i].session = true;
3391 		if (pid)
3392 			uprobes[i].consumer.filter = uprobe_multi_link_filter;
3393 	}
3394 
3395 	link->cnt = cnt;
3396 	link->uprobes = uprobes;
3397 	link->path = path;
3398 	link->task = task;
3399 	link->link.flags = flags;
3400 
3401 	bpf_link_init(&link->link, BPF_LINK_TYPE_UPROBE_MULTI,
3402 		      &bpf_uprobe_multi_link_lops, prog, attr->link_create.attach_type);
3403 
3404 	for (i = 0; i < cnt; i++) {
3405 		uprobes[i].uprobe = uprobe_register(d_real_inode(link->path.dentry),
3406 						    uprobes[i].offset,
3407 						    uprobes[i].ref_ctr_offset,
3408 						    &uprobes[i].consumer);
3409 		if (IS_ERR(uprobes[i].uprobe)) {
3410 			err = PTR_ERR(uprobes[i].uprobe);
3411 			link->cnt = i;
3412 			goto error_unregister;
3413 		}
3414 	}
3415 
3416 	err = bpf_link_prime(&link->link, &link_primer);
3417 	if (err)
3418 		goto error_unregister;
3419 
3420 	return bpf_link_settle(&link_primer);
3421 
3422 error_unregister:
3423 	bpf_uprobe_unregister(uprobes, link->cnt);
3424 
3425 error_free:
3426 	kvfree(uprobes);
3427 	kfree(link);
3428 	if (task)
3429 		put_task_struct(task);
3430 error_path_put:
3431 	path_put(&path);
3432 	return err;
3433 }
3434 #else /* !CONFIG_UPROBES */
3435 int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
3436 {
3437 	return -EOPNOTSUPP;
3438 }
3439 static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx)
3440 {
3441 	return 0;
3442 }
3443 static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
3444 {
3445 	return 0;
3446 }
3447 #endif /* CONFIG_UPROBES */
3448 
3449 __bpf_kfunc_start_defs();
3450 
3451 __bpf_kfunc bool bpf_session_is_return(void *ctx)
3452 {
3453 	struct bpf_session_run_ctx *session_ctx;
3454 
3455 	session_ctx = container_of(current->bpf_ctx, struct bpf_session_run_ctx, run_ctx);
3456 	return session_ctx->is_return;
3457 }
3458 
3459 __bpf_kfunc __u64 *bpf_session_cookie(void *ctx)
3460 {
3461 	struct bpf_session_run_ctx *session_ctx;
3462 
3463 	session_ctx = container_of(current->bpf_ctx, struct bpf_session_run_ctx, run_ctx);
3464 	return session_ctx->data;
3465 }
3466 
3467 __bpf_kfunc_end_defs();
3468 
3469 BTF_KFUNCS_START(session_kfunc_set_ids)
3470 BTF_ID_FLAGS(func, bpf_session_is_return)
3471 BTF_ID_FLAGS(func, bpf_session_cookie)
3472 BTF_KFUNCS_END(session_kfunc_set_ids)
3473 
3474 static int bpf_session_filter(const struct bpf_prog *prog, u32 kfunc_id)
3475 {
3476 	if (!btf_id_set8_contains(&session_kfunc_set_ids, kfunc_id))
3477 		return 0;
3478 
3479 	if (!is_kprobe_session(prog) && !is_uprobe_session(prog) && !is_trace_fsession(prog))
3480 		return -EACCES;
3481 
3482 	return 0;
3483 }
3484 
3485 static const struct btf_kfunc_id_set bpf_session_kfunc_set = {
3486 	.owner = THIS_MODULE,
3487 	.set = &session_kfunc_set_ids,
3488 	.filter = bpf_session_filter,
3489 };
3490 
3491 static int __init bpf_trace_kfuncs_init(void)
3492 {
3493 	int err = 0;
3494 
3495 	err = err ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_KPROBE, &bpf_session_kfunc_set);
3496 	err = err ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_session_kfunc_set);
3497 
3498 	return err;
3499 }
3500 
3501 late_initcall(bpf_trace_kfuncs_init);
3502 
3503 typedef int (*copy_fn_t)(void *dst, const void *src, u32 size, struct task_struct *tsk);
3504 
3505 /*
3506  * The __always_inline is to make sure the compiler doesn't
3507  * generate indirect calls into callbacks, which is expensive,
3508  * on some kernel configurations. This allows compiler to put
3509  * direct calls into all the specific callback implementations
3510  * (copy_user_data_sleepable, copy_user_data_nofault, and so on)
3511  */
3512 static __always_inline int __bpf_dynptr_copy_str(const struct bpf_dynptr *dptr, u64 doff, u64 size,
3513 						 const void *unsafe_src,
3514 						 copy_fn_t str_copy_fn,
3515 						 struct task_struct *tsk)
3516 {
3517 	const struct bpf_dynptr_kern *dst;
3518 	u64 chunk_sz, off;
3519 	void *dst_slice;
3520 	int cnt, err;
3521 	char buf[256];
3522 
3523 	dst_slice = bpf_dynptr_slice_rdwr(dptr, doff, NULL, size);
3524 	if (likely(dst_slice))
3525 		return str_copy_fn(dst_slice, unsafe_src, size, tsk);
3526 
3527 	dst = (struct bpf_dynptr_kern *)dptr;
3528 	if (bpf_dynptr_check_off_len(dst, doff, size))
3529 		return -E2BIG;
3530 
3531 	for (off = 0; off < size; off += chunk_sz - 1) {
3532 		chunk_sz = min_t(u64, sizeof(buf), size - off);
3533 		/* Expect str_copy_fn to return count of copied bytes, including
3534 		 * zero terminator. Next iteration increment off by chunk_sz - 1 to
3535 		 * overwrite NUL.
3536 		 */
3537 		cnt = str_copy_fn(buf, unsafe_src + off, chunk_sz, tsk);
3538 		if (cnt < 0)
3539 			return cnt;
3540 		err = __bpf_dynptr_write(dst, doff + off, buf, cnt, 0);
3541 		if (err)
3542 			return err;
3543 		if (cnt < chunk_sz || chunk_sz == 1) /* we are done */
3544 			return off + cnt;
3545 	}
3546 	return off;
3547 }
3548 
3549 static __always_inline int __bpf_dynptr_copy(const struct bpf_dynptr *dptr, u64 doff,
3550 					     u64 size, const void *unsafe_src,
3551 					     copy_fn_t copy_fn, struct task_struct *tsk)
3552 {
3553 	const struct bpf_dynptr_kern *dst;
3554 	void *dst_slice;
3555 	char buf[256];
3556 	u64 off, chunk_sz;
3557 	int err;
3558 
3559 	dst_slice = bpf_dynptr_slice_rdwr(dptr, doff, NULL, size);
3560 	if (likely(dst_slice))
3561 		return copy_fn(dst_slice, unsafe_src, size, tsk);
3562 
3563 	dst = (struct bpf_dynptr_kern *)dptr;
3564 	if (bpf_dynptr_check_off_len(dst, doff, size))
3565 		return -E2BIG;
3566 
3567 	for (off = 0; off < size; off += chunk_sz) {
3568 		chunk_sz = min_t(u64, sizeof(buf), size - off);
3569 		err = copy_fn(buf, unsafe_src + off, chunk_sz, tsk);
3570 		if (err)
3571 			return err;
3572 		err = __bpf_dynptr_write(dst, doff + off, buf, chunk_sz, 0);
3573 		if (err)
3574 			return err;
3575 	}
3576 	return 0;
3577 }
3578 
3579 static __always_inline int copy_user_data_nofault(void *dst, const void *unsafe_src,
3580 						  u32 size, struct task_struct *tsk)
3581 {
3582 	return copy_from_user_nofault(dst, (const void __user *)unsafe_src, size);
3583 }
3584 
3585 static __always_inline int copy_user_data_sleepable(void *dst, const void *unsafe_src,
3586 						    u32 size, struct task_struct *tsk)
3587 {
3588 	int ret;
3589 
3590 	if (!tsk) { /* Read from the current task */
3591 		ret = copy_from_user(dst, (const void __user *)unsafe_src, size);
3592 		if (ret)
3593 			return -EFAULT;
3594 		return 0;
3595 	}
3596 
3597 	ret = access_process_vm(tsk, (unsigned long)unsafe_src, dst, size, 0);
3598 	if (ret != size)
3599 		return -EFAULT;
3600 	return 0;
3601 }
3602 
3603 static __always_inline int copy_kernel_data_nofault(void *dst, const void *unsafe_src,
3604 						    u32 size, struct task_struct *tsk)
3605 {
3606 	return copy_from_kernel_nofault(dst, unsafe_src, size);
3607 }
3608 
3609 static __always_inline int copy_user_str_nofault(void *dst, const void *unsafe_src,
3610 						 u32 size, struct task_struct *tsk)
3611 {
3612 	return strncpy_from_user_nofault(dst, (const void __user *)unsafe_src, size);
3613 }
3614 
3615 static __always_inline int copy_user_str_sleepable(void *dst, const void *unsafe_src,
3616 						   u32 size, struct task_struct *tsk)
3617 {
3618 	int ret;
3619 
3620 	if (unlikely(size == 0))
3621 		return 0;
3622 
3623 	if (tsk) {
3624 		ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_src, dst, size, 0);
3625 	} else {
3626 		ret = strncpy_from_user(dst, (const void __user *)unsafe_src, size - 1);
3627 		/* strncpy_from_user does not guarantee NUL termination */
3628 		if (ret >= 0)
3629 			((char *)dst)[ret] = '\0';
3630 	}
3631 
3632 	if (ret < 0)
3633 		return ret;
3634 	return ret + 1;
3635 }
3636 
3637 static __always_inline int copy_kernel_str_nofault(void *dst, const void *unsafe_src,
3638 						   u32 size, struct task_struct *tsk)
3639 {
3640 	return strncpy_from_kernel_nofault(dst, unsafe_src, size);
3641 }
3642 
3643 __bpf_kfunc_start_defs();
3644 
3645 __bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type,
3646 				     u64 value)
3647 {
3648 	if (type != PIDTYPE_PID && type != PIDTYPE_TGID)
3649 		return -EINVAL;
3650 
3651 	return bpf_send_signal_common(sig, type, task, value);
3652 }
3653 
3654 __bpf_kfunc int bpf_probe_read_user_dynptr(const struct bpf_dynptr *dptr, u64 off,
3655 					   u64 size, const void __user *unsafe_ptr__ign)
3656 {
3657 	return __bpf_dynptr_copy(dptr, off, size, (const void __force *)unsafe_ptr__ign,
3658 				 copy_user_data_nofault, NULL);
3659 }
3660 
3661 __bpf_kfunc int bpf_probe_read_kernel_dynptr(const struct bpf_dynptr *dptr, u64 off,
3662 					     u64 size, const void *unsafe_ptr__ign)
3663 {
3664 	return __bpf_dynptr_copy(dptr, off, size, unsafe_ptr__ign,
3665 				 copy_kernel_data_nofault, NULL);
3666 }
3667 
3668 __bpf_kfunc int bpf_probe_read_user_str_dynptr(const struct bpf_dynptr *dptr, u64 off,
3669 					       u64 size, const void __user *unsafe_ptr__ign)
3670 {
3671 	return __bpf_dynptr_copy_str(dptr, off, size, (const void __force *)unsafe_ptr__ign,
3672 				     copy_user_str_nofault, NULL);
3673 }
3674 
3675 __bpf_kfunc int bpf_probe_read_kernel_str_dynptr(const struct bpf_dynptr *dptr, u64 off,
3676 						 u64 size, const void *unsafe_ptr__ign)
3677 {
3678 	return __bpf_dynptr_copy_str(dptr, off, size, unsafe_ptr__ign,
3679 				     copy_kernel_str_nofault, NULL);
3680 }
3681 
3682 __bpf_kfunc int bpf_copy_from_user_dynptr(const struct bpf_dynptr *dptr, u64 off,
3683 					  u64 size, const void __user *unsafe_ptr__ign)
3684 {
3685 	return __bpf_dynptr_copy(dptr, off, size, (const void __force *)unsafe_ptr__ign,
3686 				 copy_user_data_sleepable, NULL);
3687 }
3688 
3689 __bpf_kfunc int bpf_copy_from_user_str_dynptr(const struct bpf_dynptr *dptr, u64 off,
3690 					      u64 size, const void __user *unsafe_ptr__ign)
3691 {
3692 	return __bpf_dynptr_copy_str(dptr, off, size, (const void __force *)unsafe_ptr__ign,
3693 				     copy_user_str_sleepable, NULL);
3694 }
3695 
3696 __bpf_kfunc int bpf_copy_from_user_task_dynptr(const struct bpf_dynptr *dptr, u64 off,
3697 					       u64 size, const void __user *unsafe_ptr__ign,
3698 					       struct task_struct *tsk)
3699 {
3700 	return __bpf_dynptr_copy(dptr, off, size, (const void __force *)unsafe_ptr__ign,
3701 				 copy_user_data_sleepable, tsk);
3702 }
3703 
3704 __bpf_kfunc int bpf_copy_from_user_task_str_dynptr(const struct bpf_dynptr *dptr, u64 off,
3705 						   u64 size, const void __user *unsafe_ptr__ign,
3706 						   struct task_struct *tsk)
3707 {
3708 	return __bpf_dynptr_copy_str(dptr, off, size, (const void __force *)unsafe_ptr__ign,
3709 				     copy_user_str_sleepable, tsk);
3710 }
3711 
3712 __bpf_kfunc_end_defs();
3713 
3714 #if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) && \
3715     defined(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS)
3716 
3717 static void bpf_tracing_multi_link_release(struct bpf_link *link)
3718 {
3719 	struct bpf_tracing_multi_link *tr_link =
3720 		container_of(link, struct bpf_tracing_multi_link, link);
3721 
3722 	bpf_trampoline_multi_detach(link->prog, tr_link);
3723 }
3724 
3725 static void bpf_tracing_multi_link_dealloc(struct bpf_link *link)
3726 {
3727 	struct bpf_tracing_multi_link *tr_link =
3728 		container_of(link, struct bpf_tracing_multi_link, link);
3729 
3730 	kvfree(tr_link->fexits);
3731 	kvfree(tr_link->cookies);
3732 	kvfree(tr_link);
3733 }
3734 
3735 static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link,
3736 						 struct bpf_link_info *info)
3737 {
3738 	u64 __user *ucookies = u64_to_user_ptr(info->tracing_multi.cookies);
3739 	u64 __user *uaddrs = u64_to_user_ptr(info->tracing_multi.addrs);
3740 	u32 __user *uids = u64_to_user_ptr(info->tracing_multi.ids);
3741 	struct bpf_tracing_multi_link *tr_link;
3742 	u32 ucount = info->tracing_multi.count;
3743 	bool has_cookies, show_addrs;
3744 	int err = 0;
3745 
3746 	if ((uids || ucookies || uaddrs) && !ucount)
3747 		return -EINVAL;
3748 
3749 	tr_link = container_of(link, struct bpf_tracing_multi_link, link);
3750 
3751 	info->tracing_multi.attach_type = tr_link->link.attach_type;
3752 	info->tracing_multi.count = tr_link->nodes_cnt;
3753 	info->tracing_multi.btf_obj_id = btf_obj_id(tr_link->link.prog->aux->attach_btf);
3754 
3755 	if (!uids && !ucookies && !uaddrs)
3756 		return 0;
3757 
3758 	if (ucount < tr_link->nodes_cnt)
3759 		err = -ENOSPC;
3760 	else
3761 		ucount = tr_link->nodes_cnt;
3762 
3763 	has_cookies = !!tr_link->cookies;
3764 	show_addrs = kallsyms_show_value(current_cred());
3765 
3766 	for (int i = 0; i < ucount; i++) {
3767 		struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i];
3768 		u64 addr, cookie;
3769 		u32 id;
3770 
3771 		bpf_trampoline_unpack_key(mnode->trampoline->key, NULL, &id);
3772 
3773 		addr = show_addrs ? mnode->trampoline->ip : 0;
3774 		cookie = has_cookies ? tr_link->cookies[i] : 0;
3775 
3776 		if (uids && put_user(id, uids + i))
3777 			return -EFAULT;
3778 		if (uaddrs && put_user(addr, uaddrs + i))
3779 			return -EFAULT;
3780 		if (ucookies && put_user(cookie, ucookies + i))
3781 			return -EFAULT;
3782 
3783 		cond_resched();
3784 	}
3785 
3786 	return err;
3787 }
3788 
3789 #ifdef CONFIG_PROC_FS
3790 static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link,
3791 					  struct seq_file *seq)
3792 {
3793 	struct bpf_tracing_multi_link *tr_link =
3794 		container_of(link, struct bpf_tracing_multi_link, link);
3795 	bool has_cookies = !!tr_link->cookies;
3796 
3797 	seq_printf(seq, "attach_type:\t%u\n", tr_link->link.attach_type);
3798 	seq_printf(seq, "cnt:\t%u\n", tr_link->nodes_cnt);
3799 
3800 	seq_printf(seq, "%s\t %s\t %s\t %s\n", "obj-id", "btf-id", "cookie", "func");
3801 	for (int i = 0; i < tr_link->nodes_cnt; i++) {
3802 		struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i];
3803 		u32 btf_id, obj_id;
3804 
3805 		bpf_trampoline_unpack_key(mnode->trampoline->key, &obj_id, &btf_id);
3806 		seq_printf(seq, "%u\t %u\t %llu\t %pS\n",
3807 			   obj_id, btf_id,
3808 			   has_cookies ? tr_link->cookies[i] : 0,
3809 			   (void *) mnode->trampoline->ip);
3810 
3811 		cond_resched();
3812 	}
3813 }
3814 #endif
3815 
3816 static const struct bpf_link_ops bpf_tracing_multi_link_lops = {
3817 	.release = bpf_tracing_multi_link_release,
3818 	.dealloc_deferred = bpf_tracing_multi_link_dealloc,
3819 	.fill_link_info = bpf_tracing_multi_link_fill_link_info,
3820 #ifdef CONFIG_PROC_FS
3821 	.show_fdinfo = bpf_tracing_multi_show_fdinfo,
3822 #endif
3823 };
3824 
3825 static int ids_cmp_r(const void *pa, const void *pb, const void *priv __maybe_unused)
3826 {
3827 	u32 a = *(u32 *) pa;
3828 	u32 b = *(u32 *) pb;
3829 
3830 	return (a > b) - (a < b);
3831 }
3832 
3833 static void ids_swap_r(void *a, void *b, int size __maybe_unused,
3834 		       const void *priv __maybe_unused)
3835 {
3836 	u64 *cookie_a, *cookie_b, *cookies;
3837 	u32 *id_a = a, *id_b = b, *ids;
3838 	void **data = (void **) priv;
3839 
3840 	ids     = data[0];
3841 	cookies = data[1];
3842 
3843 	if (cookies) {
3844 		cookie_a = cookies + (id_a - ids);
3845 		cookie_b = cookies + (id_b - ids);
3846 		swap(*cookie_a, *cookie_b);
3847 	}
3848 	swap(*id_a, *id_b);
3849 }
3850 
3851 static int check_dup_ids(u32 *ids, u64 *cookies, u32 cnt)
3852 {
3853 	void *data[2] = { ids, cookies };
3854 	int err = 0;
3855 
3856 	/*
3857 	 * Sort ids array (together with cookies array if defined)
3858 	 * and check it for duplicates. The ids and cookies arrays
3859 	 * are left sorted.
3860 	 */
3861 	sort_r_nonatomic(ids, cnt, sizeof(ids[0]), ids_cmp_r, ids_swap_r, data);
3862 
3863 	for (int i = 1; i < cnt; i++) {
3864 		if (ids[i] == ids[i - 1]) {
3865 			err = -EINVAL;
3866 			break;
3867 		}
3868 	}
3869 	return err;
3870 }
3871 
3872 int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr)
3873 {
3874 	struct bpf_tracing_multi_link *link = NULL;
3875 	struct bpf_tramp_node *fexits = NULL;
3876 	struct bpf_link_primer link_primer;
3877 	u32 cnt, *ids = NULL;
3878 	u64 __user *ucookies;
3879 	u64 *cookies = NULL;
3880 	u32 __user *uids;
3881 	int err;
3882 
3883 	uids = u64_to_user_ptr(attr->link_create.tracing_multi.ids);
3884 	cnt = attr->link_create.tracing_multi.cnt;
3885 
3886 	if (!cnt || !uids)
3887 		return -EINVAL;
3888 	if (cnt > MAX_TRACING_MULTI_CNT)
3889 		return -E2BIG;
3890 	if (attr->link_create.flags || attr->link_create.target_fd)
3891 		return -EINVAL;
3892 
3893 	ids = kvmalloc_objs(*ids, cnt);
3894 	if (!ids)
3895 		return -ENOMEM;
3896 
3897 	if (copy_from_user(ids, uids, cnt * sizeof(*ids))) {
3898 		err = -EFAULT;
3899 		goto error;
3900 	}
3901 
3902 	ucookies = u64_to_user_ptr(attr->link_create.tracing_multi.cookies);
3903 	if (ucookies) {
3904 		cookies = kvmalloc_objs(*cookies, cnt);
3905 		if (!cookies) {
3906 			err = -ENOMEM;
3907 			goto error;
3908 		}
3909 		if (copy_from_user(cookies, ucookies, cnt * sizeof(*cookies))) {
3910 			err = -EFAULT;
3911 			goto error;
3912 		}
3913 	}
3914 
3915 	err = check_dup_ids(ids, cookies, cnt);
3916 	if (err)
3917 		goto error;
3918 
3919 	if (prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) {
3920 		fexits = kvmalloc_objs(*fexits, cnt);
3921 		if (!fexits) {
3922 			err = -ENOMEM;
3923 			goto error;
3924 		}
3925 	}
3926 
3927 	link = kvzalloc_flex(*link, nodes, cnt);
3928 	if (!link) {
3929 		err = -ENOMEM;
3930 		goto error;
3931 	}
3932 
3933 	bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING_MULTI,
3934 		      &bpf_tracing_multi_link_lops, prog, prog->expected_attach_type);
3935 
3936 	err = bpf_link_prime(&link->link, &link_primer);
3937 	if (err)
3938 		goto error;
3939 
3940 	link->nodes_cnt = cnt;
3941 	link->cookies = cookies;
3942 	link->fexits = fexits;
3943 
3944 	err = bpf_trampoline_multi_attach(prog, ids, link);
3945 	kvfree(ids);
3946 	if (err) {
3947 		bpf_link_cleanup(&link_primer);
3948 		return err;
3949 	}
3950 	return bpf_link_settle(&link_primer);
3951 
3952 error:
3953 	kvfree(fexits);
3954 	kvfree(cookies);
3955 	kvfree(ids);
3956 	kvfree(link);
3957 	return err;
3958 }
3959 
3960 #else
3961 
3962 int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr)
3963 {
3964 	return -EOPNOTSUPP;
3965 }
3966 
3967 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS && CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */
3968