xref: /linux/kernel/trace/bpf_trace.c (revision 50647a1176b7abd1b4ae55b491eb2fbbeef89db9)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com
3  * Copyright (c) 2016 Facebook
4  */
5 #include <linux/kernel.h>
6 #include <linux/types.h>
7 #include <linux/slab.h>
8 #include <linux/bpf.h>
9 #include <linux/bpf_verifier.h>
10 #include <linux/bpf_perf_event.h>
11 #include <linux/btf.h>
12 #include <linux/filter.h>
13 #include <linux/uaccess.h>
14 #include <linux/ctype.h>
15 #include <linux/kprobes.h>
16 #include <linux/spinlock.h>
17 #include <linux/syscalls.h>
18 #include <linux/error-injection.h>
19 #include <linux/btf_ids.h>
20 #include <linux/bpf_lsm.h>
21 #include <linux/fprobe.h>
22 #include <linux/bsearch.h>
23 #include <linux/sort.h>
24 #include <linux/key.h>
25 #include <linux/namei.h>
26 
27 #include <net/bpf_sk_storage.h>
28 
29 #include <uapi/linux/bpf.h>
30 #include <uapi/linux/btf.h>
31 
32 #include <asm/tlb.h>
33 
34 #include "trace_probe.h"
35 #include "trace.h"
36 
37 #define CREATE_TRACE_POINTS
38 #include "bpf_trace.h"
39 
40 #define bpf_event_rcu_dereference(p)					\
41 	rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex))
42 
43 #define MAX_UPROBE_MULTI_CNT (1U << 20)
44 #define MAX_KPROBE_MULTI_CNT (1U << 20)
45 
46 #ifdef CONFIG_MODULES
47 struct bpf_trace_module {
48 	struct module *module;
49 	struct list_head list;
50 };
51 
52 static LIST_HEAD(bpf_trace_modules);
53 static DEFINE_MUTEX(bpf_module_mutex);
54 
bpf_get_raw_tracepoint_module(const char * name)55 static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name)
56 {
57 	struct bpf_raw_event_map *btp, *ret = NULL;
58 	struct bpf_trace_module *btm;
59 	unsigned int i;
60 
61 	mutex_lock(&bpf_module_mutex);
62 	list_for_each_entry(btm, &bpf_trace_modules, list) {
63 		for (i = 0; i < btm->module->num_bpf_raw_events; ++i) {
64 			btp = &btm->module->bpf_raw_events[i];
65 			if (!strcmp(btp->tp->name, name)) {
66 				if (try_module_get(btm->module))
67 					ret = btp;
68 				goto out;
69 			}
70 		}
71 	}
72 out:
73 	mutex_unlock(&bpf_module_mutex);
74 	return ret;
75 }
76 #else
bpf_get_raw_tracepoint_module(const char * name)77 static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name)
78 {
79 	return NULL;
80 }
81 #endif /* CONFIG_MODULES */
82 
83 u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
84 u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
85 
86 static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,
87 				  u64 flags, const struct btf **btf,
88 				  s32 *btf_id);
89 static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx);
90 static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx);
91 
92 static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx);
93 static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx);
94 
95 /**
96  * trace_call_bpf - invoke BPF program
97  * @call: tracepoint event
98  * @ctx: opaque context pointer
99  *
100  * kprobe handlers execute BPF programs via this helper.
101  * Can be used from static tracepoints in the future.
102  *
103  * Return: BPF programs always return an integer which is interpreted by
104  * kprobe handler as:
105  * 0 - return from kprobe (event is filtered out)
106  * 1 - store kprobe event into ring buffer
107  * Other values are reserved and currently alias to 1
108  */
trace_call_bpf(struct trace_event_call * call,void * ctx)109 unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
110 {
111 	unsigned int ret;
112 
113 	cant_sleep();
114 
115 	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
116 		/*
117 		 * since some bpf program is already running on this cpu,
118 		 * don't call into another bpf program (same or different)
119 		 * and don't send kprobe event into ring-buffer,
120 		 * so return zero here
121 		 */
122 		rcu_read_lock();
123 		bpf_prog_inc_misses_counters(rcu_dereference(call->prog_array));
124 		rcu_read_unlock();
125 		ret = 0;
126 		goto out;
127 	}
128 
129 	/*
130 	 * Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock
131 	 * to all call sites, we did a bpf_prog_array_valid() there to check
132 	 * whether call->prog_array is empty or not, which is
133 	 * a heuristic to speed up execution.
134 	 *
135 	 * If bpf_prog_array_valid() fetched prog_array was
136 	 * non-NULL, we go into trace_call_bpf() and do the actual
137 	 * proper rcu_dereference() under RCU lock.
138 	 * If it turns out that prog_array is NULL then, we bail out.
139 	 * For the opposite, if the bpf_prog_array_valid() fetched pointer
140 	 * was NULL, you'll skip the prog_array with the risk of missing
141 	 * out of events when it was updated in between this and the
142 	 * rcu_dereference() which is accepted risk.
143 	 */
144 	rcu_read_lock();
145 	ret = bpf_prog_run_array(rcu_dereference(call->prog_array),
146 				 ctx, bpf_prog_run);
147 	rcu_read_unlock();
148 
149  out:
150 	__this_cpu_dec(bpf_prog_active);
151 
152 	return ret;
153 }
154 
155 #ifdef CONFIG_BPF_KPROBE_OVERRIDE
BPF_CALL_2(bpf_override_return,struct pt_regs *,regs,unsigned long,rc)156 BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc)
157 {
158 	regs_set_return_value(regs, rc);
159 	override_function_with_return(regs);
160 	return 0;
161 }
162 
163 static const struct bpf_func_proto bpf_override_return_proto = {
164 	.func		= bpf_override_return,
165 	.gpl_only	= true,
166 	.ret_type	= RET_INTEGER,
167 	.arg1_type	= ARG_PTR_TO_CTX,
168 	.arg2_type	= ARG_ANYTHING,
169 };
170 #endif
171 
172 static __always_inline int
bpf_probe_read_user_common(void * dst,u32 size,const void __user * unsafe_ptr)173 bpf_probe_read_user_common(void *dst, u32 size, const void __user *unsafe_ptr)
174 {
175 	int ret;
176 
177 	ret = copy_from_user_nofault(dst, unsafe_ptr, size);
178 	if (unlikely(ret < 0))
179 		memset(dst, 0, size);
180 	return ret;
181 }
182 
BPF_CALL_3(bpf_probe_read_user,void *,dst,u32,size,const void __user *,unsafe_ptr)183 BPF_CALL_3(bpf_probe_read_user, void *, dst, u32, size,
184 	   const void __user *, unsafe_ptr)
185 {
186 	return bpf_probe_read_user_common(dst, size, unsafe_ptr);
187 }
188 
189 const struct bpf_func_proto bpf_probe_read_user_proto = {
190 	.func		= bpf_probe_read_user,
191 	.gpl_only	= true,
192 	.ret_type	= RET_INTEGER,
193 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
194 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
195 	.arg3_type	= ARG_ANYTHING,
196 };
197 
198 static __always_inline int
bpf_probe_read_user_str_common(void * dst,u32 size,const void __user * unsafe_ptr)199 bpf_probe_read_user_str_common(void *dst, u32 size,
200 			       const void __user *unsafe_ptr)
201 {
202 	int ret;
203 
204 	/*
205 	 * NB: We rely on strncpy_from_user() not copying junk past the NUL
206 	 * terminator into `dst`.
207 	 *
208 	 * strncpy_from_user() does long-sized strides in the fast path. If the
209 	 * strncpy does not mask out the bytes after the NUL in `unsafe_ptr`,
210 	 * then there could be junk after the NUL in `dst`. If user takes `dst`
211 	 * and keys a hash map with it, then semantically identical strings can
212 	 * occupy multiple entries in the map.
213 	 */
214 	ret = strncpy_from_user_nofault(dst, unsafe_ptr, size);
215 	if (unlikely(ret < 0))
216 		memset(dst, 0, size);
217 	return ret;
218 }
219 
BPF_CALL_3(bpf_probe_read_user_str,void *,dst,u32,size,const void __user *,unsafe_ptr)220 BPF_CALL_3(bpf_probe_read_user_str, void *, dst, u32, size,
221 	   const void __user *, unsafe_ptr)
222 {
223 	return bpf_probe_read_user_str_common(dst, size, unsafe_ptr);
224 }
225 
226 const struct bpf_func_proto bpf_probe_read_user_str_proto = {
227 	.func		= bpf_probe_read_user_str,
228 	.gpl_only	= true,
229 	.ret_type	= RET_INTEGER,
230 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
231 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
232 	.arg3_type	= ARG_ANYTHING,
233 };
234 
BPF_CALL_3(bpf_probe_read_kernel,void *,dst,u32,size,const void *,unsafe_ptr)235 BPF_CALL_3(bpf_probe_read_kernel, void *, dst, u32, size,
236 	   const void *, unsafe_ptr)
237 {
238 	return bpf_probe_read_kernel_common(dst, size, unsafe_ptr);
239 }
240 
241 const struct bpf_func_proto bpf_probe_read_kernel_proto = {
242 	.func		= bpf_probe_read_kernel,
243 	.gpl_only	= true,
244 	.ret_type	= RET_INTEGER,
245 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
246 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
247 	.arg3_type	= ARG_ANYTHING,
248 };
249 
250 static __always_inline int
bpf_probe_read_kernel_str_common(void * dst,u32 size,const void * unsafe_ptr)251 bpf_probe_read_kernel_str_common(void *dst, u32 size, const void *unsafe_ptr)
252 {
253 	int ret;
254 
255 	/*
256 	 * The strncpy_from_kernel_nofault() call will likely not fill the
257 	 * entire buffer, but that's okay in this circumstance as we're probing
258 	 * arbitrary memory anyway similar to bpf_probe_read_*() and might
259 	 * as well probe the stack. Thus, memory is explicitly cleared
260 	 * only in error case, so that improper users ignoring return
261 	 * code altogether don't copy garbage; otherwise length of string
262 	 * is returned that can be used for bpf_perf_event_output() et al.
263 	 */
264 	ret = strncpy_from_kernel_nofault(dst, unsafe_ptr, size);
265 	if (unlikely(ret < 0))
266 		memset(dst, 0, size);
267 	return ret;
268 }
269 
BPF_CALL_3(bpf_probe_read_kernel_str,void *,dst,u32,size,const void *,unsafe_ptr)270 BPF_CALL_3(bpf_probe_read_kernel_str, void *, dst, u32, size,
271 	   const void *, unsafe_ptr)
272 {
273 	return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr);
274 }
275 
276 const struct bpf_func_proto bpf_probe_read_kernel_str_proto = {
277 	.func		= bpf_probe_read_kernel_str,
278 	.gpl_only	= true,
279 	.ret_type	= RET_INTEGER,
280 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
281 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
282 	.arg3_type	= ARG_ANYTHING,
283 };
284 
285 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
BPF_CALL_3(bpf_probe_read_compat,void *,dst,u32,size,const void *,unsafe_ptr)286 BPF_CALL_3(bpf_probe_read_compat, void *, dst, u32, size,
287 	   const void *, unsafe_ptr)
288 {
289 	if ((unsigned long)unsafe_ptr < TASK_SIZE) {
290 		return bpf_probe_read_user_common(dst, size,
291 				(__force void __user *)unsafe_ptr);
292 	}
293 	return bpf_probe_read_kernel_common(dst, size, unsafe_ptr);
294 }
295 
296 static const struct bpf_func_proto bpf_probe_read_compat_proto = {
297 	.func		= bpf_probe_read_compat,
298 	.gpl_only	= true,
299 	.ret_type	= RET_INTEGER,
300 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
301 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
302 	.arg3_type	= ARG_ANYTHING,
303 };
304 
BPF_CALL_3(bpf_probe_read_compat_str,void *,dst,u32,size,const void *,unsafe_ptr)305 BPF_CALL_3(bpf_probe_read_compat_str, void *, dst, u32, size,
306 	   const void *, unsafe_ptr)
307 {
308 	if ((unsigned long)unsafe_ptr < TASK_SIZE) {
309 		return bpf_probe_read_user_str_common(dst, size,
310 				(__force void __user *)unsafe_ptr);
311 	}
312 	return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr);
313 }
314 
315 static const struct bpf_func_proto bpf_probe_read_compat_str_proto = {
316 	.func		= bpf_probe_read_compat_str,
317 	.gpl_only	= true,
318 	.ret_type	= RET_INTEGER,
319 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
320 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
321 	.arg3_type	= ARG_ANYTHING,
322 };
323 #endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */
324 
BPF_CALL_3(bpf_probe_write_user,void __user *,unsafe_ptr,const void *,src,u32,size)325 BPF_CALL_3(bpf_probe_write_user, void __user *, unsafe_ptr, const void *, src,
326 	   u32, size)
327 {
328 	/*
329 	 * Ensure we're in user context which is safe for the helper to
330 	 * run. This helper has no business in a kthread.
331 	 *
332 	 * access_ok() should prevent writing to non-user memory, but in
333 	 * some situations (nommu, temporary switch, etc) access_ok() does
334 	 * not provide enough validation, hence the check on KERNEL_DS.
335 	 *
336 	 * nmi_uaccess_okay() ensures the probe is not run in an interim
337 	 * state, when the task or mm are switched. This is specifically
338 	 * required to prevent the use of temporary mm.
339 	 */
340 
341 	if (unlikely(in_interrupt() ||
342 		     current->flags & (PF_KTHREAD | PF_EXITING)))
343 		return -EPERM;
344 	if (unlikely(!nmi_uaccess_okay()))
345 		return -EPERM;
346 
347 	return copy_to_user_nofault(unsafe_ptr, src, size);
348 }
349 
350 static const struct bpf_func_proto bpf_probe_write_user_proto = {
351 	.func		= bpf_probe_write_user,
352 	.gpl_only	= true,
353 	.ret_type	= RET_INTEGER,
354 	.arg1_type	= ARG_ANYTHING,
355 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
356 	.arg3_type	= ARG_CONST_SIZE,
357 };
358 
359 #define MAX_TRACE_PRINTK_VARARGS	3
360 #define BPF_TRACE_PRINTK_SIZE		1024
361 
BPF_CALL_5(bpf_trace_printk,char *,fmt,u32,fmt_size,u64,arg1,u64,arg2,u64,arg3)362 BPF_CALL_5(bpf_trace_printk, char *, fmt, u32, fmt_size, u64, arg1,
363 	   u64, arg2, u64, arg3)
364 {
365 	u64 args[MAX_TRACE_PRINTK_VARARGS] = { arg1, arg2, arg3 };
366 	struct bpf_bprintf_data data = {
367 		.get_bin_args	= true,
368 		.get_buf	= true,
369 	};
370 	int ret;
371 
372 	ret = bpf_bprintf_prepare(fmt, fmt_size, args,
373 				  MAX_TRACE_PRINTK_VARARGS, &data);
374 	if (ret < 0)
375 		return ret;
376 
377 	ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt, data.bin_args);
378 
379 	trace_bpf_trace_printk(data.buf);
380 
381 	bpf_bprintf_cleanup(&data);
382 
383 	return ret;
384 }
385 
386 static const struct bpf_func_proto bpf_trace_printk_proto = {
387 	.func		= bpf_trace_printk,
388 	.gpl_only	= true,
389 	.ret_type	= RET_INTEGER,
390 	.arg1_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
391 	.arg2_type	= ARG_CONST_SIZE,
392 };
393 
__set_printk_clr_event(struct work_struct * work)394 static void __set_printk_clr_event(struct work_struct *work)
395 {
396 	/*
397 	 * This program might be calling bpf_trace_printk,
398 	 * so enable the associated bpf_trace/bpf_trace_printk event.
399 	 * Repeat this each time as it is possible a user has
400 	 * disabled bpf_trace_printk events.  By loading a program
401 	 * calling bpf_trace_printk() however the user has expressed
402 	 * the intent to see such events.
403 	 */
404 	if (trace_set_clr_event("bpf_trace", "bpf_trace_printk", 1))
405 		pr_warn_ratelimited("could not enable bpf_trace_printk events");
406 }
407 static DECLARE_WORK(set_printk_work, __set_printk_clr_event);
408 
bpf_get_trace_printk_proto(void)409 const struct bpf_func_proto *bpf_get_trace_printk_proto(void)
410 {
411 	schedule_work(&set_printk_work);
412 	return &bpf_trace_printk_proto;
413 }
414 
BPF_CALL_4(bpf_trace_vprintk,char *,fmt,u32,fmt_size,const void *,args,u32,data_len)415 BPF_CALL_4(bpf_trace_vprintk, char *, fmt, u32, fmt_size, const void *, args,
416 	   u32, data_len)
417 {
418 	struct bpf_bprintf_data data = {
419 		.get_bin_args	= true,
420 		.get_buf	= true,
421 	};
422 	int ret, num_args;
423 
424 	if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 ||
425 	    (data_len && !args))
426 		return -EINVAL;
427 	num_args = data_len / 8;
428 
429 	ret = bpf_bprintf_prepare(fmt, fmt_size, args, num_args, &data);
430 	if (ret < 0)
431 		return ret;
432 
433 	ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt, data.bin_args);
434 
435 	trace_bpf_trace_printk(data.buf);
436 
437 	bpf_bprintf_cleanup(&data);
438 
439 	return ret;
440 }
441 
442 static const struct bpf_func_proto bpf_trace_vprintk_proto = {
443 	.func		= bpf_trace_vprintk,
444 	.gpl_only	= true,
445 	.ret_type	= RET_INTEGER,
446 	.arg1_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
447 	.arg2_type	= ARG_CONST_SIZE,
448 	.arg3_type	= ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
449 	.arg4_type	= ARG_CONST_SIZE_OR_ZERO,
450 };
451 
bpf_get_trace_vprintk_proto(void)452 const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void)
453 {
454 	schedule_work(&set_printk_work);
455 	return &bpf_trace_vprintk_proto;
456 }
457 
BPF_CALL_5(bpf_seq_printf,struct seq_file *,m,char *,fmt,u32,fmt_size,const void *,args,u32,data_len)458 BPF_CALL_5(bpf_seq_printf, struct seq_file *, m, char *, fmt, u32, fmt_size,
459 	   const void *, args, u32, data_len)
460 {
461 	struct bpf_bprintf_data data = {
462 		.get_bin_args	= true,
463 	};
464 	int err, num_args;
465 
466 	if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 ||
467 	    (data_len && !args))
468 		return -EINVAL;
469 	num_args = data_len / 8;
470 
471 	err = bpf_bprintf_prepare(fmt, fmt_size, args, num_args, &data);
472 	if (err < 0)
473 		return err;
474 
475 	seq_bprintf(m, fmt, data.bin_args);
476 
477 	bpf_bprintf_cleanup(&data);
478 
479 	return seq_has_overflowed(m) ? -EOVERFLOW : 0;
480 }
481 
482 BTF_ID_LIST_SINGLE(btf_seq_file_ids, struct, seq_file)
483 
484 static const struct bpf_func_proto bpf_seq_printf_proto = {
485 	.func		= bpf_seq_printf,
486 	.gpl_only	= true,
487 	.ret_type	= RET_INTEGER,
488 	.arg1_type	= ARG_PTR_TO_BTF_ID,
489 	.arg1_btf_id	= &btf_seq_file_ids[0],
490 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
491 	.arg3_type	= ARG_CONST_SIZE,
492 	.arg4_type      = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
493 	.arg5_type      = ARG_CONST_SIZE_OR_ZERO,
494 };
495 
BPF_CALL_3(bpf_seq_write,struct seq_file *,m,const void *,data,u32,len)496 BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len)
497 {
498 	return seq_write(m, data, len) ? -EOVERFLOW : 0;
499 }
500 
501 static const struct bpf_func_proto bpf_seq_write_proto = {
502 	.func		= bpf_seq_write,
503 	.gpl_only	= true,
504 	.ret_type	= RET_INTEGER,
505 	.arg1_type	= ARG_PTR_TO_BTF_ID,
506 	.arg1_btf_id	= &btf_seq_file_ids[0],
507 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
508 	.arg3_type	= ARG_CONST_SIZE_OR_ZERO,
509 };
510 
BPF_CALL_4(bpf_seq_printf_btf,struct seq_file *,m,struct btf_ptr *,ptr,u32,btf_ptr_size,u64,flags)511 BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr,
512 	   u32, btf_ptr_size, u64, flags)
513 {
514 	const struct btf *btf;
515 	s32 btf_id;
516 	int ret;
517 
518 	ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id);
519 	if (ret)
520 		return ret;
521 
522 	return btf_type_seq_show_flags(btf, btf_id, ptr->ptr, m, flags);
523 }
524 
525 static const struct bpf_func_proto bpf_seq_printf_btf_proto = {
526 	.func		= bpf_seq_printf_btf,
527 	.gpl_only	= true,
528 	.ret_type	= RET_INTEGER,
529 	.arg1_type	= ARG_PTR_TO_BTF_ID,
530 	.arg1_btf_id	= &btf_seq_file_ids[0],
531 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
532 	.arg3_type	= ARG_CONST_SIZE_OR_ZERO,
533 	.arg4_type	= ARG_ANYTHING,
534 };
535 
536 static __always_inline int
get_map_perf_counter(struct bpf_map * map,u64 flags,u64 * value,u64 * enabled,u64 * running)537 get_map_perf_counter(struct bpf_map *map, u64 flags,
538 		     u64 *value, u64 *enabled, u64 *running)
539 {
540 	struct bpf_array *array = container_of(map, struct bpf_array, map);
541 	unsigned int cpu = smp_processor_id();
542 	u64 index = flags & BPF_F_INDEX_MASK;
543 	struct bpf_event_entry *ee;
544 
545 	if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
546 		return -EINVAL;
547 	if (index == BPF_F_CURRENT_CPU)
548 		index = cpu;
549 	if (unlikely(index >= array->map.max_entries))
550 		return -E2BIG;
551 
552 	ee = READ_ONCE(array->ptrs[index]);
553 	if (!ee)
554 		return -ENOENT;
555 
556 	return perf_event_read_local(ee->event, value, enabled, running);
557 }
558 
BPF_CALL_2(bpf_perf_event_read,struct bpf_map *,map,u64,flags)559 BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags)
560 {
561 	u64 value = 0;
562 	int err;
563 
564 	err = get_map_perf_counter(map, flags, &value, NULL, NULL);
565 	/*
566 	 * this api is ugly since we miss [-22..-2] range of valid
567 	 * counter values, but that's uapi
568 	 */
569 	if (err)
570 		return err;
571 	return value;
572 }
573 
574 const struct bpf_func_proto bpf_perf_event_read_proto = {
575 	.func		= bpf_perf_event_read,
576 	.gpl_only	= true,
577 	.ret_type	= RET_INTEGER,
578 	.arg1_type	= ARG_CONST_MAP_PTR,
579 	.arg2_type	= ARG_ANYTHING,
580 };
581 
BPF_CALL_4(bpf_perf_event_read_value,struct bpf_map *,map,u64,flags,struct bpf_perf_event_value *,buf,u32,size)582 BPF_CALL_4(bpf_perf_event_read_value, struct bpf_map *, map, u64, flags,
583 	   struct bpf_perf_event_value *, buf, u32, size)
584 {
585 	int err = -EINVAL;
586 
587 	if (unlikely(size != sizeof(struct bpf_perf_event_value)))
588 		goto clear;
589 	err = get_map_perf_counter(map, flags, &buf->counter, &buf->enabled,
590 				   &buf->running);
591 	if (unlikely(err))
592 		goto clear;
593 	return 0;
594 clear:
595 	memset(buf, 0, size);
596 	return err;
597 }
598 
599 static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
600 	.func		= bpf_perf_event_read_value,
601 	.gpl_only	= true,
602 	.ret_type	= RET_INTEGER,
603 	.arg1_type	= ARG_CONST_MAP_PTR,
604 	.arg2_type	= ARG_ANYTHING,
605 	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,
606 	.arg4_type	= ARG_CONST_SIZE,
607 };
608 
bpf_get_perf_event_read_value_proto(void)609 const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void)
610 {
611 	return &bpf_perf_event_read_value_proto;
612 }
613 
614 static __always_inline u64
__bpf_perf_event_output(struct pt_regs * regs,struct bpf_map * map,u64 flags,struct perf_raw_record * raw,struct perf_sample_data * sd)615 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
616 			u64 flags, struct perf_raw_record *raw,
617 			struct perf_sample_data *sd)
618 {
619 	struct bpf_array *array = container_of(map, struct bpf_array, map);
620 	unsigned int cpu = smp_processor_id();
621 	u64 index = flags & BPF_F_INDEX_MASK;
622 	struct bpf_event_entry *ee;
623 	struct perf_event *event;
624 
625 	if (index == BPF_F_CURRENT_CPU)
626 		index = cpu;
627 	if (unlikely(index >= array->map.max_entries))
628 		return -E2BIG;
629 
630 	ee = READ_ONCE(array->ptrs[index]);
631 	if (!ee)
632 		return -ENOENT;
633 
634 	event = ee->event;
635 	if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE ||
636 		     event->attr.config != PERF_COUNT_SW_BPF_OUTPUT))
637 		return -EINVAL;
638 
639 	if (unlikely(event->oncpu != cpu))
640 		return -EOPNOTSUPP;
641 
642 	perf_sample_save_raw_data(sd, event, raw);
643 
644 	return perf_event_output(event, sd, regs);
645 }
646 
647 /*
648  * Support executing tracepoints in normal, irq, and nmi context that each call
649  * bpf_perf_event_output
650  */
651 struct bpf_trace_sample_data {
652 	struct perf_sample_data sds[3];
653 };
654 
655 static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds);
656 static DEFINE_PER_CPU(int, bpf_trace_nest_level);
BPF_CALL_5(bpf_perf_event_output,struct pt_regs *,regs,struct bpf_map *,map,u64,flags,void *,data,u64,size)657 BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
658 	   u64, flags, void *, data, u64, size)
659 {
660 	struct bpf_trace_sample_data *sds;
661 	struct perf_raw_record raw = {
662 		.frag = {
663 			.size = size,
664 			.data = data,
665 		},
666 	};
667 	struct perf_sample_data *sd;
668 	int nest_level, err;
669 
670 	preempt_disable();
671 	sds = this_cpu_ptr(&bpf_trace_sds);
672 	nest_level = this_cpu_inc_return(bpf_trace_nest_level);
673 
674 	if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds))) {
675 		err = -EBUSY;
676 		goto out;
677 	}
678 
679 	sd = &sds->sds[nest_level - 1];
680 
681 	if (unlikely(flags & ~(BPF_F_INDEX_MASK))) {
682 		err = -EINVAL;
683 		goto out;
684 	}
685 
686 	perf_sample_data_init(sd, 0, 0);
687 
688 	err = __bpf_perf_event_output(regs, map, flags, &raw, sd);
689 out:
690 	this_cpu_dec(bpf_trace_nest_level);
691 	preempt_enable();
692 	return err;
693 }
694 
695 static const struct bpf_func_proto bpf_perf_event_output_proto = {
696 	.func		= bpf_perf_event_output,
697 	.gpl_only	= true,
698 	.ret_type	= RET_INTEGER,
699 	.arg1_type	= ARG_PTR_TO_CTX,
700 	.arg2_type	= ARG_CONST_MAP_PTR,
701 	.arg3_type	= ARG_ANYTHING,
702 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
703 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
704 };
705 
706 static DEFINE_PER_CPU(int, bpf_event_output_nest_level);
707 struct bpf_nested_pt_regs {
708 	struct pt_regs regs[3];
709 };
710 static DEFINE_PER_CPU(struct bpf_nested_pt_regs, bpf_pt_regs);
711 static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_misc_sds);
712 
bpf_event_output(struct bpf_map * map,u64 flags,void * meta,u64 meta_size,void * ctx,u64 ctx_size,bpf_ctx_copy_t ctx_copy)713 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
714 		     void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)
715 {
716 	struct perf_raw_frag frag = {
717 		.copy		= ctx_copy,
718 		.size		= ctx_size,
719 		.data		= ctx,
720 	};
721 	struct perf_raw_record raw = {
722 		.frag = {
723 			{
724 				.next	= ctx_size ? &frag : NULL,
725 			},
726 			.size	= meta_size,
727 			.data	= meta,
728 		},
729 	};
730 	struct perf_sample_data *sd;
731 	struct pt_regs *regs;
732 	int nest_level;
733 	u64 ret;
734 
735 	preempt_disable();
736 	nest_level = this_cpu_inc_return(bpf_event_output_nest_level);
737 
738 	if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bpf_misc_sds.sds))) {
739 		ret = -EBUSY;
740 		goto out;
741 	}
742 	sd = this_cpu_ptr(&bpf_misc_sds.sds[nest_level - 1]);
743 	regs = this_cpu_ptr(&bpf_pt_regs.regs[nest_level - 1]);
744 
745 	perf_fetch_caller_regs(regs);
746 	perf_sample_data_init(sd, 0, 0);
747 
748 	ret = __bpf_perf_event_output(regs, map, flags, &raw, sd);
749 out:
750 	this_cpu_dec(bpf_event_output_nest_level);
751 	preempt_enable();
752 	return ret;
753 }
754 
BPF_CALL_0(bpf_get_current_task)755 BPF_CALL_0(bpf_get_current_task)
756 {
757 	return (long) current;
758 }
759 
760 const struct bpf_func_proto bpf_get_current_task_proto = {
761 	.func		= bpf_get_current_task,
762 	.gpl_only	= true,
763 	.ret_type	= RET_INTEGER,
764 };
765 
BPF_CALL_0(bpf_get_current_task_btf)766 BPF_CALL_0(bpf_get_current_task_btf)
767 {
768 	return (unsigned long) current;
769 }
770 
771 const struct bpf_func_proto bpf_get_current_task_btf_proto = {
772 	.func		= bpf_get_current_task_btf,
773 	.gpl_only	= true,
774 	.ret_type	= RET_PTR_TO_BTF_ID_TRUSTED,
775 	.ret_btf_id	= &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
776 };
777 
BPF_CALL_1(bpf_task_pt_regs,struct task_struct *,task)778 BPF_CALL_1(bpf_task_pt_regs, struct task_struct *, task)
779 {
780 	return (unsigned long) task_pt_regs(task);
781 }
782 
783 BTF_ID_LIST_SINGLE(bpf_task_pt_regs_ids, struct, pt_regs)
784 
785 const struct bpf_func_proto bpf_task_pt_regs_proto = {
786 	.func		= bpf_task_pt_regs,
787 	.gpl_only	= true,
788 	.arg1_type	= ARG_PTR_TO_BTF_ID,
789 	.arg1_btf_id	= &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
790 	.ret_type	= RET_PTR_TO_BTF_ID,
791 	.ret_btf_id	= &bpf_task_pt_regs_ids[0],
792 };
793 
794 struct send_signal_irq_work {
795 	struct irq_work irq_work;
796 	struct task_struct *task;
797 	u32 sig;
798 	enum pid_type type;
799 	bool has_siginfo;
800 	struct kernel_siginfo info;
801 };
802 
803 static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work);
804 
do_bpf_send_signal(struct irq_work * entry)805 static void do_bpf_send_signal(struct irq_work *entry)
806 {
807 	struct send_signal_irq_work *work;
808 	struct kernel_siginfo *siginfo;
809 
810 	work = container_of(entry, struct send_signal_irq_work, irq_work);
811 	siginfo = work->has_siginfo ? &work->info : SEND_SIG_PRIV;
812 
813 	group_send_sig_info(work->sig, siginfo, work->task, work->type);
814 	put_task_struct(work->task);
815 }
816 
bpf_send_signal_common(u32 sig,enum pid_type type,struct task_struct * task,u64 value)817 static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value)
818 {
819 	struct send_signal_irq_work *work = NULL;
820 	struct kernel_siginfo info;
821 	struct kernel_siginfo *siginfo;
822 
823 	if (!task) {
824 		task = current;
825 		siginfo = SEND_SIG_PRIV;
826 	} else {
827 		clear_siginfo(&info);
828 		info.si_signo = sig;
829 		info.si_errno = 0;
830 		info.si_code = SI_KERNEL;
831 		info.si_pid = 0;
832 		info.si_uid = 0;
833 		info.si_value.sival_ptr = (void *)(unsigned long)value;
834 		siginfo = &info;
835 	}
836 
837 	/* Similar to bpf_probe_write_user, task needs to be
838 	 * in a sound condition and kernel memory access be
839 	 * permitted in order to send signal to the current
840 	 * task.
841 	 */
842 	if (unlikely(task->flags & (PF_KTHREAD | PF_EXITING)))
843 		return -EPERM;
844 	if (unlikely(!nmi_uaccess_okay()))
845 		return -EPERM;
846 	/* Task should not be pid=1 to avoid kernel panic. */
847 	if (unlikely(is_global_init(task)))
848 		return -EPERM;
849 
850 	if (preempt_count() != 0 || irqs_disabled()) {
851 		/* Do an early check on signal validity. Otherwise,
852 		 * the error is lost in deferred irq_work.
853 		 */
854 		if (unlikely(!valid_signal(sig)))
855 			return -EINVAL;
856 
857 		work = this_cpu_ptr(&send_signal_work);
858 		if (irq_work_is_busy(&work->irq_work))
859 			return -EBUSY;
860 
861 		/* Add the current task, which is the target of sending signal,
862 		 * to the irq_work. The current task may change when queued
863 		 * irq works get executed.
864 		 */
865 		work->task = get_task_struct(task);
866 		work->has_siginfo = siginfo == &info;
867 		if (work->has_siginfo)
868 			copy_siginfo(&work->info, &info);
869 		work->sig = sig;
870 		work->type = type;
871 		irq_work_queue(&work->irq_work);
872 		return 0;
873 	}
874 
875 	return group_send_sig_info(sig, siginfo, task, type);
876 }
877 
BPF_CALL_1(bpf_send_signal,u32,sig)878 BPF_CALL_1(bpf_send_signal, u32, sig)
879 {
880 	return bpf_send_signal_common(sig, PIDTYPE_TGID, NULL, 0);
881 }
882 
883 const struct bpf_func_proto bpf_send_signal_proto = {
884 	.func		= bpf_send_signal,
885 	.gpl_only	= false,
886 	.ret_type	= RET_INTEGER,
887 	.arg1_type	= ARG_ANYTHING,
888 };
889 
BPF_CALL_1(bpf_send_signal_thread,u32,sig)890 BPF_CALL_1(bpf_send_signal_thread, u32, sig)
891 {
892 	return bpf_send_signal_common(sig, PIDTYPE_PID, NULL, 0);
893 }
894 
895 const struct bpf_func_proto bpf_send_signal_thread_proto = {
896 	.func		= bpf_send_signal_thread,
897 	.gpl_only	= false,
898 	.ret_type	= RET_INTEGER,
899 	.arg1_type	= ARG_ANYTHING,
900 };
901 
BPF_CALL_3(bpf_d_path,const struct path *,path,char *,buf,u32,sz)902 BPF_CALL_3(bpf_d_path, const struct path *, path, char *, buf, u32, sz)
903 {
904 	struct path copy;
905 	long len;
906 	char *p;
907 
908 	if (!sz)
909 		return 0;
910 
911 	/*
912 	 * The path pointer is verified as trusted and safe to use,
913 	 * but let's double check it's valid anyway to workaround
914 	 * potentially broken verifier.
915 	 */
916 	len = copy_from_kernel_nofault(&copy, path, sizeof(*path));
917 	if (len < 0)
918 		return len;
919 
920 	p = d_path(&copy, buf, sz);
921 	if (IS_ERR(p)) {
922 		len = PTR_ERR(p);
923 	} else {
924 		len = buf + sz - p;
925 		memmove(buf, p, len);
926 	}
927 
928 	return len;
929 }
930 
931 BTF_SET_START(btf_allowlist_d_path)
932 #ifdef CONFIG_SECURITY
BTF_ID(func,security_file_permission)933 BTF_ID(func, security_file_permission)
934 BTF_ID(func, security_inode_getattr)
935 BTF_ID(func, security_file_open)
936 #endif
937 #ifdef CONFIG_SECURITY_PATH
938 BTF_ID(func, security_path_truncate)
939 #endif
940 BTF_ID(func, vfs_truncate)
941 BTF_ID(func, vfs_fallocate)
942 BTF_ID(func, dentry_open)
943 BTF_ID(func, vfs_getattr)
944 BTF_ID(func, filp_close)
945 BTF_SET_END(btf_allowlist_d_path)
946 
947 static bool bpf_d_path_allowed(const struct bpf_prog *prog)
948 {
949 	if (prog->type == BPF_PROG_TYPE_TRACING &&
950 	    prog->expected_attach_type == BPF_TRACE_ITER)
951 		return true;
952 
953 	if (prog->type == BPF_PROG_TYPE_LSM)
954 		return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id);
955 
956 	return btf_id_set_contains(&btf_allowlist_d_path,
957 				   prog->aux->attach_btf_id);
958 }
959 
960 BTF_ID_LIST_SINGLE(bpf_d_path_btf_ids, struct, path)
961 
962 static const struct bpf_func_proto bpf_d_path_proto = {
963 	.func		= bpf_d_path,
964 	.gpl_only	= false,
965 	.ret_type	= RET_INTEGER,
966 	.arg1_type	= ARG_PTR_TO_BTF_ID,
967 	.arg1_btf_id	= &bpf_d_path_btf_ids[0],
968 	.arg2_type	= ARG_PTR_TO_MEM,
969 	.arg3_type	= ARG_CONST_SIZE_OR_ZERO,
970 	.allowed	= bpf_d_path_allowed,
971 };
972 
973 #define BTF_F_ALL	(BTF_F_COMPACT  | BTF_F_NONAME | \
974 			 BTF_F_PTR_RAW | BTF_F_ZERO)
975 
bpf_btf_printf_prepare(struct btf_ptr * ptr,u32 btf_ptr_size,u64 flags,const struct btf ** btf,s32 * btf_id)976 static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,
977 				  u64 flags, const struct btf **btf,
978 				  s32 *btf_id)
979 {
980 	const struct btf_type *t;
981 
982 	if (unlikely(flags & ~(BTF_F_ALL)))
983 		return -EINVAL;
984 
985 	if (btf_ptr_size != sizeof(struct btf_ptr))
986 		return -EINVAL;
987 
988 	*btf = bpf_get_btf_vmlinux();
989 
990 	if (IS_ERR_OR_NULL(*btf))
991 		return IS_ERR(*btf) ? PTR_ERR(*btf) : -EINVAL;
992 
993 	if (ptr->type_id > 0)
994 		*btf_id = ptr->type_id;
995 	else
996 		return -EINVAL;
997 
998 	if (*btf_id > 0)
999 		t = btf_type_by_id(*btf, *btf_id);
1000 	if (*btf_id <= 0 || !t)
1001 		return -ENOENT;
1002 
1003 	return 0;
1004 }
1005 
BPF_CALL_5(bpf_snprintf_btf,char *,str,u32,str_size,struct btf_ptr *,ptr,u32,btf_ptr_size,u64,flags)1006 BPF_CALL_5(bpf_snprintf_btf, char *, str, u32, str_size, struct btf_ptr *, ptr,
1007 	   u32, btf_ptr_size, u64, flags)
1008 {
1009 	const struct btf *btf;
1010 	s32 btf_id;
1011 	int ret;
1012 
1013 	ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id);
1014 	if (ret)
1015 		return ret;
1016 
1017 	return btf_type_snprintf_show(btf, btf_id, ptr->ptr, str, str_size,
1018 				      flags);
1019 }
1020 
1021 const struct bpf_func_proto bpf_snprintf_btf_proto = {
1022 	.func		= bpf_snprintf_btf,
1023 	.gpl_only	= false,
1024 	.ret_type	= RET_INTEGER,
1025 	.arg1_type	= ARG_PTR_TO_MEM,
1026 	.arg2_type	= ARG_CONST_SIZE,
1027 	.arg3_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
1028 	.arg4_type	= ARG_CONST_SIZE,
1029 	.arg5_type	= ARG_ANYTHING,
1030 };
1031 
BPF_CALL_1(bpf_get_func_ip_tracing,void *,ctx)1032 BPF_CALL_1(bpf_get_func_ip_tracing, void *, ctx)
1033 {
1034 	/* This helper call is inlined by verifier. */
1035 	return ((u64 *)ctx)[-2];
1036 }
1037 
1038 static const struct bpf_func_proto bpf_get_func_ip_proto_tracing = {
1039 	.func		= bpf_get_func_ip_tracing,
1040 	.gpl_only	= true,
1041 	.ret_type	= RET_INTEGER,
1042 	.arg1_type	= ARG_PTR_TO_CTX,
1043 };
1044 
get_entry_ip(unsigned long fentry_ip)1045 static inline unsigned long get_entry_ip(unsigned long fentry_ip)
1046 {
1047 #ifdef CONFIG_X86_KERNEL_IBT
1048 	if (is_endbr((void *)(fentry_ip - ENDBR_INSN_SIZE)))
1049 		fentry_ip -= ENDBR_INSN_SIZE;
1050 #endif
1051 	return fentry_ip;
1052 }
1053 
BPF_CALL_1(bpf_get_func_ip_kprobe,struct pt_regs *,regs)1054 BPF_CALL_1(bpf_get_func_ip_kprobe, struct pt_regs *, regs)
1055 {
1056 	struct bpf_trace_run_ctx *run_ctx __maybe_unused;
1057 	struct kprobe *kp;
1058 
1059 #ifdef CONFIG_UPROBES
1060 	run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx);
1061 	if (run_ctx->is_uprobe)
1062 		return ((struct uprobe_dispatch_data *)current->utask->vaddr)->bp_addr;
1063 #endif
1064 
1065 	kp = kprobe_running();
1066 
1067 	if (!kp || !(kp->flags & KPROBE_FLAG_ON_FUNC_ENTRY))
1068 		return 0;
1069 
1070 	return get_entry_ip((uintptr_t)kp->addr);
1071 }
1072 
1073 static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe = {
1074 	.func		= bpf_get_func_ip_kprobe,
1075 	.gpl_only	= true,
1076 	.ret_type	= RET_INTEGER,
1077 	.arg1_type	= ARG_PTR_TO_CTX,
1078 };
1079 
BPF_CALL_1(bpf_get_func_ip_kprobe_multi,struct pt_regs *,regs)1080 BPF_CALL_1(bpf_get_func_ip_kprobe_multi, struct pt_regs *, regs)
1081 {
1082 	return bpf_kprobe_multi_entry_ip(current->bpf_ctx);
1083 }
1084 
1085 static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe_multi = {
1086 	.func		= bpf_get_func_ip_kprobe_multi,
1087 	.gpl_only	= false,
1088 	.ret_type	= RET_INTEGER,
1089 	.arg1_type	= ARG_PTR_TO_CTX,
1090 };
1091 
BPF_CALL_1(bpf_get_attach_cookie_kprobe_multi,struct pt_regs *,regs)1092 BPF_CALL_1(bpf_get_attach_cookie_kprobe_multi, struct pt_regs *, regs)
1093 {
1094 	return bpf_kprobe_multi_cookie(current->bpf_ctx);
1095 }
1096 
1097 static const struct bpf_func_proto bpf_get_attach_cookie_proto_kmulti = {
1098 	.func		= bpf_get_attach_cookie_kprobe_multi,
1099 	.gpl_only	= false,
1100 	.ret_type	= RET_INTEGER,
1101 	.arg1_type	= ARG_PTR_TO_CTX,
1102 };
1103 
BPF_CALL_1(bpf_get_func_ip_uprobe_multi,struct pt_regs *,regs)1104 BPF_CALL_1(bpf_get_func_ip_uprobe_multi, struct pt_regs *, regs)
1105 {
1106 	return bpf_uprobe_multi_entry_ip(current->bpf_ctx);
1107 }
1108 
1109 static const struct bpf_func_proto bpf_get_func_ip_proto_uprobe_multi = {
1110 	.func		= bpf_get_func_ip_uprobe_multi,
1111 	.gpl_only	= false,
1112 	.ret_type	= RET_INTEGER,
1113 	.arg1_type	= ARG_PTR_TO_CTX,
1114 };
1115 
BPF_CALL_1(bpf_get_attach_cookie_uprobe_multi,struct pt_regs *,regs)1116 BPF_CALL_1(bpf_get_attach_cookie_uprobe_multi, struct pt_regs *, regs)
1117 {
1118 	return bpf_uprobe_multi_cookie(current->bpf_ctx);
1119 }
1120 
1121 static const struct bpf_func_proto bpf_get_attach_cookie_proto_umulti = {
1122 	.func		= bpf_get_attach_cookie_uprobe_multi,
1123 	.gpl_only	= false,
1124 	.ret_type	= RET_INTEGER,
1125 	.arg1_type	= ARG_PTR_TO_CTX,
1126 };
1127 
BPF_CALL_1(bpf_get_attach_cookie_trace,void *,ctx)1128 BPF_CALL_1(bpf_get_attach_cookie_trace, void *, ctx)
1129 {
1130 	struct bpf_trace_run_ctx *run_ctx;
1131 
1132 	run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx);
1133 	return run_ctx->bpf_cookie;
1134 }
1135 
1136 static const struct bpf_func_proto bpf_get_attach_cookie_proto_trace = {
1137 	.func		= bpf_get_attach_cookie_trace,
1138 	.gpl_only	= false,
1139 	.ret_type	= RET_INTEGER,
1140 	.arg1_type	= ARG_PTR_TO_CTX,
1141 };
1142 
BPF_CALL_1(bpf_get_attach_cookie_pe,struct bpf_perf_event_data_kern *,ctx)1143 BPF_CALL_1(bpf_get_attach_cookie_pe, struct bpf_perf_event_data_kern *, ctx)
1144 {
1145 	return ctx->event->bpf_cookie;
1146 }
1147 
1148 static const struct bpf_func_proto bpf_get_attach_cookie_proto_pe = {
1149 	.func		= bpf_get_attach_cookie_pe,
1150 	.gpl_only	= false,
1151 	.ret_type	= RET_INTEGER,
1152 	.arg1_type	= ARG_PTR_TO_CTX,
1153 };
1154 
BPF_CALL_1(bpf_get_attach_cookie_tracing,void *,ctx)1155 BPF_CALL_1(bpf_get_attach_cookie_tracing, void *, ctx)
1156 {
1157 	struct bpf_trace_run_ctx *run_ctx;
1158 
1159 	run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx);
1160 	return run_ctx->bpf_cookie;
1161 }
1162 
1163 static const struct bpf_func_proto bpf_get_attach_cookie_proto_tracing = {
1164 	.func		= bpf_get_attach_cookie_tracing,
1165 	.gpl_only	= false,
1166 	.ret_type	= RET_INTEGER,
1167 	.arg1_type	= ARG_PTR_TO_CTX,
1168 };
1169 
BPF_CALL_3(bpf_get_branch_snapshot,void *,buf,u32,size,u64,flags)1170 BPF_CALL_3(bpf_get_branch_snapshot, void *, buf, u32, size, u64, flags)
1171 {
1172 	static const u32 br_entry_size = sizeof(struct perf_branch_entry);
1173 	u32 entry_cnt = size / br_entry_size;
1174 
1175 	entry_cnt = static_call(perf_snapshot_branch_stack)(buf, entry_cnt);
1176 
1177 	if (unlikely(flags))
1178 		return -EINVAL;
1179 
1180 	if (!entry_cnt)
1181 		return -ENOENT;
1182 
1183 	return entry_cnt * br_entry_size;
1184 }
1185 
1186 const struct bpf_func_proto bpf_get_branch_snapshot_proto = {
1187 	.func		= bpf_get_branch_snapshot,
1188 	.gpl_only	= true,
1189 	.ret_type	= RET_INTEGER,
1190 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
1191 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
1192 };
1193 
BPF_CALL_3(get_func_arg,void *,ctx,u32,n,u64 *,value)1194 BPF_CALL_3(get_func_arg, void *, ctx, u32, n, u64 *, value)
1195 {
1196 	/* This helper call is inlined by verifier. */
1197 	u64 nr_args = ((u64 *)ctx)[-1];
1198 
1199 	if ((u64) n >= nr_args)
1200 		return -EINVAL;
1201 	*value = ((u64 *)ctx)[n];
1202 	return 0;
1203 }
1204 
1205 static const struct bpf_func_proto bpf_get_func_arg_proto = {
1206 	.func		= get_func_arg,
1207 	.ret_type	= RET_INTEGER,
1208 	.arg1_type	= ARG_PTR_TO_CTX,
1209 	.arg2_type	= ARG_ANYTHING,
1210 	.arg3_type	= ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED,
1211 	.arg3_size	= sizeof(u64),
1212 };
1213 
BPF_CALL_2(get_func_ret,void *,ctx,u64 *,value)1214 BPF_CALL_2(get_func_ret, void *, ctx, u64 *, value)
1215 {
1216 	/* This helper call is inlined by verifier. */
1217 	u64 nr_args = ((u64 *)ctx)[-1];
1218 
1219 	*value = ((u64 *)ctx)[nr_args];
1220 	return 0;
1221 }
1222 
1223 static const struct bpf_func_proto bpf_get_func_ret_proto = {
1224 	.func		= get_func_ret,
1225 	.ret_type	= RET_INTEGER,
1226 	.arg1_type	= ARG_PTR_TO_CTX,
1227 	.arg2_type	= ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED,
1228 	.arg2_size	= sizeof(u64),
1229 };
1230 
BPF_CALL_1(get_func_arg_cnt,void *,ctx)1231 BPF_CALL_1(get_func_arg_cnt, void *, ctx)
1232 {
1233 	/* This helper call is inlined by verifier. */
1234 	return ((u64 *)ctx)[-1];
1235 }
1236 
1237 static const struct bpf_func_proto bpf_get_func_arg_cnt_proto = {
1238 	.func		= get_func_arg_cnt,
1239 	.ret_type	= RET_INTEGER,
1240 	.arg1_type	= ARG_PTR_TO_CTX,
1241 };
1242 
1243 static const struct bpf_func_proto *
bpf_tracing_func_proto(enum bpf_func_id func_id,const struct bpf_prog * prog)1244 bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1245 {
1246 	const struct bpf_func_proto *func_proto;
1247 
1248 	switch (func_id) {
1249 	case BPF_FUNC_get_smp_processor_id:
1250 		return &bpf_get_smp_processor_id_proto;
1251 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
1252 	case BPF_FUNC_probe_read:
1253 		return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
1254 		       NULL : &bpf_probe_read_compat_proto;
1255 	case BPF_FUNC_probe_read_str:
1256 		return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
1257 		       NULL : &bpf_probe_read_compat_str_proto;
1258 #endif
1259 	case BPF_FUNC_get_func_ip:
1260 		return &bpf_get_func_ip_proto_tracing;
1261 	default:
1262 		break;
1263 	}
1264 
1265 	func_proto = bpf_base_func_proto(func_id, prog);
1266 	if (func_proto)
1267 		return func_proto;
1268 
1269 	if (!bpf_token_capable(prog->aux->token, CAP_SYS_ADMIN))
1270 		return NULL;
1271 
1272 	switch (func_id) {
1273 	case BPF_FUNC_probe_write_user:
1274 		return security_locked_down(LOCKDOWN_BPF_WRITE_USER) < 0 ?
1275 		       NULL : &bpf_probe_write_user_proto;
1276 	default:
1277 		return NULL;
1278 	}
1279 }
1280 
is_kprobe_multi(const struct bpf_prog * prog)1281 static bool is_kprobe_multi(const struct bpf_prog *prog)
1282 {
1283 	return prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI ||
1284 	       prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;
1285 }
1286 
is_kprobe_session(const struct bpf_prog * prog)1287 static inline bool is_kprobe_session(const struct bpf_prog *prog)
1288 {
1289 	return prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;
1290 }
1291 
is_uprobe_multi(const struct bpf_prog * prog)1292 static inline bool is_uprobe_multi(const struct bpf_prog *prog)
1293 {
1294 	return prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI ||
1295 	       prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1296 }
1297 
is_uprobe_session(const struct bpf_prog * prog)1298 static inline bool is_uprobe_session(const struct bpf_prog *prog)
1299 {
1300 	return prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1301 }
1302 
1303 static const struct bpf_func_proto *
kprobe_prog_func_proto(enum bpf_func_id func_id,const struct bpf_prog * prog)1304 kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1305 {
1306 	switch (func_id) {
1307 	case BPF_FUNC_perf_event_output:
1308 		return &bpf_perf_event_output_proto;
1309 	case BPF_FUNC_get_stackid:
1310 		return &bpf_get_stackid_proto;
1311 	case BPF_FUNC_get_stack:
1312 		return prog->sleepable ? &bpf_get_stack_sleepable_proto : &bpf_get_stack_proto;
1313 #ifdef CONFIG_BPF_KPROBE_OVERRIDE
1314 	case BPF_FUNC_override_return:
1315 		return &bpf_override_return_proto;
1316 #endif
1317 	case BPF_FUNC_get_func_ip:
1318 		if (is_kprobe_multi(prog))
1319 			return &bpf_get_func_ip_proto_kprobe_multi;
1320 		if (is_uprobe_multi(prog))
1321 			return &bpf_get_func_ip_proto_uprobe_multi;
1322 		return &bpf_get_func_ip_proto_kprobe;
1323 	case BPF_FUNC_get_attach_cookie:
1324 		if (is_kprobe_multi(prog))
1325 			return &bpf_get_attach_cookie_proto_kmulti;
1326 		if (is_uprobe_multi(prog))
1327 			return &bpf_get_attach_cookie_proto_umulti;
1328 		return &bpf_get_attach_cookie_proto_trace;
1329 	default:
1330 		return bpf_tracing_func_proto(func_id, prog);
1331 	}
1332 }
1333 
1334 /* bpf+kprobe programs can access fields of 'struct pt_regs' */
kprobe_prog_is_valid_access(int off,int size,enum bpf_access_type type,const struct bpf_prog * prog,struct bpf_insn_access_aux * info)1335 static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type type,
1336 					const struct bpf_prog *prog,
1337 					struct bpf_insn_access_aux *info)
1338 {
1339 	if (off < 0 || off >= sizeof(struct pt_regs))
1340 		return false;
1341 	if (off % size != 0)
1342 		return false;
1343 	/*
1344 	 * Assertion for 32 bit to make sure last 8 byte access
1345 	 * (BPF_DW) to the last 4 byte member is disallowed.
1346 	 */
1347 	if (off + size > sizeof(struct pt_regs))
1348 		return false;
1349 
1350 	if (type == BPF_WRITE)
1351 		prog->aux->kprobe_write_ctx = true;
1352 
1353 	return true;
1354 }
1355 
1356 const struct bpf_verifier_ops kprobe_verifier_ops = {
1357 	.get_func_proto  = kprobe_prog_func_proto,
1358 	.is_valid_access = kprobe_prog_is_valid_access,
1359 };
1360 
1361 const struct bpf_prog_ops kprobe_prog_ops = {
1362 };
1363 
BPF_CALL_5(bpf_perf_event_output_tp,void *,tp_buff,struct bpf_map *,map,u64,flags,void *,data,u64,size)1364 BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map,
1365 	   u64, flags, void *, data, u64, size)
1366 {
1367 	struct pt_regs *regs = *(struct pt_regs **)tp_buff;
1368 
1369 	/*
1370 	 * r1 points to perf tracepoint buffer where first 8 bytes are hidden
1371 	 * from bpf program and contain a pointer to 'struct pt_regs'. Fetch it
1372 	 * from there and call the same bpf_perf_event_output() helper inline.
1373 	 */
1374 	return ____bpf_perf_event_output(regs, map, flags, data, size);
1375 }
1376 
1377 static const struct bpf_func_proto bpf_perf_event_output_proto_tp = {
1378 	.func		= bpf_perf_event_output_tp,
1379 	.gpl_only	= true,
1380 	.ret_type	= RET_INTEGER,
1381 	.arg1_type	= ARG_PTR_TO_CTX,
1382 	.arg2_type	= ARG_CONST_MAP_PTR,
1383 	.arg3_type	= ARG_ANYTHING,
1384 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
1385 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
1386 };
1387 
BPF_CALL_3(bpf_get_stackid_tp,void *,tp_buff,struct bpf_map *,map,u64,flags)1388 BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map,
1389 	   u64, flags)
1390 {
1391 	struct pt_regs *regs = *(struct pt_regs **)tp_buff;
1392 
1393 	/*
1394 	 * Same comment as in bpf_perf_event_output_tp(), only that this time
1395 	 * the other helper's function body cannot be inlined due to being
1396 	 * external, thus we need to call raw helper function.
1397 	 */
1398 	return bpf_get_stackid((unsigned long) regs, (unsigned long) map,
1399 			       flags, 0, 0);
1400 }
1401 
1402 static const struct bpf_func_proto bpf_get_stackid_proto_tp = {
1403 	.func		= bpf_get_stackid_tp,
1404 	.gpl_only	= true,
1405 	.ret_type	= RET_INTEGER,
1406 	.arg1_type	= ARG_PTR_TO_CTX,
1407 	.arg2_type	= ARG_CONST_MAP_PTR,
1408 	.arg3_type	= ARG_ANYTHING,
1409 };
1410 
BPF_CALL_4(bpf_get_stack_tp,void *,tp_buff,void *,buf,u32,size,u64,flags)1411 BPF_CALL_4(bpf_get_stack_tp, void *, tp_buff, void *, buf, u32, size,
1412 	   u64, flags)
1413 {
1414 	struct pt_regs *regs = *(struct pt_regs **)tp_buff;
1415 
1416 	return bpf_get_stack((unsigned long) regs, (unsigned long) buf,
1417 			     (unsigned long) size, flags, 0);
1418 }
1419 
1420 static const struct bpf_func_proto bpf_get_stack_proto_tp = {
1421 	.func		= bpf_get_stack_tp,
1422 	.gpl_only	= true,
1423 	.ret_type	= RET_INTEGER,
1424 	.arg1_type	= ARG_PTR_TO_CTX,
1425 	.arg2_type	= ARG_PTR_TO_UNINIT_MEM,
1426 	.arg3_type	= ARG_CONST_SIZE_OR_ZERO,
1427 	.arg4_type	= ARG_ANYTHING,
1428 };
1429 
1430 static const struct bpf_func_proto *
tp_prog_func_proto(enum bpf_func_id func_id,const struct bpf_prog * prog)1431 tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1432 {
1433 	switch (func_id) {
1434 	case BPF_FUNC_perf_event_output:
1435 		return &bpf_perf_event_output_proto_tp;
1436 	case BPF_FUNC_get_stackid:
1437 		return &bpf_get_stackid_proto_tp;
1438 	case BPF_FUNC_get_stack:
1439 		return &bpf_get_stack_proto_tp;
1440 	case BPF_FUNC_get_attach_cookie:
1441 		return &bpf_get_attach_cookie_proto_trace;
1442 	default:
1443 		return bpf_tracing_func_proto(func_id, prog);
1444 	}
1445 }
1446 
tp_prog_is_valid_access(int off,int size,enum bpf_access_type type,const struct bpf_prog * prog,struct bpf_insn_access_aux * info)1447 static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type,
1448 				    const struct bpf_prog *prog,
1449 				    struct bpf_insn_access_aux *info)
1450 {
1451 	if (off < sizeof(void *) || off >= PERF_MAX_TRACE_SIZE)
1452 		return false;
1453 	if (type != BPF_READ)
1454 		return false;
1455 	if (off % size != 0)
1456 		return false;
1457 
1458 	BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(__u64));
1459 	return true;
1460 }
1461 
1462 const struct bpf_verifier_ops tracepoint_verifier_ops = {
1463 	.get_func_proto  = tp_prog_func_proto,
1464 	.is_valid_access = tp_prog_is_valid_access,
1465 };
1466 
1467 const struct bpf_prog_ops tracepoint_prog_ops = {
1468 };
1469 
BPF_CALL_3(bpf_perf_prog_read_value,struct bpf_perf_event_data_kern *,ctx,struct bpf_perf_event_value *,buf,u32,size)1470 BPF_CALL_3(bpf_perf_prog_read_value, struct bpf_perf_event_data_kern *, ctx,
1471 	   struct bpf_perf_event_value *, buf, u32, size)
1472 {
1473 	int err = -EINVAL;
1474 
1475 	if (unlikely(size != sizeof(struct bpf_perf_event_value)))
1476 		goto clear;
1477 	err = perf_event_read_local(ctx->event, &buf->counter, &buf->enabled,
1478 				    &buf->running);
1479 	if (unlikely(err))
1480 		goto clear;
1481 	return 0;
1482 clear:
1483 	memset(buf, 0, size);
1484 	return err;
1485 }
1486 
1487 static const struct bpf_func_proto bpf_perf_prog_read_value_proto = {
1488          .func           = bpf_perf_prog_read_value,
1489          .gpl_only       = true,
1490          .ret_type       = RET_INTEGER,
1491          .arg1_type      = ARG_PTR_TO_CTX,
1492          .arg2_type      = ARG_PTR_TO_UNINIT_MEM,
1493          .arg3_type      = ARG_CONST_SIZE,
1494 };
1495 
BPF_CALL_4(bpf_read_branch_records,struct bpf_perf_event_data_kern *,ctx,void *,buf,u32,size,u64,flags)1496 BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx,
1497 	   void *, buf, u32, size, u64, flags)
1498 {
1499 	static const u32 br_entry_size = sizeof(struct perf_branch_entry);
1500 	struct perf_branch_stack *br_stack = ctx->data->br_stack;
1501 	u32 to_copy;
1502 
1503 	if (unlikely(flags & ~BPF_F_GET_BRANCH_RECORDS_SIZE))
1504 		return -EINVAL;
1505 
1506 	if (unlikely(!(ctx->data->sample_flags & PERF_SAMPLE_BRANCH_STACK)))
1507 		return -ENOENT;
1508 
1509 	if (unlikely(!br_stack))
1510 		return -ENOENT;
1511 
1512 	if (flags & BPF_F_GET_BRANCH_RECORDS_SIZE)
1513 		return br_stack->nr * br_entry_size;
1514 
1515 	if (!buf || (size % br_entry_size != 0))
1516 		return -EINVAL;
1517 
1518 	to_copy = min_t(u32, br_stack->nr * br_entry_size, size);
1519 	memcpy(buf, br_stack->entries, to_copy);
1520 
1521 	return to_copy;
1522 }
1523 
1524 static const struct bpf_func_proto bpf_read_branch_records_proto = {
1525 	.func           = bpf_read_branch_records,
1526 	.gpl_only       = true,
1527 	.ret_type       = RET_INTEGER,
1528 	.arg1_type      = ARG_PTR_TO_CTX,
1529 	.arg2_type      = ARG_PTR_TO_MEM_OR_NULL,
1530 	.arg3_type      = ARG_CONST_SIZE_OR_ZERO,
1531 	.arg4_type      = ARG_ANYTHING,
1532 };
1533 
1534 static const struct bpf_func_proto *
pe_prog_func_proto(enum bpf_func_id func_id,const struct bpf_prog * prog)1535 pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1536 {
1537 	switch (func_id) {
1538 	case BPF_FUNC_perf_event_output:
1539 		return &bpf_perf_event_output_proto_tp;
1540 	case BPF_FUNC_get_stackid:
1541 		return &bpf_get_stackid_proto_pe;
1542 	case BPF_FUNC_get_stack:
1543 		return &bpf_get_stack_proto_pe;
1544 	case BPF_FUNC_perf_prog_read_value:
1545 		return &bpf_perf_prog_read_value_proto;
1546 	case BPF_FUNC_read_branch_records:
1547 		return &bpf_read_branch_records_proto;
1548 	case BPF_FUNC_get_attach_cookie:
1549 		return &bpf_get_attach_cookie_proto_pe;
1550 	default:
1551 		return bpf_tracing_func_proto(func_id, prog);
1552 	}
1553 }
1554 
1555 /*
1556  * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp
1557  * to avoid potential recursive reuse issue when/if tracepoints are added
1558  * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack.
1559  *
1560  * Since raw tracepoints run despite bpf_prog_active, support concurrent usage
1561  * in normal, irq, and nmi context.
1562  */
1563 struct bpf_raw_tp_regs {
1564 	struct pt_regs regs[3];
1565 };
1566 static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs);
1567 static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level);
get_bpf_raw_tp_regs(void)1568 static struct pt_regs *get_bpf_raw_tp_regs(void)
1569 {
1570 	struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs);
1571 	int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level);
1572 
1573 	if (nest_level > ARRAY_SIZE(tp_regs->regs)) {
1574 		this_cpu_dec(bpf_raw_tp_nest_level);
1575 		return ERR_PTR(-EBUSY);
1576 	}
1577 
1578 	return &tp_regs->regs[nest_level - 1];
1579 }
1580 
put_bpf_raw_tp_regs(void)1581 static void put_bpf_raw_tp_regs(void)
1582 {
1583 	this_cpu_dec(bpf_raw_tp_nest_level);
1584 }
1585 
BPF_CALL_5(bpf_perf_event_output_raw_tp,struct bpf_raw_tracepoint_args *,args,struct bpf_map *,map,u64,flags,void *,data,u64,size)1586 BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args,
1587 	   struct bpf_map *, map, u64, flags, void *, data, u64, size)
1588 {
1589 	struct pt_regs *regs = get_bpf_raw_tp_regs();
1590 	int ret;
1591 
1592 	if (IS_ERR(regs))
1593 		return PTR_ERR(regs);
1594 
1595 	perf_fetch_caller_regs(regs);
1596 	ret = ____bpf_perf_event_output(regs, map, flags, data, size);
1597 
1598 	put_bpf_raw_tp_regs();
1599 	return ret;
1600 }
1601 
1602 static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
1603 	.func		= bpf_perf_event_output_raw_tp,
1604 	.gpl_only	= true,
1605 	.ret_type	= RET_INTEGER,
1606 	.arg1_type	= ARG_PTR_TO_CTX,
1607 	.arg2_type	= ARG_CONST_MAP_PTR,
1608 	.arg3_type	= ARG_ANYTHING,
1609 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
1610 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
1611 };
1612 
1613 extern const struct bpf_func_proto bpf_skb_output_proto;
1614 extern const struct bpf_func_proto bpf_xdp_output_proto;
1615 extern const struct bpf_func_proto bpf_xdp_get_buff_len_trace_proto;
1616 
BPF_CALL_3(bpf_get_stackid_raw_tp,struct bpf_raw_tracepoint_args *,args,struct bpf_map *,map,u64,flags)1617 BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args,
1618 	   struct bpf_map *, map, u64, flags)
1619 {
1620 	struct pt_regs *regs = get_bpf_raw_tp_regs();
1621 	int ret;
1622 
1623 	if (IS_ERR(regs))
1624 		return PTR_ERR(regs);
1625 
1626 	perf_fetch_caller_regs(regs);
1627 	/* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */
1628 	ret = bpf_get_stackid((unsigned long) regs, (unsigned long) map,
1629 			      flags, 0, 0);
1630 	put_bpf_raw_tp_regs();
1631 	return ret;
1632 }
1633 
1634 static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = {
1635 	.func		= bpf_get_stackid_raw_tp,
1636 	.gpl_only	= true,
1637 	.ret_type	= RET_INTEGER,
1638 	.arg1_type	= ARG_PTR_TO_CTX,
1639 	.arg2_type	= ARG_CONST_MAP_PTR,
1640 	.arg3_type	= ARG_ANYTHING,
1641 };
1642 
BPF_CALL_4(bpf_get_stack_raw_tp,struct bpf_raw_tracepoint_args *,args,void *,buf,u32,size,u64,flags)1643 BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args,
1644 	   void *, buf, u32, size, u64, flags)
1645 {
1646 	struct pt_regs *regs = get_bpf_raw_tp_regs();
1647 	int ret;
1648 
1649 	if (IS_ERR(regs))
1650 		return PTR_ERR(regs);
1651 
1652 	perf_fetch_caller_regs(regs);
1653 	ret = bpf_get_stack((unsigned long) regs, (unsigned long) buf,
1654 			    (unsigned long) size, flags, 0);
1655 	put_bpf_raw_tp_regs();
1656 	return ret;
1657 }
1658 
1659 static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = {
1660 	.func		= bpf_get_stack_raw_tp,
1661 	.gpl_only	= true,
1662 	.ret_type	= RET_INTEGER,
1663 	.arg1_type	= ARG_PTR_TO_CTX,
1664 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
1665 	.arg3_type	= ARG_CONST_SIZE_OR_ZERO,
1666 	.arg4_type	= ARG_ANYTHING,
1667 };
1668 
1669 static const struct bpf_func_proto *
raw_tp_prog_func_proto(enum bpf_func_id func_id,const struct bpf_prog * prog)1670 raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1671 {
1672 	switch (func_id) {
1673 	case BPF_FUNC_perf_event_output:
1674 		return &bpf_perf_event_output_proto_raw_tp;
1675 	case BPF_FUNC_get_stackid:
1676 		return &bpf_get_stackid_proto_raw_tp;
1677 	case BPF_FUNC_get_stack:
1678 		return &bpf_get_stack_proto_raw_tp;
1679 	case BPF_FUNC_get_attach_cookie:
1680 		return &bpf_get_attach_cookie_proto_tracing;
1681 	default:
1682 		return bpf_tracing_func_proto(func_id, prog);
1683 	}
1684 }
1685 
1686 const struct bpf_func_proto *
tracing_prog_func_proto(enum bpf_func_id func_id,const struct bpf_prog * prog)1687 tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1688 {
1689 	const struct bpf_func_proto *fn;
1690 
1691 	switch (func_id) {
1692 #ifdef CONFIG_NET
1693 	case BPF_FUNC_skb_output:
1694 		return &bpf_skb_output_proto;
1695 	case BPF_FUNC_xdp_output:
1696 		return &bpf_xdp_output_proto;
1697 	case BPF_FUNC_skc_to_tcp6_sock:
1698 		return &bpf_skc_to_tcp6_sock_proto;
1699 	case BPF_FUNC_skc_to_tcp_sock:
1700 		return &bpf_skc_to_tcp_sock_proto;
1701 	case BPF_FUNC_skc_to_tcp_timewait_sock:
1702 		return &bpf_skc_to_tcp_timewait_sock_proto;
1703 	case BPF_FUNC_skc_to_tcp_request_sock:
1704 		return &bpf_skc_to_tcp_request_sock_proto;
1705 	case BPF_FUNC_skc_to_udp6_sock:
1706 		return &bpf_skc_to_udp6_sock_proto;
1707 	case BPF_FUNC_skc_to_unix_sock:
1708 		return &bpf_skc_to_unix_sock_proto;
1709 	case BPF_FUNC_skc_to_mptcp_sock:
1710 		return &bpf_skc_to_mptcp_sock_proto;
1711 	case BPF_FUNC_sk_storage_get:
1712 		return &bpf_sk_storage_get_tracing_proto;
1713 	case BPF_FUNC_sk_storage_delete:
1714 		return &bpf_sk_storage_delete_tracing_proto;
1715 	case BPF_FUNC_sock_from_file:
1716 		return &bpf_sock_from_file_proto;
1717 	case BPF_FUNC_get_socket_cookie:
1718 		return &bpf_get_socket_ptr_cookie_proto;
1719 	case BPF_FUNC_xdp_get_buff_len:
1720 		return &bpf_xdp_get_buff_len_trace_proto;
1721 #endif
1722 	case BPF_FUNC_seq_printf:
1723 		return prog->expected_attach_type == BPF_TRACE_ITER ?
1724 		       &bpf_seq_printf_proto :
1725 		       NULL;
1726 	case BPF_FUNC_seq_write:
1727 		return prog->expected_attach_type == BPF_TRACE_ITER ?
1728 		       &bpf_seq_write_proto :
1729 		       NULL;
1730 	case BPF_FUNC_seq_printf_btf:
1731 		return prog->expected_attach_type == BPF_TRACE_ITER ?
1732 		       &bpf_seq_printf_btf_proto :
1733 		       NULL;
1734 	case BPF_FUNC_d_path:
1735 		return &bpf_d_path_proto;
1736 	case BPF_FUNC_get_func_arg:
1737 		return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_proto : NULL;
1738 	case BPF_FUNC_get_func_ret:
1739 		return bpf_prog_has_trampoline(prog) ? &bpf_get_func_ret_proto : NULL;
1740 	case BPF_FUNC_get_func_arg_cnt:
1741 		return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_cnt_proto : NULL;
1742 	case BPF_FUNC_get_attach_cookie:
1743 		if (prog->type == BPF_PROG_TYPE_TRACING &&
1744 		    prog->expected_attach_type == BPF_TRACE_RAW_TP)
1745 			return &bpf_get_attach_cookie_proto_tracing;
1746 		return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto_tracing : NULL;
1747 	default:
1748 		fn = raw_tp_prog_func_proto(func_id, prog);
1749 		if (!fn && prog->expected_attach_type == BPF_TRACE_ITER)
1750 			fn = bpf_iter_get_func_proto(func_id, prog);
1751 		return fn;
1752 	}
1753 }
1754 
raw_tp_prog_is_valid_access(int off,int size,enum bpf_access_type type,const struct bpf_prog * prog,struct bpf_insn_access_aux * info)1755 static bool raw_tp_prog_is_valid_access(int off, int size,
1756 					enum bpf_access_type type,
1757 					const struct bpf_prog *prog,
1758 					struct bpf_insn_access_aux *info)
1759 {
1760 	return bpf_tracing_ctx_access(off, size, type);
1761 }
1762 
tracing_prog_is_valid_access(int off,int size,enum bpf_access_type type,const struct bpf_prog * prog,struct bpf_insn_access_aux * info)1763 static bool tracing_prog_is_valid_access(int off, int size,
1764 					 enum bpf_access_type type,
1765 					 const struct bpf_prog *prog,
1766 					 struct bpf_insn_access_aux *info)
1767 {
1768 	return bpf_tracing_btf_ctx_access(off, size, type, prog, info);
1769 }
1770 
bpf_prog_test_run_tracing(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)1771 int __weak bpf_prog_test_run_tracing(struct bpf_prog *prog,
1772 				     const union bpf_attr *kattr,
1773 				     union bpf_attr __user *uattr)
1774 {
1775 	return -ENOTSUPP;
1776 }
1777 
1778 const struct bpf_verifier_ops raw_tracepoint_verifier_ops = {
1779 	.get_func_proto  = raw_tp_prog_func_proto,
1780 	.is_valid_access = raw_tp_prog_is_valid_access,
1781 };
1782 
1783 const struct bpf_prog_ops raw_tracepoint_prog_ops = {
1784 #ifdef CONFIG_NET
1785 	.test_run = bpf_prog_test_run_raw_tp,
1786 #endif
1787 };
1788 
1789 const struct bpf_verifier_ops tracing_verifier_ops = {
1790 	.get_func_proto  = tracing_prog_func_proto,
1791 	.is_valid_access = tracing_prog_is_valid_access,
1792 };
1793 
1794 const struct bpf_prog_ops tracing_prog_ops = {
1795 	.test_run = bpf_prog_test_run_tracing,
1796 };
1797 
raw_tp_writable_prog_is_valid_access(int off,int size,enum bpf_access_type type,const struct bpf_prog * prog,struct bpf_insn_access_aux * info)1798 static bool raw_tp_writable_prog_is_valid_access(int off, int size,
1799 						 enum bpf_access_type type,
1800 						 const struct bpf_prog *prog,
1801 						 struct bpf_insn_access_aux *info)
1802 {
1803 	if (off == 0) {
1804 		if (size != sizeof(u64) || type != BPF_READ)
1805 			return false;
1806 		info->reg_type = PTR_TO_TP_BUFFER;
1807 	}
1808 	return raw_tp_prog_is_valid_access(off, size, type, prog, info);
1809 }
1810 
1811 const struct bpf_verifier_ops raw_tracepoint_writable_verifier_ops = {
1812 	.get_func_proto  = raw_tp_prog_func_proto,
1813 	.is_valid_access = raw_tp_writable_prog_is_valid_access,
1814 };
1815 
1816 const struct bpf_prog_ops raw_tracepoint_writable_prog_ops = {
1817 };
1818 
pe_prog_is_valid_access(int off,int size,enum bpf_access_type type,const struct bpf_prog * prog,struct bpf_insn_access_aux * info)1819 static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type,
1820 				    const struct bpf_prog *prog,
1821 				    struct bpf_insn_access_aux *info)
1822 {
1823 	const int size_u64 = sizeof(u64);
1824 
1825 	if (off < 0 || off >= sizeof(struct bpf_perf_event_data))
1826 		return false;
1827 	if (type != BPF_READ)
1828 		return false;
1829 	if (off % size != 0) {
1830 		if (sizeof(unsigned long) != 4)
1831 			return false;
1832 		if (size != 8)
1833 			return false;
1834 		if (off % size != 4)
1835 			return false;
1836 	}
1837 
1838 	switch (off) {
1839 	case bpf_ctx_range(struct bpf_perf_event_data, sample_period):
1840 		bpf_ctx_record_field_size(info, size_u64);
1841 		if (!bpf_ctx_narrow_access_ok(off, size, size_u64))
1842 			return false;
1843 		break;
1844 	case bpf_ctx_range(struct bpf_perf_event_data, addr):
1845 		bpf_ctx_record_field_size(info, size_u64);
1846 		if (!bpf_ctx_narrow_access_ok(off, size, size_u64))
1847 			return false;
1848 		break;
1849 	default:
1850 		if (size != sizeof(long))
1851 			return false;
1852 	}
1853 
1854 	return true;
1855 }
1856 
pe_prog_convert_ctx_access(enum bpf_access_type type,const struct bpf_insn * si,struct bpf_insn * insn_buf,struct bpf_prog * prog,u32 * target_size)1857 static u32 pe_prog_convert_ctx_access(enum bpf_access_type type,
1858 				      const struct bpf_insn *si,
1859 				      struct bpf_insn *insn_buf,
1860 				      struct bpf_prog *prog, u32 *target_size)
1861 {
1862 	struct bpf_insn *insn = insn_buf;
1863 
1864 	switch (si->off) {
1865 	case offsetof(struct bpf_perf_event_data, sample_period):
1866 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,
1867 						       data), si->dst_reg, si->src_reg,
1868 				      offsetof(struct bpf_perf_event_data_kern, data));
1869 		*insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg,
1870 				      bpf_target_off(struct perf_sample_data, period, 8,
1871 						     target_size));
1872 		break;
1873 	case offsetof(struct bpf_perf_event_data, addr):
1874 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,
1875 						       data), si->dst_reg, si->src_reg,
1876 				      offsetof(struct bpf_perf_event_data_kern, data));
1877 		*insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg,
1878 				      bpf_target_off(struct perf_sample_data, addr, 8,
1879 						     target_size));
1880 		break;
1881 	default:
1882 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,
1883 						       regs), si->dst_reg, si->src_reg,
1884 				      offsetof(struct bpf_perf_event_data_kern, regs));
1885 		*insn++ = BPF_LDX_MEM(BPF_SIZEOF(long), si->dst_reg, si->dst_reg,
1886 				      si->off);
1887 		break;
1888 	}
1889 
1890 	return insn - insn_buf;
1891 }
1892 
1893 const struct bpf_verifier_ops perf_event_verifier_ops = {
1894 	.get_func_proto		= pe_prog_func_proto,
1895 	.is_valid_access	= pe_prog_is_valid_access,
1896 	.convert_ctx_access	= pe_prog_convert_ctx_access,
1897 };
1898 
1899 const struct bpf_prog_ops perf_event_prog_ops = {
1900 };
1901 
1902 static DEFINE_MUTEX(bpf_event_mutex);
1903 
1904 #define BPF_TRACE_MAX_PROGS 64
1905 
perf_event_attach_bpf_prog(struct perf_event * event,struct bpf_prog * prog,u64 bpf_cookie)1906 int perf_event_attach_bpf_prog(struct perf_event *event,
1907 			       struct bpf_prog *prog,
1908 			       u64 bpf_cookie)
1909 {
1910 	struct bpf_prog_array *old_array;
1911 	struct bpf_prog_array *new_array;
1912 	int ret = -EEXIST;
1913 
1914 	/*
1915 	 * Kprobe override only works if they are on the function entry,
1916 	 * and only if they are on the opt-in list.
1917 	 */
1918 	if (prog->kprobe_override &&
1919 	    (!trace_kprobe_on_func_entry(event->tp_event) ||
1920 	     !trace_kprobe_error_injectable(event->tp_event)))
1921 		return -EINVAL;
1922 
1923 	mutex_lock(&bpf_event_mutex);
1924 
1925 	if (event->prog)
1926 		goto unlock;
1927 
1928 	old_array = bpf_event_rcu_dereference(event->tp_event->prog_array);
1929 	if (old_array &&
1930 	    bpf_prog_array_length(old_array) >= BPF_TRACE_MAX_PROGS) {
1931 		ret = -E2BIG;
1932 		goto unlock;
1933 	}
1934 
1935 	ret = bpf_prog_array_copy(old_array, NULL, prog, bpf_cookie, &new_array);
1936 	if (ret < 0)
1937 		goto unlock;
1938 
1939 	/* set the new array to event->tp_event and set event->prog */
1940 	event->prog = prog;
1941 	event->bpf_cookie = bpf_cookie;
1942 	rcu_assign_pointer(event->tp_event->prog_array, new_array);
1943 	bpf_prog_array_free_sleepable(old_array);
1944 
1945 unlock:
1946 	mutex_unlock(&bpf_event_mutex);
1947 	return ret;
1948 }
1949 
perf_event_detach_bpf_prog(struct perf_event * event)1950 void perf_event_detach_bpf_prog(struct perf_event *event)
1951 {
1952 	struct bpf_prog_array *old_array;
1953 	struct bpf_prog_array *new_array;
1954 	struct bpf_prog *prog = NULL;
1955 	int ret;
1956 
1957 	mutex_lock(&bpf_event_mutex);
1958 
1959 	if (!event->prog)
1960 		goto unlock;
1961 
1962 	old_array = bpf_event_rcu_dereference(event->tp_event->prog_array);
1963 	if (!old_array)
1964 		goto put;
1965 
1966 	ret = bpf_prog_array_copy(old_array, event->prog, NULL, 0, &new_array);
1967 	if (ret < 0) {
1968 		bpf_prog_array_delete_safe(old_array, event->prog);
1969 	} else {
1970 		rcu_assign_pointer(event->tp_event->prog_array, new_array);
1971 		bpf_prog_array_free_sleepable(old_array);
1972 	}
1973 
1974 put:
1975 	prog = event->prog;
1976 	event->prog = NULL;
1977 
1978 unlock:
1979 	mutex_unlock(&bpf_event_mutex);
1980 
1981 	if (prog) {
1982 		/*
1983 		 * It could be that the bpf_prog is not sleepable (and will be freed
1984 		 * via normal RCU), but is called from a point that supports sleepable
1985 		 * programs and uses tasks-trace-RCU.
1986 		 */
1987 		synchronize_rcu_tasks_trace();
1988 
1989 		bpf_prog_put(prog);
1990 	}
1991 }
1992 
perf_event_query_prog_array(struct perf_event * event,void __user * info)1993 int perf_event_query_prog_array(struct perf_event *event, void __user *info)
1994 {
1995 	struct perf_event_query_bpf __user *uquery = info;
1996 	struct perf_event_query_bpf query = {};
1997 	struct bpf_prog_array *progs;
1998 	u32 *ids, prog_cnt, ids_len;
1999 	int ret;
2000 
2001 	if (!perfmon_capable())
2002 		return -EPERM;
2003 	if (event->attr.type != PERF_TYPE_TRACEPOINT)
2004 		return -EINVAL;
2005 	if (copy_from_user(&query, uquery, sizeof(query)))
2006 		return -EFAULT;
2007 
2008 	ids_len = query.ids_len;
2009 	if (ids_len > BPF_TRACE_MAX_PROGS)
2010 		return -E2BIG;
2011 	ids = kcalloc(ids_len, sizeof(u32), GFP_USER | __GFP_NOWARN);
2012 	if (!ids)
2013 		return -ENOMEM;
2014 	/*
2015 	 * The above kcalloc returns ZERO_SIZE_PTR when ids_len = 0, which
2016 	 * is required when user only wants to check for uquery->prog_cnt.
2017 	 * There is no need to check for it since the case is handled
2018 	 * gracefully in bpf_prog_array_copy_info.
2019 	 */
2020 
2021 	mutex_lock(&bpf_event_mutex);
2022 	progs = bpf_event_rcu_dereference(event->tp_event->prog_array);
2023 	ret = bpf_prog_array_copy_info(progs, ids, ids_len, &prog_cnt);
2024 	mutex_unlock(&bpf_event_mutex);
2025 
2026 	if (copy_to_user(&uquery->prog_cnt, &prog_cnt, sizeof(prog_cnt)) ||
2027 	    copy_to_user(uquery->ids, ids, ids_len * sizeof(u32)))
2028 		ret = -EFAULT;
2029 
2030 	kfree(ids);
2031 	return ret;
2032 }
2033 
2034 extern struct bpf_raw_event_map __start__bpf_raw_tp[];
2035 extern struct bpf_raw_event_map __stop__bpf_raw_tp[];
2036 
bpf_get_raw_tracepoint(const char * name)2037 struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name)
2038 {
2039 	struct bpf_raw_event_map *btp = __start__bpf_raw_tp;
2040 
2041 	for (; btp < __stop__bpf_raw_tp; btp++) {
2042 		if (!strcmp(btp->tp->name, name))
2043 			return btp;
2044 	}
2045 
2046 	return bpf_get_raw_tracepoint_module(name);
2047 }
2048 
bpf_put_raw_tracepoint(struct bpf_raw_event_map * btp)2049 void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp)
2050 {
2051 	struct module *mod;
2052 
2053 	guard(rcu)();
2054 	mod = __module_address((unsigned long)btp);
2055 	module_put(mod);
2056 }
2057 
2058 static __always_inline
__bpf_trace_run(struct bpf_raw_tp_link * link,u64 * args)2059 void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args)
2060 {
2061 	struct bpf_prog *prog = link->link.prog;
2062 	struct bpf_run_ctx *old_run_ctx;
2063 	struct bpf_trace_run_ctx run_ctx;
2064 
2065 	cant_sleep();
2066 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
2067 		bpf_prog_inc_misses_counter(prog);
2068 		goto out;
2069 	}
2070 
2071 	run_ctx.bpf_cookie = link->cookie;
2072 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
2073 
2074 	rcu_read_lock();
2075 	(void) bpf_prog_run(prog, args);
2076 	rcu_read_unlock();
2077 
2078 	bpf_reset_run_ctx(old_run_ctx);
2079 out:
2080 	this_cpu_dec(*(prog->active));
2081 }
2082 
2083 #define UNPACK(...)			__VA_ARGS__
2084 #define REPEAT_1(FN, DL, X, ...)	FN(X)
2085 #define REPEAT_2(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_1(FN, DL, __VA_ARGS__)
2086 #define REPEAT_3(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_2(FN, DL, __VA_ARGS__)
2087 #define REPEAT_4(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_3(FN, DL, __VA_ARGS__)
2088 #define REPEAT_5(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_4(FN, DL, __VA_ARGS__)
2089 #define REPEAT_6(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_5(FN, DL, __VA_ARGS__)
2090 #define REPEAT_7(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_6(FN, DL, __VA_ARGS__)
2091 #define REPEAT_8(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_7(FN, DL, __VA_ARGS__)
2092 #define REPEAT_9(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_8(FN, DL, __VA_ARGS__)
2093 #define REPEAT_10(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_9(FN, DL, __VA_ARGS__)
2094 #define REPEAT_11(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_10(FN, DL, __VA_ARGS__)
2095 #define REPEAT_12(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_11(FN, DL, __VA_ARGS__)
2096 #define REPEAT(X, FN, DL, ...)		REPEAT_##X(FN, DL, __VA_ARGS__)
2097 
2098 #define SARG(X)		u64 arg##X
2099 #define COPY(X)		args[X] = arg##X
2100 
2101 #define __DL_COM	(,)
2102 #define __DL_SEM	(;)
2103 
2104 #define __SEQ_0_11	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
2105 
2106 #define BPF_TRACE_DEFN_x(x)						\
2107 	void bpf_trace_run##x(struct bpf_raw_tp_link *link,		\
2108 			      REPEAT(x, SARG, __DL_COM, __SEQ_0_11))	\
2109 	{								\
2110 		u64 args[x];						\
2111 		REPEAT(x, COPY, __DL_SEM, __SEQ_0_11);			\
2112 		__bpf_trace_run(link, args);				\
2113 	}								\
2114 	EXPORT_SYMBOL_GPL(bpf_trace_run##x)
2115 BPF_TRACE_DEFN_x(1);
2116 BPF_TRACE_DEFN_x(2);
2117 BPF_TRACE_DEFN_x(3);
2118 BPF_TRACE_DEFN_x(4);
2119 BPF_TRACE_DEFN_x(5);
2120 BPF_TRACE_DEFN_x(6);
2121 BPF_TRACE_DEFN_x(7);
2122 BPF_TRACE_DEFN_x(8);
2123 BPF_TRACE_DEFN_x(9);
2124 BPF_TRACE_DEFN_x(10);
2125 BPF_TRACE_DEFN_x(11);
2126 BPF_TRACE_DEFN_x(12);
2127 
bpf_probe_register(struct bpf_raw_event_map * btp,struct bpf_raw_tp_link * link)2128 int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
2129 {
2130 	struct tracepoint *tp = btp->tp;
2131 	struct bpf_prog *prog = link->link.prog;
2132 
2133 	/*
2134 	 * check that program doesn't access arguments beyond what's
2135 	 * available in this tracepoint
2136 	 */
2137 	if (prog->aux->max_ctx_offset > btp->num_args * sizeof(u64))
2138 		return -EINVAL;
2139 
2140 	if (prog->aux->max_tp_access > btp->writable_size)
2141 		return -EINVAL;
2142 
2143 	return tracepoint_probe_register_may_exist(tp, (void *)btp->bpf_func, link);
2144 }
2145 
bpf_probe_unregister(struct bpf_raw_event_map * btp,struct bpf_raw_tp_link * link)2146 int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
2147 {
2148 	return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, link);
2149 }
2150 
bpf_get_perf_event_info(const struct perf_event * event,u32 * prog_id,u32 * fd_type,const char ** buf,u64 * probe_offset,u64 * probe_addr,unsigned long * missed)2151 int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
2152 			    u32 *fd_type, const char **buf,
2153 			    u64 *probe_offset, u64 *probe_addr,
2154 			    unsigned long *missed)
2155 {
2156 	bool is_tracepoint, is_syscall_tp;
2157 	struct bpf_prog *prog;
2158 	int flags, err = 0;
2159 
2160 	prog = event->prog;
2161 	if (!prog)
2162 		return -ENOENT;
2163 
2164 	/* not supporting BPF_PROG_TYPE_PERF_EVENT yet */
2165 	if (prog->type == BPF_PROG_TYPE_PERF_EVENT)
2166 		return -EOPNOTSUPP;
2167 
2168 	*prog_id = prog->aux->id;
2169 	flags = event->tp_event->flags;
2170 	is_tracepoint = flags & TRACE_EVENT_FL_TRACEPOINT;
2171 	is_syscall_tp = is_syscall_trace_event(event->tp_event);
2172 
2173 	if (is_tracepoint || is_syscall_tp) {
2174 		*buf = is_tracepoint ? event->tp_event->tp->name
2175 				     : event->tp_event->name;
2176 		/* We allow NULL pointer for tracepoint */
2177 		if (fd_type)
2178 			*fd_type = BPF_FD_TYPE_TRACEPOINT;
2179 		if (probe_offset)
2180 			*probe_offset = 0x0;
2181 		if (probe_addr)
2182 			*probe_addr = 0x0;
2183 	} else {
2184 		/* kprobe/uprobe */
2185 		err = -EOPNOTSUPP;
2186 #ifdef CONFIG_KPROBE_EVENTS
2187 		if (flags & TRACE_EVENT_FL_KPROBE)
2188 			err = bpf_get_kprobe_info(event, fd_type, buf,
2189 						  probe_offset, probe_addr, missed,
2190 						  event->attr.type == PERF_TYPE_TRACEPOINT);
2191 #endif
2192 #ifdef CONFIG_UPROBE_EVENTS
2193 		if (flags & TRACE_EVENT_FL_UPROBE)
2194 			err = bpf_get_uprobe_info(event, fd_type, buf,
2195 						  probe_offset, probe_addr,
2196 						  event->attr.type == PERF_TYPE_TRACEPOINT);
2197 #endif
2198 	}
2199 
2200 	return err;
2201 }
2202 
send_signal_irq_work_init(void)2203 static int __init send_signal_irq_work_init(void)
2204 {
2205 	int cpu;
2206 	struct send_signal_irq_work *work;
2207 
2208 	for_each_possible_cpu(cpu) {
2209 		work = per_cpu_ptr(&send_signal_work, cpu);
2210 		init_irq_work(&work->irq_work, do_bpf_send_signal);
2211 	}
2212 	return 0;
2213 }
2214 
2215 subsys_initcall(send_signal_irq_work_init);
2216 
2217 #ifdef CONFIG_MODULES
bpf_event_notify(struct notifier_block * nb,unsigned long op,void * module)2218 static int bpf_event_notify(struct notifier_block *nb, unsigned long op,
2219 			    void *module)
2220 {
2221 	struct bpf_trace_module *btm, *tmp;
2222 	struct module *mod = module;
2223 	int ret = 0;
2224 
2225 	if (mod->num_bpf_raw_events == 0 ||
2226 	    (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING))
2227 		goto out;
2228 
2229 	mutex_lock(&bpf_module_mutex);
2230 
2231 	switch (op) {
2232 	case MODULE_STATE_COMING:
2233 		btm = kzalloc(sizeof(*btm), GFP_KERNEL);
2234 		if (btm) {
2235 			btm->module = module;
2236 			list_add(&btm->list, &bpf_trace_modules);
2237 		} else {
2238 			ret = -ENOMEM;
2239 		}
2240 		break;
2241 	case MODULE_STATE_GOING:
2242 		list_for_each_entry_safe(btm, tmp, &bpf_trace_modules, list) {
2243 			if (btm->module == module) {
2244 				list_del(&btm->list);
2245 				kfree(btm);
2246 				break;
2247 			}
2248 		}
2249 		break;
2250 	}
2251 
2252 	mutex_unlock(&bpf_module_mutex);
2253 
2254 out:
2255 	return notifier_from_errno(ret);
2256 }
2257 
2258 static struct notifier_block bpf_module_nb = {
2259 	.notifier_call = bpf_event_notify,
2260 };
2261 
bpf_event_init(void)2262 static int __init bpf_event_init(void)
2263 {
2264 	register_module_notifier(&bpf_module_nb);
2265 	return 0;
2266 }
2267 
2268 fs_initcall(bpf_event_init);
2269 #endif /* CONFIG_MODULES */
2270 
2271 struct bpf_session_run_ctx {
2272 	struct bpf_run_ctx run_ctx;
2273 	bool is_return;
2274 	void *data;
2275 };
2276 
2277 #ifdef CONFIG_FPROBE
2278 struct bpf_kprobe_multi_link {
2279 	struct bpf_link link;
2280 	struct fprobe fp;
2281 	unsigned long *addrs;
2282 	u64 *cookies;
2283 	u32 cnt;
2284 	u32 mods_cnt;
2285 	struct module **mods;
2286 };
2287 
2288 struct bpf_kprobe_multi_run_ctx {
2289 	struct bpf_session_run_ctx session_ctx;
2290 	struct bpf_kprobe_multi_link *link;
2291 	unsigned long entry_ip;
2292 };
2293 
2294 struct user_syms {
2295 	const char **syms;
2296 	char *buf;
2297 };
2298 
2299 #ifndef CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS
2300 static DEFINE_PER_CPU(struct pt_regs, bpf_kprobe_multi_pt_regs);
2301 #define bpf_kprobe_multi_pt_regs_ptr()	this_cpu_ptr(&bpf_kprobe_multi_pt_regs)
2302 #else
2303 #define bpf_kprobe_multi_pt_regs_ptr()	(NULL)
2304 #endif
2305 
ftrace_get_entry_ip(unsigned long fentry_ip)2306 static unsigned long ftrace_get_entry_ip(unsigned long fentry_ip)
2307 {
2308 	unsigned long ip = ftrace_get_symaddr(fentry_ip);
2309 
2310 	return ip ? : fentry_ip;
2311 }
2312 
copy_user_syms(struct user_syms * us,unsigned long __user * usyms,u32 cnt)2313 static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 cnt)
2314 {
2315 	unsigned long __user usymbol;
2316 	const char **syms = NULL;
2317 	char *buf = NULL, *p;
2318 	int err = -ENOMEM;
2319 	unsigned int i;
2320 
2321 	syms = kvmalloc_array(cnt, sizeof(*syms), GFP_KERNEL);
2322 	if (!syms)
2323 		goto error;
2324 
2325 	buf = kvmalloc_array(cnt, KSYM_NAME_LEN, GFP_KERNEL);
2326 	if (!buf)
2327 		goto error;
2328 
2329 	for (p = buf, i = 0; i < cnt; i++) {
2330 		if (__get_user(usymbol, usyms + i)) {
2331 			err = -EFAULT;
2332 			goto error;
2333 		}
2334 		err = strncpy_from_user(p, (const char __user *) usymbol, KSYM_NAME_LEN);
2335 		if (err == KSYM_NAME_LEN)
2336 			err = -E2BIG;
2337 		if (err < 0)
2338 			goto error;
2339 		syms[i] = p;
2340 		p += err + 1;
2341 	}
2342 
2343 	us->syms = syms;
2344 	us->buf = buf;
2345 	return 0;
2346 
2347 error:
2348 	if (err) {
2349 		kvfree(syms);
2350 		kvfree(buf);
2351 	}
2352 	return err;
2353 }
2354 
kprobe_multi_put_modules(struct module ** mods,u32 cnt)2355 static void kprobe_multi_put_modules(struct module **mods, u32 cnt)
2356 {
2357 	u32 i;
2358 
2359 	for (i = 0; i < cnt; i++)
2360 		module_put(mods[i]);
2361 }
2362 
free_user_syms(struct user_syms * us)2363 static void free_user_syms(struct user_syms *us)
2364 {
2365 	kvfree(us->syms);
2366 	kvfree(us->buf);
2367 }
2368 
bpf_kprobe_multi_link_release(struct bpf_link * link)2369 static void bpf_kprobe_multi_link_release(struct bpf_link *link)
2370 {
2371 	struct bpf_kprobe_multi_link *kmulti_link;
2372 
2373 	kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
2374 	unregister_fprobe(&kmulti_link->fp);
2375 	kprobe_multi_put_modules(kmulti_link->mods, kmulti_link->mods_cnt);
2376 }
2377 
bpf_kprobe_multi_link_dealloc(struct bpf_link * link)2378 static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link)
2379 {
2380 	struct bpf_kprobe_multi_link *kmulti_link;
2381 
2382 	kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
2383 	kvfree(kmulti_link->addrs);
2384 	kvfree(kmulti_link->cookies);
2385 	kfree(kmulti_link->mods);
2386 	kfree(kmulti_link);
2387 }
2388 
bpf_kprobe_multi_link_fill_link_info(const struct bpf_link * link,struct bpf_link_info * info)2389 static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link,
2390 						struct bpf_link_info *info)
2391 {
2392 	u64 __user *ucookies = u64_to_user_ptr(info->kprobe_multi.cookies);
2393 	u64 __user *uaddrs = u64_to_user_ptr(info->kprobe_multi.addrs);
2394 	struct bpf_kprobe_multi_link *kmulti_link;
2395 	u32 ucount = info->kprobe_multi.count;
2396 	int err = 0, i;
2397 
2398 	if (!uaddrs ^ !ucount)
2399 		return -EINVAL;
2400 	if (ucookies && !ucount)
2401 		return -EINVAL;
2402 
2403 	kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
2404 	info->kprobe_multi.count = kmulti_link->cnt;
2405 	info->kprobe_multi.flags = kmulti_link->link.flags;
2406 	info->kprobe_multi.missed = kmulti_link->fp.nmissed;
2407 
2408 	if (!uaddrs)
2409 		return 0;
2410 	if (ucount < kmulti_link->cnt)
2411 		err = -ENOSPC;
2412 	else
2413 		ucount = kmulti_link->cnt;
2414 
2415 	if (ucookies) {
2416 		if (kmulti_link->cookies) {
2417 			if (copy_to_user(ucookies, kmulti_link->cookies, ucount * sizeof(u64)))
2418 				return -EFAULT;
2419 		} else {
2420 			for (i = 0; i < ucount; i++) {
2421 				if (put_user(0, ucookies + i))
2422 					return -EFAULT;
2423 			}
2424 		}
2425 	}
2426 
2427 	if (kallsyms_show_value(current_cred())) {
2428 		if (copy_to_user(uaddrs, kmulti_link->addrs, ucount * sizeof(u64)))
2429 			return -EFAULT;
2430 	} else {
2431 		for (i = 0; i < ucount; i++) {
2432 			if (put_user(0, uaddrs + i))
2433 				return -EFAULT;
2434 		}
2435 	}
2436 	return err;
2437 }
2438 
2439 #ifdef CONFIG_PROC_FS
bpf_kprobe_multi_show_fdinfo(const struct bpf_link * link,struct seq_file * seq)2440 static void bpf_kprobe_multi_show_fdinfo(const struct bpf_link *link,
2441 					 struct seq_file *seq)
2442 {
2443 	struct bpf_kprobe_multi_link *kmulti_link;
2444 
2445 	kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
2446 
2447 	seq_printf(seq,
2448 		   "kprobe_cnt:\t%u\n"
2449 		   "missed:\t%lu\n",
2450 		   kmulti_link->cnt,
2451 		   kmulti_link->fp.nmissed);
2452 
2453 	seq_printf(seq, "%s\t %s\n", "cookie", "func");
2454 	for (int i = 0; i < kmulti_link->cnt; i++) {
2455 		seq_printf(seq,
2456 			   "%llu\t %pS\n",
2457 			   kmulti_link->cookies[i],
2458 			   (void *)kmulti_link->addrs[i]);
2459 	}
2460 }
2461 #endif
2462 
2463 static const struct bpf_link_ops bpf_kprobe_multi_link_lops = {
2464 	.release = bpf_kprobe_multi_link_release,
2465 	.dealloc_deferred = bpf_kprobe_multi_link_dealloc,
2466 	.fill_link_info = bpf_kprobe_multi_link_fill_link_info,
2467 #ifdef CONFIG_PROC_FS
2468 	.show_fdinfo = bpf_kprobe_multi_show_fdinfo,
2469 #endif
2470 };
2471 
bpf_kprobe_multi_cookie_swap(void * a,void * b,int size,const void * priv)2472 static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv)
2473 {
2474 	const struct bpf_kprobe_multi_link *link = priv;
2475 	unsigned long *addr_a = a, *addr_b = b;
2476 	u64 *cookie_a, *cookie_b;
2477 
2478 	cookie_a = link->cookies + (addr_a - link->addrs);
2479 	cookie_b = link->cookies + (addr_b - link->addrs);
2480 
2481 	/* swap addr_a/addr_b and cookie_a/cookie_b values */
2482 	swap(*addr_a, *addr_b);
2483 	swap(*cookie_a, *cookie_b);
2484 }
2485 
bpf_kprobe_multi_addrs_cmp(const void * a,const void * b)2486 static int bpf_kprobe_multi_addrs_cmp(const void *a, const void *b)
2487 {
2488 	const unsigned long *addr_a = a, *addr_b = b;
2489 
2490 	if (*addr_a == *addr_b)
2491 		return 0;
2492 	return *addr_a < *addr_b ? -1 : 1;
2493 }
2494 
bpf_kprobe_multi_cookie_cmp(const void * a,const void * b,const void * priv)2495 static int bpf_kprobe_multi_cookie_cmp(const void *a, const void *b, const void *priv)
2496 {
2497 	return bpf_kprobe_multi_addrs_cmp(a, b);
2498 }
2499 
bpf_kprobe_multi_cookie(struct bpf_run_ctx * ctx)2500 static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx)
2501 {
2502 	struct bpf_kprobe_multi_run_ctx *run_ctx;
2503 	struct bpf_kprobe_multi_link *link;
2504 	u64 *cookie, entry_ip;
2505 	unsigned long *addr;
2506 
2507 	if (WARN_ON_ONCE(!ctx))
2508 		return 0;
2509 	run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx,
2510 			       session_ctx.run_ctx);
2511 	link = run_ctx->link;
2512 	if (!link->cookies)
2513 		return 0;
2514 	entry_ip = run_ctx->entry_ip;
2515 	addr = bsearch(&entry_ip, link->addrs, link->cnt, sizeof(entry_ip),
2516 		       bpf_kprobe_multi_addrs_cmp);
2517 	if (!addr)
2518 		return 0;
2519 	cookie = link->cookies + (addr - link->addrs);
2520 	return *cookie;
2521 }
2522 
bpf_kprobe_multi_entry_ip(struct bpf_run_ctx * ctx)2523 static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
2524 {
2525 	struct bpf_kprobe_multi_run_ctx *run_ctx;
2526 
2527 	run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx,
2528 			       session_ctx.run_ctx);
2529 	return run_ctx->entry_ip;
2530 }
2531 
2532 static int
kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link * link,unsigned long entry_ip,struct ftrace_regs * fregs,bool is_return,void * data)2533 kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,
2534 			   unsigned long entry_ip, struct ftrace_regs *fregs,
2535 			   bool is_return, void *data)
2536 {
2537 	struct bpf_kprobe_multi_run_ctx run_ctx = {
2538 		.session_ctx = {
2539 			.is_return = is_return,
2540 			.data = data,
2541 		},
2542 		.link = link,
2543 		.entry_ip = entry_ip,
2544 	};
2545 	struct bpf_run_ctx *old_run_ctx;
2546 	struct pt_regs *regs;
2547 	int err;
2548 
2549 	/*
2550 	 * graph tracer framework ensures we won't migrate, so there is no need
2551 	 * to use migrate_disable for bpf_prog_run again. The check here just for
2552 	 * __this_cpu_inc_return.
2553 	 */
2554 	cant_sleep();
2555 
2556 	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
2557 		bpf_prog_inc_misses_counter(link->link.prog);
2558 		err = 1;
2559 		goto out;
2560 	}
2561 
2562 	rcu_read_lock();
2563 	regs = ftrace_partial_regs(fregs, bpf_kprobe_multi_pt_regs_ptr());
2564 	old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx);
2565 	err = bpf_prog_run(link->link.prog, regs);
2566 	bpf_reset_run_ctx(old_run_ctx);
2567 	rcu_read_unlock();
2568 
2569  out:
2570 	__this_cpu_dec(bpf_prog_active);
2571 	return err;
2572 }
2573 
2574 static int
kprobe_multi_link_handler(struct fprobe * fp,unsigned long fentry_ip,unsigned long ret_ip,struct ftrace_regs * fregs,void * data)2575 kprobe_multi_link_handler(struct fprobe *fp, unsigned long fentry_ip,
2576 			  unsigned long ret_ip, struct ftrace_regs *fregs,
2577 			  void *data)
2578 {
2579 	struct bpf_kprobe_multi_link *link;
2580 	int err;
2581 
2582 	link = container_of(fp, struct bpf_kprobe_multi_link, fp);
2583 	err = kprobe_multi_link_prog_run(link, ftrace_get_entry_ip(fentry_ip),
2584 					 fregs, false, data);
2585 	return is_kprobe_session(link->link.prog) ? err : 0;
2586 }
2587 
2588 static void
kprobe_multi_link_exit_handler(struct fprobe * fp,unsigned long fentry_ip,unsigned long ret_ip,struct ftrace_regs * fregs,void * data)2589 kprobe_multi_link_exit_handler(struct fprobe *fp, unsigned long fentry_ip,
2590 			       unsigned long ret_ip, struct ftrace_regs *fregs,
2591 			       void *data)
2592 {
2593 	struct bpf_kprobe_multi_link *link;
2594 
2595 	link = container_of(fp, struct bpf_kprobe_multi_link, fp);
2596 	kprobe_multi_link_prog_run(link, ftrace_get_entry_ip(fentry_ip),
2597 				   fregs, true, data);
2598 }
2599 
symbols_cmp_r(const void * a,const void * b,const void * priv)2600 static int symbols_cmp_r(const void *a, const void *b, const void *priv)
2601 {
2602 	const char **str_a = (const char **) a;
2603 	const char **str_b = (const char **) b;
2604 
2605 	return strcmp(*str_a, *str_b);
2606 }
2607 
2608 struct multi_symbols_sort {
2609 	const char **funcs;
2610 	u64 *cookies;
2611 };
2612 
symbols_swap_r(void * a,void * b,int size,const void * priv)2613 static void symbols_swap_r(void *a, void *b, int size, const void *priv)
2614 {
2615 	const struct multi_symbols_sort *data = priv;
2616 	const char **name_a = a, **name_b = b;
2617 
2618 	swap(*name_a, *name_b);
2619 
2620 	/* If defined, swap also related cookies. */
2621 	if (data->cookies) {
2622 		u64 *cookie_a, *cookie_b;
2623 
2624 		cookie_a = data->cookies + (name_a - data->funcs);
2625 		cookie_b = data->cookies + (name_b - data->funcs);
2626 		swap(*cookie_a, *cookie_b);
2627 	}
2628 }
2629 
2630 struct modules_array {
2631 	struct module **mods;
2632 	int mods_cnt;
2633 	int mods_cap;
2634 };
2635 
add_module(struct modules_array * arr,struct module * mod)2636 static int add_module(struct modules_array *arr, struct module *mod)
2637 {
2638 	struct module **mods;
2639 
2640 	if (arr->mods_cnt == arr->mods_cap) {
2641 		arr->mods_cap = max(16, arr->mods_cap * 3 / 2);
2642 		mods = krealloc_array(arr->mods, arr->mods_cap, sizeof(*mods), GFP_KERNEL);
2643 		if (!mods)
2644 			return -ENOMEM;
2645 		arr->mods = mods;
2646 	}
2647 
2648 	arr->mods[arr->mods_cnt] = mod;
2649 	arr->mods_cnt++;
2650 	return 0;
2651 }
2652 
has_module(struct modules_array * arr,struct module * mod)2653 static bool has_module(struct modules_array *arr, struct module *mod)
2654 {
2655 	int i;
2656 
2657 	for (i = arr->mods_cnt - 1; i >= 0; i--) {
2658 		if (arr->mods[i] == mod)
2659 			return true;
2660 	}
2661 	return false;
2662 }
2663 
get_modules_for_addrs(struct module *** mods,unsigned long * addrs,u32 addrs_cnt)2664 static int get_modules_for_addrs(struct module ***mods, unsigned long *addrs, u32 addrs_cnt)
2665 {
2666 	struct modules_array arr = {};
2667 	u32 i, err = 0;
2668 
2669 	for (i = 0; i < addrs_cnt; i++) {
2670 		bool skip_add = false;
2671 		struct module *mod;
2672 
2673 		scoped_guard(rcu) {
2674 			mod = __module_address(addrs[i]);
2675 			/* Either no module or it's already stored  */
2676 			if (!mod || has_module(&arr, mod)) {
2677 				skip_add = true;
2678 				break; /* scoped_guard */
2679 			}
2680 			if (!try_module_get(mod))
2681 				err = -EINVAL;
2682 		}
2683 		if (skip_add)
2684 			continue;
2685 		if (err)
2686 			break;
2687 		err = add_module(&arr, mod);
2688 		if (err) {
2689 			module_put(mod);
2690 			break;
2691 		}
2692 	}
2693 
2694 	/* We return either err < 0 in case of error, ... */
2695 	if (err) {
2696 		kprobe_multi_put_modules(arr.mods, arr.mods_cnt);
2697 		kfree(arr.mods);
2698 		return err;
2699 	}
2700 
2701 	/* or number of modules found if everything is ok. */
2702 	*mods = arr.mods;
2703 	return arr.mods_cnt;
2704 }
2705 
addrs_check_error_injection_list(unsigned long * addrs,u32 cnt)2706 static int addrs_check_error_injection_list(unsigned long *addrs, u32 cnt)
2707 {
2708 	u32 i;
2709 
2710 	for (i = 0; i < cnt; i++) {
2711 		if (!within_error_injection_list(addrs[i]))
2712 			return -EINVAL;
2713 	}
2714 	return 0;
2715 }
2716 
bpf_kprobe_multi_link_attach(const union bpf_attr * attr,struct bpf_prog * prog)2717 int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
2718 {
2719 	struct bpf_kprobe_multi_link *link = NULL;
2720 	struct bpf_link_primer link_primer;
2721 	void __user *ucookies;
2722 	unsigned long *addrs;
2723 	u32 flags, cnt, size;
2724 	void __user *uaddrs;
2725 	u64 *cookies = NULL;
2726 	void __user *usyms;
2727 	int err;
2728 
2729 	/* no support for 32bit archs yet */
2730 	if (sizeof(u64) != sizeof(void *))
2731 		return -EOPNOTSUPP;
2732 
2733 	if (attr->link_create.flags)
2734 		return -EINVAL;
2735 
2736 	if (!is_kprobe_multi(prog))
2737 		return -EINVAL;
2738 
2739 	/* Writing to context is not allowed for kprobes. */
2740 	if (prog->aux->kprobe_write_ctx)
2741 		return -EINVAL;
2742 
2743 	flags = attr->link_create.kprobe_multi.flags;
2744 	if (flags & ~BPF_F_KPROBE_MULTI_RETURN)
2745 		return -EINVAL;
2746 
2747 	uaddrs = u64_to_user_ptr(attr->link_create.kprobe_multi.addrs);
2748 	usyms = u64_to_user_ptr(attr->link_create.kprobe_multi.syms);
2749 	if (!!uaddrs == !!usyms)
2750 		return -EINVAL;
2751 
2752 	cnt = attr->link_create.kprobe_multi.cnt;
2753 	if (!cnt)
2754 		return -EINVAL;
2755 	if (cnt > MAX_KPROBE_MULTI_CNT)
2756 		return -E2BIG;
2757 
2758 	size = cnt * sizeof(*addrs);
2759 	addrs = kvmalloc_array(cnt, sizeof(*addrs), GFP_KERNEL);
2760 	if (!addrs)
2761 		return -ENOMEM;
2762 
2763 	ucookies = u64_to_user_ptr(attr->link_create.kprobe_multi.cookies);
2764 	if (ucookies) {
2765 		cookies = kvmalloc_array(cnt, sizeof(*addrs), GFP_KERNEL);
2766 		if (!cookies) {
2767 			err = -ENOMEM;
2768 			goto error;
2769 		}
2770 		if (copy_from_user(cookies, ucookies, size)) {
2771 			err = -EFAULT;
2772 			goto error;
2773 		}
2774 	}
2775 
2776 	if (uaddrs) {
2777 		if (copy_from_user(addrs, uaddrs, size)) {
2778 			err = -EFAULT;
2779 			goto error;
2780 		}
2781 	} else {
2782 		struct multi_symbols_sort data = {
2783 			.cookies = cookies,
2784 		};
2785 		struct user_syms us;
2786 
2787 		err = copy_user_syms(&us, usyms, cnt);
2788 		if (err)
2789 			goto error;
2790 
2791 		if (cookies)
2792 			data.funcs = us.syms;
2793 
2794 		sort_r(us.syms, cnt, sizeof(*us.syms), symbols_cmp_r,
2795 		       symbols_swap_r, &data);
2796 
2797 		err = ftrace_lookup_symbols(us.syms, cnt, addrs);
2798 		free_user_syms(&us);
2799 		if (err)
2800 			goto error;
2801 	}
2802 
2803 	if (prog->kprobe_override && addrs_check_error_injection_list(addrs, cnt)) {
2804 		err = -EINVAL;
2805 		goto error;
2806 	}
2807 
2808 	link = kzalloc(sizeof(*link), GFP_KERNEL);
2809 	if (!link) {
2810 		err = -ENOMEM;
2811 		goto error;
2812 	}
2813 
2814 	bpf_link_init(&link->link, BPF_LINK_TYPE_KPROBE_MULTI,
2815 		      &bpf_kprobe_multi_link_lops, prog, attr->link_create.attach_type);
2816 
2817 	err = bpf_link_prime(&link->link, &link_primer);
2818 	if (err)
2819 		goto error;
2820 
2821 	if (!(flags & BPF_F_KPROBE_MULTI_RETURN))
2822 		link->fp.entry_handler = kprobe_multi_link_handler;
2823 	if ((flags & BPF_F_KPROBE_MULTI_RETURN) || is_kprobe_session(prog))
2824 		link->fp.exit_handler = kprobe_multi_link_exit_handler;
2825 	if (is_kprobe_session(prog))
2826 		link->fp.entry_data_size = sizeof(u64);
2827 
2828 	link->addrs = addrs;
2829 	link->cookies = cookies;
2830 	link->cnt = cnt;
2831 	link->link.flags = flags;
2832 
2833 	if (cookies) {
2834 		/*
2835 		 * Sorting addresses will trigger sorting cookies as well
2836 		 * (check bpf_kprobe_multi_cookie_swap). This way we can
2837 		 * find cookie based on the address in bpf_get_attach_cookie
2838 		 * helper.
2839 		 */
2840 		sort_r(addrs, cnt, sizeof(*addrs),
2841 		       bpf_kprobe_multi_cookie_cmp,
2842 		       bpf_kprobe_multi_cookie_swap,
2843 		       link);
2844 	}
2845 
2846 	err = get_modules_for_addrs(&link->mods, addrs, cnt);
2847 	if (err < 0) {
2848 		bpf_link_cleanup(&link_primer);
2849 		return err;
2850 	}
2851 	link->mods_cnt = err;
2852 
2853 	err = register_fprobe_ips(&link->fp, addrs, cnt);
2854 	if (err) {
2855 		kprobe_multi_put_modules(link->mods, link->mods_cnt);
2856 		bpf_link_cleanup(&link_primer);
2857 		return err;
2858 	}
2859 
2860 	return bpf_link_settle(&link_primer);
2861 
2862 error:
2863 	kfree(link);
2864 	kvfree(addrs);
2865 	kvfree(cookies);
2866 	return err;
2867 }
2868 #else /* !CONFIG_FPROBE */
bpf_kprobe_multi_link_attach(const union bpf_attr * attr,struct bpf_prog * prog)2869 int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
2870 {
2871 	return -EOPNOTSUPP;
2872 }
bpf_kprobe_multi_cookie(struct bpf_run_ctx * ctx)2873 static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx)
2874 {
2875 	return 0;
2876 }
bpf_kprobe_multi_entry_ip(struct bpf_run_ctx * ctx)2877 static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
2878 {
2879 	return 0;
2880 }
2881 #endif
2882 
2883 #ifdef CONFIG_UPROBES
2884 struct bpf_uprobe_multi_link;
2885 
2886 struct bpf_uprobe {
2887 	struct bpf_uprobe_multi_link *link;
2888 	loff_t offset;
2889 	unsigned long ref_ctr_offset;
2890 	u64 cookie;
2891 	struct uprobe *uprobe;
2892 	struct uprobe_consumer consumer;
2893 	bool session;
2894 };
2895 
2896 struct bpf_uprobe_multi_link {
2897 	struct path path;
2898 	struct bpf_link link;
2899 	u32 cnt;
2900 	struct bpf_uprobe *uprobes;
2901 	struct task_struct *task;
2902 };
2903 
2904 struct bpf_uprobe_multi_run_ctx {
2905 	struct bpf_session_run_ctx session_ctx;
2906 	unsigned long entry_ip;
2907 	struct bpf_uprobe *uprobe;
2908 };
2909 
bpf_uprobe_unregister(struct bpf_uprobe * uprobes,u32 cnt)2910 static void bpf_uprobe_unregister(struct bpf_uprobe *uprobes, u32 cnt)
2911 {
2912 	u32 i;
2913 
2914 	for (i = 0; i < cnt; i++)
2915 		uprobe_unregister_nosync(uprobes[i].uprobe, &uprobes[i].consumer);
2916 
2917 	if (cnt)
2918 		uprobe_unregister_sync();
2919 }
2920 
bpf_uprobe_multi_link_release(struct bpf_link * link)2921 static void bpf_uprobe_multi_link_release(struct bpf_link *link)
2922 {
2923 	struct bpf_uprobe_multi_link *umulti_link;
2924 
2925 	umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
2926 	bpf_uprobe_unregister(umulti_link->uprobes, umulti_link->cnt);
2927 	if (umulti_link->task)
2928 		put_task_struct(umulti_link->task);
2929 	path_put(&umulti_link->path);
2930 }
2931 
bpf_uprobe_multi_link_dealloc(struct bpf_link * link)2932 static void bpf_uprobe_multi_link_dealloc(struct bpf_link *link)
2933 {
2934 	struct bpf_uprobe_multi_link *umulti_link;
2935 
2936 	umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
2937 	kvfree(umulti_link->uprobes);
2938 	kfree(umulti_link);
2939 }
2940 
bpf_uprobe_multi_link_fill_link_info(const struct bpf_link * link,struct bpf_link_info * info)2941 static int bpf_uprobe_multi_link_fill_link_info(const struct bpf_link *link,
2942 						struct bpf_link_info *info)
2943 {
2944 	u64 __user *uref_ctr_offsets = u64_to_user_ptr(info->uprobe_multi.ref_ctr_offsets);
2945 	u64 __user *ucookies = u64_to_user_ptr(info->uprobe_multi.cookies);
2946 	u64 __user *uoffsets = u64_to_user_ptr(info->uprobe_multi.offsets);
2947 	u64 __user *upath = u64_to_user_ptr(info->uprobe_multi.path);
2948 	u32 upath_size = info->uprobe_multi.path_size;
2949 	struct bpf_uprobe_multi_link *umulti_link;
2950 	u32 ucount = info->uprobe_multi.count;
2951 	int err = 0, i;
2952 	char *p, *buf;
2953 	long left = 0;
2954 
2955 	if (!upath ^ !upath_size)
2956 		return -EINVAL;
2957 
2958 	if ((uoffsets || uref_ctr_offsets || ucookies) && !ucount)
2959 		return -EINVAL;
2960 
2961 	umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
2962 	info->uprobe_multi.count = umulti_link->cnt;
2963 	info->uprobe_multi.flags = umulti_link->link.flags;
2964 	info->uprobe_multi.pid = umulti_link->task ?
2965 				 task_pid_nr_ns(umulti_link->task, task_active_pid_ns(current)) : 0;
2966 
2967 	upath_size = upath_size ? min_t(u32, upath_size, PATH_MAX) : PATH_MAX;
2968 	buf = kmalloc(upath_size, GFP_KERNEL);
2969 	if (!buf)
2970 		return -ENOMEM;
2971 	p = d_path(&umulti_link->path, buf, upath_size);
2972 	if (IS_ERR(p)) {
2973 		kfree(buf);
2974 		return PTR_ERR(p);
2975 	}
2976 	upath_size = buf + upath_size - p;
2977 
2978 	if (upath)
2979 		left = copy_to_user(upath, p, upath_size);
2980 	kfree(buf);
2981 	if (left)
2982 		return -EFAULT;
2983 	info->uprobe_multi.path_size = upath_size;
2984 
2985 	if (!uoffsets && !ucookies && !uref_ctr_offsets)
2986 		return 0;
2987 
2988 	if (ucount < umulti_link->cnt)
2989 		err = -ENOSPC;
2990 	else
2991 		ucount = umulti_link->cnt;
2992 
2993 	for (i = 0; i < ucount; i++) {
2994 		if (uoffsets &&
2995 		    put_user(umulti_link->uprobes[i].offset, uoffsets + i))
2996 			return -EFAULT;
2997 		if (uref_ctr_offsets &&
2998 		    put_user(umulti_link->uprobes[i].ref_ctr_offset, uref_ctr_offsets + i))
2999 			return -EFAULT;
3000 		if (ucookies &&
3001 		    put_user(umulti_link->uprobes[i].cookie, ucookies + i))
3002 			return -EFAULT;
3003 	}
3004 
3005 	return err;
3006 }
3007 
3008 #ifdef CONFIG_PROC_FS
bpf_uprobe_multi_show_fdinfo(const struct bpf_link * link,struct seq_file * seq)3009 static void bpf_uprobe_multi_show_fdinfo(const struct bpf_link *link,
3010 					 struct seq_file *seq)
3011 {
3012 	struct bpf_uprobe_multi_link *umulti_link;
3013 	char *p, *buf;
3014 	pid_t pid;
3015 
3016 	umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
3017 
3018 	buf = kmalloc(PATH_MAX, GFP_KERNEL);
3019 	if (!buf)
3020 		return;
3021 
3022 	p = d_path(&umulti_link->path, buf, PATH_MAX);
3023 	if (IS_ERR(p)) {
3024 		kfree(buf);
3025 		return;
3026 	}
3027 
3028 	pid = umulti_link->task ?
3029 	      task_pid_nr_ns(umulti_link->task, task_active_pid_ns(current)) : 0;
3030 	seq_printf(seq,
3031 		   "uprobe_cnt:\t%u\n"
3032 		   "pid:\t%u\n"
3033 		   "path:\t%s\n",
3034 		   umulti_link->cnt, pid, p);
3035 
3036 	seq_printf(seq, "%s\t %s\t %s\n", "cookie", "offset", "ref_ctr_offset");
3037 	for (int i = 0; i < umulti_link->cnt; i++) {
3038 		seq_printf(seq,
3039 			   "%llu\t %#llx\t %#lx\n",
3040 			   umulti_link->uprobes[i].cookie,
3041 			   umulti_link->uprobes[i].offset,
3042 			   umulti_link->uprobes[i].ref_ctr_offset);
3043 	}
3044 
3045 	kfree(buf);
3046 }
3047 #endif
3048 
3049 static const struct bpf_link_ops bpf_uprobe_multi_link_lops = {
3050 	.release = bpf_uprobe_multi_link_release,
3051 	.dealloc_deferred = bpf_uprobe_multi_link_dealloc,
3052 	.fill_link_info = bpf_uprobe_multi_link_fill_link_info,
3053 #ifdef CONFIG_PROC_FS
3054 	.show_fdinfo = bpf_uprobe_multi_show_fdinfo,
3055 #endif
3056 };
3057 
uprobe_prog_run(struct bpf_uprobe * uprobe,unsigned long entry_ip,struct pt_regs * regs,bool is_return,void * data)3058 static int uprobe_prog_run(struct bpf_uprobe *uprobe,
3059 			   unsigned long entry_ip,
3060 			   struct pt_regs *regs,
3061 			   bool is_return, void *data)
3062 {
3063 	struct bpf_uprobe_multi_link *link = uprobe->link;
3064 	struct bpf_uprobe_multi_run_ctx run_ctx = {
3065 		.session_ctx = {
3066 			.is_return = is_return,
3067 			.data = data,
3068 		},
3069 		.entry_ip = entry_ip,
3070 		.uprobe = uprobe,
3071 	};
3072 	struct bpf_prog *prog = link->link.prog;
3073 	bool sleepable = prog->sleepable;
3074 	struct bpf_run_ctx *old_run_ctx;
3075 	int err;
3076 
3077 	if (link->task && !same_thread_group(current, link->task))
3078 		return 0;
3079 
3080 	if (sleepable)
3081 		rcu_read_lock_trace();
3082 	else
3083 		rcu_read_lock();
3084 
3085 	migrate_disable();
3086 
3087 	old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx);
3088 	err = bpf_prog_run(link->link.prog, regs);
3089 	bpf_reset_run_ctx(old_run_ctx);
3090 
3091 	migrate_enable();
3092 
3093 	if (sleepable)
3094 		rcu_read_unlock_trace();
3095 	else
3096 		rcu_read_unlock();
3097 	return err;
3098 }
3099 
3100 static bool
uprobe_multi_link_filter(struct uprobe_consumer * con,struct mm_struct * mm)3101 uprobe_multi_link_filter(struct uprobe_consumer *con, struct mm_struct *mm)
3102 {
3103 	struct bpf_uprobe *uprobe;
3104 
3105 	uprobe = container_of(con, struct bpf_uprobe, consumer);
3106 	return uprobe->link->task->mm == mm;
3107 }
3108 
3109 static int
uprobe_multi_link_handler(struct uprobe_consumer * con,struct pt_regs * regs,__u64 * data)3110 uprobe_multi_link_handler(struct uprobe_consumer *con, struct pt_regs *regs,
3111 			  __u64 *data)
3112 {
3113 	struct bpf_uprobe *uprobe;
3114 	int ret;
3115 
3116 	uprobe = container_of(con, struct bpf_uprobe, consumer);
3117 	ret = uprobe_prog_run(uprobe, instruction_pointer(regs), regs, false, data);
3118 	if (uprobe->session)
3119 		return ret ? UPROBE_HANDLER_IGNORE : 0;
3120 	return 0;
3121 }
3122 
3123 static int
uprobe_multi_link_ret_handler(struct uprobe_consumer * con,unsigned long func,struct pt_regs * regs,__u64 * data)3124 uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, struct pt_regs *regs,
3125 			      __u64 *data)
3126 {
3127 	struct bpf_uprobe *uprobe;
3128 
3129 	uprobe = container_of(con, struct bpf_uprobe, consumer);
3130 	uprobe_prog_run(uprobe, func, regs, true, data);
3131 	return 0;
3132 }
3133 
bpf_uprobe_multi_entry_ip(struct bpf_run_ctx * ctx)3134 static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
3135 {
3136 	struct bpf_uprobe_multi_run_ctx *run_ctx;
3137 
3138 	run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3139 			       session_ctx.run_ctx);
3140 	return run_ctx->entry_ip;
3141 }
3142 
bpf_uprobe_multi_cookie(struct bpf_run_ctx * ctx)3143 static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx)
3144 {
3145 	struct bpf_uprobe_multi_run_ctx *run_ctx;
3146 
3147 	run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3148 			       session_ctx.run_ctx);
3149 	return run_ctx->uprobe->cookie;
3150 }
3151 
bpf_uprobe_multi_link_attach(const union bpf_attr * attr,struct bpf_prog * prog)3152 int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
3153 {
3154 	struct bpf_uprobe_multi_link *link = NULL;
3155 	unsigned long __user *uref_ctr_offsets;
3156 	struct bpf_link_primer link_primer;
3157 	struct bpf_uprobe *uprobes = NULL;
3158 	struct task_struct *task = NULL;
3159 	unsigned long __user *uoffsets;
3160 	u64 __user *ucookies;
3161 	void __user *upath;
3162 	u32 flags, cnt, i;
3163 	struct path path;
3164 	char *name;
3165 	pid_t pid;
3166 	int err;
3167 
3168 	/* no support for 32bit archs yet */
3169 	if (sizeof(u64) != sizeof(void *))
3170 		return -EOPNOTSUPP;
3171 
3172 	if (attr->link_create.flags)
3173 		return -EINVAL;
3174 
3175 	if (!is_uprobe_multi(prog))
3176 		return -EINVAL;
3177 
3178 	flags = attr->link_create.uprobe_multi.flags;
3179 	if (flags & ~BPF_F_UPROBE_MULTI_RETURN)
3180 		return -EINVAL;
3181 
3182 	/*
3183 	 * path, offsets and cnt are mandatory,
3184 	 * ref_ctr_offsets and cookies are optional
3185 	 */
3186 	upath = u64_to_user_ptr(attr->link_create.uprobe_multi.path);
3187 	uoffsets = u64_to_user_ptr(attr->link_create.uprobe_multi.offsets);
3188 	cnt = attr->link_create.uprobe_multi.cnt;
3189 	pid = attr->link_create.uprobe_multi.pid;
3190 
3191 	if (!upath || !uoffsets || !cnt || pid < 0)
3192 		return -EINVAL;
3193 	if (cnt > MAX_UPROBE_MULTI_CNT)
3194 		return -E2BIG;
3195 
3196 	uref_ctr_offsets = u64_to_user_ptr(attr->link_create.uprobe_multi.ref_ctr_offsets);
3197 	ucookies = u64_to_user_ptr(attr->link_create.uprobe_multi.cookies);
3198 
3199 	name = strndup_user(upath, PATH_MAX);
3200 	if (IS_ERR(name)) {
3201 		err = PTR_ERR(name);
3202 		return err;
3203 	}
3204 
3205 	err = kern_path(name, LOOKUP_FOLLOW, &path);
3206 	kfree(name);
3207 	if (err)
3208 		return err;
3209 
3210 	if (!d_is_reg(path.dentry)) {
3211 		err = -EBADF;
3212 		goto error_path_put;
3213 	}
3214 
3215 	if (pid) {
3216 		rcu_read_lock();
3217 		task = get_pid_task(find_vpid(pid), PIDTYPE_TGID);
3218 		rcu_read_unlock();
3219 		if (!task) {
3220 			err = -ESRCH;
3221 			goto error_path_put;
3222 		}
3223 	}
3224 
3225 	err = -ENOMEM;
3226 
3227 	link = kzalloc(sizeof(*link), GFP_KERNEL);
3228 	uprobes = kvcalloc(cnt, sizeof(*uprobes), GFP_KERNEL);
3229 
3230 	if (!uprobes || !link)
3231 		goto error_free;
3232 
3233 	for (i = 0; i < cnt; i++) {
3234 		if (__get_user(uprobes[i].offset, uoffsets + i)) {
3235 			err = -EFAULT;
3236 			goto error_free;
3237 		}
3238 		if (uprobes[i].offset < 0) {
3239 			err = -EINVAL;
3240 			goto error_free;
3241 		}
3242 		if (uref_ctr_offsets && __get_user(uprobes[i].ref_ctr_offset, uref_ctr_offsets + i)) {
3243 			err = -EFAULT;
3244 			goto error_free;
3245 		}
3246 		if (ucookies && __get_user(uprobes[i].cookie, ucookies + i)) {
3247 			err = -EFAULT;
3248 			goto error_free;
3249 		}
3250 
3251 		uprobes[i].link = link;
3252 
3253 		if (!(flags & BPF_F_UPROBE_MULTI_RETURN))
3254 			uprobes[i].consumer.handler = uprobe_multi_link_handler;
3255 		if (flags & BPF_F_UPROBE_MULTI_RETURN || is_uprobe_session(prog))
3256 			uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3257 		if (is_uprobe_session(prog))
3258 			uprobes[i].session = true;
3259 		if (pid)
3260 			uprobes[i].consumer.filter = uprobe_multi_link_filter;
3261 	}
3262 
3263 	link->cnt = cnt;
3264 	link->uprobes = uprobes;
3265 	link->path = path;
3266 	link->task = task;
3267 	link->link.flags = flags;
3268 
3269 	bpf_link_init(&link->link, BPF_LINK_TYPE_UPROBE_MULTI,
3270 		      &bpf_uprobe_multi_link_lops, prog, attr->link_create.attach_type);
3271 
3272 	for (i = 0; i < cnt; i++) {
3273 		uprobes[i].uprobe = uprobe_register(d_real_inode(link->path.dentry),
3274 						    uprobes[i].offset,
3275 						    uprobes[i].ref_ctr_offset,
3276 						    &uprobes[i].consumer);
3277 		if (IS_ERR(uprobes[i].uprobe)) {
3278 			err = PTR_ERR(uprobes[i].uprobe);
3279 			link->cnt = i;
3280 			goto error_unregister;
3281 		}
3282 	}
3283 
3284 	err = bpf_link_prime(&link->link, &link_primer);
3285 	if (err)
3286 		goto error_unregister;
3287 
3288 	return bpf_link_settle(&link_primer);
3289 
3290 error_unregister:
3291 	bpf_uprobe_unregister(uprobes, link->cnt);
3292 
3293 error_free:
3294 	kvfree(uprobes);
3295 	kfree(link);
3296 	if (task)
3297 		put_task_struct(task);
3298 error_path_put:
3299 	path_put(&path);
3300 	return err;
3301 }
3302 #else /* !CONFIG_UPROBES */
bpf_uprobe_multi_link_attach(const union bpf_attr * attr,struct bpf_prog * prog)3303 int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
3304 {
3305 	return -EOPNOTSUPP;
3306 }
bpf_uprobe_multi_cookie(struct bpf_run_ctx * ctx)3307 static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx)
3308 {
3309 	return 0;
3310 }
bpf_uprobe_multi_entry_ip(struct bpf_run_ctx * ctx)3311 static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
3312 {
3313 	return 0;
3314 }
3315 #endif /* CONFIG_UPROBES */
3316 
3317 __bpf_kfunc_start_defs();
3318 
bpf_session_is_return(void)3319 __bpf_kfunc bool bpf_session_is_return(void)
3320 {
3321 	struct bpf_session_run_ctx *session_ctx;
3322 
3323 	session_ctx = container_of(current->bpf_ctx, struct bpf_session_run_ctx, run_ctx);
3324 	return session_ctx->is_return;
3325 }
3326 
bpf_session_cookie(void)3327 __bpf_kfunc __u64 *bpf_session_cookie(void)
3328 {
3329 	struct bpf_session_run_ctx *session_ctx;
3330 
3331 	session_ctx = container_of(current->bpf_ctx, struct bpf_session_run_ctx, run_ctx);
3332 	return session_ctx->data;
3333 }
3334 
3335 __bpf_kfunc_end_defs();
3336 
3337 BTF_KFUNCS_START(kprobe_multi_kfunc_set_ids)
BTF_ID_FLAGS(func,bpf_session_is_return)3338 BTF_ID_FLAGS(func, bpf_session_is_return)
3339 BTF_ID_FLAGS(func, bpf_session_cookie)
3340 BTF_KFUNCS_END(kprobe_multi_kfunc_set_ids)
3341 
3342 static int bpf_kprobe_multi_filter(const struct bpf_prog *prog, u32 kfunc_id)
3343 {
3344 	if (!btf_id_set8_contains(&kprobe_multi_kfunc_set_ids, kfunc_id))
3345 		return 0;
3346 
3347 	if (!is_kprobe_session(prog) && !is_uprobe_session(prog))
3348 		return -EACCES;
3349 
3350 	return 0;
3351 }
3352 
3353 static const struct btf_kfunc_id_set bpf_kprobe_multi_kfunc_set = {
3354 	.owner = THIS_MODULE,
3355 	.set = &kprobe_multi_kfunc_set_ids,
3356 	.filter = bpf_kprobe_multi_filter,
3357 };
3358 
bpf_kprobe_multi_kfuncs_init(void)3359 static int __init bpf_kprobe_multi_kfuncs_init(void)
3360 {
3361 	return register_btf_kfunc_id_set(BPF_PROG_TYPE_KPROBE, &bpf_kprobe_multi_kfunc_set);
3362 }
3363 
3364 late_initcall(bpf_kprobe_multi_kfuncs_init);
3365 
3366 typedef int (*copy_fn_t)(void *dst, const void *src, u32 size, struct task_struct *tsk);
3367 
3368 /*
3369  * The __always_inline is to make sure the compiler doesn't
3370  * generate indirect calls into callbacks, which is expensive,
3371  * on some kernel configurations. This allows compiler to put
3372  * direct calls into all the specific callback implementations
3373  * (copy_user_data_sleepable, copy_user_data_nofault, and so on)
3374  */
__bpf_dynptr_copy_str(struct bpf_dynptr * dptr,u32 doff,u32 size,const void * unsafe_src,copy_fn_t str_copy_fn,struct task_struct * tsk)3375 static __always_inline int __bpf_dynptr_copy_str(struct bpf_dynptr *dptr, u32 doff, u32 size,
3376 						 const void *unsafe_src,
3377 						 copy_fn_t str_copy_fn,
3378 						 struct task_struct *tsk)
3379 {
3380 	struct bpf_dynptr_kern *dst;
3381 	u32 chunk_sz, off;
3382 	void *dst_slice;
3383 	int cnt, err;
3384 	char buf[256];
3385 
3386 	dst_slice = bpf_dynptr_slice_rdwr(dptr, doff, NULL, size);
3387 	if (likely(dst_slice))
3388 		return str_copy_fn(dst_slice, unsafe_src, size, tsk);
3389 
3390 	dst = (struct bpf_dynptr_kern *)dptr;
3391 	if (bpf_dynptr_check_off_len(dst, doff, size))
3392 		return -E2BIG;
3393 
3394 	for (off = 0; off < size; off += chunk_sz - 1) {
3395 		chunk_sz = min_t(u32, sizeof(buf), size - off);
3396 		/* Expect str_copy_fn to return count of copied bytes, including
3397 		 * zero terminator. Next iteration increment off by chunk_sz - 1 to
3398 		 * overwrite NUL.
3399 		 */
3400 		cnt = str_copy_fn(buf, unsafe_src + off, chunk_sz, tsk);
3401 		if (cnt < 0)
3402 			return cnt;
3403 		err = __bpf_dynptr_write(dst, doff + off, buf, cnt, 0);
3404 		if (err)
3405 			return err;
3406 		if (cnt < chunk_sz || chunk_sz == 1) /* we are done */
3407 			return off + cnt;
3408 	}
3409 	return off;
3410 }
3411 
__bpf_dynptr_copy(const struct bpf_dynptr * dptr,u32 doff,u32 size,const void * unsafe_src,copy_fn_t copy_fn,struct task_struct * tsk)3412 static __always_inline int __bpf_dynptr_copy(const struct bpf_dynptr *dptr, u32 doff,
3413 					     u32 size, const void *unsafe_src,
3414 					     copy_fn_t copy_fn, struct task_struct *tsk)
3415 {
3416 	struct bpf_dynptr_kern *dst;
3417 	void *dst_slice;
3418 	char buf[256];
3419 	u32 off, chunk_sz;
3420 	int err;
3421 
3422 	dst_slice = bpf_dynptr_slice_rdwr(dptr, doff, NULL, size);
3423 	if (likely(dst_slice))
3424 		return copy_fn(dst_slice, unsafe_src, size, tsk);
3425 
3426 	dst = (struct bpf_dynptr_kern *)dptr;
3427 	if (bpf_dynptr_check_off_len(dst, doff, size))
3428 		return -E2BIG;
3429 
3430 	for (off = 0; off < size; off += chunk_sz) {
3431 		chunk_sz = min_t(u32, sizeof(buf), size - off);
3432 		err = copy_fn(buf, unsafe_src + off, chunk_sz, tsk);
3433 		if (err)
3434 			return err;
3435 		err = __bpf_dynptr_write(dst, doff + off, buf, chunk_sz, 0);
3436 		if (err)
3437 			return err;
3438 	}
3439 	return 0;
3440 }
3441 
copy_user_data_nofault(void * dst,const void * unsafe_src,u32 size,struct task_struct * tsk)3442 static __always_inline int copy_user_data_nofault(void *dst, const void *unsafe_src,
3443 						  u32 size, struct task_struct *tsk)
3444 {
3445 	return copy_from_user_nofault(dst, (const void __user *)unsafe_src, size);
3446 }
3447 
copy_user_data_sleepable(void * dst,const void * unsafe_src,u32 size,struct task_struct * tsk)3448 static __always_inline int copy_user_data_sleepable(void *dst, const void *unsafe_src,
3449 						    u32 size, struct task_struct *tsk)
3450 {
3451 	int ret;
3452 
3453 	if (!tsk) { /* Read from the current task */
3454 		ret = copy_from_user(dst, (const void __user *)unsafe_src, size);
3455 		if (ret)
3456 			return -EFAULT;
3457 		return 0;
3458 	}
3459 
3460 	ret = access_process_vm(tsk, (unsigned long)unsafe_src, dst, size, 0);
3461 	if (ret != size)
3462 		return -EFAULT;
3463 	return 0;
3464 }
3465 
copy_kernel_data_nofault(void * dst,const void * unsafe_src,u32 size,struct task_struct * tsk)3466 static __always_inline int copy_kernel_data_nofault(void *dst, const void *unsafe_src,
3467 						    u32 size, struct task_struct *tsk)
3468 {
3469 	return copy_from_kernel_nofault(dst, unsafe_src, size);
3470 }
3471 
copy_user_str_nofault(void * dst,const void * unsafe_src,u32 size,struct task_struct * tsk)3472 static __always_inline int copy_user_str_nofault(void *dst, const void *unsafe_src,
3473 						 u32 size, struct task_struct *tsk)
3474 {
3475 	return strncpy_from_user_nofault(dst, (const void __user *)unsafe_src, size);
3476 }
3477 
copy_user_str_sleepable(void * dst,const void * unsafe_src,u32 size,struct task_struct * tsk)3478 static __always_inline int copy_user_str_sleepable(void *dst, const void *unsafe_src,
3479 						   u32 size, struct task_struct *tsk)
3480 {
3481 	int ret;
3482 
3483 	if (unlikely(size == 0))
3484 		return 0;
3485 
3486 	if (tsk) {
3487 		ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_src, dst, size, 0);
3488 	} else {
3489 		ret = strncpy_from_user(dst, (const void __user *)unsafe_src, size - 1);
3490 		/* strncpy_from_user does not guarantee NUL termination */
3491 		if (ret >= 0)
3492 			((char *)dst)[ret] = '\0';
3493 	}
3494 
3495 	if (ret < 0)
3496 		return ret;
3497 	return ret + 1;
3498 }
3499 
copy_kernel_str_nofault(void * dst,const void * unsafe_src,u32 size,struct task_struct * tsk)3500 static __always_inline int copy_kernel_str_nofault(void *dst, const void *unsafe_src,
3501 						   u32 size, struct task_struct *tsk)
3502 {
3503 	return strncpy_from_kernel_nofault(dst, unsafe_src, size);
3504 }
3505 
3506 __bpf_kfunc_start_defs();
3507 
bpf_send_signal_task(struct task_struct * task,int sig,enum pid_type type,u64 value)3508 __bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type,
3509 				     u64 value)
3510 {
3511 	if (type != PIDTYPE_PID && type != PIDTYPE_TGID)
3512 		return -EINVAL;
3513 
3514 	return bpf_send_signal_common(sig, type, task, value);
3515 }
3516 
bpf_probe_read_user_dynptr(struct bpf_dynptr * dptr,u32 off,u32 size,const void __user * unsafe_ptr__ign)3517 __bpf_kfunc int bpf_probe_read_user_dynptr(struct bpf_dynptr *dptr, u32 off,
3518 					   u32 size, const void __user *unsafe_ptr__ign)
3519 {
3520 	return __bpf_dynptr_copy(dptr, off, size, (const void *)unsafe_ptr__ign,
3521 				 copy_user_data_nofault, NULL);
3522 }
3523 
bpf_probe_read_kernel_dynptr(struct bpf_dynptr * dptr,u32 off,u32 size,const void * unsafe_ptr__ign)3524 __bpf_kfunc int bpf_probe_read_kernel_dynptr(struct bpf_dynptr *dptr, u32 off,
3525 					     u32 size, const void *unsafe_ptr__ign)
3526 {
3527 	return __bpf_dynptr_copy(dptr, off, size, unsafe_ptr__ign,
3528 				 copy_kernel_data_nofault, NULL);
3529 }
3530 
bpf_probe_read_user_str_dynptr(struct bpf_dynptr * dptr,u32 off,u32 size,const void __user * unsafe_ptr__ign)3531 __bpf_kfunc int bpf_probe_read_user_str_dynptr(struct bpf_dynptr *dptr, u32 off,
3532 					       u32 size, const void __user *unsafe_ptr__ign)
3533 {
3534 	return __bpf_dynptr_copy_str(dptr, off, size, (const void *)unsafe_ptr__ign,
3535 				     copy_user_str_nofault, NULL);
3536 }
3537 
bpf_probe_read_kernel_str_dynptr(struct bpf_dynptr * dptr,u32 off,u32 size,const void * unsafe_ptr__ign)3538 __bpf_kfunc int bpf_probe_read_kernel_str_dynptr(struct bpf_dynptr *dptr, u32 off,
3539 						 u32 size, const void *unsafe_ptr__ign)
3540 {
3541 	return __bpf_dynptr_copy_str(dptr, off, size, unsafe_ptr__ign,
3542 				     copy_kernel_str_nofault, NULL);
3543 }
3544 
bpf_copy_from_user_dynptr(struct bpf_dynptr * dptr,u32 off,u32 size,const void __user * unsafe_ptr__ign)3545 __bpf_kfunc int bpf_copy_from_user_dynptr(struct bpf_dynptr *dptr, u32 off,
3546 					  u32 size, const void __user *unsafe_ptr__ign)
3547 {
3548 	return __bpf_dynptr_copy(dptr, off, size, (const void *)unsafe_ptr__ign,
3549 				 copy_user_data_sleepable, NULL);
3550 }
3551 
bpf_copy_from_user_str_dynptr(struct bpf_dynptr * dptr,u32 off,u32 size,const void __user * unsafe_ptr__ign)3552 __bpf_kfunc int bpf_copy_from_user_str_dynptr(struct bpf_dynptr *dptr, u32 off,
3553 					      u32 size, const void __user *unsafe_ptr__ign)
3554 {
3555 	return __bpf_dynptr_copy_str(dptr, off, size, (const void *)unsafe_ptr__ign,
3556 				     copy_user_str_sleepable, NULL);
3557 }
3558 
bpf_copy_from_user_task_dynptr(struct bpf_dynptr * dptr,u32 off,u32 size,const void __user * unsafe_ptr__ign,struct task_struct * tsk)3559 __bpf_kfunc int bpf_copy_from_user_task_dynptr(struct bpf_dynptr *dptr, u32 off,
3560 					       u32 size, const void __user *unsafe_ptr__ign,
3561 					       struct task_struct *tsk)
3562 {
3563 	return __bpf_dynptr_copy(dptr, off, size, (const void *)unsafe_ptr__ign,
3564 				 copy_user_data_sleepable, tsk);
3565 }
3566 
bpf_copy_from_user_task_str_dynptr(struct bpf_dynptr * dptr,u32 off,u32 size,const void __user * unsafe_ptr__ign,struct task_struct * tsk)3567 __bpf_kfunc int bpf_copy_from_user_task_str_dynptr(struct bpf_dynptr *dptr, u32 off,
3568 						   u32 size, const void __user *unsafe_ptr__ign,
3569 						   struct task_struct *tsk)
3570 {
3571 	return __bpf_dynptr_copy_str(dptr, off, size, (const void *)unsafe_ptr__ign,
3572 				     copy_user_str_sleepable, tsk);
3573 }
3574 
3575 __bpf_kfunc_end_defs();
3576